diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..8d05c0dc --- /dev/null +++ b/.dockerignore @@ -0,0 +1,67 @@ +# Dependencies — installed fresh inside the image +node_modules/ + +# Git history +.git/ +.github/ + +# TypeScript build output — rebuilt inside the image +dist/ +build/ +*.tsbuildinfo + +# Test artifacts +coverage/ +tests/ +benchmarks/ + +# Logs and runtime data +logs/ +*.log + +# SQLite databases — created at runtime via volume mounts +*.db +*.db-wal +*.db-shm + +# WhatsApp session data — mounted as a volume +.wwebjs_auth/ +.wwebjs_cache/ + +# OpenBridge runtime state +.openbridge/ + +# Config with secrets — passed via ENV vars in Docker +config.json +config.local.json +config.production.json + +# Environment files +.env +.env.local +.env.*.local + +# Development files +.vscode/ +.idea/ +*.swp +*.swo + +# OS files +.DS_Store +Thumbs.db + +# npm pack artifacts +*.tgz + +# Smoke test directory +ob-smoke-test/ + +# Test workspace directories +test-workspace-*/ + +# Docs (not needed at runtime) +docs/ + +# Scripts (not needed at runtime) +scripts/ diff --git a/.env.example b/.env.example index e8dfc040..06a592cf 100644 --- a/.env.example +++ b/.env.example @@ -12,3 +12,10 @@ CONFIG_PATH=./config.json # Claude Code provider options # CLAUDE_CODE_WORKSPACE=~/projects/my-project + +# OpenAI API key — optional, shared with Codex +# Used for two features: +# 1. Voice message transcription via OpenAI Whisper API ($0.006/min, zero local setup) +# 2. Codex AI provider (if you use Codex as your AI tool instead of Claude Code) +# If unset, voice transcription falls back to local whisper CLI, then a placeholder message. +# OPENAI_API_KEY=sk-... diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..4efb832a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,33 @@ +version: 2 +updates: + - package-ecosystem: 'npm' + directory: '/' + target-branch: 'develop' + schedule: + interval: 'monthly' + open-pull-requests-limit: 5 + reviewers: + - 'medomar' + groups: + minor-and-patch: + update-types: + - 'minor' + - 'patch' + major: + update-types: + - 'major' + ignore: + # @types/node major must match our engines.node (>=22) — don't auto-bump + - dependency-name: '@types/node' + update-types: ['version-update:semver-major'] + # @vitest/coverage-v8 must stay in sync with vitest — upgrade together manually + - dependency-name: '@vitest/coverage-v8' + update-types: ['version-update:semver-major'] + - dependency-name: 'vitest' + update-types: ['version-update:semver-major'] + # Zod 4 requires dedicated migration — don't auto-bump + - dependency-name: 'zod' + update-types: ['version-update:semver-major'] + # ESLint 10 needs typescript-eslint compatibility — don't auto-bump + - dependency-name: 'eslint' + update-types: ['version-update:semver-major'] diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..1e5b6154 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,174 @@ +name: Release + +on: + push: + tags: + - 'v*' + +permissions: + contents: write + id-token: write + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + cache: 'npm' + + - run: npm ci + + - name: Run ESLint + run: npm run lint + + - name: Check formatting + run: npm run format:check + + typecheck: + name: Type Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + cache: 'npm' + + - run: npm ci + + - name: TypeScript type check + run: npm run typecheck + + test: + name: Test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + cache: 'npm' + + - run: npm ci + + - name: Run tests + run: npm run test + + build: + name: Build + runs-on: ubuntu-latest + needs: [lint, typecheck, test] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + cache: 'npm' + + - run: npm ci + + - name: Build + run: npm run build + + - name: Verify dist output + run: ls -la dist/ + + - name: Upload dist artifact + uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + retention-days: 1 + + publish: + name: Publish to npm + GitHub Release + runs-on: ubuntu-latest + needs: [build] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + cache: 'npm' + registry-url: 'https://registry.npmjs.org' + + - run: npm ci + + - name: Download dist artifact + uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + + - name: Publish to npm + run: npm publish --provenance --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + - name: Extract version from tag + id: version + run: echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" + + - name: Generate release notes + id: changelog + run: | + version="${{ steps.version.outputs.version }}" + tag="${{ github.ref_name }}" + + # Try CHANGELOG.md first (hand-curated notes take priority) + notes=$(awk "/^## \[$version\]/{found=1; next} found && /^## \[/{exit} found{print}" CHANGELOG.md) + + if [ -z "$notes" ] || echo "$notes" | grep -q "No unreleased changes"; then + # Fall back to auto-generated notes from conventional commits + echo "No CHANGELOG.md entry found for $version — generating from commits" >&2 + # Find the previous tag + prev_tag=$(git tag --sort=-v:refname | grep -v "^${tag}$" | head -1 || echo "") + if [ -n "$prev_tag" ]; then + range="${prev_tag}..${tag}" + else + range="${tag}" + fi + + notes="## What's Changed"$'\n\n' + + # Group by conventional commit type + feats=$(git log "$range" --pretty=format:"%s" --no-merges | grep "^feat" | sed 's/^feat(\([^)]*\)): /- **\1**: /' | sed 's/^feat: /- /' || true) + fixes=$(git log "$range" --pretty=format:"%s" --no-merges | grep "^fix" | sed 's/^fix(\([^)]*\)): /- **\1**: /' | sed 's/^fix: /- /' || true) + others=$(git log "$range" --pretty=format:"%s" --no-merges | grep -v "^feat\|^fix\|^Merge" | sed 's/^[a-z]*(\([^)]*\)): /- **\1**: /' | sed 's/^[a-z]*: /- /' || true) + + [ -n "$feats" ] && notes+="### Added"$'\n'"${feats}"$'\n\n' + [ -n "$fixes" ] && notes+="### Fixed"$'\n'"${fixes}"$'\n\n' + [ -n "$others" ] && notes+="### Changed"$'\n'"${others}"$'\n\n' + + notes+=$'\n'"**Full Changelog**: https://github.com/${{ github.repository }}/compare/${prev_tag}...${tag}" + fi + + # Write to file to handle multiline safely + echo "$notes" > release-notes.txt + + - name: Create GitHub Release + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const notes = fs.readFileSync('release-notes.txt', 'utf8').trim(); + const tag = context.ref.replace('refs/tags/', ''); + await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: tag, + name: tag, + body: notes || `Release ${tag}`, + draft: false, + prerelease: tag.includes('-'), + generate_release_notes: notes === '', + }); diff --git a/.gitignore b/.gitignore index 38506760..439d84d8 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,9 @@ npm-debug.log* .wwebjs_auth/ .wwebjs_cache/ +# npm pack artifacts +*.tgz + # Runtime data pids/ *.pid @@ -57,5 +60,26 @@ config.json config.local.json config.production.json +# OpenBridge runtime state (generated when running against itself) +.openbridge/ + # Script runtime state docs/audit/.current_task + +# Test workspaces (created by integration tests) +test-workspace-*/ + +# Smoke test directory (created by OB-621 fresh install verification) +ob-smoke-test/ + +# SQLite database files +*.db +*.db-wal +*.db-shm + +# Temporary/scratch files +tmp_* +*.traineddata + +# Demo/experiment folders (not part of open-source distribution) +demos/ diff --git a/CHANGELOG.md b/CHANGELOG.md index b3826462..e2152a63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,468 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +_No pending changes._ + +## [0.0.15] — 2026-03-09 + +### Added + +#### Prompt Budget & Assembly (Phase 105) + +- **PromptAssembler** — priority-ranked section assembly with per-section `maxChars` caps and provider-aware total budget +- **`getPromptBudget()`** on CLIAdapter — each adapter declares model-aware prompt/systemPrompt limits +- **Budget-aware Master prompts** — `buildMasterSpawnOptions()` uses PromptAssembler instead of raw concatenation +- **Adapter-specific limits** — ClaudeAdapter (180K system), CodexAdapter (merged budget), AiderAdapter (conservative) + +#### Classification Fixes (Phase 107) + +- **Attachment-aware classification** — messages with file attachments auto-escalate from `quick-answer` to `tool-use` +- **File-reference keywords** — "xl", "pdf", "csv", "spreadsheet", "attachment" trigger `tool-use` classification + +#### Worker & Exploration Cleanup (Phase 108) + +- **Pending-worker watchdog** — workers stuck in `pending` for >5 min auto-cancelled +- **Stale exploration cleanup** — old `exploration_progress` rows deleted on new exploration start +- **Post-exploration memory.md** — `memory.md` seeded immediately after exploration completes + +#### Monorepo Awareness (Phase 109) + +- **`detectMonorepoPattern()`** — scans for multiple `package.json`/`.git`/`pom.xml`/`go.mod` at depth 1-2 +- **Per-sub-project exploration** — each sub-project gets independent classification and directory dive +- **Monorepo workspace map** — Phase 4 assembly produces `{ type: "monorepo", subProjects: [...] }` structure + +#### God-Class Refactoring (Phase 110) + +- **8 new focused modules** extracted from 3 god-class files (16,291 LOC → manageable modules): + - `classification-engine.ts` (942 LOC) — from master-manager.ts + - `exploration-manager.ts` (1539 LOC) — from master-manager.ts + - `worker-orchestrator.ts` (2065 LOC) — from master-manager.ts + - `prompt-context-builder.ts` (560 LOC) — from master-manager.ts + - `command-handlers.ts` (2936 LOC) — from router.ts + - `output-marker-processor.ts` — from router.ts + - `error-classifier.ts` — from agent-runner.ts + - `cost-manager.ts` — from agent-runner.ts + +#### Memory Leak Fixes (Phase 113) + +- **Queue recursion → while loop** — prevents stack overflow with large queues +- **Rate limiter cleanup** — periodic 5-min sweep removes stale sender entries +- **Classification cache eviction** — LRU eviction at 10,000 entries (oldest 20% removed) +- **Connector session cleanup** — WebChat, Discord, WhatsApp Maps/Sets with TTL-based purge + +### Fixed + +- Process kill race condition — `killed` flag prevents double SIGKILL (OB-F162) +- Session checkpoint/resume race — `finally` block ensures `resumeSession()` always called (OB-F163) +- Memory init null pointer — eviction interval guarded by `if (this.memory)` (OB-F164) +- Config watcher unhandled rejection — `.catch()` on fire-and-forget reload (OB-F167) +- Spawn confirmation timer leak — clear existing timer before overwriting (OB-F168) +- Batch timers not cleaned on shutdown — iterate and clear all handles (OB-F170) +- Worker abort handle leak — `try/finally` ensures cleanup on pre-spawn failure (OB-F171) +- Pending messages dropped on drain failure — try-catch per message with user notification (OB-F172) +- Cancellation notifications re-injected — clear array after injection (OB-F173) +- DotFolderManager silent I/O errors — `logger.warn()` in all catch blocks (OB-F174) +- JSON.parse crashes on corrupt data — try-catch in observation-store, access-store, memory index (OB-F175) +- Connector Maps/Sets unbounded growth — periodic cleanup intervals with TTL (OB-F176) +- WhatsApp reconnect timer not cleared on shutdown (OB-F177) +- Self-improvement prompt growth unbounded — 45K char cap on `createPromptVersion()` (OB-F149) +- Workspace map duplicated in exploration prompts — `skipWorkspaceContext` flag (OB-F150) +- Prompt version seed duplicates — existence check + dedup migration (OB-F151) +- Quick-answer timeout too tight — 60s CLI startup budget floor (OB-F144) +- Idle self-improvement runs every 5 min — exponential backoff up to 2h (OB-F145) +- Phase 4 Assembly fails on large workspaces — budget-capped data + markdown fallback (OB-F146) +- Exit code 143 (SIGTERM) now classified as transient (retryable) instead of permanent +- 29 test regressions across 12 files restored to green (Phase 115) + +## [0.0.14] — 2026-03-07 + +### Added + +#### Skill Pack System Extensions (Phase 98) + +- **SkillPack type + SKILLPACK.md format** — reusable domain-specific instruction sets for workers +- **5 built-in skill packs** — `security-audit`, `code-review`, `test-writer`, `data-analysis`, `documentation` +- **Master auto-selects skill packs** — task type determines which pack injects into worker system prompt +- **User-defined skill packs** — `.openbridge/skill-packs/` overrides built-in defaults +- **`/skill-packs` chat + CLI command** — list available packs with descriptions + +#### Design & Creative Output (Phase 100) + +- **6 creative skill packs** — `diagram-maker`, `chart-generator`, `web-designer`, `slide-designer`, `generative-art`, `brand-assets` +- **HTML-to-image pipeline** — Puppeteer rendering for sending visual outputs via messaging +- **SVG + Mermaid rendering** — diagram-to-image conversion via mermaid-cli +- **Creative task auto-detection** — intent classification extended with design/visual/chart intents +- **Image delivery** — rendered PNG/SVG sent as media via WhatsApp/Telegram + +#### Agent Orchestration Patterns (Phase 101) + +- **PlanningGate** — Master enters read-only analysis phase before code-edit execution phase +- **WorkerSwarm** — named worker groups (research, implement, review, test) with handoff protocol +- **Test protection** — workers cannot modify test files unless explicitly authorized by Master +- **Iteration cap** — max 3 lint/test fix attempts before escalating to Master (configurable) +- **Parallel spawning within swarms** — independent workers in same swarm run concurrently + +## [0.0.13] — 2026-03-07 + +### Added + +#### Structured Observations & Worker Summaries (Phase 93) + +- **Observations table + FTS5** — typed facts, concepts, files_read, files_modified from every worker +- **Observation extractor** — auto-parses worker output into structured observations +- **WorkerSummary schema** — `request/investigated/completed/learned/next_steps` per worker +- **Content-hash deduplication** — SHA-256 dedup for workspace chunks with 30s write window +- **Master context injection** — `next_steps` from recent workers fed into system prompt + +#### Session Compaction & Token Economics (Phase 95) + +- **SessionCompactor** — auto-summarizes when Master session exceeds 80% of max-turns +- **Identifier preservation** — file paths, function names, finding IDs survive compaction +- **Token economics tracking** — discovery vs retrieval token costs, ROI calculation +- **`/stats` chat + CLI command** — exploration ROI visibility + +#### Channel Role Management UX Fix (Phase 96d) + +- **`auth.defaultRole` config** — whitelisted users default to `owner` (was `viewer`) +- **`auth.channelRoles`** — per-channel role overrides +- **`/whoami` + `/role` commands** — in-chat role visibility and management +- **Softened classification** — `chat` action default for conversational messages, allowed for all roles +- **Improved denial messages** — show role, classified action, and allowed actions + +#### Document Generation Skills (Phase 99) + +- **4 document skill packs** — `document-writer` (DOCX), `presentation-maker` (PPTX), `spreadsheet-builder` (XLSX), `report-generator` (PDF/HTML) +- **Skill pack loader** — discovers built-in + `.openbridge/skill-packs/` custom packs +- **Document task auto-detection** — intent classification extended with document/report/presentation intents +- **File attachment delivery** — generated documents sent via WhatsApp/Telegram/WebChat + +#### Vector Search & Hybrid Retrieval (Phase 94) + +- **sqlite-vec integration** — vector similarity search with cosine distance +- **Hybrid scoring** — 0.4 vector + 0.4 FTS5 + 0.2 temporal decay +- **MMR diversity** — prevents result clustering from same file +- **Progressive disclosure** — `searchIndex()` returns compact results, `getDetails()` loads full content +- **Graceful fallback** — FTS5-only when no embedding provider configured + +#### Developer Experience (Phases 96a–96c) + +- **`openbridge doctor`** — self-diagnostic command checking Node.js, AI tools, config, DB, channels +- **Pairing-based auth** — 6-digit codes for self-service user onboarding via DM +- **Skills directory** — `SkillManager` discovers SKILL.md files, Master auto-creates skills from patterns +- **`/doctor`, `/approve`, `/skills` chat commands** + +### Fixed + +- 7 data integrity findings (OB-F89–F95) — audit log, QA cache, sessions, turns, prompts, sub-masters, memory.md + +## [0.0.12] — 2026-03-05 + +### Added + +#### Deep Mode — 5-Phase Analysis (Phase Deep) + +- **DeepModeManager** — investigate → report → plan → execute → verify state machine +- **Per-phase model selection** — each phase picks the best model for its task type +- **Interactive commands** — `/deep`, phase navigation, task model overrides via chat +- **Plan task parsing** — dependency-ordered batching with BFS topological sort +- **Session persistence** — completed sessions saved to `.openbridge/deep-mode/` + +#### Output Delivery Pipeline (Phases 82–84) + +- **Tunnel integration** — auto-detects cloudflared/ngrok, generates public URLs +- **Ephemeral app server** — scaffold detection, port allocation, idle timeout +- **Interaction relay** — WebSocket bidirectional app↔Master communication + +#### WebChat Modernization (Phases 88–92) + +- **Extracted frontend** — modular JS/CSS, dark mode, markdown, syntax highlighting +- **Authentication** — token/password auth, sessions, rate limiting +- **Mobile PWA** — LAN/tunnel access, QR codes, responsive, service worker +- **Conversation history** — sidebar, file upload, voice input, autocomplete +- **Settings panel** — gear panel, stepper, phase cards + +#### Runtime Features (Phases 97–104) + +- **Permission escalation** — `/allow`, `/deny`, persistent tool grants +- **Batch task continuation** — self-messaging loop with safety rails (iteration + cost + time limits) +- **Docker sandbox** — container isolation, resource limits, cleanup +- **Worker watchdog** — orphan detection, state audit, `/workers` command +- **Cost controls** — per-profile cost caps, partial status, adaptive maxTurns + +### Fixed + +- Codex streaming output parsing (RWT phase) +- RAG zero-result fallback (RWT phase) +- Classifier strategic keywords and menu-selection handling (Phase 100) +- Batch timer cleanup and .catch handlers (Phase 101) +- Whitelist diagnostics on startup (Phase 103) +- 14 stale test mocks updated (Phase 104) + +## [0.0.11] — 2026-03-03 + +### Added + +- **Master output sharing** — `[SHARE:*]` markers, file-server URL delivery, routing guidelines (Phase 81) +- **User consent** — risk classification, confirmation prompts for high-risk operations, cost estimation (Phase 86) + +## [0.0.10] — 2026-03-02 + +### Added + +- **RAG knowledge retrieval** — FTS5 queries, workspace map context, dir-dive results, Q&A cache infrastructure (Phases 74–77) +- **Environment variable protection** — deny-list, allow-list, per-adapter sanitization strips CLAUDECODE/CLAUDE*CODE*_/CLAUDE*AGENT_SDK*_ vars (Phase 85) + +## [0.0.9] — 2026-03-02 + +### Fixed + +- **Classification fixes** — strategic keywords, SPAWN response parsing, text-generation keyword classifier (Phase 78a) +- **Code-audit profile** — new read-only + analysis tool profile for security audits (Phase 78b) +- **Exploration bugs** — 8 bugs fixed: JSON fallbacks, chunk dedup, stale detection (Phase 79) +- **.openbridge data cleanup** — orphaned entries, corrupted state files (Phase 80) + +## [0.0.8] — 2026-03-02 + +### Added + +#### Voice Transcription with API Fallback (Phase 70) + +- **Voice message support** — WhatsApp and Telegram voice messages automatically transcribed +- **API fallback pipeline** — tries local `ffmpeg` first, falls back to OpenAI Whisper API +- **Graceful degradation** — clear error message when neither transcription method is available + +### Scaffolded (not yet validated) + +- **Enhanced CLI Setup Wizard (Phase 71)** — guided `npx openbridge init` flow (needs testing) + +### Changed + +- grammy 1.40 → 1.41 (webhook JSON parse fix) +- eslint 9.39.2 → 9.39.3 (TypeScript compatibility) +- lint-staged 16.2 → 16.3 (switched from nano-spawn to tinyexec) +- @types/node patch update + +## [0.0.7] — 2026-02-28 + +### Fixed + +- **Telegram message splitting** — long responses now split at paragraph boundaries instead of mid-sentence; each chunk respects Telegram's 4096-char limit +- **Discord message splitting** — same splitting logic applied; respects Discord's 2000-char limit +- **Live context fixes** — worker progress events now correctly display elapsed time and turn counts across all connectors + +## [0.0.6] — 2026-02-27 + +### Fixed + +- **WhatsApp media handling** — voice messages, images, and documents now properly downloaded and forwarded to Master AI with MIME type metadata +- **Telegram media handling** — photos, documents, and voice messages extracted via grammY file API + +## [0.0.5] — 2026-02-27 + +### Fixed + +- **FTS5 syntax errors** — `searchConversations()` now calls `sanitizeFts5Query()` before MATCH; queries with `'`, `"`, `(`, `)`, `*`, `AND`, `OR`, `NOT` no longer crash (OB-F38) +- **memory.md never updates** — `triggerMemoryUpdate()` now injects last 20 messages from SQLite so the stateless AI has context to write meaningful notes (OB-F39) +- **Memory-update prompt hardcodes tool name** — uses generic language instead of `Use the Write tool` — works with both Claude and Codex (OB-F39) +- **`getRecentMessages()` missing** — added to `conversation-store.ts` and exposed on `MemoryManager` facade (OB-F39) +- **Ungraceful shutdown** — `shutdown()` now prints status, wraps `bridge.stop()` in 10s timeout, exits cleanly (OB-F40) +- **Session state lost on rapid shutdown** — `saveMasterSessionToStore()` (fast, <100ms) runs before `triggerMemoryUpdate()` (slow, 10–30s) (OB-F40) + +## [0.0.4] — 2026-02-26 + +### Added + +#### Codex Provider + Adapter Fixes (Phases 57–59) + +- **`CodexProvider`** — implements `AIProvider` interface using `AgentRunner` + `CodexAdapter` +- **`CodexConfig` schema** — Zod schema with `workspacePath`, `timeout`, `model?`, `sandbox?` +- **`CodexSessionManager`** — session state for multi-turn Codex conversations +- **`--skip-git-repo-check`** always present in `CodexAdapter` — fixes non-git directory failures +- **Default sandbox `read-only`** — empty `allowedTools` → `--sandbox read-only` +- **`OPENAI_API_KEY` validation** before spawn — clear error instead of confusing timeout +- **`--json` flag** for structured JSONL output parsing +- **`-o` output file** for reliable result capture with temp file cleanup + +#### MCP Integration — Model Context Protocol (Phases 60–62, scaffolded) + +- **`MCPServerSchema` + `MCPConfigSchema`** in config types +- **Per-worker MCP isolation** — temp JSON configs with only requested servers, `--strict-mcp-config` +- **Global MCP config writer** — validates and writes `.openbridge/mcp-config.json` on startup +- **MCP server health checks** — verifies server commands exist on PATH +- **MCP step in CLI wizard** — optional MCP server configuration during `npx openbridge init` +- **Master MCP awareness** — system prompt includes available MCP servers; Master decides assignment +- **Status: scaffolded, not yet fully validated** + +## [0.0.3] — 2026-02-26 + +### Added + +#### Exploration Overhaul (Phase 50) + +- **`explore` command** — users can trigger workspace re-exploration from any messaging channel. Syntax: `explore` (quick refresh), `explore full` (5-phase re-exploration), `explore status` (show progress) +- **Large directory splitting (OB-F26)** — directories exceeding 25 files are automatically split into subdirectories before Phase 3 dives +- **Per-directory timeout scaling** — dive timeout scales with file count: `max(180s, min(600s, fileCount * 4s))` + +#### Prompt Library (Phase 51) + +- **Prompt manifest** — `.openbridge/prompts/manifest.json` with usage tracking, success rates, versioning +- **7 prompt management methods** on `DotFolderManager` — read/write manifests, templates, usage recording, low-performer detection + +#### Conversation Continuity — memory.md (Phase 52) + +- **`memory.md` pattern** — Master reads a curated 200-line knowledge file on every session start, updates on session end +- **Cross-session context** — FTS5 search fallback when memory doesn't cover the current topic + +#### /history Command (Phase 53) + +- **`/history`** — lists last 10 sessions with title, date, message count +- **`/history search `** — FTS5 search across past conversations +- **`/history `** — full transcript for a session +- **REST endpoints** — `/api/sessions` and `/api/sessions/:id` for WebChat + +#### Schema Versioning (Phase 54) + +- **`schema_versions` table** — numbered migrations with transaction safety and idempotency + +#### Worker Streaming & Checkpointing (Phase 55) + +- **Worker progress streaming** — `execOnceStreaming()` streams stdout chunks from active workers +- **Session checkpointing** — `checkpointSession()` / `resumeSession()` for crash recovery +- **Priority queue integration** — urgent messages trigger checkpoint-handle-resume cycle + +### Fixed + +- `exploration_progress` always empty — `explorationId` now wired through all 5 phases (OB-F23) +- Workers hitting max-turns silently succeeded — now detected, logged, and retried (OB-F24) +- Worker failures not retried — default retries changed 0 → 2, with error classification (OB-F25) +- Prompt library methods unimplemented — fixed 39 test failures, 20 TS errors, 264 lint errors (OB-F32–F34) +- `AuditLogger` missing JSONL output — `write()` now appends to disk (OB-F27) +- No DB schema versioning — `schema_versions` table added (OB-F28) +- Session checkpointing not wired (OB-F31) + +## [0.0.2] — 2026-02-25 + +### Added + +#### Memory System — SQLite + FTS5 + +- **`src/memory/` module** — full SQLite + FTS5 layer with WAL mode +- **Workspace chunks** — ~500 token chunks with FTS5 full-text search +- **Conversation store** — message recording with FTS5 and 30/90-day eviction +- **Task store** — execution records + model performance learnings +- **Worker briefing** — context packages for each worker before spawn +- **Activity store** — real-time agent dashboard data (PID, status, turns) +- **Access control store** — role-based access (owner/admin/developer/viewer) + +#### Worker Resilience (Phases 45–47) + +- **Max-turns exhaustion detection** — stdout scan + `turnsExhausted` flag +- **Adaptive turn budget** — `baselineTurns + ⌈promptLength / 1000⌉` capped at 50 +- **Auto-retry** on turns exhaustion with 1.5x budget +- **Error classification** — `rate-limit | auth | timeout | crash | context-overflow | unknown` +- **Master-driven re-delegation** — persistent failures reported back to Master + +#### Worker Control (Phase 48) + +- **`stop` / `stop all` / `stop `** commands with confirmation flow +- **Real PID capture** via `spawnWithHandle()` +- **Cross-channel broadcast** on worker cancellation + +#### Responsive Master (Phases 49–50) + +- **Queue depth acknowledgment** — "You're #N in queue (~Xs)" +- **Message priority classification** — quick-answer/tool-use/complex-task +- **Fast-path responder** — lightweight `claude --print` for quick answers during Master processing +- **Enhanced `status` command** — current task, queue depth, worker count, exploration progress + +#### Content Publishing & Sharing (scaffolded) + +- **`[SHARE:channel]` markers** — parsing logic for file delivery via WhatsApp/WebChat/email (scaffolded — Master doesn't use these yet, see OB-F68) +- **File server** — local HTTP server for generated content (scaffolded) +- **Email sender** — SMTP integration (scaffolded) +- **GitHub publisher** — push to `gh-pages` (scaffolded) + +#### Access Control + +- **Role-based access** — per-user roles enforced in auth layer +- **`npx openbridge access`** — CLI for managing user access +- **`stop` command restricted** to owner/admin roles + +## [0.0.1] — 2026-02-23 + +### Added + +#### Core Bridge + +- **AI Tool Discovery** — auto-detects AI CLI tools (Claude Code, Codex, Aider, Cursor, Cody) and VS Code extensions +- **Master AI Manager** — autonomous agent lifecycle (idle → exploring → ready) +- **Incremental 5-pass exploration** — structure scan, classification, directory dives, assembly, finalization +- **`.openbridge/` folder** — the AI's brain inside the target project +- **Session continuity** — multi-turn Master conversations via `--session-id`/`--resume` +- **Multi-AI delegation** — Master assigns subtasks via SPAWN markers +- **V2 config format** — simplified to 3 fields; V0 auto-detected for backward compatibility +- **Config watcher** — hot-reload without restart + +#### Connectors (5 total) + +- **Console** — reference implementation for rapid testing +- **WebChat** — browser UI on `localhost:3000` with Markdown rendering +- **WhatsApp** — via whatsapp-web.js with session persistence and auto-reconnect +- **Telegram** — via grammY; DM and group @mention support +- **Discord** — via discord.js v14; DM and guild channel support + +#### Agent Runner + +- **`AgentRunner` class** — unified CLI executor with `--allowedTools`, `--max-turns`, `--model`, retries, disk logging +- **Streaming support** — `stream()` yields stdout chunks in real time +- **Model fallback chain** — opus → sonnet → haiku on rate-limit + +#### Tool Profiles + +- **Built-in profiles** — `read-only`, `code-edit`, `full-access`, `master` +- **Custom profile registry** — stored in `.openbridge/` +- **Model selector** — recommends model based on profile and task keywords + +#### Self-Governing Master AI + +- **Master system prompt** — generated from workspace context, self-editable +- **Task decomposition** — `[SPAWN:profile]{JSON}[/SPAWN]` markers trigger workers +- **Worker result injection** — structured formatting fed back to Master session + +#### Worker Orchestration + +- **Worker registry** — tracks all workers with configurable concurrency (default 5) +- **Parallel spawning** — up to concurrency limit +- **Depth limiting** — workers cannot spawn other workers + +#### Production Readiness + +- **npm packaging** — `files` field, `exports` map +- **Global error handlers** — `unhandledRejection`, `uncaughtException`, `SIGHUP` +- **Release workflow** — lint → typecheck → test → build → npm publish → GitHub Release +- **Dependabot** — weekly npm dependency updates +- **1,218 tests passing** across 60 test files + +### Changed + +- Documentation rewrite for autonomous AI vision +- Router with Master AI routing path +- CLI executor generalized from Claude-only to any AI tool + +### Removed + +- Old knowledge layer (archived to `src/_archived/`) +- `--dangerously-skip-permissions` dead code + +### Fixed + +- `tsx watch` killing process on file changes +- Master session ID using invalid UUID format +- `maxTurns: 3` blocking all non-Q&A tasks +- `pino` `MaxListenersExceededWarning` + +## [0.1.0] — 2026-02-19 + ### Added - Initial project scaffolding diff --git a/CLAUDE.md b/CLAUDE.md index 7ed56f61..c16a94f6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,7 +15,7 @@ npm run format:check # Check Prettier formatting ## What is OpenBridge? -An open-source **autonomous AI bridge** — connects messaging channels to AI agents that **explore your workspace, discover your project, and execute tasks**. The AI auto-discovers tools on your machine (Claude Code, Codex, Aider), picks the best one as Master, and silently learns your project on startup. You interact through messaging (WhatsApp). Zero API keys. Zero extra cost. +An open-source **autonomous AI bridge** — connects messaging channels to AI agents that **explore your workspace, discover your project, and execute tasks**. The AI auto-discovers tools on your machine (Claude Code, Codex, Aider), picks the best one as Master, and silently learns your project on startup through 5 incremental passes (never times out). Session continuity enables multi-turn conversations. You interact through messaging (WhatsApp or Console). Zero API keys. Zero extra cost. ## How to Use OpenBridge @@ -52,8 +52,13 @@ On startup, OpenBridge: 1. Scans your machine for AI tools (`which claude`, `which codex`, etc.) 2. Picks the most capable one as Master -3. Master silently explores the target workspace -4. Creates `.openbridge/` folder with a git repo to track everything +3. Master silently explores the target workspace in 5 incremental passes: + - Pass 1: Structure scan (list top-level files/dirs, count files per dir) + - Pass 2: Classification (detect project type, frameworks, commands) + - Pass 3: Directory dives (explore each significant directory) + - Pass 4: Assembly (merge results into workspace-map.json) + - Pass 5: Finalization (create agents.json, git commit) +4. Creates `.openbridge/` folder with a git repo and exploration/ subfolder to track everything 5. Waits for your messages ### Step 3: Send a command @@ -71,12 +76,12 @@ The bridge will: 3. Route to the Master AI (which already explored your workspace) 4. Send the AI's response back to your WhatsApp -## Architecture (4 layers) +## Architecture (6 layers) ``` ┌──────────────────────────────────────────────────────────────────┐ │ CHANNELS │ -│ WhatsApp · Console · Telegram (planned) · Discord (planned) │ +│ WhatsApp · Console · WebChat · Telegram · Discord │ │ Connectors translate between messaging APIs and OpenBridge │ └──────────────────────┬────────────────────────────────────────────┘ │ @@ -96,49 +101,111 @@ The bridge will: │ ▼ ┌──────────────────────────────────────────────────────────────────┐ +│ AGENT RUNNER │ +│ Unified CLI executor — --allowedTools, --max-turns, --model │ +│ Retries, model fallback, disk logging, tool profiles │ +└──────────────────────┬────────────────────────────────────────────┘ + │ + ▼ +┌──────────────────────────────────────────────────────────────────┐ │ MASTER AI │ -│ Master Manager · .openbridge/ Folder · Delegation Coordinator │ -│ Autonomous exploration, task execution, multi-AI delegation │ +│ Master Manager · .openbridge/ Folder · Worker Orchestration │ +│ Self-governing exploration, worker spawning, self-improvement │ +└──────────────────────┬────────────────────────────────────────────┘ + │ + ▼ +┌──────────────────────────────────────────────────────────────────┐ +│ MCP (Model Context Protocol) │ +│ Per-worker temp config · --mcp-config · --strict-mcp-config │ +│ Claude workers access external services (Gmail, Canva, Slack) │ +│ Master AI decides which workers need which MCP servers │ └──────────────────────────────────────────────────────────────────┘ ``` ### Key Files -| File | Purpose | -| --------------------------------------------------- | ---------------------------------------------------------- | -| `config.json` | Your runtime config (gitignored) | -| `src/index.ts` | Entry point — V0 + V2 startup flows | -| `src/core/bridge.ts` | Orchestrator — wires connectors, auth, queue, Master AI | -| `src/core/router.ts` | Routes messages: connector → Master AI → connector | -| `src/core/auth.ts` | Phone whitelist + prefix + command allow/deny filters | -| `src/core/queue.ts` | Per-user sequential processing, retry, DLQ | -| `src/core/registry.ts` | Plugin registry — auto-discovers connectors | -| `src/core/config.ts` | Config loader — V2 detection + V0 fallback (Zod validated) | -| `src/core/config-watcher.ts` | Config hot-reload (file watcher) | -| `src/core/health.ts` | Health check HTTP endpoint | -| `src/core/metrics.ts` | Message count, latency, error rate metrics | -| `src/core/audit-logger.ts` | Structured audit trail of all message events | -| `src/core/rate-limiter.ts` | Per-user rate limiting | -| `src/core/logger.ts` | Pino logger | -| `src/types/connector.ts` | Interface every connector must implement | -| `src/types/provider.ts` | Interface every AI provider must implement | -| `src/types/message.ts` | InboundMessage / OutboundMessage types | -| `src/types/config.ts` | Zod config schemas (V0 + V2) | -| `src/types/discovery.ts` | DiscoveredTool, ScanResult Zod schemas | -| `src/types/master.ts` | MasterState, ExplorationSummary Zod schemas | -| `src/types/common.ts` | Shared types | -| `src/discovery/tool-scanner.ts` | CLI tool detection (`which claude`, `which codex`, etc.) | -| `src/discovery/vscode-scanner.ts` | VS Code AI extension detection | -| `src/discovery/index.ts` | `scanForAITools()` export — combines CLI + VS Code scans | -| `src/master/master-manager.ts` | Master AI lifecycle (idle → exploring → ready) + messaging | -| `src/master/dotfolder-manager.ts` | `.openbridge/` folder CRUD + git operations | -| `src/master/exploration-prompt.ts` | System prompt for autonomous workspace exploration | -| `src/master/delegation.ts` | Multi-AI task delegation coordinator | -| `src/connectors/whatsapp/` | WhatsApp connector — auto-reconnect, sessions, typing | -| `src/connectors/console/` | Console connector (reference implementation) | -| `src/providers/claude-code/` | Claude Code CLI provider — streaming, sessions, errors | -| `src/providers/claude-code/claude-code-executor.ts` | Generalized CLI executor (any AI tool) | -| `src/cli/init.ts` | CLI config generator — 3 questions for V2 config | +| File | Purpose | +| ---------------------------------------- | -------------------------------------------------------------------------------------------------- | +| `config.json` | Your runtime config (gitignored) | +| `src/index.ts` | Entry point — V0 + V2 startup flows | +| `src/core/bridge.ts` | Orchestrator — wires connectors, auth, queue, Master AI | +| `src/core/router.ts` | Message routing + `/history` + stop/status/confirm commands (2069 LOC) | +| `src/core/auth.ts` | Phone whitelist + prefix + command allow/deny filters | +| `src/core/queue.ts` | Per-user sequential processing, retry, DLQ | +| `src/core/registry.ts` | Plugin registry — auto-discovers connectors | +| `src/core/config.ts` | Config loader — V2 detection + V0 fallback (Zod validated) | +| `src/core/config-watcher.ts` | Config hot-reload (file watcher) | +| `src/core/health.ts` | Health check HTTP endpoint | +| `src/core/metrics.ts` | Message count, latency, error rate metrics | +| `src/core/audit-logger.ts` | Structured audit trail of all message events | +| `src/core/rate-limiter.ts` | Per-user rate limiting | +| `src/core/logger.ts` | Pino logger | +| `src/types/connector.ts` | Interface every connector must implement | +| `src/types/provider.ts` | Interface every AI provider must implement | +| `src/types/message.ts` | InboundMessage / OutboundMessage types | +| `src/types/config.ts` | Zod config schemas (V0 + V2) — includes `MCPServerSchema`, `MCPConfigSchema`, `getMcpConfigPath()` | +| `src/types/discovery.ts` | DiscoveredTool, ScanResult Zod schemas | +| `src/types/master.ts` | MasterState, ExplorationSummary Zod schemas | +| `src/types/common.ts` | Shared types | +| `src/discovery/tool-scanner.ts` | CLI tool detection (`which claude`, `which codex`, etc.) | +| `src/discovery/vscode-scanner.ts` | VS Code AI extension detection | +| `src/discovery/index.ts` | `scanForAITools()` export — combines CLI + VS Code scans | +| `src/core/agent-runner.ts` | Unified CLI executor (--allowedTools, --max-turns, --model, retries, error classification) | +| `src/core/command-handlers.ts` | 40+ `/command` handler functions extracted from router.ts (2936 LOC) | +| `src/core/output-marker-processor.ts` | SHARE/VOICE/APP output marker detection and routing extracted from router.ts | +| `src/core/error-classifier.ts` | Error classification and retry detection extracted from agent-runner.ts | +| `src/core/cost-manager.ts` | Cost tracking and budget management extracted from agent-runner.ts | +| `src/core/model-selector.ts` | Model recommendation per task type | +| `src/core/model-registry.ts` | Provider-agnostic model tier resolution (fast/balanced/powerful → concrete model IDs) | +| `src/core/cli-adapter.ts` | CLIAdapter interface — translates SpawnOptions to tool-specific binary + args + env | +| `src/core/adapter-registry.ts` | Maps discovered tool names to CLIAdapter instances (lazy-loads built-ins) | +| `src/core/adapters/` | CLIAdapter implementations: ClaudeAdapter, CodexAdapter, AiderAdapter | +| `src/core/agent-orchestrator.ts` | Agent orchestration layer — manages TaskAgent lifecycle, wired into Bridge + Router | +| `src/core/fast-path-responder.ts` | Quick-answer agent pool for low-latency responses during Master processing | +| `src/core/email-sender.ts` | Outbound email delivery (SHARE email outputs) | +| `src/core/file-server.ts` | Static file server for media/file outputs | +| `src/core/github-publisher.ts` | GitHub Pages publishing for HTML/report outputs | +| `src/core/workspace-manager.ts` | Workspace path validation and helper utilities | +| `src/memory/database.ts` | SQLite DB init, schema creation, migration runner | +| `src/memory/migration.ts` | Schema migration runner (ALTER TABLE sequences) | +| `src/memory/activity-store.ts` | agent_activity CRUD — PID, status, turn counts, explorationId | +| `src/memory/task-store.ts` | tasks + learnings tables (model success rates, retry patterns) | +| `src/memory/conversation-store.ts` | conversation_messages + FTS5 full-text search | +| `src/memory/chunk-store.ts` | workspace_chunks + FTS5 semantic retrieval | +| `src/memory/prompt-store.ts` | prompts + prompt_versions — effectiveness tracking | +| `src/memory/access-store.ts` | access_control table — role-based permissions per sender | +| `src/memory/worker-briefing.ts` | Per-worker context injection from DB before spawn | +| `src/memory/retrieval.ts` | Semantic + FTS5 search helpers for context assembly | +| `src/memory/eviction.ts` | LRU eviction policy for workspace chunk cache | +| `src/memory/sub-master-store.ts` | Sub-master session state persistence | +| `src/memory/index.ts` | MemoryManager facade — unified API over all store modules | +| `src/master/master-manager.ts` | Master AI lifecycle + self-governing session + worker spawning + checkpoint/resume (6677 LOC) | +| `src/master/master-system-prompt.ts` | Master AI system prompt builder — includes "Available MCP Servers" section when configured | +| `src/master/dotfolder-manager.ts` | `.openbridge/` CRUD + exploration state + context/memory.md + prompt library (1109 LOC) | +| `src/master/worker-registry.ts` | Active worker tracking + concurrency limits | +| `src/master/exploration-coordinator.ts` | 5-phase incremental exploration orchestrator with checkpointing + resumability | +| `src/master/exploration-prompts.ts` | Focused prompts (structure scan, classification, directory dive, assembly) | +| `src/master/result-parser.ts` | Robust JSON extraction from AI output with progressive fallbacks | +| `src/master/spawn-parser.ts` | Parse worker spawn requests from Master output | +| `src/master/worker-result-formatter.ts` | Format worker results for Master consumption | +| `src/master/workspace-change-tracker.ts` | Git-based workspace change detection | +| `src/master/prompt-evolver.ts` | Prompt effectiveness tracking + self-improvement refinement | +| `src/master/sub-master-detector.ts` | Sub-master capability detection and selection | +| `src/master/sub-master-manager.ts` | Sub-master session pool management | +| `src/master/delegation.ts` | Multi-AI task delegation coordinator | +| `src/master/classification-engine.ts` | Message/intent classification engine extracted from master-manager.ts (942 LOC) | +| `src/master/exploration-manager.ts` | Exploration lifecycle management extracted from master-manager.ts (1539 LOC) | +| `src/master/worker-orchestrator.ts` | Worker spawn and result handling extracted from master-manager.ts (2065 LOC) | +| `src/master/prompt-context-builder.ts` | Prompt and conversation context assembly extracted from master-manager.ts (560 LOC) | +| `src/connectors/whatsapp/` | WhatsApp connector — auto-reconnect, sessions, typing | +| `src/connectors/console/` | Console connector (reference implementation) | +| `src/connectors/webchat/` | WebChat connector — HTTP + WebSocket, browser UI | +| `src/connectors/telegram/` | Telegram connector | +| `src/connectors/discord/` | Discord connector | +| `src/providers/claude-code/` | Claude Code CLI provider — streaming, sessions, errors (uses AgentRunner) | +| `src/providers/codex/` | Codex provider — `CodexProvider`, `CodexConfig` schema, session manager (ephemeral + resume) | +| `src/cli/init.ts` | CLI config generator — 3 questions for V2 config + optional MCP server setup step | +| `src/cli/access.ts` | CLI access control tool — `openbridge access add/remove/list` (role management) | ### How `workspacePath` Works @@ -159,13 +226,37 @@ my-app/ ├── src/ ├── package.json └── .openbridge/ ← Created by Master AI - ├── .git/ ← Local git repo (AI's changes only) - ├── workspace-map.json ← Auto-generated project understanding - ├── exploration.log ← Timestamped scan history - ├── agents.json ← Discovered AI tools + their roles - └── tasks/ ← Task history (one JSON per task) + ├── exploration/ ← Intermediate exploration state (for resumability) + │ ├── exploration-state.json ← Phase progress tracker + │ ├── structure-scan.json ← Pass 1 output + │ ├── classification.json ← Pass 2 output + │ └── dirs/ ← Pass 3 outputs (one per directory) + │ ├── src.json + │ ├── tests.json + │ └── docs.json + ├── context/ ← Cross-session memory + │ └── memory.md ← Master-curated memory (≤200 lines, updated each session) + ├── workspace-map.json ← Auto-generated project understanding (legacy) + ├── agents.json ← Discovered AI tools + their roles (legacy) + ├── exploration.log ← Timestamped scan history (legacy) + ├── tasks/ ← Task history (legacy JSON; superseded by SQLite) + └── openbridge.db ← SQLite memory (workspace chunks, conversations, + tasks, learnings, agent activity, prompts) ``` +> **Note:** `openbridge.db` (SQLite + FTS5) ships in v0.0.2 and progressively replaces JSON files. See [docs/ROADMAP.md](docs/ROADMAP.md). + +### memory.md Pattern + +The **`memory.md` pattern** is how the Master AI maintains continuity across sessions without bloating the context window: + +- **File:** `.openbridge/context/memory.md` (≤200 lines, plain Markdown) +- **On session start:** Loaded by `buildConversationContext()` in `MasterManager` and injected into the system prompt as primary context. +- **During session:** Master updates the file when asked to remember something, or when key decisions/findings are made. +- **On session end:** `triggerMemoryUpdate()` fetches the last 20 conversation entries from SQLite and injects them into the prompt (as `## Recent conversation history:`), then spawns a stateless `--print` agent to write updated notes. This ensures the agent has real conversation context to produce meaningful memory updates (fixed in v0.0.5, OB-F39). +- **FTS5 fallback:** When `memory.md` is empty or missing, `buildConversationContext()` falls back to `searchConversations()` FTS5 search for cross-session results. All FTS5 queries are sanitized via `sanitizeFts5Query()` to prevent `SqliteError` on special characters (fixed in v0.0.5, OB-F38). +- **Why this works:** The Master AI is the curator — no complex summarizer pipelines, no topic clustering workers. Same pattern used by Claude Code's own `MEMORY.md`. + ### Adding a New Connector 1. Create `src/connectors/your-connector/` @@ -181,24 +272,41 @@ src/ ├── index.ts Entry point (V0 + V2 startup flows) ├── cli/ CLI tools │ ├── index.ts CLI entry -│ └── init.ts Config generator (3 questions for V2) +│ ├── init.ts Config generator (3 questions for V2 + optional MCP step) +│ └── access.ts Access control management (openbridge access) ├── types/ Interfaces + Zod schemas │ ├── connector.ts Connector interface │ ├── provider.ts AIProvider interface │ ├── message.ts Message types -│ ├── config.ts Config schemas (V0 + V2) +│ ├── config.ts Config schemas (V0 + V2) + MCPServerSchema/MCPConfigSchema │ ├── common.ts Shared types -│ ├── agent.ts Agent / TaskAgent types +│ ├── agent.ts Agent / TaskAgent types (TaskManifest has mcpServers field) │ ├── discovery.ts DiscoveredTool, ScanResult schemas │ └── master.ts MasterState, ExplorationSummary schemas ├── core/ Bridge engine │ ├── bridge.ts Main orchestrator (setMaster + lifecycle) -│ ├── router.ts Message routing (Master → provider fallback) -│ ├── auth.ts Authentication -│ ├── queue.ts Message queues +│ ├── router.ts Message routing + /history + stop/status/confirm commands (2069 LOC) +│ ├── auth.ts Authentication + role-based access control +│ ├── queue.ts Message queues (priority queue, fast-path classification) │ ├── registry.ts Plugin registry -│ ├── config.ts Config loader (V2 detection + V0 fallback) +│ ├── config.ts Config loader (V2 detection + V0 fallback) + MCP config writer │ ├── config-watcher.ts Hot-reload +│ ├── agent-runner.ts Unified CLI executor (retries, error classification, adaptive turns, per-worker MCP isolation) +│ ├── command-handlers.ts 40+ /command handler functions (2936 LOC) +│ ├── output-marker-processor.ts SHARE/VOICE/APP output marker routing +│ ├── error-classifier.ts Error classification + retry detection +│ ├── cost-manager.ts Cost tracking + budget management +│ ├── model-selector.ts Model recommendation per task type +│ ├── model-registry.ts Provider-agnostic model tiers (fast/balanced/powerful) +│ ├── cli-adapter.ts CLIAdapter interface (SpawnOptions → binary + args + env) +│ ├── adapter-registry.ts Maps tool names to CLIAdapter instances +│ ├── adapters/ CLIAdapter implementations (claude, codex, aider) +│ ├── agent-orchestrator.ts TaskAgent lifecycle management (Bridge + Router) +│ ├── fast-path-responder.ts Quick-answer agent pool (max 2 concurrent, maxTurns=3) +│ ├── email-sender.ts Outbound email for SHARE outputs +│ ├── file-server.ts Static file server for media outputs +│ ├── github-publisher.ts GitHub Pages publishing for HTML outputs +│ ├── workspace-manager.ts Workspace path validation utilities │ ├── health.ts Health checks │ ├── metrics.ts Metrics │ ├── audit-logger.ts Audit logging @@ -206,21 +314,59 @@ src/ │ └── logger.ts Pino logger ├── connectors/ │ ├── index.ts Registry -│ ├── whatsapp/ WhatsApp (V0) -│ └── console/ Console (reference) +│ ├── whatsapp/ WhatsApp (whatsapp-web.js) +│ ├── console/ Console (reference) +│ ├── webchat/ WebChat (HTTP + WebSocket) +│ ├── telegram/ Telegram +│ └── discord/ Discord ├── providers/ │ ├── index.ts Registry -│ └── claude-code/ Claude Code (V0) + generalized executor +│ ├── claude-code/ Claude Code CLI provider (uses AgentRunner) +│ └── codex/ Codex provider (CodexProvider, CodexConfig, session-manager) ├── discovery/ AI tool auto-discovery │ ├── index.ts scanForAITools() export │ ├── tool-scanner.ts CLI tool detection │ └── vscode-scanner.ts VS Code extension detection +├── memory/ SQLite memory system (openbridge.db) +│ ├── index.ts MemoryManager facade +│ ├── database.ts DB init + schema +│ ├── migration.ts Schema migration runner +│ ├── activity-store.ts agent_activity CRUD (PID, status, turns) +│ ├── task-store.ts tasks + learnings tables +│ ├── conversation-store.ts conversation_messages + FTS5 +│ ├── chunk-store.ts workspace_chunks + FTS5 +│ ├── prompt-store.ts prompts + prompt_versions +│ ├── access-store.ts access_control (role-based permissions) +│ ├── worker-briefing.ts per-worker context injection +│ ├── retrieval.ts semantic + FTS5 search helpers +│ ├── eviction.ts LRU eviction for chunk cache +│ └── sub-master-store.ts sub-master session state +├── orchestrator/ Task orchestration layer (experimental) +│ ├── index.ts Exports TaskAgentRuntime + ScriptCoordinator +│ ├── task-agent-runtime.ts Runs TaskAgent step-by-step (AI + API execution) +│ └── script-coordinator.ts Multi-step script execution with step tracking └── master/ Master AI management ├── index.ts Module exports - ├── master-manager.ts Master AI lifecycle + message routing - ├── dotfolder-manager.ts .openbridge/ folder CRUD + git - ├── exploration-prompt.ts Workspace exploration prompt - └── delegation.ts Multi-AI task delegation + ├── master-manager.ts Master AI lifecycle + self-governing + worker spawning + checkpoint/resume (6677 LOC) + ├── master-system-prompt.ts Master AI system prompt builder + ├── worker-registry.ts Active worker tracking + concurrency limits + ├── dotfolder-manager.ts .openbridge/ CRUD + exploration state + context/memory.md + prompt library (1109 LOC) + ├── exploration-coordinator.ts 5-phase incremental exploration orchestrator + ├── exploration-prompts.ts Focused prompts (structure, classification, dive, assembly) + ├── result-parser.ts Robust JSON extraction from AI output + ├── spawn-parser.ts Parse worker spawn requests from Master output + ├── worker-result-formatter.ts Format worker results for Master + ├── workspace-change-tracker.ts Git-based workspace change detection + ├── seed-prompts.ts Initial prompt templates + ├── exploration-prompt.ts Legacy monolithic exploration prompt (V0) + ├── prompt-evolver.ts Prompt effectiveness tracking + refinement + ├── sub-master-detector.ts Sub-master capability detection + ├── sub-master-manager.ts Sub-master session pool management + ├── delegation.ts Multi-AI task delegation + ├── classification-engine.ts Message/intent classification engine (942 LOC) + ├── exploration-manager.ts Exploration lifecycle management (1539 LOC) + ├── worker-orchestrator.ts Worker spawn + result handling (2065 LOC) + └── prompt-context-builder.ts Prompt + conversation context assembly (560 LOC) tests/ Vitest test suite ├── core/ Unit tests for each core module @@ -238,6 +384,22 @@ docs/ Documentation + audit tracking scripts/ Task runner utilities ``` +## Current Version + +**v0.1.3** — 230 findings resolved. 1672 tasks shipped across Phases 1–169. + +Key additions since v0.0.5: + +- v0.0.6–v0.0.8: Media handling, message splitting, voice transcription, CLI wizard +- v0.0.9–v0.0.11: Classification fixes, RAG knowledge retrieval, output sharing, user consent +- v0.0.12: Deep Mode, WebChat modernization, tunnel/relay, Docker sandbox, batch continuation +- v0.0.13: Structured observations, session compaction, vector search, document generation, role UX fix, doctor, pairing auth, skills directory +- v0.0.14: Skill pack extensions, creative output (diagrams/charts/art), agent orchestration (planning gate, swarms, test protection, iteration caps) +- v0.0.15: Deep stability audit — prompt budget, god-class refactoring (8 modules extracted), classification fixes, memory leak fixes, process safety, monorepo awareness +- v0.1.0: Business platform — document intelligence, DocType engine, integration hub, workflow engine, business document generation, API adapter framework, industry templates, skill pack extensions, Agent SDK permission relay +- v0.1.1–v0.1.2: Real-world testing fixes, model budgets, WebChat session isolation, trust level system, workspace boundaries +- v0.1.3: Real-world fixes — escalation loop, DLQ error response, classification escalation, worker boundaries, headless safety, message queueing, remote delivery + ## Conventions - Conventional commits: `feat(scope): description` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bff0a3bf..4fcc3d8c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,6 +25,35 @@ npm run lint # Run linter npm run dev # Start in development mode with hot reload ``` +### Development Tips + +**Ctrl+C / Shutdown behavior** + +When running `npm run dev`, press Ctrl+C **once** and wait for the message: + +``` +Shutting down gracefully... please wait +``` + +Do not press Ctrl+C again. If you press it multiple times, `tsx` force-kills the process before +shutdown completes. + +**Graceful shutdown takes up to 10 seconds.** During this window, OpenBridge: + +1. Saves the Master AI session state to SQLite (fast — always completes) +2. Triggers a memory update — runs a short AI agent to update `.openbridge/context/memory.md` + with recent conversation context (slow — may take up to 10 seconds) + +If the 10-second timeout is exceeded, OpenBridge logs a warning and calls `process.exit(1)`. + +**If force-killed:** + +- Session state is still saved (it runs first — critical-first ordering) +- The `memory.md` update may be lost (it runs last and is interruptible) + +This means a force-kill loses at most the current session's memory notes — the SQLite state +(tasks, conversations, agent activity) is always safe. + ## Branch Strategy | Branch | Purpose | @@ -38,6 +67,21 @@ npm run dev # Start in development mode with hot reload Always branch from `develop`, not `main`. +## Branch Protection + +The `main` and `develop` branches are protected. Maintainers should configure the following settings in **GitHub → Settings → Branches → Branch protection rules**: + +| Rule | `main` | `develop` | +| --------------------------------- | :--------------------------: | :--------------------------: | +| Require pull request before merge | ✅ | ✅ | +| Required approving reviews | 1 | 1 | +| Require status checks to pass | ✅ | ✅ | +| Required status checks | lint, typecheck, test, build | lint, typecheck, test, build | +| Restrict direct pushes | ✅ | ✅ | +| Allow force pushes | ❌ | ❌ | + +All changes to `main` and `develop` must go through a pull request. Direct commits and force-pushes are not permitted. If your push is rejected, open a PR from your feature branch instead. + ## Commit Convention We use [Conventional Commits](https://www.conventionalcommits.org/). Commits are @@ -47,7 +91,7 @@ Format: `(): ` **Types**: feat, fix, docs, style, refactor, perf, test, build, ci, chore, revert -**Scopes**: core, whatsapp, claude, connector, provider, config, deps +**Scopes**: core, whatsapp, claude, connector, provider, config, discovery, master, runner, deps, ci, docs Examples: @@ -73,6 +117,36 @@ docs(readme): add installation instructions - Prefer explicit types over `any` - Use interfaces for plugin contracts (connectors, providers) +## Release Process + +Releases are automated via the `.github/workflows/release.yml` workflow. When a version tag is pushed to `main`, the workflow runs the full CI pipeline (lint → typecheck → test → build) and then publishes to npm and creates a GitHub Release. + +### Repository Secrets + +Maintainers must configure the following secret in the GitHub repository settings before releases can be published: + +| Secret | Description | +| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | +| `NPM_TOKEN` | An npm automation token with publish access to the `openbridge` package. Generate at npmjs.com → Access Tokens → Generate New Token → Automation. | + +### Tagging a Release + +```bash +# Ensure you are on main and up to date +git checkout main && git pull + +# Create and push the version tag (triggers the release workflow) +git tag v0.0.1 +git push origin v0.0.1 +``` + +The workflow will: + +1. Run lint, type check, and tests +2. Build `dist/` +3. Publish to npm (`npm publish --provenance --access public`) +4. Create a GitHub Release with changelog notes extracted from `CHANGELOG.md` + ## Adding a Connector 1. Create a new directory: `src/connectors/your-connector/` diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..8d9452bb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,80 @@ +# OpenBridge — Docker Container Image +# +# Build: +# docker build -t openbridge . +# +# Run (with ENV vars): +# docker run -d \ +# -e OPENBRIDGE_WORKSPACE_PATH=/workspace \ +# -e OPENBRIDGE_AUTH_WHITELIST="+1234567890" \ +# -e OPENBRIDGE_HEADLESS=true \ +# -v /path/to/your/project:/workspace \ +# -v openbridge-data:/app/.wwebjs_auth \ +# -p 3000:3000 \ +# openbridge +# +# Or use docker-compose (see docker-compose.yml). + +# ─── Stage 1: Builder ──────────────────────────────────────────────────────── +FROM node:22-slim AS builder + +# Install build dependencies required for better-sqlite3 native module +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 \ + make \ + g++ \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Copy package manifests first for layer caching +COPY package.json package-lock.json ./ + +# Install all dependencies (including devDependencies for build) +RUN npm ci + +# Copy source files +COPY tsconfig.json ./ +COPY src/ ./src/ + +# Compile TypeScript → dist/ +RUN npm run build + +# ─── Stage 2: Production image ─────────────────────────────────────────────── +FROM node:22-slim AS production + +# Install build dependencies needed to rebuild better-sqlite3 in production +# and chromium deps for whatsapp-web.js +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 \ + make \ + g++ \ + git \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Copy package manifests +COPY package.json package-lock.json ./ + +# Install production dependencies only (rebuilds native modules for this image) +RUN npm ci --omit=dev + +# Copy compiled output from builder +COPY --from=builder /app/dist ./dist + +# Copy example config for reference +COPY config.example.json ./ + +# Create workspace mount point and data directories +RUN mkdir -p /workspace /app/logs + +# OpenBridge listens on port 3000 (WebChat connector) +EXPOSE 3000 + +# Default environment — override at runtime +ENV NODE_ENV=production +ENV OPENBRIDGE_HEADLESS=true + +# Start the compiled bridge +CMD ["node", "dist/index.js"] diff --git a/OVERVIEW.md b/OVERVIEW.md index f8fb6cc7..dfd10f23 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -2,22 +2,23 @@ ## What is OpenBridge? -OpenBridge is an open-source platform that turns AI into an **autonomous worker for your project**. Point it at any workspace, connect your messaging app, and the AI explores your project, learns its structure, and executes tasks on your behalf — all using the AI tools already installed on your machine. +OpenBridge is an open-source platform that turns your installed AI tools into a **coordinated team that works on your project from any messaging app**. Point it at any workspace, connect WhatsApp or Telegram, and a lead AI explores your project, spawns worker agents to execute tasks, and continuously improves — all using the AI tools already on your machine. -There are no API keys to configure. No map files to write. No complex setup. OpenBridge auto-discovers the AI tools on your system (Claude Code, Codex, Aider, etc.), picks the most capable one as the "Master", and lets it work. +No API keys. No per-request fees. No complex setup. OpenBridge auto-discovers Claude Code, Codex, Gemini, Aider, and any other AI tool on your system, then coordinates them automatically. ## Why OpenBridge? -**The problem:** AI tools are powerful but isolated. You open Claude Code, type a question, get an answer, then manually relay instructions. You can't trigger AI work from your phone. You can't coordinate multiple AI tools. You can't give an AI persistent knowledge of your project that survives across sessions. +**The problem:** You have powerful AI tools installed, but they're isolated. You open Claude Code, type a question, close it, open Codex for something else. You can't coordinate them. You can't trigger AI work from your phone. And every new session starts from scratch — no memory of your project. -**The solution:** OpenBridge bridges the gap between you and your AI tools: +**The solution:** OpenBridge makes your AI tools work together: -- **Message from anywhere** — send a WhatsApp message, the AI handles it in your workspace -- **Zero setup** — auto-discovers installed AI tools, no API keys, no config files to study -- **Autonomous exploration** — the Master AI silently learns your project on startup -- **Persistent knowledge** — everything the AI learns is stored in `.openbridge/` with git tracking -- **Multi-AI** — the Master can delegate tasks to other AI tools found on your machine -- **Your subscription** — runs locally, uses your existing AI tools, zero extra cost +- **Multi-AI orchestration** — Claude reads the code, Codex writes the fix, another AI runs the tests. One message from you, coordinated automatically. +- **Manage AI from your phone** — send a WhatsApp message and a team of AI agents gets to work. Check progress, ask follow-ups, approve changes — all from your phone. +- **You control AI access** — three levels (read-only, code-edit, full-access), workspace-scoped, phone whitelist. The AI only touches what you allow. +- **Always up-to-date context** — OpenBridge explores your workspace, detects changes, and keeps its knowledge current. Multi-turn conversations remember everything. +- **Zero extra cost** — runs locally, uses your existing AI subscriptions. No API keys, no new bills. + +**Under the hood:** A self-governing Master AI picks the best model, tools, and strategy for each task. It spawns short-lived worker agents with bounded permissions. Everything the AI learns is stored in `openbridge.db` (SQLite + FTS5) inside `.openbridge/`. Workers receive context briefings from the database before each task. The Master refines its own prompts over time. ## How It Works @@ -34,16 +35,15 @@ openbridge init ``` 1. Load config (workspace path + channel + whitelist) -2. Connect WhatsApp (restore session or scan QR) +2. Connect channel (Console / WebChat / WhatsApp / Telegram / Discord) 3. Auto-discover AI tools: - Scan: claude? codex? aider? cursor? - Pick Master (most capable) - - Register others as delegates -4. Launch Master AI silently: - - Explore target workspace - - Create .openbridge/ folder - - Generate workspace understanding - - Init local git repo for tracking + - Register others as worker candidates +4. Launch Master AI as long-lived session: + - Explore workspace via worker agents (read-only, haiku model) + - Create .openbridge/ folder with openbridge.db (SQLite) + - Generate workspace understanding (stored as context chunks) 5. Ready. Waiting for messages. ``` @@ -54,8 +54,8 @@ You (WhatsApp) OpenBridge (your machine) | | | "/ai what's in my project?" | | ──────────────────────────────────> | - | | Master AI already explored - | | Replies from its knowledge + | | Master AI replies from + | | its workspace knowledge | "Your project is a Node.js | | API with 12 routes, PostgreSQL | | database, React frontend..." | @@ -64,21 +64,56 @@ You (WhatsApp) OpenBridge (your machine) | "/ai add input validation | | to the login endpoint" | | ──────────────────────────────────> | - | | Master AI modifies files - | | Changes tracked in .openbridge/.git + | | Master spawns workers: + | | → Worker 1 (haiku): read code + | | → Worker 2 (sonnet): add validation + | | → Worker 3 (haiku): run tests | "Done. Added zod validation | | to POST /auth/login. Changes | - | committed." | + | committed. All tests pass." | | <────────────────────────────────── | ``` +### How the Master Governs Workers + +``` +User sends: "/ai refactor auth to use JWT" + │ + ▼ +Master AI (long-lived session, opus) + │ thinks: "Complex task. I need to: + │ 1. Read current auth code + │ 2. Implement JWT + │ 3. Run tests" + │ queries openbridge.db: similar past tasks, workspace chunks, model learnings + │ + ├──► Worker 1: { model: "haiku", profile: "read-only", task: "read auth files" } + │ briefed with: relevant workspace chunks from openbridge.db (FTS5) + │ └──► returns file contents → stored as context_chunks in DB + │ + ├──► Master analyzes, plans the change + │ + ├──► Worker 2: { model: "sonnet", profile: "code-edit", task: "implement JWT" } + │ briefed with: auth file contents + past similar task results + │ └──► returns diff + result → task record saved to DB + │ + ├──► Worker 3: { model: "haiku", profile: "code-edit", task: "run tests" } + │ └──► returns test output + │ + ▼ +Master: "Done. Refactored to JWT. 4 files modified, all tests pass." + │ records learnings: model/task-type performance updated in DB + ▼ +User (WhatsApp) ← response +``` + ## Real-World Use Cases ### Manage Your Projects from Your Phone > _"/ai what changed since yesterday?"_ -The Master checks `.openbridge/.git` and your project's git log, then summarizes recent changes — files modified, commits made, current branch status. +The Master queries `openbridge.db` for recent task history and checks your project's git log, then summarizes recent changes — files modified, commits made, current branch status. ### Explore Unfamiliar Codebases @@ -90,22 +125,36 @@ The Master already explored the workspace on startup. It knows the file structur > _"/ai run the tests and fix any failures"_ -The Master runs your test suite, identifies failures, reads the failing code, applies fixes, and reports back. All changes tracked in `.openbridge/.git`. +The Master spawns a worker to run tests, another to read failing code, another to fix it. Each worker receives a focused briefing from `openbridge.db` with relevant workspace context. ### Multi-AI Collaboration > _"/ai refactor the database layer to use Prisma"_ -The Master delegates subtasks — one AI tool analyzes the current schema, another generates Prisma models, the Master coordinates and verifies the result. +The Master delegates subtasks to workers — one analyzes the current schema, another generates Prisma models, the Master coordinates and verifies the result. + +### Multi-Turn Conversations + +> _You: "/ai which invoices are overdue?"_ +> _AI: "3 invoices: Client A ($1,200), Client B ($850), Client C ($2,400)"_ +> _You: "/ai send reminder emails to those clients"_ + +Session continuity preserves context across messages. The AI remembers "those clients" refers to A, B, and C from the previous question. + +### Non-Code Workspaces + +> _You: "/ai what ingredients are running low this week?"_ + +Point OpenBridge at a folder of spreadsheets — the Master reads your files and answers business questions. Works for cafes, law firms, real estate, accounting, and any business with files to query. See [USE_CASES.md](docs/USE_CASES.md). ## Architecture -OpenBridge has 4 layers: +OpenBridge has 5 layers: ``` ┌──────────────────────────────────────────────────────────────────┐ │ CHANNELS │ -│ WhatsApp · Telegram · Discord · Web Chat │ +│ Console · WebChat · WhatsApp · Telegram · Discord │ │ Messaging adapters that translate between platforms and bridge │ └──────────────────────┬────────────────────────────────────────────┘ │ @@ -125,10 +174,18 @@ OpenBridge has 4 layers: │ ▼ ┌──────────────────────────────────────────────────────────────────┐ -│ MASTER AI │ -│ Master Manager · .openbridge/ Folder · Delegation Coordinator │ -│ Autonomous workspace exploration, task execution, multi-AI │ -│ delegation, git-tracked knowledge in .openbridge/ │ +│ AGENT RUNNER │ +│ Unified executor: --allowedTools · --max-turns · --model │ +│ Retries · Disk logging · Streaming · Tool profiles │ +│ Spawns worker processes with bounded permissions │ +└──────────────────────┬────────────────────────────────────────────┘ + │ + ▼ +┌──────────────────────────────────────────────────────────────────┐ +│ MASTER AI │ +│ Self-governing: picks model, tools, strategy per task │ +│ Long-lived session · Worker spawning · Task decomposition │ +│ .openbridge/ knowledge · Self-improvement · Learnings │ └──────────────────────────────────────────────────────────────────┘ ``` @@ -138,10 +195,11 @@ Messaging platform adapters. Each implements the `Connector` interface. | Channel | Status | Library | | -------- | :----: | ----------------- | -| WhatsApp | V0 | `whatsapp-web.js` | -| Console | V0 | built-in (stdin) | -| Telegram | -- | planned | -| Discord | -- | planned | +| Console | ✅ | built-in (stdin) | +| WebChat | ✅ | built-in (ws) | +| WhatsApp | ✅ | `whatsapp-web.js` | +| Telegram | ✅ | `grammy` | +| Discord | ✅ | `discord.js` v14 | ### Layer 2: Bridge Core @@ -163,21 +221,43 @@ Auto-detects AI tools on the machine at startup: - **Auto-Selection** — ranks tools by capability, picks the best as Master - **No API keys needed** — uses tools that are already authenticated via your terminal/IDE -### Layer 4: Master AI +### Layer 4: Agent Runner + +Unified executor for all AI CLI calls. Inspired by the project's bash scripts (`scripts/run-tasks.sh`): + +- **`--allowedTools`** — restricts what the AI can do (no `--dangerously-skip-permissions`) +- **`--max-turns`** — bounds agent execution to prevent runaway processes +- **`--model`** — selects the model per task (haiku for mechanical work, opus for reasoning) +- **Retry logic** — configurable retries with backoff (default: 3 attempts, 10s delay) +- **Disk logging** — full stdout/stderr written to `.openbridge/logs/` +- **Tool profiles** — `read-only`, `code-edit`, `full-access` — Master picks per task + +### Layer 5: Master AI -The autonomous agent that knows your project: +The self-governing autonomous agent: -- **Master Manager** — launches the Master AI, manages its lifecycle (idle → exploring → ready) +- **Long-lived session** — maintains context across messages (not single-turn `--print` calls) +- **Task decomposition** — breaks complex user requests into worker subtasks +- **Worker spawning** — creates short-lived worker agents with specific model + tools + turn limits +- **Auto-announcement** — workers report results back to Master (no polling) - **`.openbridge/` Folder** — the AI's brain, stored inside your target project: ``` .openbridge/ - ├── .git/ ← tracks all AI changes - ├── workspace-map.json ← auto-generated project understanding - ├── exploration.log ← scan history - ├── agents.json ← discovered AI tools + roles - └── tasks/ ← task history + ├── openbridge.db ← SQLite database (all persistent state) + │ ├── context_chunks ← workspace knowledge, FTS5-indexed for fast retrieval + │ ├── conversations ← every user↔Master message, FTS5-indexed + │ ├── tasks ← full task history (type, model, result, cost, timing) + │ ├── learnings ← model/task-type performance stats (drives model selection) + │ ├── prompts ← versioned prompt library with effectiveness tracking + │ ├── sessions ← Master session state (replaces master-session.json) + │ ├── agent_activity ← real-time worker status with PID tracking + │ ├── exploration_progress ← per-phase exploration tracking + │ ├── access_control ← per-user RBAC (owner/admin/developer/viewer) + │ └── system_config ← discovered AI tools, tool profiles (key-value) + ├── exploration/ ← incremental exploration state (JSON checkpoints) + └── logs/ ← full worker execution logs ``` -- **Delegation** — Master can assign subtasks to other discovered AI tools +- **Self-improvement** — Master tracks prompt effectiveness, refines strategies, creates custom profiles - **Silent by default** — only speaks when the user sends a message ## Business Model @@ -192,16 +272,26 @@ OpenBridge is open source (Apache 2.0). The tool is free; the expertise to confi ## Current Status -| Component | Status | -| ---------------- | ---------------------------------------------------------- | -| WhatsApp | V0 — auto-reconnect, sessions, chunking, typing indicators | -| Claude Code | V0 — streaming, sessions, error classification | -| Bridge Core | V0 — router, auth, queue, metrics, health, audit | -| AI Discovery | Planned — Phase 6 | -| Master AI | Planned — Phase 7 | -| V2 Config | Planned — Phase 8 | -| Multi-AI | Planned — Phase 10 | -| Telegram/Discord | Planned — Phase 14 | +| Component | Status | +| ----------------------- | ----------------------------------------------------------------------------------------- | +| Console | ✅ Stable — simplest path; E2E verified, no external accounts required | +| WebChat | ✅ Stable — localhost:3000 UI, markdown rendering, connection status, typing indicator | +| WhatsApp | ✅ Stable — auto-reconnect, sessions, chunking, typing indicators, local web cache | +| Telegram | ✅ Stable — grammY, DM + group @mention support, typing indicator | +| Discord | ✅ Stable — discord.js v14, DM + guild channel, bot message filtering | +| Bridge Core | ✅ Stable — router, auth, queue, metrics, health, audit, rate limiting | +| AI Discovery | ✅ Stable — CLI scanner, VS Code scanner, auto-selection, capability ranking | +| V2 Config | ✅ Stable — 3-field setup, V0 backward compatibility, CLI init, tilde expansion | +| Agent Runner | ✅ Stable — `--allowedTools`, `--max-turns`, `--model`, retries, streaming, disk logging | +| Tool Profiles | ✅ Stable — read-only, code-edit, full-access, master; custom profiles registry | +| Smart Orchestration | ✅ Stable — task classifier (quick/tool-use/complex), auto-delegation, progress feedback | +| Self-Governing Master | ✅ Stable — persistent session, task decomposition, worker spawning, session recovery | +| Worker Orchestration | ✅ Stable — parallel workers, registry, depth limiting, task history, timeout + cleanup | +| Incremental Exploration | ✅ Stable — 5-pass with checkpointing, git + timestamp change detection, freshness track | +| Self-Improvement | ✅ Stable — prompt library, learnings store, effectiveness tracking, idle self-refinement | +| Memory System | ✅ Stable — SQLite + FTS5, context chunks, conversation history, worker briefings, RBAC | +| Worker Control | ✅ Stable — PID capture, kill/stop commands, confirmation flow, cross-channel broadcast | +| Responsive Master | ✅ Stable — priority queue, fast-path responder, queue depth reporting, wait estimates | ## Tech Stack @@ -213,4 +303,5 @@ OpenBridge is open source (Apache 2.0). The tool is free; the expertise to confi - **Git hooks:** Husky v9 + lint-staged + commitlint (conventional commits) - **Config validation:** Zod - **Logging:** Pino +- **Database:** better-sqlite3 (SQLite + FTS5) - **License:** Apache 2.0 diff --git a/README.md b/README.md index 2dcf0976..509cbec5 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # OpenBridge -**Your AI, one message away.** +**Your AI team, one message away.** [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE) [![Node.js](https://img.shields.io/badge/node-%3E%3D22.0.0-brightgreen.svg)](https://nodejs.org/) @@ -10,116 +10,70 @@ [![CI](https://github.com/medomar/OpenBridge/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/medomar/OpenBridge/actions/workflows/ci.yml) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](CONTRIBUTING.md) -An open-source bridge that connects messaging channels to AI agents that **autonomously explore your workspace and execute tasks** — using the AI tools already on your machine. Zero API keys. Zero extra cost. +Connect your messaging app to the AI tools on your machine. Send a message from your phone, and OpenBridge coordinates Claude, Codex, and Gemini to explore your workspace and execute tasks — using your existing subscriptions, at zero extra cost. +[Features](#features) | [Quick Start](#quick-start) | +[Prerequisites](#prerequisites) | +[Examples](#see-it-in-action) | [How It Works](#how-it-works) | -[Examples](#examples) | -[Documentation](#documentation) | -[Contributing](#contributing) +[Documentation](#documentation) --- -## Why OpenBridge? +## Features -You have AI tools installed — Claude Code, Codex, Aider. But they're stuck in your terminal. OpenBridge lets you **control them from your phone** through WhatsApp. +### Multi-AI Orchestration -- **Auto-discovers** AI tools on your machine — no API keys, no manual config -- **Master AI** explores your workspace silently on startup and learns the project -- **Execute tasks** from anywhere — code changes, test runs, project analysis -- **Multi-AI delegation** — the Master assigns subtasks to other discovered tools -- **`.openbridge/` folder** — the AI's brain, git-tracked inside your project -- **Silent worker** — only speaks when you ask +Claude, Codex, Gemini — working together on your tasks. OpenBridge discovers every AI tool installed on your machine and coordinates them automatically. One AI reads your codebase, another writes the fix, a third runs the tests. You send one message; the system figures out which AI handles which part. ---- +### Manage Your AI From Your Phone -## Examples +Send a WhatsApp message, and a team of AI agents gets to work on your project. OpenBridge supports **5 channels** — WhatsApp, Telegram, Discord, WebChat, and Console. Break complex tasks into subtasks, check progress, ask follow-up questions — all from your phone. Session continuity means the AI remembers every previous conversation. -### Explore Your Project +### Persistent Memory Across Sessions -``` -You: /ai what's in this project? -Master: This is a Node.js REST API (Express + TypeScript) with: - - 12 routes in src/routes/ - - PostgreSQL via Prisma ORM - - JWT authentication in src/middleware/auth.ts - - 47 tests (all passing) - - Docker setup with docker-compose.yml -``` +The Master AI maintains a curated `memory.md` file (≤ 200 lines) inside your workspace. It is loaded on every session start so the AI picks up exactly where you left off — no re-explaining your project, no lost context. The AI updates the file itself: merging topics, discarding stale info, keeping it focused. Use `/history` to browse past conversations, search by keyword, or retrieve a full transcript. -### Execute Tasks +### You Control What AI Can Access -``` -You: /ai run the tests and fix any failures -Master: Running tests... 45/47 passing. 2 failures in auth.test.ts. - → Reading failing tests... - → Fixed: expired mock token + missing header check - → Re-running... 47/47 passing. - Changes committed to .openbridge/.git -``` +Three access levels keep you in control: **read-only** (browse files), **code-edit** (modify files and run tests), and **full-access** (everything). The AI only touches the workspace folder you point it at — nothing else on your machine. A phone whitelist ensures only authorized users can send commands. -### Understand Code +### Always Up-to-Date Project Context -``` -You: /ai how does the payment flow work? -Master: Payment flow (src/services/payment.ts): - 1. POST /checkout → validates cart → creates Stripe session - 2. Stripe webhook → POST /webhooks/stripe → updates order status - 3. Order service notifies user via email (src/services/email.ts) - Key files: payment.ts, order.ts, stripe-webhook.ts -``` +On startup, OpenBridge explores your workspace and builds a knowledge base inside your project. It detects git commits and file changes, then re-explores incrementally — so the AI's understanding is always current. Multi-turn conversations maintain context across messages: ask a question, get an answer, follow up — the AI remembers. Across sessions, persistent `memory.md` ensures key decisions, findings, and preferences survive restarts without bloating the context window. -### Delegate Work +### Connect to External Services via MCP -``` -You: /ai refactor the user model to add role-based access -Master: Breaking this into subtasks... - → Analyzing current user model (Master) - → Generating migration script (Codex) - → Updating middleware + tests (Master) - Done. 4 files modified, migration ready, tests passing. -``` +Workers connect to external services (Gmail, Slack, Canva, databases) via the Model Context Protocol. Each worker gets an isolated MCP config with only the servers it needs — no cross-contamination of API keys. The Master AI autonomously decides which workers get which MCP servers. ---- +### Document & Creative Output -## Architecture +Generate Word documents, PowerPoint presentations, Excel spreadsheets, and PDF reports from a chat message. Create diagrams (Mermaid/PlantUML), charts (D3.js/Chart.js), landing pages, slide decks, generative art, and brand assets. Generated files are delivered as attachments via WhatsApp/Telegram or download links in WebChat. -``` -┌─────────────┐ ┌──────────────────────────────────┐ ┌──────────────┐ -│ CHANNELS │ │ BRIDGE CORE │ │ MASTER AI │ -│ │ │ │ │ │ -│ WhatsApp ──┼────>│ Auth → Queue → Router ───────────┼────>│ Explores │ -│ Telegram │ │ │ │ workspace │ -│ Discord │ │ Discovery: scans for AI tools │ │ Delegates │ -│ │<────┼── Health · Metrics · Audit │<────│ tasks │ -└─────────────┘ └──────────────────────────────────┘ └──────────────┘ - .openbridge/ - ├── .git/ - ├── workspace-map.json - ├── agents.json - └── tasks/ -``` +### Smart Agent Orchestration + +The Master AI plans before it acts — spawning read-only analysis workers before committing to code changes. Workers are grouped into coordinated swarms (research → implement → review → test) with automatic handoff. Test files are protected from unauthorized modification. Fix loops are capped at 3 iterations before escalating. + +### Skill Packs + +Domain-specific instruction sets that make workers experts at their task. Built-in packs for security audits, code review, test writing, data analysis, and documentation. Create custom skill packs in `.openbridge/skill-packs/` or let the Master auto-create them from successful task patterns. + +### Zero Extra Cost -| Layer | What it does | -| ---------------- | --------------------------------------------------------------- | -| **Channels** | Messaging adapters (WhatsApp, Telegram, Discord) | -| **Bridge Core** | Routing, auth, queuing, config, metrics, health | -| **AI Discovery** | Scans machine for AI CLIs + VS Code extensions, picks Master | -| **Master AI** | Explores workspace, executes tasks, delegates to other AI tools | +No API keys. No per-request fees. No new subscriptions. OpenBridge runs locally on your machine and uses whatever AI tools you already have installed — Claude Code, Codex, Aider, or anything else. Your subscription, your machine, your data. --- ## Quick Start -### Prerequisites +### Try It Now — Console Mode -- Node.js >= 22 -- A WhatsApp account -- At least one AI CLI tool installed (e.g. [Claude Code](https://docs.anthropic.com/en/docs/claude-code)) +No external accounts needed. Use the built-in Console connector to try OpenBridge immediately. -### Install +**Prerequisites:** Node.js >= 22, [Claude Code](https://docs.anthropic.com/en/docs/claude-code) installed ```bash git clone https://github.com/medomar/OpenBridge.git @@ -127,30 +81,35 @@ cd OpenBridge npm install ``` -### Configure - -```bash -npx openbridge init -``` - -Or create `config.json` manually: +Create `config.json`: ```json { "workspacePath": "/absolute/path/to/your/project", - "channels": [{ "type": "whatsapp", "enabled": true }], + "channels": [{ "type": "console", "enabled": true }], "auth": { - "whitelist": ["+1234567890"], + "whitelist": ["console-user"], "prefix": "/ai" } } ``` -That's it. Three fields. +```bash +npm run dev +``` -### Run +Type a message in the terminal: + +``` +/ai what's in this project? +``` + +### Connect WhatsApp + +**Prerequisites:** Node.js >= 22, a WhatsApp account, Claude Code installed. ```bash +npx openbridge init npm run dev ``` @@ -160,74 +119,158 @@ Scan the QR code with WhatsApp. Then from your phone: /ai what's in this project? ``` +### Other Channels + +OpenBridge supports 5 channels: Console, WebChat, WhatsApp, Telegram, and Discord. See the [Connector Setup Guide](docs/CONNECTORS.md) for all options. + --- -## How It Works +## Prerequisites + +**Required** + +- **Node.js >= 22** — [nodejs.org](https://nodejs.org/) + +**Optional — by connector / feature** + +- **WhatsApp:** Chrome/Chromium — auto-installed by Puppeteer, no extra steps needed +- **Voice messages (recommended):** `OPENAI_API_KEY` env var — uses the OpenAI Whisper API ($0.006/min, zero local setup; same key Codex uses) +- **Voice messages (offline):** `pip install openai-whisper` or `brew install whisper-cpp` — free, runs locally +- **Telegram:** Bot token from [@BotFather](https://t.me/BotFather) +- **Discord:** Bot token + application ID from the [Discord Developer Portal](https://discord.com/developers/applications) + +--- + +## See It In Action + +### Code Project — Explore and Execute + +``` +You: /ai what's in this project? +AI: This is a Node.js REST API (Express + TypeScript) with: + - 12 routes in src/routes/ + - PostgreSQL via Prisma ORM + - JWT authentication in src/middleware/auth.ts + - 47 tests (all passing) + - Docker setup with docker-compose.yml + +You: /ai run the tests and fix any failures +AI: On it... + → Reading test output... 45/47 passing + → Fixing 2 failures in auth.test.ts + → Re-running tests... 47/47 passing + Changes committed. All tests pass. + +You: /ai how does the payment flow work? +AI: Payment flow (src/services/payment.ts): + 1. POST /checkout → validates cart → creates Stripe session + 2. Stripe webhook → POST /webhooks/stripe → updates order status + 3. Order service notifies user via email (src/services/email.ts) + Key files: payment.ts, order.ts, stripe-webhook.ts +``` + +### Business Workspace — No Code Required + +``` +You: /ai what's our current coffee inventory? +AI: Based on inventory.xlsx: + - Arabica beans: 12kg (reorder threshold: 10kg) + - Robusta beans: 8kg + - Espresso blend: 15kg + You're below the reorder threshold on Robusta. + +You: /ai which items sold best last week? +AI: Top 5 from sales_weekly.csv: + 1. Iced Latte (142 sold) + 2. Cappuccino (128 sold) + 3. Americano (97 sold) + 4. Cold Brew (89 sold) + 5. Mocha (76 sold) +``` + +### Conversation History + +``` +You: /history +AI: Recent conversations (last 10): + 1. [2026-02-25] "Refactor auth to JWT" — 8 messages + 2. [2026-02-24] "Fix failing tests in checkout" — 12 messages + 3. [2026-02-20] "What's in this project?" — 4 messages + +You: /history search authentication +AI: Found 3 sessions matching "authentication": + 1. [2026-02-25] "Refactor auth to JWT" + 2. [2026-02-18] "Add OAuth support" — 15 messages + 3. [2026-02-10] "JWT token expiry bug" — 6 messages + +You: /history 2026-02-25-auth-refactor +AI: [Full conversation transcript...] +``` + +### Multi-AI Task Delegation ``` -Your Phone Your Machine -────────────────────────────────────────────── +You: /ai refactor auth to use JWT - "/ai what's new?" - │ - ▼ - WhatsApp Connector ──> Auth (whitelist + prefix) - │ - ▼ - Queue ──> Router - │ - ▼ - Master AI (already explored your workspace) - │ - Reads .openbridge/workspace-map.json - Checks project git log - Builds response - │ - WhatsApp <──── Response <──────────┘ +AI: Complex task. Breaking it down... - "3 commits today: added user roles, - fixed payment bug, updated tests." + → AI 1 (fast, read-only): reading current auth code + → AI 2 (powerful, code-edit): implementing JWT authentication + → AI 3 (fast, code-edit): running tests and verifying + +AI: Done. Refactored to JWT. 4 files modified, all tests pass. ``` -**On startup:** +One message from you. Three AI agents coordinated automatically. Each with the right capabilities for its subtask. -1. OpenBridge scans your machine for AI tools (`which claude`, `which codex`, etc.) -2. Picks the best one as Master -3. Master silently explores the target workspace -4. Creates `.openbridge/` folder with a git repo to track everything -5. Waits for your messages +More examples: [Use Cases](docs/USE_CASES.md) — software teams, cafes, law firms, real estate, and more. --- -## Current Status +## How It Works -| Component | Status | -| ---------------- | ---------------------------------------------------- | -| WhatsApp | Stable — auto-reconnect, sessions, chunking, typing | -| Claude Code | Stable — streaming, sessions, error classification | -| Bridge Core | Stable — router, auth, queue, metrics, health, audit | -| AI Discovery | In development | -| Master AI | In development | -| Multi-AI | Planned | -| Telegram/Discord | Planned | +``` + Your Phone / Browser Your Machine Your Workspace + ───────────────────── ───────────────── ───────────────── + WhatsApp · Telegram OpenBridge .openbridge/ + Discord · WebChat ──────> authenticates, ──────> workspace map + Console routes messages, learnings + <────── coordinates AI <────── session state + workers task history +``` + +1. You send a message from any channel. +2. OpenBridge authenticates it and routes it to the lead AI, which decides how to handle it. +3. For complex tasks, the AI spawns focused workers — each with specific access permissions and capabilities — then synthesizes the results and responds. + +Deep dive: [Architecture](docs/ARCHITECTURE.md) | [Project Overview](OVERVIEW.md) | [API Reference](docs/API_REFERENCE.md) --- ## Documentation -| Guide | Description | -| -------------------------------------------------- | ----------------------------------- | -| [Project Overview](OVERVIEW.md) | Vision, architecture, roadmap | -| [Architecture](docs/ARCHITECTURE.md) | System design, message flow, layers | -| [Configuration Guide](docs/CONFIGURATION.md) | All config options explained | -| [API Reference](docs/API_REFERENCE.md) | Interfaces, types, module APIs | -| [Writing a Connector](docs/WRITING_A_CONNECTOR.md) | How to add a new messaging channel | -| [Writing a Provider](docs/WRITING_A_PROVIDER.md) | How to add a new AI backend | -| [Deployment Guide](docs/DEPLOYMENT.md) | Docker, PM2, systemd setup | -| [Troubleshooting](docs/TROUBLESHOOTING.md) | Common issues and solutions | +| Guide | Description | +| -------------------------------------------- | ----------------------------------- | +| [Documentation Hub](docs/README.md) | All docs in one place | +| [Project Overview](OVERVIEW.md) | Vision, architecture, roadmap | +| [Configuration Guide](docs/CONFIGURATION.md) | All config options explained | +| [Connector Setup](docs/CONNECTORS.md) | Setup guides for all 5 channels | +| [Use Cases](docs/USE_CASES.md) | Examples for every industry | +| [Architecture](docs/ARCHITECTURE.md) | System design, message flow, layers | +| [Deployment Guide](docs/DEPLOYMENT.md) | Docker, PM2, systemd setup | +| [Troubleshooting](docs/TROUBLESHOOTING.md) | Common issues and solutions | --- +## Contributors + + + + + + +

Sayadi Med Omar
Founder & Lead Developer
Haifa BEN LETAIFA
Founder & Lead Developer
+ ## Contributing We welcome contributions! Whether it's a new connector, AI tool integration, bug fix, or documentation improvement — see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines. diff --git a/SECURITY.md b/SECURITY.md index c7c14542..73909a0a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -10,30 +10,51 @@ **Please do NOT report security vulnerabilities through public GitHub issues.** -Instead, please report them via email to the project maintainers. +Instead, use one of the following private channels: -You should receive a response within 48 hours. If the issue is confirmed, we will -release a patch as soon as possible depending on complexity. +- **GitHub Security Advisories (preferred):** [Report a vulnerability](https://github.com/openbridge-ai/openbridge/security/advisories/new) +- **Email:** security@openbridge.dev -Please include: +### Responsible Disclosure Process -- Description of the vulnerability -- Steps to reproduce -- Potential impact -- Suggested fix (if any) +1. **Submit your report** via GitHub Security Advisories or the email above. Include: + - Description of the vulnerability + - Steps to reproduce + - Potential impact + - Suggested fix (if any) + +2. **Acknowledgement:** You will receive an acknowledgement within **48 hours** confirming we received your report. + +3. **Assessment:** We will assess the severity and scope within **7 days** and keep you updated on our progress. + +4. **Fix & Release:** Once confirmed, we will release a patch as soon as possible depending on complexity. Critical issues target a fix within **14 days**; high-severity issues within **30 days**. + +5. **Disclosure:** We coordinate public disclosure with the reporter. We ask for a **90-day embargo** from report to public disclosure to give users time to update. + +6. **Credit:** Reporters who responsibly disclose vulnerabilities will be credited in the release notes and CHANGELOG unless they prefer to remain anonymous. Please let us know your preference when submitting. ## Security Considerations OpenBridge handles sensitive data including: - **WhatsApp session tokens** (stored locally in `.wwebjs_auth/`) +- **Telegram bot tokens** (set via `channels[].token` in `config.json` — treat as a secret) +- **Discord bot tokens** (set via `channels[].token` in `config.json` — treat as a secret) - **Message content** routed between platforms and AI providers - **AI provider credentials** (API keys for non-local providers) - **Workspace access** (the AI provider operates within the project workspace) +### Token Handling — Telegram & Discord + +- Telegram and Discord bot tokens grant full control over the bot. Store them in environment variables or a secrets manager rather than directly in `config.json`. +- If using environment variables, reference them in your config via `"token": "${TELEGRAM_BOT_TOKEN}"` and load them before starting OpenBridge (e.g. via a `.env` file with `dotenv`). +- Never commit `config.json` containing real tokens to version control. Add `config.json` to `.gitignore`. +- Rotate tokens immediately if they are accidentally exposed in a commit, log file, or public channel. + ### Best Practices for Users -- Never commit `.env`, `config.local.json`, or `.wwebjs_auth/` to version control -- Use the phone number whitelist to restrict who can send commands +- Never commit `.env`, `config.json`, `config.local.json`, or `.wwebjs_auth/` to version control +- Use the phone/user whitelist to restrict who can send commands - Run OpenBridge in a dedicated workspace with appropriate scope - Review AI provider permissions before granting workspace access +- Set `NODE_ENV=production` in production deployments to disable development connectors diff --git a/_parse_xls.mjs b/_parse_xls.mjs new file mode 100644 index 00000000..ee4243b7 --- /dev/null +++ b/_parse_xls.mjs @@ -0,0 +1,101 @@ +import XLSX from 'xlsx'; + +const filePath = '/Users/sayadimohamedomar/Desktop/AI-Bridge/OpenBridge/.openbridge/media/1772920977196-624de47c-5d32-481b-806e-27c38ce90390.xls'; + +const workbook = XLSX.readFile(filePath); + +console.log('=== FILE STRUCTURE ==='); +console.log('Sheet count:', workbook.SheetNames.length); +console.log('Sheet names:', workbook.SheetNames); +console.log(''); + +for (const sheetName of workbook.SheetNames) { + const sheet = workbook.Sheets[sheetName]; + const range = XLSX.utils.decode_range(sheet['!ref'] || 'A1'); + + console.log(`\n${'='.repeat(80)}`); + console.log(`SHEET: "${sheetName}"`); + console.log(`Range: ${sheet['!ref']}`); + console.log(`Rows: ${range.e.r - range.s.r + 1} (from ${range.s.r} to ${range.e.r})`); + console.log(`Columns: ${range.e.c - range.s.c + 1} (from ${XLSX.utils.encode_col(range.s.c)} to ${XLSX.utils.encode_col(range.e.c)})`); + + // Get merged cells + if (sheet['!merges'] && sheet['!merges'].length > 0) { + console.log(`\nMerged cells (${sheet['!merges'].length}):`); + for (const merge of sheet['!merges']) { + console.log(` ${XLSX.utils.encode_range(merge)}`); + } + } + + // Convert to JSON (all rows) + const jsonData = XLSX.utils.sheet_to_json(sheet, { header: 1, defval: '' }); + + console.log(`\nTotal rows in data: ${jsonData.length}`); + + // Print ALL rows (raw cell data) + console.log('\n--- ALL DATA (row by row) ---'); + for (let i = 0; i < jsonData.length; i++) { + const row = jsonData[i]; + // Skip completely empty rows + const hasData = row.some(cell => cell !== '' && cell !== null && cell !== undefined); + if (hasData) { + console.log(`Row ${i}: ${JSON.stringify(row)}`); + } + } + + // Also output with headers for easier reading + const jsonWithHeaders = XLSX.utils.sheet_to_json(sheet, { defval: '' }); + if (jsonWithHeaders.length > 0) { + console.log('\n--- DATA AS OBJECTS (with detected headers) ---'); + const headers = Object.keys(jsonWithHeaders[0]); + console.log('Detected headers:', JSON.stringify(headers)); + console.log(`Data rows (excluding header): ${jsonWithHeaders.length}`); + + for (let i = 0; i < jsonWithHeaders.length; i++) { + console.log(`Row ${i + 1}:`, JSON.stringify(jsonWithHeaders[i])); + } + } + + // Raw cell inspection for first 5 rows to understand data types + console.log('\n--- RAW CELL TYPES (first 10 data rows) ---'); + for (let r = range.s.r; r <= Math.min(range.s.r + 10, range.e.r); r++) { + for (let c = range.s.c; c <= range.e.c; c++) { + const cellRef = XLSX.utils.encode_cell({ r, c }); + const cell = sheet[cellRef]; + if (cell) { + console.log(` ${cellRef}: type=${cell.t}, value=${JSON.stringify(cell.v)}, formatted=${JSON.stringify(cell.w)}`); + } + } + } + + // Column statistics + console.log('\n--- COLUMN ANALYSIS ---'); + if (jsonData.length > 1) { + const headerRow = jsonData[0]; + for (let c = 0; c < headerRow.length; c++) { + const colName = headerRow[c] || `Col_${c}`; + const values = jsonData.slice(1).map(row => row[c]).filter(v => v !== '' && v !== null && v !== undefined); + const numericValues = values.filter(v => typeof v === 'number'); + + let analysis = ` Column "${colName}": ${values.length} non-empty values`; + if (numericValues.length > 0) { + const sum = numericValues.reduce((a, b) => a + b, 0); + const min = Math.min(...numericValues); + const max = Math.max(...numericValues); + analysis += ` | Numeric: min=${min}, max=${max}, sum=${sum.toFixed(2)}, avg=${(sum/numericValues.length).toFixed(2)}`; + } + const uniqueTypes = [...new Set(values.map(v => typeof v))]; + analysis += ` | Types: ${uniqueTypes.join(', ')}`; + console.log(analysis); + } + } +} + +// Final summary +console.log('\n\n=== COMPLETE CSV OUTPUT (for readability) ==='); +for (const sheetName of workbook.SheetNames) { + const sheet = workbook.Sheets[sheetName]; + console.log(`\n--- Sheet: "${sheetName}" ---`); + const csv = XLSX.utils.sheet_to_csv(sheet); + console.log(csv); +} diff --git a/commitlint.config.js b/commitlint.config.js index 4109b93b..fa18418d 100644 --- a/commitlint.config.js +++ b/commitlint.config.js @@ -4,7 +4,20 @@ export default { 'scope-enum': [ 2, 'always', - ['core', 'whatsapp', 'claude', 'connector', 'provider', 'config', 'discovery', 'master', 'deps', 'ci', 'docs', 'scripts'], + [ + 'core', + 'whatsapp', + 'claude', + 'connector', + 'provider', + 'config', + 'discovery', + 'master', + 'deps', + 'ci', + 'docs', + 'scripts', + ], ], 'subject-case': [2, 'never', ['start-case', 'pascal-case', 'upper-case']], }, diff --git a/config.example.json b/config.example.json index d2528092..fd142c0f 100644 --- a/config.example.json +++ b/config.example.json @@ -1,13 +1,370 @@ { + "_doc": "OpenBridge Configuration Reference — Every parameter explained. Remove '_*' comment fields before use.", + + "__________ REQUIRED __________": "The 3 fields below are mandatory", + "workspacePath": "/absolute/path/to/your/project", + "_workspacePath_note": "Absolute path to the TARGET project (not the OpenBridge folder). The Master AI runs inside this directory with full access to its files, git, and terminal. On startup, it creates .openbridge/ here to store exploration data and memory.", + "channels": [ + { + "type": "console", + "enabled": true, + "_note": "Terminal-based chat. No setup needed. Good for testing." + }, { "type": "whatsapp", - "enabled": true + "enabled": false, + "_note": "WhatsApp via whatsapp-web.js. Displays QR code on first run — scan with WhatsApp > Linked Devices.", + "options": { + "sessionName": "openbridge-default", + "_sessionName_note": "Identifier for persistent login. Change to run multiple WhatsApp accounts.", + "headless": true, + "_headless_note": "Run browser in headless mode. Set false to see the browser window (debugging).", + "reconnect": { + "enabled": true, + "maxAttempts": 10, + "initialDelayMs": 2000, + "maxDelayMs": 60000, + "backoffFactor": 2, + "_note": "Exponential backoff reconnection. initialDelayMs * backoffFactor^attempt, capped at maxDelayMs." + } + } + }, + { + "type": "telegram", + "enabled": false, + "_note": "Telegram bot. Get a token from @BotFather.", + "options": { + "token": "YOUR_TELEGRAM_BOT_TOKEN_HERE", + "botUsername": "your_bot_name", + "_botUsername_note": "Without the @. Used for group mention detection (e.g. @your_bot_name in group chats)." + } + }, + { + "type": "discord", + "enabled": false, + "_note": "Discord bot. Get a token from the Discord Developer Portal.", + "options": { + "token": "YOUR_DISCORD_BOT_TOKEN_HERE", + "applicationId": "YOUR_APPLICATION_ID", + "_applicationId_note": "Optional. Application ID from the Developer Portal, for advanced features like slash commands." + } + }, + { + "type": "webchat", + "enabled": false, + "_note": "Browser-based chat UI served over HTTP + WebSocket.", + "options": { + "port": 3000, + "_port_note": "HTTP server port for the WebChat UI.", + "host": "0.0.0.0", + "_host_note": "Bind address. Use '0.0.0.0' for all interfaces or 'localhost' to restrict to local-only access.", + "_password_note": "Optional: set 'password' to enable login-screen auth instead of the default token-based auth. Token auth auto-generates a 64-char hex token saved to .openbridge/webchat-token — share the URL with ?token=. Password auth shows a login screen; sessions last 24 hours; 5 failed attempts in 15 min = 30 min IP block. Remove this field to use token auth (default).", + "password": "your-secure-webchat-password" + } } ], + "auth": { "whitelist": ["+1234567890"], - "prefix": "/ai" + "_whitelist_note": "Phone numbers (E.164 format) or user IDs with access. Required. At least one entry.", + + "prefix": "/ai", + "_prefix_note": "Command prefix for messages. Messages starting with this prefix are routed to the AI. Default: '/ai'.", + + "defaultRole": "owner", + "_defaultRole_note": "Role auto-assigned to whitelisted users. Options: owner (full access), admin (manage users + tasks), developer (read + write code), viewer (read-only). Default: 'owner'.", + + "channelRoles": { + "webchat": "owner", + "telegram": "admin" + }, + "_channelRoles_note": "Per-channel role overrides — takes precedence over defaultRole. Useful when webchat users should have different access than WhatsApp users.", + + "pairingEnabled": true, + "_pairingEnabled_note": "Enable 6-digit pairing code for unknown users. When true, unauthenticated users can request a code and be added to the whitelist. Default: true.", + + "rateLimit": { + "enabled": true, + "maxMessages": 10, + "windowMs": 60000, + "_note": "Per-user rate limiting. maxMessages per windowMs (in milliseconds). Default: 10 messages per 60 seconds." + }, + + "commandFilter": { + "allowPatterns": [], + "denyPatterns": [], + "denyMessage": "That command is not allowed.", + "_note": "Regex patterns to allow or deny specific commands. denyPatterns are checked first. If allowPatterns is non-empty, only matching commands are allowed." + } + }, + + "__________ OPTIONAL __________": "Everything below is optional with sensible defaults", + + "security": { + "trustLevel": "standard", + "_trustLevel_note": "AI autonomy level. Options: 'sandbox' (read-only agents, safest), 'standard' (confirmation gates for risky ops, default), 'trusted' (full AI autonomy within workspace, auto-sets confirmHighRisk=false).", + + "confirmHighRisk": true, + "_confirmHighRisk_note": "Require user confirmation before high-risk operations (file deletion, git push, etc.). Auto-set to false when trustLevel is 'trusted'. Default: true.", + + "envDenyPatterns": [ + "AWS_*", + "GITHUB_*", + "GH_*", + "TOKEN*", + "*_TOKEN", + "SECRET*", + "*_SECRET", + "PASSWORD*", + "*_PASSWORD", + "PRIVATE_*", + "DB_*", + "DATABASE_*", + "SMTP_*", + "OPENAI_*", + "ANTHROPIC_*", + "API*KEY*", + "*_CREDENTIAL", + "REDIS_*", + "MONGO_*", + "MYSQL_*", + "POSTGRES_*" + ], + "_envDenyPatterns_note": "Glob patterns for env vars to strip from worker processes. Prevents API keys and secrets from leaking to AI agents. These are the defaults — override to customize.", + + "envAllowPatterns": ["GITHUB_ACTIONS", "GITHUB_WORKSPACE"], + "_envAllowPatterns_note": "Env vars to always allow even if they match a deny pattern. Whitelist overrides deny.", + + "sensitiveFileExceptions": [".env.example", ".env.sample", ".env.template"], + "_sensitiveFileExceptions_note": "File basename patterns excluded from sensitive file detection. Workers can read these even though they look like .env files.", + + "sandbox": { + "mode": "none", + "_mode_note": "Worker isolation mode. 'none' (no isolation, default), 'docker' (Docker containers), 'bubblewrap' (Linux namespaces, Linux-only).", + + "network": "none", + "_network_note": "Worker network access inside sandbox. 'none' (blocked), 'host' (full host network), 'bridge' (Docker bridge only).", + + "memoryMB": 512, + "_memoryMB_note": "Memory limit per worker container in MB. Default: 512.", + + "cpus": 1, + "_cpus_note": "CPU limit per worker. Supports fractions (e.g., 0.5 for half a core). Default: 1." + } + }, + + "master": { + "tool": "claude", + "_tool_note": "Force a specific AI tool as Master. Options: 'claude', 'codex', 'aider'. If omitted, auto-detected from installed tools (picks most capable).", + + "excludeTools": [], + "_excludeTools_note": "Tools to exclude from auto-discovery. Example: ['claude'] to force Codex as Master.", + + "explorationPrompt": "", + "_explorationPrompt_note": "Custom prompt for workspace exploration. If empty, uses the built-in 5-phase exploration prompt.", + + "sessionTtlMs": 1800000, + "_sessionTtlMs_note": "Master session timeout in milliseconds. Default: 1,800,000 (30 minutes). After this, session is renewed.", + + "workerWatchdogMinutes": { + "readOnly": 10, + "codeEdit": 30, + "_note": "Force-kill timeout for stuck workers, in minutes. readOnly: simple read tasks. codeEdit: code modification + full-access tasks." + }, + + "workerCostCaps": { + "read-only": 0.5, + "code-edit": 1.0, + "code-audit": 1.0, + "full-access": 2.0, + "_note": "Per-worker cost cap in USD. Worker receives SIGTERM if it exceeds its cap. Override individual profiles as needed." + } + }, + + "workspace": { + "pullInterval": 300, + "_pullInterval_note": "Auto-pull interval in seconds for git-based workspaces. Set to 0 to disable. Default: 300 (5 minutes).", + + "include": [], + "_include_note": "Glob patterns for files to include. If non-empty, only matching files are visible to AI. Example: ['src/**', 'docs/**'].", + + "exclude": [], + "_exclude_note": "Glob patterns for files to exclude. Combined with built-in excludes (.env, *.pem, *.key, node_modules/, .git/objects/, etc.). Example: ['tests/**', '*.log']." + }, + + "mcp": { + "enabled": false, + "_enabled_note": "Enable MCP (Model Context Protocol) integration. Allows workers to access external services like Gmail, Canva, Slack via MCP servers. Claude-only feature.", + + "servers": [ + { + "name": "filesystem", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-filesystem", "/tmp"], + "_note": "Each server needs: name (identifier), command (executable), args (arguments). Optional: env (environment variables)." + }, + { + "name": "my-service", + "command": "npx", + "args": ["-y", "my-mcp-server"], + "env": { + "MY_SERVICE_API_KEY": "your-api-key-here" + } + }, + { + "name": "gmail", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-gmail"], + "env": { + "GMAIL_OAUTH_CLIENT_ID": "", + "GMAIL_OAUTH_CLIENT_SECRET": "", + "GMAIL_OAUTH_REFRESH_TOKEN": "" + } + } + ], + + "configPath": "~/.claude/claude_desktop_config.json", + "_configPath_note": "Path to an external MCP config file (Claude Desktop format). If both servers[] and configPath are set, external config is loaded first, then inline servers override same-name imports." + }, + + "memory": { + "embedding": { + "provider": "none", + "_provider_note": "Embedding provider for vector search. 'none' (FTS5-only, zero dependencies, default), 'local' (Ollama with nomic-embed-text, 768 dims), 'openai' (text-embedding-3-small, 1536 dims, requires OPENAI_API_KEY env var).", + + "model": "", + "_model_note": "Model override. Defaults: local → 'nomic-embed-text', openai → 'text-embedding-3-small'. Leave empty to use default.", + + "batchSize": 50, + "_batchSize_note": "Number of chunks to embed per batch call. Default: 50.", + + "dimensions": 0, + "_dimensions_note": "Vector dimensions. Auto-inferred from provider/model if 0 or omitted. local=768, openai=1536." + } + }, + + "tunnel": { + "enabled": false, + "_enabled_note": "Expose the local file server to the internet so the Master AI can send public URLs to mobile users (e.g., generated reports, HTML apps).", + + "provider": "auto", + "_provider_note": "Tunnel provider. 'auto' (detect installed tool), 'cloudflared' (free, no signup, preferred), 'ngrok' (requires auth token). Requires the tool to be installed (e.g., brew install cloudflared).", + + "subdomain": "my-openbridge", + "_subdomain_note": "Preferred subdomain hint. Supported by some providers. Remove to use a randomly assigned URL." + }, + + "deep": { + "defaultProfile": "fast", + "_defaultProfile_note": "Deep Mode execution profile. 'fast' (skip Deep Mode, default), 'thorough' (run all 5 phases: investigate → report → plan → execute → verify), 'manual' (pause between phases for user review).", + + "phaseModels": { + "investigate": "powerful", + "report": "balanced", + "plan": "balanced", + "execute": "balanced", + "verify": "fast", + "_note": "Per-phase model tier override. Options: 'fast' (cheapest/fastest), 'balanced' (default), 'powerful' (most capable). Unset phases use 'balanced'." + } + }, + + "batch": { + "maxBatchIterations": 20, + "_maxBatchIterations_note": "Max iterations before pausing batch execution. Default: 20.", + + "batchBudgetUsd": 5.0, + "_batchBudgetUsd_note": "Cumulative cost limit for batch operations in USD. Default: $5.00.", + + "batchTimeoutMinutes": 120, + "_batchTimeoutMinutes_note": "Max elapsed time for batch operations in minutes. Default: 120 (2 hours)." + }, + + "apps": { + "maxConcurrent": 5, + "_maxConcurrent_note": "Max concurrent app processes (e.g., generated web apps). Default: 5.", + + "maxMemoryMB": 256, + "_maxMemoryMB_note": "Memory limit per app process in MB. Default: 256.", + + "idleTimeoutMinutes": 30, + "_idleTimeoutMinutes_note": "Auto-stop app after this many minutes of inactivity. Default: 30." + }, + + "email": { + "host": "smtp.example.com", + "_host_note": "SMTP server hostname for outbound email (used by SHARE email outputs).", + + "port": 587, + "_port_note": "SMTP port. Common values: 587 (TLS), 465 (SSL), 25 (plain). Default: 587.", + + "user": "your-email@example.com", + "pass": "your-smtp-password", + + "from": "openbridge@example.com", + "_from_note": "Sender email address. Must be a valid email format.", + + "allowlist": [], + "_allowlist_note": "Recipient email allowlist. If non-empty, only these addresses can receive emails. Empty = allow all." + }, + + "worker": { + "maxFixIterations": 3, + "_maxFixIterations_note": "Max lint/test fix iterations before a worker escalates back to Master. Set to 0 to disable fix attempts. Default: 3." + }, + + "queue": { + "maxRetries": 3, + "_maxRetries_note": "Max retry attempts for failed messages in the queue. Default: 3.", + + "retryDelayMs": 1000, + "_retryDelayMs_note": "Delay between retries in milliseconds. Default: 1000 (1 second)." + }, + + "router": { + "progressIntervalMs": 15000, + "_progressIntervalMs_note": "How often to send progress updates to the user while waiting for AI response, in milliseconds. Default: 15000 (15 seconds).", + + "escalationTimeoutMs": 300000, + "_escalationTimeoutMs_note": "Timeout before escalating a stuck request to the Master AI, in milliseconds. Default: 300000 (5 minutes)." + }, + + "audit": { + "enabled": true, + "_enabled_note": "Enable audit logging of all message events. Default: true.", + + "logPath": "audit.log", + "_logPath_note": "Path to the audit log file (relative to workspace or absolute). Default: 'audit.log'." + }, + + "health": { + "enabled": false, + "_enabled_note": "Enable health check HTTP endpoint. Useful for monitoring/alerting. Default: false.", + + "port": 8080, + "_port_note": "Port for the health check endpoint. Default: 8080." + }, + + "metrics": { + "enabled": false, + "_enabled_note": "Enable metrics collection endpoint (message counts, latency, error rates). Default: false.", + + "port": 9090, + "_port_note": "Port for the metrics endpoint. Default: 9090." + }, + + "logLevel": "info", + "_logLevel_note": "Log verbosity. Options: 'trace' (most verbose), 'debug', 'info' (default), 'warn', 'error', 'fatal' (least verbose).", + + "__________ ENV OVERRIDES __________": "These environment variables override config.json values", + "_env_overrides": { + "OPENBRIDGE_WORKSPACE_PATH": "Override workspacePath", + "OPENBRIDGE_CHANNELS": "JSON array override (e.g., '[{\"type\":\"console\",\"enabled\":true}]')", + "OPENBRIDGE_AUTH_WHITELIST": "Comma-separated phone numbers (e.g., '+1234567890,+0987654321')", + "OPENBRIDGE_AUTH_PREFIX": "Override prefix (e.g., '/bot')", + "OPENBRIDGE_LOG_LEVEL": "Override log level", + "CONFIG_PATH": "Path to config.json file (default: ./config.json)", + "NODE_ENV": "Set to 'production' to disable auto-injection of WebChat in dev mode" } } diff --git a/config.v0.backup.json b/config.v0.backup.json new file mode 100644 index 00000000..8f37e310 --- /dev/null +++ b/config.v0.backup.json @@ -0,0 +1,58 @@ +{ + "connectors": [ + { + "type": "whatsapp", + "enabled": true, + "options": { + "sessionName": "openbridge-default", + "sessionPath": ".wwebjs_auth" + } + } + ], + "providers": [ + { + "type": "claude-code", + "enabled": true, + "options": { + "workspacePath": "~/Desktop/AI-Bridge/OpenBridge", + "maxTokens": 4096, + "timeout": 600000 + } + } + ], + "defaultProvider": "claude-code", + "workspaces": [ + { + "name": "Social-Media-Automation-Platform", + "path": "~/Desktop/Social-Media-Automation-Platform" + } + ], + "defaultWorkspace": "Social-Media-Automation-Platform", + "auth": { + "whitelist": ["+33617491603", "+33629539495"], + "prefix": "/ai", + "rateLimit": { + "enabled": true, + "maxMessages": 10, + "windowMs": 60000 + }, + "commandFilter": { + "denyPatterns": ["rm\\s+-rf", "drop\\s+table", "format\\s+disk"], + "allowPatterns": [], + "denyMessage": "That command is not allowed." + } + }, + "queue": { + "maxRetries": 3, + "retryDelayMs": 1000 + }, + "audit": { + "enabled": false, + "logPath": "audit.log" + }, + "health": { + "enabled": false, + "port": 8080 + }, + "logLevel": "info" +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..bb430fb7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,92 @@ +version: '3.9' + +# OpenBridge — docker-compose +# +# Quick start: +# OPENBRIDGE_WORKSPACE_PATH=/path/to/project \ +# OPENBRIDGE_AUTH_WHITELIST="+1234567890" \ +# docker compose up -d +# +# View logs: +# docker compose logs -f openbridge +# +# Stop: +# docker compose down + +services: + openbridge: + build: + context: . + dockerfile: Dockerfile + target: production + image: openbridge:latest + container_name: openbridge + restart: unless-stopped + + environment: + # ── Required ────────────────────────────────────────────────────────── + # Path inside the container where your project is mounted (see volumes) + OPENBRIDGE_WORKSPACE_PATH: /workspace + + # Comma-separated list of whitelisted phone numbers (WhatsApp) or + # usernames (Telegram/Discord). E.g. "+1234567890,+0987654321" + OPENBRIDGE_AUTH_WHITELIST: ${OPENBRIDGE_AUTH_WHITELIST:-} + + # ── Optional overrides ──────────────────────────────────────────────── + OPENBRIDGE_AUTH_PREFIX: ${OPENBRIDGE_AUTH_PREFIX:-/ai} + OPENBRIDGE_LOG_LEVEL: ${OPENBRIDGE_LOG_LEVEL:-info} + OPENBRIDGE_HEADLESS: 'true' + NODE_ENV: production + + # ── Channel tokens (set these for Telegram / Discord) ───────────────── + # These are read by connectors; set via .env file or shell env + TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:-} + DISCORD_BOT_TOKEN: ${DISCORD_BOT_TOKEN:-} + + volumes: + # Mount your target project (read-write — AI will modify files) + - ${OPENBRIDGE_WORKSPACE_PATH:-/tmp/workspace}:/workspace + + # Persist WhatsApp session so you only scan QR once + - openbridge-whatsapp:/app/.wwebjs_auth + - openbridge-whatsapp-cache:/app/.wwebjs_cache + + # Persist logs + - openbridge-logs:/app/logs + + ports: + # WebChat UI + health/metrics endpoints + - '${OPENBRIDGE_PORT:-3000}:3000' + + # Health check uses the /health endpoint added in OB-764 + healthcheck: + test: + [ + 'CMD', + 'node', + '-e', + "fetch('http://localhost:3000/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))", + ] + interval: 30s + timeout: 10s + retries: 3 + start_period: 20s + + # ── Optional: Chromium service for WhatsApp Web ────────────────────────── + # Uncomment if you want to run Chrome in a separate container. + # whatsapp-web.js can connect to a remote Chromium via PUPPETEER_WS_ENDPOINT. + # + # chromium: + # image: browserless/chrome:latest + # container_name: openbridge-chromium + # restart: unless-stopped + # environment: + # - MAX_CONCURRENT_SESSIONS=1 + # - CONNECTION_TIMEOUT=60000 + # ports: + # - "3001:3000" + +volumes: + openbridge-whatsapp: + openbridge-whatsapp-cache: + openbridge-logs: diff --git a/docker/Dockerfile.worker b/docker/Dockerfile.worker new file mode 100644 index 00000000..a4629183 --- /dev/null +++ b/docker/Dockerfile.worker @@ -0,0 +1,46 @@ +# OpenBridge Worker Image +# +# Based on node:22-slim to keep image size under 500 MB. +# Installs the Claude CLI via npm, sets /workspace as the working directory, +# and runs as a non-root user for reduced attack surface. +# +# Build: +# docker build -f docker/Dockerfile.worker -t openbridge-worker:latest . +# +# @see OB-1547 + +FROM node:22-slim + +# Install system deps needed by the claude CLI and common AI tools. +# git is required so agents can inspect repos inside /workspace. +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git \ + ca-certificates \ + curl && \ + rm -rf /var/lib/apt/lists/* + +# Install the Claude CLI globally. +# Using --prefer-offline avoids re-downloading registry metadata on each build +# layer if the npm cache is warm. +RUN npm install -g @anthropic-ai/claude-code && \ + npm cache clean --force + +# Create a non-root user to run the agent. +RUN useradd --create-home --shell /bin/bash --uid 1001 worker + +# Create the workspace directory and grant ownership to the worker user. +# Containers mount the host workspace at /workspace at runtime; /workspace +# itself must exist in the image so Docker can bind-mount into it. +RUN mkdir -p /workspace && chown worker:worker /workspace + +# Switch to the non-root user for all subsequent instructions and at runtime. +USER worker + +# Set /workspace as the default working directory so that commands executed +# via `docker exec` or `docker run` start in the right place. +WORKDIR /workspace + +# Default command — kept minimal; the caller (DockerSandbox) will invoke the +# agent via `docker exec` rather than relying on CMD. +CMD ["sleep", "infinity"] diff --git a/docs/API_REFERENCE.md b/docs/API_REFERENCE.md index 2207f1f6..02303bec 100644 --- a/docs/API_REFERENCE.md +++ b/docs/API_REFERENCE.md @@ -14,12 +14,24 @@ This document covers every public interface, class, and configuration schema in - [ConnectorEvents](#connectorevents) - [AIProvider](#aiprovider) - [ProviderResult](#providerresult) + - [SpawnOptions](#spawnoptions) +- [CLI Adapter Layer](#cli-adapter-layer) + - [CLIAdapter](#cliadapter) + - [CLISpawnConfig](#clispawnconfig) + - [CapabilityLevel](#capabilitylevel) + - [AdapterRegistry](#adapterregistry) + - [CodexAdapter](#codexadapter) +- [Built-in Providers](#built-in-providers) + - [CodexProvider](#codexprovider) + - [CodexConfig](#codexconfig) - [Discovery Types](#discovery-types) - [DiscoveredTool](#discoveredtool) - [ScanResult](#scanresult) - [Master AI Types](#master-ai-types) - [MasterState](#masterstate) - [ExplorationSummary](#explorationsummary) + - [TaskManifest](#taskmanifest) + - [MasterSystemPromptContext](#mastersystempromptcontext) - [Core Classes](#core-classes) - [Bridge](#bridge) - [Router](#router) @@ -46,13 +58,30 @@ This document covers every public interface, class, and configuration schema in - [AuditConfig](#auditconfig) - [HealthConfig](#healthconfig) - [MetricsConfig](#metricsconfig) + - [MCPServer](#mcpserver) + - [MCPConfig](#mcpconfig) +- [Memory Classes](#memory-classes) + - [ConversationStore](#conversationstore) + - [listSessions()](#listsessions) + - [searchSessions()](#searchsessions) + - [searchConversations()](#searchconversations) + - [getSessionHistory()](#getsessionhistory) + - [getRecentMessages()](#getrecentmessages) + - [MemoryManager](#memorymanager) +- [Master AI Classes](#master-ai-classes) + - [MasterManager](#mastermanager) + - [DotFolderManager](#dotfoldermanager) - [Utility Functions](#utility-functions) - [loadConfig()](#loadconfig) + - [getMcpConfigPath()](#getmcpconfigpath) - [scanForAITools()](#scanforaitools) - [createLogger()](#createlogger) +- [Built-in Commands](#built-in-commands) + - [/history](#history-command) - [HTTP Endpoints](#http-endpoints) - [Health Check](#health-check-endpoint) - [Metrics](#metrics-endpoint) + - [Sessions (WebChat)](#sessions-endpoints-webchat) --- @@ -150,6 +179,201 @@ Result returned by an AI provider after processing. | `usage?` | `Record` | Token count or similar usage data | | `[key]` | `unknown` | Provider-specific data | +### SpawnOptions + +_Source: `src/core/agent-runner.ts`_ + +Options accepted by `AgentRunner.spawn()`. Adapter-specific fields are silently dropped by adapters that don't support them. + +| Property | Type | Default | Description | +| ------------------ | ---------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `prompt` | `string` | — | The prompt to send to the AI agent | +| `workspacePath` | `string` | — | Working directory for the agent process | +| `model?` | `string` | — | Tier alias (`'haiku'`, `'sonnet'`, `'opus'`) or a full model ID | +| `allowedTools?` | `string[]` | — | Tools the agent may use. Mapped to `--allowedTools` (Claude) or sandbox mode (Codex) | +| `maxTurns?` | `number` | — | Max agentic turns before stopping. Claude only — dropped by Codex/Aider adapters | +| `timeout?` | `number` | — | Timeout in milliseconds per attempt | +| `retries?` | `number` | `3` | Number of retry attempts on non-zero exit codes | +| `retryDelay?` | `number` | `10000` | Delay in milliseconds between retry attempts | +| `logFile?` | `string` | — | Path to write the full agent log output | +| `sessionId?` | `string` | — | Start a new named session (`--session-id` for Claude, named session for Codex) | +| `resumeSessionId?` | `string` | — | Resume a prior session (`--resume` for Claude, `exec resume --last` for Codex) | +| `systemPrompt?` | `string` | — | System prompt injected at the top of the agent context | +| `maxBudgetUsd?` | `number` | — | Max spend in USD (`--max-budget-usd` for Claude; dropped by Codex/Aider) | +| `mcpConfigPath?` | `string` | — | Path to MCP config JSON (`--mcp-config` for Claude, `-c` for Codex; Aider drops it) | +| `strictMcpConfig?` | `boolean` | — | When `true`, passes `--strict-mcp-config` to Claude — isolates the worker from any global MCP configs (e.g. `~/.claude/claude_desktop_config.json`). Always set to `true` by `manifestToSpawnOptions()` when a per-worker temp MCP config is generated. | + +--- + +## CLI Adapter Layer + +_Source: `src/core/cli-adapter.ts`, `src/core/adapter-registry.ts`, `src/core/adapters/`_ + +The CLI Adapter layer translates provider-neutral `SpawnOptions` into tool-specific binary, args, and env for each supported AI CLI (`claude`, `codex`, `aider`). This abstraction lets the rest of OpenBridge remain tool-agnostic. + +``` +SpawnOptions → CLIAdapter.buildSpawnConfig() → CLISpawnConfig → child_process.spawn() +``` + +### CLIAdapter + +Interface that every CLI adapter must implement. + +| Member | Type | Description | +| --------------------------- | ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | `readonly string` | Provider name matching `DiscoveredTool.name` (e.g. `'claude'`, `'codex'`, `'aider'`) | +| `buildSpawnConfig(opts)` | `(opts: SpawnOptions) => CLISpawnConfig` | Translate `SpawnOptions` into the binary, args, and env for this CLI | +| `cleanEnv(env)` | `(env: Record) => Record<...>` | Strip env vars that would cause conflicts for this CLI | +| `mapCapabilityLevel(level)` | `(level: CapabilityLevel) => string[] \| undefined` | Map capability level to CLI-specific access restrictions. Returns `undefined` if the CLI doesn't use tool lists (e.g. Codex uses sandbox modes instead) | +| `isValidModel(model)` | `(model: string) => boolean` | Return `true` if the model string is recognized by this CLI | + +### CLISpawnConfig + +The output of `CLIAdapter.buildSpawnConfig()` — everything needed to call `child_process.spawn()`. + +| Property | Type | Description | +| -------------- | ------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `binary` | `string` | Command name or absolute path to the binary | +| `args` | `string[]` | CLI arguments array | +| `env` | `Record` | Environment variables (cleaned of conflicting vars) | +| `stdin?` | `'ignore' \| 'pipe'` | stdin behavior. `'ignore'` closes stdin (default for Claude/Codex). `'pipe'` provides a writable stream for CLIs that require TTY detection. | +| `parseOutput?` | `(stdout: string) => string` | Optional post-processor for raw stdout. Used by Codex to extract the final message from `--json` JSONL. Falls back to raw stdout if absent or if the function throws. | + +### CapabilityLevel + +```typescript +type CapabilityLevel = 'read-only' | 'code-edit' | 'full-access'; +``` + +Maps tool profiles to CLI-specific access mechanisms: + +| Level | Claude (`--allowedTools`) | Codex (`--sandbox`) | +| --------------- | --------------------------------------------------------- | --------------------------- | +| `'read-only'` | `Read, Glob, Grep` | `read-only` | +| `'code-edit'` | `Read, Edit, Write, Glob, Grep, Bash(git:*), Bash(npm:*)` | `workspace-write` | +| `'full-access'` | All tools | `--full-auto` (danger mode) | + +### AdapterRegistry + +_Source: `src/core/adapter-registry.ts`_ + +Maps discovered tool names to `CLIAdapter` instances. Built-in adapters (`claude`, `codex`, `aider`) are lazy-loaded on first access. Custom adapters registered via `register()` take priority over built-ins. + +**Methods:** + +| Method | Returns | Description | +| ------------------------- | ------------------------- | ------------------------------------------------------------------- | +| `register(name, adapter)` | `void` | Register a `CLIAdapter` for a tool name (overrides built-ins) | +| `get(name)` | `CLIAdapter \| undefined` | Get the adapter for a tool name, creating from built-ins if needed | +| `getForTool(tool)` | `CLIAdapter \| undefined` | Get the adapter for a `DiscoveredTool` | +| `has(name)` | `boolean` | Check if an adapter exists for a tool name (registered or built-in) | + +**Factory:** + +```typescript +function createAdapterRegistry(): AdapterRegistry; +``` + +Creates an `AdapterRegistry` pre-loaded with the `ClaudeAdapter`. + +**Built-in adapters:** + +| Tool name | Adapter | Source | +| --------- | --------------- | ------------------------------------- | +| `claude` | `ClaudeAdapter` | `src/core/adapters/claude-adapter.ts` | +| `codex` | `CodexAdapter` | `src/core/adapters/codex-adapter.ts` | +| `aider` | `AiderAdapter` | `src/core/adapters/aider-adapter.ts` | + +### CodexAdapter + +_Source: `src/core/adapters/codex-adapter.ts`_ + +Translates `SpawnOptions` into `codex exec` arguments for non-interactive Codex CLI execution. + +**Flags always included:** + +| Flag | Value | Reason | +| ----------------------- | ------------------------ | ------------------------------------------------------------------------------ | +| `--skip-git-repo-check` | (boolean flag) | Required for non-git or untrusted workspaces — Codex refuses to run without it | +| `--json` | (boolean flag) | Enables JSONL structured output (one JSON event per line to stdout) | +| `-o ` | Auto-generated temp path | Reliable final-answer capture. Codex writes the last message to this file | +| `--ephemeral` | (worker spawns only) | Suppresses session persistence for short-lived worker invocations | + +**Flags set conditionally:** + +| Flag | Condition | Value | +| ------------------ | ------------------------------------- | ---------------------------------------------------------- | +| `--model ` | `opts.model` is set | Model string passed through | +| `--sandbox ` | `allowedTools` present, not `Bash(*)` | `read-only` or `workspace-write` | +| `--full-auto` | `Bash(*)` in `allowedTools` | Enables auto-approve + full sandbox (`danger-full-access`) | +| `-c ` | `opts.mcpConfigPath` is set | MCP config file for Codex-native MCP passthrough | + +**Sandbox inference** from `allowedTools`: + +| `allowedTools` content | Sandbox mode | +| ------------------------------------ | ---------------------------------------- | +| `Bash(*)` present | `danger-full-access` (via `--full-auto`) | +| `Edit` or `Write` present | `workspace-write` | +| Empty, undefined, or read-only tools | `read-only` (safe default) | + +**Output parsing priority:** + +1. Read the `-o` temp file — Codex's most reliable output path +2. Fall back to `--json` JSONL parsing (`parseCodexJsonlOutput()`) if the temp file is absent +3. Fall back to raw stdout if no parseable `type: "message"` event is found + +**OPENAI_API_KEY validation:** `buildSpawnConfig()` throws immediately if `OPENAI_API_KEY` is not set, preventing confusing downstream auth failures. + +**Valid models** (Codex CLI v0.104.0): `gpt-5.2-codex` (default), `o3`, `o4-mini`. Any model ID matching `/^(gpt-|o[0-9]|codex)/` is also accepted for forward compatibility. + +--- + +## Built-in Providers + +### CodexProvider + +_Source: `src/providers/codex/codex-provider.ts`_ + +`AIProvider` implementation for the OpenAI Codex CLI. Uses `AgentRunner` + `CodexAdapter` internally — the same pattern as `ClaudeCodeProvider`. + +```typescript +import { CodexProvider } from './providers/codex/index.js'; + +const provider = new CodexProvider({ + workspacePath: '/path/to/project', + timeout: 120000, + model: 'gpt-5.2-codex', +}); + +await provider.initialize(); +``` + +**Methods:** + +| Method | Returns | Description | +| ------------------------- | ---------------------------------------- | ---------------------------------------------------------------------------------------------------- | +| `initialize()` | `Promise` | Validates `workspacePath` is accessible | +| `processMessage(message)` | `Promise` | Runs `codex exec` with session wiring and returns the AI response | +| `streamMessage(message)` | `AsyncGenerator` | Falls back to `processMessage()` — yields the result in one chunk (Codex has no real-time streaming) | +| `isAvailable()` | `Promise` | Returns `true` if `codex` binary is on PATH and `OPENAI_API_KEY` is set | +| `shutdown()` | `Promise` | Clears all active sessions | + +**Session management:** Sessions are scoped by `sender:workspacePath`. The first message in a session window starts a new Codex session; follow-up messages use `codex exec resume --last`. Sessions expire after the configured TTL (default 30 minutes). + +### CodexConfig + +_Source: `src/providers/codex/codex-config.ts`_ + +Zod schema for `CodexProvider` constructor options. + +| Property | Type | Default | Description | +| --------------- | -------- | ------------------ | ---------------------------------------------------------------------------------- | +| `workspacePath` | `string` | `'.'` | Working directory for Codex invocations. Supports `~/` home directory expansion | +| `timeout` | `number` | `120000` (2 min) | Timeout per invocation in milliseconds | +| `model?` | `string` | — | Codex model override. Defaults to the Codex CLI's built-in default | +| `sandbox?` | `string` | — | Sandbox mode override (`'read-only'`, `'workspace-write'`, `'danger-full-access'`) | +| `sessionTtlMs` | `number` | `1800000` (30 min) | Session inactivity TTL in milliseconds | + --- ## Discovery Types @@ -205,6 +429,41 @@ Summary of the Master AI's workspace exploration. | `frameworks` | `string[]` | Detected frameworks | | `exploredAt` | `string` | ISO 8601 timestamp of exploration | +### TaskManifest + +_Source: `src/types/agent.ts`_ + +Describes everything needed to spawn a worker agent. The Master AI produces these; `AgentRunner` consumes them via `manifestToSpawnOptions()`. + +| Property | Type | Description | +| --------------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | +| `prompt` | `string` | The prompt to send to the worker agent (required) | +| `workspacePath` | `string` | Working directory for the worker (required) | +| `model?` | `string` | Tier alias (`'haiku'`, `'sonnet'`, `'opus'`) or a full model ID | +| `profile?` | `string` | Named tool profile — resolved to `allowedTools` by `AgentRunner` | +| `allowedTools?` | `string[]` | Explicit tools list — overrides `profile` if both are provided | +| `maxTurns?` | `number` | Maximum number of agentic turns | +| `timeout?` | `number` | Timeout in milliseconds per attempt | +| `retries?` | `number` | Number of retry attempts on failure | +| `retryDelay?` | `number` | Delay in milliseconds between retries | +| `maxBudgetUsd?` | `number` | Maximum spend in USD (`--max-budget-usd` for Claude; dropped by Codex/Aider) | +| `mcpServers?` | `MCPServer[]` | MCP servers this worker may use. `manifestToSpawnOptions()` writes a per-worker temp config with only these servers and sets `strictMcpConfig: true`. | + +### MasterSystemPromptContext + +_Source: `src/master/master-system-prompt.ts`_ + +Context object passed to `generateMasterSystemPrompt()` to build the Master AI's system prompt. + +| Property | Type | Description | +| ------------------ | ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `workspacePath` | `string` | Absolute path to the target workspace | +| `masterToolName` | `string` | The Master AI tool's name (e.g. `'claude'`, `'codex'`) | +| `discoveredTools?` | `DiscoveredTool[]` | All AI tools found on the machine | +| `customProfiles?` | `object` | Custom tool profiles for the Master to assign to workers | +| `modelRegistry?` | `ModelRegistry` | Provider-agnostic model registry for resolving tier aliases | +| `mcpServers?` | `MCPServer[]` | MCP servers available for workers (sourced from `V2Config.mcp.servers`). When non-empty, the system prompt gains an "Available MCP Servers" section listing each server name. The Master can then assign servers to workers via the `mcpServers` field of `TaskManifest`. Omit or pass `[]` to suppress the section. | + --- ## Core Classes @@ -240,6 +499,35 @@ await bridge.stop(); | `start()` | `Promise` | Initialize all plugins, start processing | | `stop()` | `Promise` | Drain queue, shut down Master + plugins gracefully | +#### Shutdown Behavior + +_Source: `src/index.ts`_ + +OpenBridge registers `SIGINT` and `SIGTERM` handlers that trigger a graceful shutdown sequence: + +1. **Double-shutdown guard** — a module-level `shutdownInProgress` flag is set on the first signal. Subsequent signals (e.g., double Ctrl+C) are ignored with a WARN log rather than force-killing the process. +2. **User-facing message** — `console.log('\nShutting down gracefully... please wait')` is printed immediately so users know not to press Ctrl+C again. +3. **10-second timeout** — `bridge.stop()` races against a 10-second `setTimeout`. If the timeout fires first, `console.error('Shutdown timeout exceeded (10s) — forcing exit')` is printed and `process.exit(1)` is called. +4. **Critical-first ordering** — inside `MasterManager.shutdown()`, `saveMasterSessionToStore()` (fast SQLite write, <100ms) runs **before** `triggerMemoryUpdate()` (slow AI spawn, 10-30s). This ensures session state is always persisted even if the memory update is interrupted by the timeout. +5. **Memory update failure isolation** — `triggerMemoryUpdate()` on shutdown is wrapped in `try/catch` so its failure cannot block or throw past `MasterManager.shutdown()`. + +**Full shutdown sequence:** + +``` +SIGINT / SIGTERM + → shutdownInProgress guard (no-op if already set) + → console.log "Shutting down gracefully..." + → workspaceManager.stopPolling() + → Promise.race([bridge.stop(), 10s timeout]) + → bridge.stop() + → queue.drain() + → MasterManager.shutdown() + → saveMasterSessionToStore() ← always runs (critical) + → triggerMemoryUpdate() ← may be skipped by timeout + → connectors[].shutdown() + → process.exit(0) +``` + --- ### Router @@ -385,13 +673,35 @@ HTTP server exposing bridge health status as JSON. **HealthStatus** (response shape): -| Property | Type | Description | -| ------------ | ---------------------------------------- | ---------------------------- | -| `status` | `'healthy' \| 'degraded' \| 'unhealthy'` | Overall bridge status | -| `uptime` | `number` | Seconds since bridge started | -| `timestamp` | `string` | ISO 8601 timestamp | -| `connectors` | `ComponentStatus[]` | Status of each connector | -| `queue` | `object` | Queue state | +| Property | Type | Description | +| ----------------- | ---------------------------------------- | -------------------------------------------------------------------------------------- | +| `status` | `'healthy' \| 'degraded' \| 'unhealthy'` | Overall bridge status | +| `uptime_seconds` | `number` | Seconds since bridge started | +| `memory_mb` | `number` | Resident memory usage in MB | +| `active_workers` | `number` | Current number of active worker agents | +| `master_status` | `string` | Master AI lifecycle state | +| `db_status` | `'connected' \| 'disconnected'` | SQLite memory system status | +| `last_message_at` | `string \| null` | ISO 8601 timestamp of the most recent message | +| `timestamp` | `string` | ISO 8601 timestamp of this health check | +| `connectors` | `ComponentStatus[]` | Status of each connector | +| `providers` | `ComponentStatus[]` | Status of each provider | +| `queue` | `object` | Queue state (`pending`, `processing`, `deadLetterSize`) | +| `mcp?` | `object` | MCP server health (present only when MCP servers are configured via `setMcpServers()`) | + +**`mcp` sub-object:** + +| Property | Type | Description | +| --------- | ------------------- | ------------------------------------------- | +| `enabled` | `boolean` | Whether MCP is enabled | +| `servers` | `McpServerStatus[]` | Health status of each configured MCP server | + +**`McpServerStatus`:** + +| Property | Type | Description | +| --------- | ------------------------- | --------------------------------------------------------- | +| `name` | `string` | MCP server name (from config) | +| `status` | `'configured' \| 'error'` | `'configured'` if command found on PATH, `'error'` if not | +| `command` | `string` | The server command being checked (e.g. `npx`) | --- @@ -465,18 +775,19 @@ All configuration is validated at startup using [Zod](https://zod.dev) schemas. The simplified config format. Three required fields. -| Property | Type | Default | Description | -| --------------- | -------- | --------- | ---------------------------------------------- | -| `workspacePath` | `string` | — | Absolute path to the target project (required) | -| `channels` | `array` | — | At least one channel config (required) | -| `auth` | `object` | — | Authentication settings (required) | -| `master?` | `object` | `{}` | Override auto-detected Master AI settings | -| `queue?` | `object` | see below | Queue retry configuration | -| `router?` | `object` | see below | Router configuration | -| `audit?` | `object` | see below | Audit logging configuration | -| `health?` | `object` | see below | Health check endpoint config | -| `metrics?` | `object` | see below | Metrics endpoint configuration | -| `logLevel?` | `string` | `'info'` | One of: trace, debug, info, warn, error, fatal | +| Property | Type | Default | Description | +| --------------- | ----------- | --------- | ------------------------------------------------------ | +| `workspacePath` | `string` | — | Absolute path to the target project (required) | +| `channels` | `array` | — | At least one channel config (required) | +| `auth` | `object` | — | Authentication settings (required) | +| `master?` | `object` | `{}` | Override auto-detected Master AI settings | +| `mcp?` | `MCPConfig` | — | MCP server configuration (see [MCPConfig](#mcpconfig)) | +| `queue?` | `object` | see below | Queue retry configuration | +| `router?` | `object` | see below | Router configuration | +| `audit?` | `object` | see below | Audit logging configuration | +| `health?` | `object` | see below | Health check endpoint config | +| `metrics?` | `object` | see below | Metrics endpoint configuration | +| `logLevel?` | `string` | `'info'` | One of: trace, debug, info, warn, error, fatal | **Channel config:** @@ -562,6 +873,326 @@ The original config format. Still fully supported — auto-detected by the confi | `enabled?` | `boolean` | `false` | Enable/disable metrics endpoint | | `port?` | `number` | `9090` | HTTP port for the metrics endpoint | +### MCPServer + +_Source: `src/types/config.ts`_ + +A single MCP server definition, used in the `mcp.servers` array of `AppConfigV2`. + +| Property | Type | Description | +| --------- | ------------------------ | ----------------------------------------------------------------------------------- | +| `name` | `string` | Unique server name — used by the Master AI to reference the server | +| `command` | `string` | Command to launch the MCP server (e.g. `npx`, `/usr/local/bin/my-mcp`) | +| `args?` | `string[]` | CLI arguments for the server command (e.g. `['-y', '@anthropic/canva-mcp-server']`) | +| `env?` | `Record` | Environment variables injected when the MCP server process is started | + +### MCPConfig + +_Source: `src/types/config.ts`_ + +The `mcp` section of `AppConfigV2`. Controls which MCP servers are available to workers. + +| Property | Type | Default | Description | +| ------------- | ------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | +| `enabled?` | `boolean` | `true` | Enable or disable MCP support. When `false`, no MCP config is written at startup | +| `servers?` | `MCPServer[]` | `[]` | Inline server definitions | +| `configPath?` | `string` | — | Path to an existing Claude Desktop or Claude Code MCP config file to import. Inline `servers` override same-name entries from the imported file. | + +At bridge startup, `loadConfig()` transforms the `mcp` section into a global `.openbridge/mcp-config.json` file in the Claude CLI format. Use `getMcpConfigPath()` to retrieve the resulting file path. + +--- + +## Memory Classes + +### ConversationStore + +_Source: `src/memory/conversation-store.ts`_ + +Functions for querying conversation history stored in the SQLite memory database. + +#### listSessions() + +```typescript +function listSessions(db: Database.Database, limit?: number, offset?: number): SessionSummary[]; +``` + +Returns a paginated list of conversation sessions ordered by most recent activity. + +| Parameter | Type | Default | Description | +| --------- | ------------------- | ------- | -------------------------- | +| `db` | `Database.Database` | — | SQLite database instance | +| `limit` | `number` | `20` | Maximum sessions to return | +| `offset` | `number` | `0` | Pagination offset | + +**Returns:** `SessionSummary[]` — sessions ordered by `last_message_at DESC`. + +**SessionSummary:** + +| Property | Type | Description | +| ------------------ | ---------------- | --------------------------------------------- | +| `session_id` | `string` | Unique session identifier | +| `title` | `string \| null` | Session title (first user message, ≤50 chars) | +| `first_message_at` | `string` | ISO 8601 timestamp of first message | +| `last_message_at` | `string` | ISO 8601 timestamp of most recent message | +| `message_count` | `number` | Total messages in the session | +| `channel` | `string \| null` | Channel that originated the session | +| `user_id` | `string \| null` | Sender identifier | + +#### searchSessions() + +```typescript +function searchSessions(db: Database.Database, query: string, limit?: number): SessionSummary[]; +``` + +Full-text search over conversation content returning session-level results via FTS5. + +| Parameter | Type | Default | Description | +| --------- | ------------------- | ------- | -------------------------- | +| `db` | `Database.Database` | — | SQLite database instance | +| `query` | `string` | — | Search query string | +| `limit` | `number` | `10` | Maximum sessions to return | + +**Returns:** `SessionSummary[]` ranked by number of matching messages, then by recency. Returns `[]` if query is empty or no matches found. The query is passed through `sanitizeFts5Query()` before the FTS5 `MATCH` clause — special characters (`'`, `"`, `*`, `AND`, `OR`, `NOT`, `(`, `)`, etc.) are stripped and each token is quoted to prevent `SqliteError: fts5: syntax error` (Phase 63 fix). + +#### getSessionHistory() + +```typescript +function getSessionHistory( + db: Database.Database, + sessionId: string, + limit?: number, +): ConversationEntry[]; +``` + +Retrieves the full message transcript for one conversation session. + +| Parameter | Type | Default | Description | +| ----------- | ------------------- | ------- | ----------------------------------------------------------- | +| `db` | `Database.Database` | — | SQLite database instance | +| `sessionId` | `string` | — | Session ID to retrieve | +| `limit` | `number` | `50` | Maximum messages to return (most recent, then oldest-first) | + +**Returns:** `ConversationEntry[]` in chronological order (oldest first). Returns `[]` if session not found. + +**ConversationEntry:** + +| Property | Type | Description | +| ------------ | -------------------------------------------- | -------------------------------- | +| `id` | `number` | Auto-increment row ID | +| `session_id` | `string` | Parent session identifier | +| `role` | `'user' \| 'master' \| 'worker' \| 'system'` | Who sent this message | +| `content` | `string` | Message text | +| `channel` | `string \| undefined` | Channel that carried the message | +| `user_id` | `string \| undefined` | Sender identifier | +| `created_at` | `string` | ISO 8601 timestamp | + +#### searchConversations() + +_Source: `src/memory/retrieval.ts`_ + +```typescript +function searchConversations( + db: Database.Database, + query: string, + limit?: number, +): ConversationEntry[]; +``` + +Full-text search over individual conversation messages using the `conversations_fts` FTS5 index. Results are BM25-ranked for relevance, then sorted by recency within equal-relevance groups. + +| Parameter | Type | Default | Description | +| --------- | ------------------- | ------- | -------------------------- | +| `db` | `Database.Database` | — | SQLite database instance | +| `query` | `string` | — | Search query string | +| `limit` | `number` | `10` | Maximum messages to return | + +**Returns:** `ConversationEntry[]` BM25-ranked, then by `created_at DESC`. Returns `[]` if query is empty or all special characters. The query is passed through `sanitizeFts5Query()` before the FTS5 `MATCH` clause — this prevents `SqliteError: fts5: syntax error` for messages containing `'`, `"`, `*`, `AND`, `OR`, `NOT`, `(`, `)`, etc. (Phase 63 fix). Previously, special characters in user messages silently caused cross-session context injection to fail. + +#### getRecentMessages() + +_Source: `src/memory/conversation-store.ts`_ + +```typescript +function getRecentMessages(db: Database.Database, limit?: number): ConversationEntry[]; +``` + +Returns the most recent messages across all sessions, filtered to `user` and `master` roles only, in chronological order (oldest → newest). Used by `triggerMemoryUpdate()` to give the stateless `--print` agent conversation context to write meaningful memory notes. + +| Parameter | Type | Default | Description | +| --------- | ------------------- | ------- | ------------------------------------------ | +| `db` | `Database.Database` | — | SQLite database instance | +| `limit` | `number` | `20` | Maximum messages to return (most recent N) | + +**Returns:** `ConversationEntry[]` in chronological order (oldest first). The query selects by `created_at DESC` then `.reverse()` is applied to restore chronological order. Only `role IN ('user', 'master')` — `worker` and `system` messages are excluded. + +--- + +### MemoryManager + +_Source: `src/memory/index.ts`_ + +Facade class over all SQLite store modules. Provides a unified async API for the rest of OpenBridge. Initialized with a `Database.Database` instance; all methods throw if the DB is not initialized. + +#### getRecentMessages() + +```typescript +async getRecentMessages(limit?: number): Promise +``` + +Returns the most recent user + master messages across all sessions in chronological order (oldest → newest). Delegates to the `getRecentMessages()` store function. Called by `MasterManager.triggerMemoryUpdate()` to inject conversation context into the memory-update prompt. + +| Parameter | Type | Default | Description | +| --------- | -------- | ------- | -------------------------- | +| `limit` | `number` | `20` | Maximum messages to return | + +**Returns:** `Promise` — rejects with `Error('MemoryManager not initialised')` if `db` is null. + +--- + +## Master AI Classes + +### MasterManager + +_Source: `src/master/master-manager.ts`_ + +The core Master AI lifecycle manager. Handles session initialization, worker spawning, exploration coordination, and graceful shutdown. 6155 LOC. + +#### triggerMemoryUpdate() + +```typescript +private async triggerMemoryUpdate(): Promise +``` + +Spawns a stateless `--print` mode AI agent that reads recent conversation history from SQLite and writes updated notes to `.openbridge/context/memory.md`. Non-blocking — callers fire-and-forget with `void`. + +**When triggered:** + +| Trigger | Condition | +| -------- | ----------------------------------------------------------------- | +| Periodic | Every `MEMORY_UPDATE_INTERVAL` (10) completed tasks | +| Shutdown | In `MasterManager.shutdown()`, after `saveMasterSessionToStore()` | + +**Behavior:** + +1. Returns early if no master session or state is `'shutdown'`. +2. Calls `this.memory?.getRecentMessages(20)` — fetches the last 20 `user`/`master` messages from SQLite. +3. Logs `{ messageCount }` at INFO level. +4. Formats each entry as `[YYYY-MM-DD HH:MM] Role: content` (content truncated to 300 chars with `…`). +5. If messages exist, prepends a `## Recent conversation history:` section to the prompt. +6. Spawns an agent in `--print` mode with `allowedTools: ['Write']` pointing at `memoryPath`. +7. Prompt uses generic language ("Write your updated notes to …") — not Claude-specific tool names — so it works with Codex as Master. + +**Error handling:** Failures are logged as WARN and do not block the caller. The `triggerMemoryUpdate()` call in `shutdown()` is wrapped in `try/catch` so a failed memory update cannot block session state persistence. + +### DotFolderManager + +_Source: `src/master/dotfolder-manager.ts`_ + +Manages the `.openbridge/` folder in the target workspace: exploration state, context memory, and the prompt library. + +#### Prompt Library Methods + +##### readPromptManifest() + +```typescript +readPromptManifest(): Promise +``` + +Reads `.openbridge/prompts/manifest.json` and returns the parsed manifest, or `null` if the file does not exist or fails to parse. + +##### writePromptManifest() + +```typescript +writePromptManifest(manifest: PromptManifest): Promise +``` + +Validates and writes the manifest to `.openbridge/prompts/manifest.json`. Creates the `prompts/` directory if needed. + +##### writePromptTemplate() + +```typescript +writePromptTemplate( + filename: string, + content: string, + metadata: Omit, +): Promise +``` + +Writes a prompt template `.md` file to `.openbridge/prompts/` and creates or updates its entry in the manifest. Preserves `createdAt` when overwriting; sets `previousVersion` and `previousSuccessRate` when updating an existing entry. + +##### getPromptTemplate() + +```typescript +getPromptTemplate(id: string): Promise +``` + +Looks up a prompt template by ID in the manifest. Returns `null` if the manifest does not exist or the ID is not found. + +##### recordPromptUsage() + +```typescript +recordPromptUsage(id: string, success: boolean): Promise +``` + +Increments `usageCount`, conditionally increments `successCount`, recalculates `successRate = successCount / usageCount`, and updates `lastUsedAt`. No-op if the prompt ID is not found. + +##### getLowPerformingPrompts() + +```typescript +getLowPerformingPrompts(threshold: number): Promise +``` + +Returns all prompts where `usageCount >= 3` AND `successRate < threshold`. Used by the prompt evolver to identify candidates for refinement. + +##### resetPromptStats() + +```typescript +resetPromptStats(id: string): Promise +``` + +Zeros `usageCount`, `successCount`, and `successRate`. Preserves the previous value in `previousSuccessRate` before resetting. No-op if the ID is not found. + +#### Memory File Methods + +##### readMemoryFile() + +```typescript +readMemoryFile(): Promise +``` + +Reads `.openbridge/context/memory.md` and returns its content as a string, or `null` if the file does not exist. This file is the Master AI's curated cross-session memory. + +##### writeMemoryFile() + +```typescript +writeMemoryFile(content: string): Promise +``` + +Writes content to `.openbridge/context/memory.md`. Validates that content is at most 200 lines — throws if exceeded. Creates the `context/` directory if it doesn't exist. + +**Prompt Types** — _Source: `src/types/master.ts`_ + +| Type | Description | +| ---------------- | --------------------------------------------- | +| `PromptManifest` | `{ prompts: Record }` | +| `PromptTemplate` | Full metadata object for one prompt template | + +**PromptTemplate fields:** + +| Property | Type | Description | +| --------------------- | ---------------- | ---------------------------------------------- | +| `id` | `string` | Unique prompt identifier | +| `name` | `string` | Human-readable name | +| `filePath` | `string` | Path to the `.md` file | +| `usageCount` | `number` | Total times this prompt was used | +| `successCount` | `number` | Times the prompt produced a successful outcome | +| `successRate` | `number` | `successCount / usageCount` (0–1) | +| `createdAt` | `string` | ISO 8601 creation timestamp | +| `updatedAt` | `string` | ISO 8601 last-modified timestamp | +| `lastUsedAt` | `string \| null` | ISO 8601 last-used timestamp | +| `previousVersion` | `string \| null` | Content of the previous version (on update) | +| `previousSuccessRate` | `number \| null` | Success rate before last reset | + --- ## Utility Functions @@ -576,6 +1207,18 @@ function loadConfig(configPath?: string): Promise; Reads and validates `config.json`. Tries V2 schema first, falls back to V0. Falls back to `CONFIG_PATH` env var, then `./config.json`. +When loading a V2 config with an `mcp` section, `loadConfig()` also writes the global MCP config file (`.openbridge/mcp-config.json`) in the Claude CLI format (`{ mcpServers: { [name]: { command, args?, env? } } }`). If `mcp.configPath` is set, that file is imported and merged with inline `servers` (inline entries win on name conflicts). + +### getMcpConfigPath() + +_Source: `src/core/config.ts`_ + +```typescript +function getMcpConfigPath(): string | null; +``` + +Returns the absolute path to the global MCP config file written by `loadConfig()`, or `null` if no MCP config was written (no `mcp` section, `enabled: false`, or no servers defined). Used by `MasterManager` to pass the global MCP config to workers that don't specify per-worker servers. + ### scanForAITools() _Source: `src/discovery/index.ts`_ @@ -598,6 +1241,77 @@ Creates a [Pino](https://getpino.io) logger instance. Uses `pino-pretty` in non- --- +## Built-in Commands + +Built-in commands are intercepted by the Router before any message reaches the Master AI. They require the memory system to be initialized. + +### /history Command + +_Source: `src/core/router.ts`_ + +Provides access to past conversation sessions. Three subcommands: + +#### `/history` (bare) + +Lists the last 10 conversation sessions with title, message count, and date. + +``` +/history +``` + +**Response (WhatsApp/Telegram/Discord):** + +``` +1. What is the project structure? — 12 msgs — 2026-01-15 +2. Fix the auth bug — 4 msgs — 2026-01-14 +... +``` + +**Response (Console):** ASCII table with aligned columns. + +**Response (WebChat):** HTML table with `/` cells. + +#### `/history search ` + +Full-text search across all past sessions by keyword. Returns up to 10 matching sessions ranked by relevance. + +``` +/history search authentication +``` + +**Error:** Returns an error message if `query` is empty. + +#### `/history ` + +Displays the full conversation transcript for one session (up to 50 messages, oldest first). + +``` +/history a1b2c3d4-... +``` + +**Response (WhatsApp/Telegram/Discord):** + +``` +[2026-01-15 10:30] User: What is the project structure? +[2026-01-15 10:30] Master: The project has src/, tests/, and docs/ directories... +``` + +**Response (Console):** Plain text with separator lines. + +**Response (WebChat):** HTML `
` bubbles with time and role. + +**Error responses:** + +| Condition | Message | +| --------------------------------- | ------------------------------------------------------------------------ | +| Memory not initialized | `History not available — memory system not initialized` | +| DB query failure | `History search/list temporarily unavailable — could not query sessions` | +| No sessions found (bare) | `No past sessions found` | +| No sessions found (search) | `No sessions found matching ` | +| Session ID not found (transcript) | `No conversation found for session: ` | + +--- + ## HTTP Endpoints ### Health Check Endpoint @@ -651,3 +1365,94 @@ Enabled via `metrics.enabled: true` in config. Default port: `9090`. "errors": { "total": 3, "transient": 2, "permanent": 1 } } ``` + +--- + +### Sessions Endpoints (WebChat) + +_Source: `src/connectors/webchat/webchat-connector.ts`_ + +Available when the WebChat connector is enabled. Requires the memory system to be initialized (memory must be wired via `setMemory()` on the connector). + +#### GET /api/sessions + +Returns a paginated list of conversation sessions. + +**Query Parameters:** + +| Parameter | Type | Default | Description | +| --------- | -------- | ------- | ----------------------------------------- | +| `limit` | `number` | `20` | Max sessions to return (clamped to 1–100) | +| `offset` | `number` | `0` | Pagination offset (min 0) | + +**Response (200 OK):** + +```json +[ + { + "session_id": "uuid-or-sender-id", + "title": "What is the project structure?", + "first_message_at": "2026-01-15T10:30:00.000Z", + "last_message_at": "2026-01-15T11:45:30.000Z", + "message_count": 12, + "channel": "whatsapp", + "user_id": "+1234567890" + } +] +``` + +**Error Responses:** + +| Status | Body | Condition | +| ------ | ----------------------------------- | ----------------------- | +| `503` | `{"error":"Memory not available"}` | MemoryManager not wired | +| `500` | `{"error":"Internal server error"}` | DB query error | + +#### GET /api/sessions/:id + +Returns the full conversation transcript for one session. + +**URL Parameter:** `id` — the session ID (URL-decoded). + +**Query Parameters:** + +| Parameter | Type | Default | Description | +| --------- | -------- | ------- | ----------------------------------------- | +| `limit` | `number` | `100` | Max messages to return (clamped to 1–500) | + +**Response (200 OK):** + +```json +{ + "session_id": "uuid-or-sender-id", + "messages": [ + { + "id": 1, + "session_id": "uuid-or-sender-id", + "role": "user", + "content": "What is the project structure?", + "channel": "whatsapp", + "user_id": "+1234567890", + "created_at": "2026-01-15T10:30:00.000Z" + }, + { + "id": 2, + "session_id": "uuid-or-sender-id", + "role": "master", + "content": "The project has src/, tests/, and docs/ directories...", + "channel": null, + "user_id": null, + "created_at": "2026-01-15T10:30:15.000Z" + } + ] +} +``` + +Messages are in chronological order (oldest first). + +**Error Responses:** + +| Status | Body | Condition | +| ------ | ----------------------------------- | ----------------------- | +| `503` | `{"error":"Memory not available"}` | MemoryManager not wired | +| `500` | `{"error":"Internal server error"}` | DB query error | diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ccfeede7..ead17f77 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,17 +1,17 @@ # OpenBridge — Architecture -> **Last Updated:** 2026-02-20 +> **Last Updated:** 2026-02-27 --- ## Overview -OpenBridge is a 4-layer system that connects messaging channels to an autonomous AI Master that explores and operates on your workspace. +OpenBridge is a 6-layer autonomous AI bridge that connects messaging channels to AI agents. The system auto-discovers AI tools on your machine, picks the most capable one as "Master", and launches it to autonomously explore and operate on your workspace using an incremental, resumable exploration strategy. Workers can access external services (Gmail, Canva, Slack, GitHub, etc.) via the MCP (Model Context Protocol) ecosystem. ``` ┌──────────────────────────────────────────────────────────────────┐ │ CHANNELS │ -│ WhatsApp · Telegram · Discord · Web Chat │ +│ WhatsApp · Console · WebChat · Telegram · Discord │ │ Connectors translate between messaging APIs and OpenBridge │ └──────────────────────┬────────────────────────────────────────────┘ │ @@ -31,11 +31,39 @@ OpenBridge is a 4-layer system that connects messaging channels to an autonomous │ ▼ ┌──────────────────────────────────────────────────────────────────┐ +│ CLIADAPTER LAYER │ +│ AdapterRegistry · ClaudeAdapter · CodexAdapter · AiderAdapter │ +│ Maps discovered tool names to CLI-specific spawn configurations │ +│ Translates SpawnOptions → binary + args + env per tool │ +└──────────────────────┬────────────────────────────────────────────┘ + │ + ▼ +┌──────────────────────────────────────────────────────────────────┐ +│ AGENT RUNNER │ +│ AgentRunner · Model Selector · Tool Profiles │ +│ Unified CLI executor: --allowedTools, --max-turns, --model, │ +│ retries, disk logging, model fallback, worker orchestration │ +└──────────────────────┬────────────────────────────────────────────┘ + │ + ▼ +┌──────────────────────────────────────────────────────────────────┐ │ MASTER AI │ -│ Master Manager · .openbridge/ Folder · Delegation Coordinator │ -│ Autonomous exploration, task execution, multi-AI delegation, │ -│ git-tracked knowledge base in target workspace │ -└──────────────────────────────────────────────────────────────────┘ +│ Master Manager · Worker Registry · Exploration Coordinator │ +│ Self-governing Master, AI task classification, auto-delegation │ +│ via SPAWN markers, worker orchestration, session continuity, │ +│ self-improvement, git-tracked knowledge in .openbridge/ │ +└──────────────────────┬────────────────────────────────────────────┘ + │ spawns workers with mcpServers in TaskManifest + ▼ +┌──────────────────────────────────────────────────────────────────┐ +│ MCP (MODEL CONTEXT PROTOCOL) │ +│ Per-worker temp config · --mcp-config · --strict-mcp-config │ +│ Gmail · Canva · Slack · GitHub · Filesystem · any MCP server │ +│ Claude CLI spawns MCP servers per invocation (bounded lifecycle) │ +└──────────────────────┬────────────────────────────────────────────┘ + │ + ▼ + External Services ``` --- @@ -63,8 +91,11 @@ interface Connector { | Connector | Directory | Library | Features | | --------- | -------------------------- | ----------------- | ------------------------------------------------------------------------------------------------------ | +| Console | `src/connectors/console/` | built-in (stdin) | Rapid local testing without any external service dependency | +| WebChat | `src/connectors/webchat/` | built-in (ws) | Browser chat UI on localhost, markdown rendering, live progress bar, Thinking animation | | WhatsApp | `src/connectors/whatsapp/` | `whatsapp-web.js` | QR auth, session persistence, auto-reconnect, message chunking, typing indicators, markdown formatting | -| Console | `src/connectors/console/` | built-in (stdin) | Reference implementation for testing | +| Telegram | `src/connectors/telegram/` | `grammy` | DM and group @mention support, in-place progress editing via editMessageText | +| Discord | `src/connectors/discord/` | `discord.js` v14 | DM and guild channel support, bot message filtering, in-place progress editing | ### WhatsApp Connector Details @@ -82,20 +113,20 @@ The engine that wires everything together. Lives in `src/core/`. ### Components -| Component | File | Purpose | -| ------------------ | ------------------- | -------------------------------------------------------------------------------- | -| **Bridge** | `bridge.ts` | Main orchestrator — wires connectors, providers, auth, queue, Master AI | -| **Router** | `router.ts` | Routes messages: connector → Master AI → connector. Sends ack + progress updates | -| **AuthService** | `auth.ts` | Phone whitelist, `/ai` prefix check, command allow/deny filters | -| **MessageQueue** | `queue.ts` | Per-user sequential processing, retry with backoff, dead-letter queue | -| **PluginRegistry** | `registry.ts` | Factory pattern for connectors. Auto-discovery from directories | -| **Config** | `config.ts` | Loads and validates `config.json` via Zod. Supports V0 and V2 formats | -| **ConfigWatcher** | `config-watcher.ts` | Hot-reload config on file change (auth + rate limit updates) | -| **HealthServer** | `health.ts` | HTTP `/health` endpoint with uptime, connector/queue status | -| **Metrics** | `metrics.ts` | Message counts, latency histograms, error rates | -| **AuditLogger** | `audit-logger.ts` | Structured audit trail of all message events | -| **RateLimiter** | `rate-limiter.ts` | Per-user sliding window rate limiting | -| **Logger** | `logger.ts` | Pino logger with child logger factory | +| Component | File | Purpose | +| ------------------ | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Bridge** | `bridge.ts` | Main orchestrator — wires connectors, providers, auth, queue, Master AI | +| **Router** | `router.ts` | Routes messages: connector → Master AI → connector. Sends ack + progress updates. Built-in commands: `/history`, `/history search `, `/history `, `/stop`, `/stop-all`, `/status` | +| **AuthService** | `auth.ts` | Phone whitelist, `/ai` prefix check, command allow/deny filters | +| **MessageQueue** | `queue.ts` | Per-user sequential processing, retry with backoff, dead-letter queue | +| **PluginRegistry** | `registry.ts` | Factory pattern for connectors. Auto-discovery from directories | +| **Config** | `config.ts` | Loads and validates `config.json` via Zod. Supports V0 and V2 formats | +| **ConfigWatcher** | `config-watcher.ts` | Hot-reload config on file change (auth + rate limit updates) | +| **HealthServer** | `health.ts` | HTTP `/health` endpoint with uptime, connector/queue status | +| **Metrics** | `metrics.ts` | Message counts, latency histograms, error rates | +| **AuditLogger** | `audit-logger.ts` | Structured audit trail of all message events | +| **RateLimiter** | `rate-limiter.ts` | Per-user sliding window rate limiting | +| **Logger** | `logger.ts` | Pino logger with child logger factory | ### Message Flow (V2 with Master AI) @@ -119,6 +150,9 @@ The engine that wires everything together. Lives in `src/core/`. ├─ Start progress timer (every 15s) ├─ Route to Master AI: │ └─ master.processMessage(message) → ProviderResult + │ ├─ Session continuity: --resume for existing conversation + │ ├─ New session: --session-id for first message from sender + │ └─ Timeout: 30-minute TTL per session ├─ Stop progress timer └─ Send result back to user via connector │ @@ -188,6 +222,122 @@ interface ScanResult { | Cursor | `cursor` | 4 | code-gen, file-editing | | Cody | `cody` | 5 | code-gen, conversation | +### Adapter Resolution + +After discovery, each `DiscoveredTool` is matched to a `CLIAdapter` in the `AdapterRegistry`. Only tools with a registered adapter can be used to spawn workers. Built-in adapters are provided for `claude`, `codex`, and `aider`. Tools without an adapter (e.g. `cursor`, `cody`) are recorded in `agents.json` but cannot be delegated to at runtime. + +``` +scanForAITools() + → ScanResult { tools: DiscoveredTool[], master: DiscoveredTool } + → adapterRegistry.getForTool(master) + → CLIAdapter (ClaudeAdapter | CodexAdapter | AiderAdapter) + → used by AgentRunner for every spawn() call +``` + +--- + +## CLIAdapter Layer + +Lives in `src/core/cli-adapter.ts`, `src/core/adapter-registry.ts`, and `src/core/adapters/`. + +The `CLIAdapter` interface decouples `AgentRunner` from the specific CLI invocation details of each AI tool. When `AgentRunner.spawn()` is called, it asks the appropriate adapter to build a `CLISpawnConfig` and then passes that config directly to `child_process.spawn()`. + +### CLIAdapter Interface + +```typescript +interface CLIAdapter { + /** Provider name matching DiscoveredTool.name ('claude', 'codex', 'aider') */ + readonly name: string; + + /** + * Translate provider-neutral SpawnOptions into the binary, args, and env + * for this CLI. Called once per spawn() or stream() invocation. + */ + buildSpawnConfig(opts: SpawnOptions): CLISpawnConfig; + + /** + * Clean the process environment before spawning. + * Removes vars that would cause nested-session conflicts + * (e.g. CLAUDECODE, CLAUDE_CODE_*, CLAUDE_AGENT_SDK_*). + */ + cleanEnv(env: Record): Record; + + /** + * Map a CapabilityLevel to CLI-specific access restrictions. + * Claude → tool name lists for --allowedTools. + * Codex → sandbox mode strings (handled in buildSpawnConfig). + * Aider → flags like --yes. + * Returns undefined if the CLI has no restriction mechanism. + */ + mapCapabilityLevel(level: CapabilityLevel): string[] | undefined; + + /** + * Validate a model string for this provider. + * Returns true if recognized or safely passable to the CLI. + */ + isValidModel(model: string): boolean; +} +``` + +### CLISpawnConfig + +The output of `buildSpawnConfig()` — everything `child_process.spawn()` needs: + +```typescript +interface CLISpawnConfig { + binary: string; // e.g. 'claude', 'codex' + args: string[]; // CLI argument array + env: Record; // Cleaned environment + stdin?: 'ignore' | 'pipe'; // stdin behavior (default: 'ignore') + parseOutput?: (stdout: string) => string; // Optional post-processor for raw stdout +} +``` + +The `parseOutput` hook lets adapters that emit structured output (e.g. Codex `--json` JSONL) extract the final human-readable message before `AgentRunner` returns the result. + +### CapabilityLevel + +Maps tool profiles to CLI-specific access mechanisms: + +| Level | Claude (`--allowedTools`) | Codex (`--sandbox`) | +| ------------- | ------------------------------------------ | -------------------- | +| `read-only` | `Read`, `Glob`, `Grep` | `read-only` | +| `code-edit` | `Read`, `Edit`, `Write`, `Glob`, ... | `workspace-write` | +| `full-access` | `Read`, `Edit`, `Write`, `Glob`, `Bash(*)` | `danger-full-access` | + +### Built-in Adapters + +| Adapter | File | Tool | Key Flags | +| --------------- | ---------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `ClaudeAdapter` | `adapters/claude-adapter.ts` | `claude` | `--print`, `--session-id`, `--resume`, `--model`, `--max-turns`, `--allowedTools`, `--append-system-prompt`, `--max-budget-usd` | +| `CodexAdapter` | `adapters/codex-adapter.ts` | `codex` | `exec`, `--skip-git-repo-check`, `--model`, `--sandbox`, `--full-auto`, `--ephemeral`, `--json`, `-o `, `-c ` | +| `AiderAdapter` | `adapters/aider-adapter.ts` | `aider` | `--no-auto-commits`, `--yes`, `--model`, `--message` | + +**ClaudeAdapter** translates `allowedTools` directly to repeated `--allowedTools ` flags. Session management uses `--print` (worker/stateless), `--session-id ` (new session), or `--resume ` (continue session). + +**CodexAdapter** uses the `exec` subcommand for non-interactive execution. Key behaviors: + +- Always pushes `--skip-git-repo-check` (required for non-git or untrusted workspaces) +- Validates `OPENAI_API_KEY` at build time — throws if missing +- Maps `allowedTools` → sandbox mode heuristically (`Bash(*)` → `danger-full-access`, `Edit/Write` → `workspace-write`, otherwise `read-only`) +- Emits `--json` (JSONL structured output) + `-o ` (reliable final answer capture) +- Session resume uses `exec resume --last`; new sessions omit `--ephemeral` so Codex saves state +- MCP passthrough: when `opts.mcpConfigPath` is set, passes it via `-c ` + +### AdapterRegistry + +`AdapterRegistry` (`src/core/adapter-registry.ts`) maps tool names to `CLIAdapter` instances. Built-in adapters are lazy-loaded on first access; custom adapters registered via `register()` take priority. + +```typescript +const registry = createAdapterRegistry(); // pre-registers ClaudeAdapter +const adapter = registry.get('codex'); // lazy-loads CodexAdapter +const adapter2 = registry.getForTool(tool); // looks up by DiscoveredTool.name +registry.has('aider'); // true (built-in exists) +registry.register('my-tool', myAdapter); // register a custom adapter +``` + +`createAdapterRegistry()` is the production factory — it pre-registers `ClaudeAdapter` so the most common case never incurs lazy-load overhead. + --- ## Layer 4: Master AI @@ -196,23 +346,109 @@ The autonomous agent that knows your project. Lives in `src/master/`. ### Master Manager -The central component that manages the Master AI lifecycle: +The central component that manages the Master AI lifecycle and session continuity: ``` States: idle → exploring → ready → error - │ - startExploration() fires on startup - │ - Sends exploration prompt to Master AI CLI - │ - Master AI reads project files, creates .openbridge/ - │ - State transitions to 'ready' - │ - processMessage(msg) handles user requests + +Lifecycle: + 1. startExploration() fires on startup + 2. Delegates to ExplorationCoordinator.explore() + 3. State transitions to 'ready' when all 5 passes complete + 4. processMessage(msg) handles user requests with session continuity + +Session Continuity: + - Maps sender → sessionId with 30-minute TTL + - First message from sender: creates new session with --session-id + - Subsequent messages: resumes with --resume + - Preserves context across multi-turn conversations + - Cleans up expired sessions automatically + +Conversation Context (memory.md pattern — v0.0.2): + - On session start: buildConversationContext() loads .openbridge/context/memory.md + into the Master's system prompt (always small, always relevant) + - Fallback: findRelevantHistory() FTS5 search (sanitized via sanitizeFts5Query()) + when memory.md is empty/missing + - Memory updates: triggerMemoryUpdate() fires every 10 completed tasks and on + shutdown. It fetches the last 20 user+master messages from SQLite and injects + them into the prompt so the stateless --print agent has real context to write + meaningful notes (v0.0.5 fix for OB-F39). + - Dual-layer: memory.md = curated brain; SQLite conversations = raw archive + - See "memory.md Update Mechanism" in the SQLite Memory System section for details. + +Session Checkpointing (v0.0.2): + - checkpointSession(): serializes pending workers + accumulated results to DB + - resumeSession(): restores state and continues processing + - Integrated with priority queue: urgent messages trigger checkpoint-handle-resume +``` + +### Incremental Exploration Architecture + +The exploration is split into **5 short passes** to avoid timeouts on large projects. Each pass is independently checkpointed and resumable. + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ INCREMENTAL 5-PASS EXPLORATION │ +└─────────────────────────────────────────────────────────────────┘ + +Pass 1: Structure Scan (90s timeout) + ├─ List top-level files/dirs + ├─ Count files per directory + ├─ Detect config files (package.json, tsconfig.json, etc.) + ├─ Skip: node_modules, .git, dist, build + └─ Output: exploration/structure-scan.json + +Pass 2: Classification (90s timeout) + ├─ Read config files from Pass 1 + ├─ Detect project type (Node.js, Python, Go, etc.) + ├─ Identify frameworks (Express, React, Django, etc.) + ├─ Extract dependencies and scripts + └─ Output: exploration/classification.json + +Pass 3: Directory Dives (90s timeout per directory, batched) + ├─ For each significant directory (src, tests, docs, etc.): + │ ├─ Explore contents (purpose, key files, subdirs) + │ └─ Output: exploration/dirs/.json + ├─ Process in batches of 3 via Promise.allSettled() + ├─ Retry failed dives up to 3 times with backoff + └─ Checkpoint after each batch + +Pass 4: Assembly (60s timeout) + ├─ Merge partial results from Passes 1-3 + ├─ Generate human-readable summary field + └─ Output: workspace-map.json (final assembled map) + +Pass 5: Finalization (no AI call, pure code) + ├─ Create agents.json (discovered tools + roles) + ├─ Git commit all files in .openbridge/ + └─ Write log entry to exploration.log ``` -### `.openbridge/` Folder +**Resumability:** The `exploration-state.json` file tracks which passes are complete. On restart, the coordinator loads this file and skips completed phases. + +```json +{ + "currentPhase": "directory_dives", + "status": "in_progress", + "startedAt": "2026-02-21T10:00:00.000Z", + "phases": { + "structure_scan": "completed", + "classification": "completed", + "directory_dives": "in_progress", + "assembly": "pending", + "finalization": "pending" + }, + "directoryDives": [ + { "path": "src", "status": "completed", "outputFile": "dirs/src.json" }, + { "path": "tests", "status": "pending" }, + { "path": "docs", "status": "failed", "attempts": 1 } + ], + "totalCalls": 5, + "totalAITimeMs": 45000 +} +``` + +### `.openbridge/` Folder Specification Created by the Master AI inside the target workspace. This is the AI's persistent knowledge base. @@ -223,44 +459,105 @@ target-project/ ├── ... └── .openbridge/ ← Created by Master AI ├── .git/ ← Local git repo (Master's changes only) - ├── workspace-map.json ← Auto-generated project understanding + │ ├── HEAD + │ ├── objects/ + │ └── refs/ + ├── exploration/ ← Incremental exploration state (Phase 11) + │ ├── exploration-state.json ← Single source of truth for resumability + │ │ { + │ │ "currentPhase": "assembly", + │ │ "status": "in_progress", + │ │ "startedAt": "2026-02-21T10:00:00.000Z", + │ │ "phases": { + │ │ "structure_scan": "completed", + │ │ "classification": "completed", + │ │ "directory_dives": "completed", + │ │ "assembly": "in_progress", + │ │ "finalization": "pending" + │ │ }, + │ │ "directoryDives": [...], + │ │ "totalCalls": 12, + │ │ "totalAITimeMs": 108000 + │ │ } + │ ├── structure-scan.json ← Pass 1 output + │ │ { + │ │ "topLevelFiles": ["package.json", "README.md", ...], + │ │ "directories": [ + │ │ { "path": "src", "fileCount": 42 }, + │ │ { "path": "tests", "fileCount": 18 } + │ │ ], + │ │ "configFiles": ["package.json", "tsconfig.json", ...] + │ │ } + │ ├── classification.json ← Pass 2 output + │ │ { + │ │ "projectType": "Node.js", + │ │ "languages": ["typescript"], + │ │ "frameworks": ["express"], + │ │ "dependencies": { "express": "^4.18.0", ... }, + │ │ "scripts": { "dev": "tsx src/index.ts", ... } + │ │ } + │ └── dirs/ ← Pass 3 outputs (one per directory) + │ ├── src.json + │ │ { + │ │ "path": "src", + │ │ "purpose": "Main application source code", + │ │ "keyFiles": ["index.ts", "server.ts", ...], + │ │ "subdirs": ["routes", "middleware", "services"] + │ │ } + │ ├── tests.json + │ └── docs.json + ├── workspace-map.json ← Final assembled map (Pass 4) │ { │ "name": "my-project", - │ "description": "Node.js REST API", - │ "languages": ["typescript", "sql"], + │ "description": "Node.js REST API with Express and TypeScript", + │ "summary": "A production-ready API server with 12 routes...", + │ "languages": ["typescript"], │ "frameworks": ["express", "prisma"], - │ "structure": { ... }, - │ "exploredAt": "2026-02-20T13:00:00Z" + │ "structure": { + │ "src": { "purpose": "Main application source", ... }, + │ "tests": { "purpose": "Vitest test suite", ... } + │ }, + │ "exploredAt": "2026-02-21T10:05:30.000Z" │ } + ├── context/ ← Master AI memory (added in v0.0.2) + │ └── memory.md ← Master's curated brain — decisions, preferences, + │ project state, active threads. Cap: 200 lines. + │ Read on every session start. Updated by Master + │ at session end via "update memory" prompt. + ├── prompts/ ← Prompt library (added in v0.0.2) + │ ├── manifest.json ← Prompt registry: id, filename, usageCount, + │ │ successRate, lastUsedAt, previousVersion + │ └── *.md ← Individual prompt template files ├── exploration.log ← Timestamped scan history - ├── agents.json ← Discovered AI tools + their roles + │ 2026-02-21T10:00:00Z | Exploration started + │ 2026-02-21T10:01:30Z | Pass 1 (structure_scan) completed in 90s + │ 2026-02-21T10:03:00Z | Pass 2 (classification) completed in 90s + │ ... + ├── agents.json ← Discovered AI tools + their roles (Pass 5) │ { │ "master": { "name": "claude", "path": "/usr/local/bin/claude" }, - │ "delegates": [ { "name": "codex", "path": "..." } ] + │ "delegates": [ + │ { "name": "codex", "path": "/usr/local/bin/codex" } + │ ] │ } - └── tasks/ ← Task history (one JSON per task) - ├── task-001.json - └── task-002.json + ├── tasks/ ← Task history JSON (superseded by openbridge.db) + │ ├── task-001.json + │ └── task-002.json + └── openbridge.db ← SQLite memory (shipped in v0.0.2) + Tables: workspace_chunks (FTS5), conversation_messages (FTS5), + tasks, learnings, agent_activity (PID tracking), prompts, + prompt_versions, access_control, sessions, schema_versions ``` -### Exploration Prompt - -On startup, the Master Manager sends a carefully crafted prompt to the Master AI CLI: - -``` -You are the Master AI for the project at /path/to/workspace. +### Exploration Components -Your job: -1. Silently explore the workspace — read key files (package.json, README, src/, etc.) -2. Create a .openbridge/ folder at the workspace root -3. Inside .openbridge/, create workspace-map.json with your findings -4. Initialize a git repo in .openbridge/ and commit your findings -5. Do NOT send any messages to the user — work silently - -IMPORTANT: Only create files inside .openbridge/. Do not modify existing project files. -``` - -The AI does the exploring — we don't write framework detectors or file parsers. +| Module | File | Purpose | +| -------------------------- | ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **ExplorationCoordinator** | `exploration-coordinator.ts` | Orchestrates the 5-pass flow, loads/saves state, skips completed phases, checkpoints progress | +| **Exploration Prompts** | `exploration-prompts.ts` | 4 focused prompt generators (structure scan, classification, directory dive, summary) | +| **Result Parser** | `result-parser.ts` | Robust JSON extraction with fallbacks (direct parse → markdown fence → regex → retry) | +| **DotFolderManager** | `dotfolder-manager.ts` | `.openbridge/` CRUD, exploration state, git ops, prompt library (readPromptManifest, writePromptManifest, writePromptTemplate, getPromptTemplate, recordPromptUsage, getLowPerformingPrompts, resetPromptStats), memory file (readMemoryFile, writeMemoryFile) | +| **Exploration Types** | `types/master.ts` | Zod schemas for all exploration data structures | ### Delegation @@ -281,22 +578,247 @@ When the Master needs help from another AI tool: 5. Task recorded in .openbridge/tasks/ and committed to git ``` -### Generalized CLI Executor +**Delegation features:** -The `claude-code-executor.ts` module supports any CLI tool via the `command` option: +- Concurrent delegation limit (max 3 at once) +- Timeout handling per delegate (default 120s) +- Result aggregation and error handling +- Git commit per completed task + +### Agent Runner (Unified CLI Executor) + +The `agent-runner.ts` module (`src/core/agent-runner.ts`) is the production-grade CLI executor that uses `AdapterRegistry` to resolve the correct `CLIAdapter` for each spawn call. It supports: ```typescript -// Run claude (default) -await executeClaudeCode({ prompt: '...', workspacePath: '...', timeout: 120000 }); +const runner = new AgentRunner(); + +// Single-turn execution with tool restrictions +const result = await runner.spawn({ + prompt: 'Fix the auth bug', + workspacePath: '/path/to/project', + model: 'sonnet', + allowedTools: ['Read', 'Edit', 'Write', 'Glob', 'Grep'], + maxTurns: 25, + timeout: 120_000, + retries: 3, +}); + +// Streaming execution +const stream = runner.stream({ prompt: '...', workspacePath: '...' }); +for await (const chunk of stream) { + /* ... */ +} +``` + +Internally, `spawn()` calls `adapterRegistry.get(toolName).buildSpawnConfig(opts)` to produce the binary + args + env for the selected tool. The result is passed to `child_process.spawn()`. If `CLISpawnConfig.parseOutput` is set, `AgentRunner` applies it to accumulated stdout after the process exits — this is how `CodexAdapter` extracts the final message from `--json` JSONL output. + +Features: `--allowedTools` for tool restrictions (via adapter), `--max-turns` for bounded execution, `--model` for model selection, automatic retries with model fallback chain (opus → sonnet → haiku), session support (`--session-id`, `--resume`), prompt sanitization, disk logging, tool profiles (`read-only`, `code-edit`, `full-access`). + +--- -// Run codex -await executeClaudeCode({ prompt: '...', workspacePath: '...', timeout: 120000, command: 'codex' }); +## SQLite Memory System (v0.0.2) -// Run aider -await executeClaudeCode({ prompt: '...', workspacePath: '...', timeout: 120000, command: 'aider' }); +The memory system lives at `.openbridge/openbridge.db` inside the target workspace. It provides persistent storage across sessions via 13 store modules, all exposed through a single `MemoryManager` facade (`src/memory/index.ts`). + +### Store Modules + +| Module | File | Tables | Purpose | +| --------------------- | ----------------------- | ----------------------------------- | ------------------------------------------------- | +| **ActivityStore** | `activity-store.ts` | `agent_activity` | Worker PID tracking, status, turn counts | +| **TaskStore** | `task-store.ts` | `tasks`, `learnings` | Task history, model success rates, retry patterns | +| **ConversationStore** | `conversation-store.ts` | `conversation_messages` (FTS5) | Full conversation archive, FTS5 search | +| **ChunkStore** | `chunk-store.ts` | `workspace_chunks` (FTS5) | Workspace file chunks, semantic retrieval | +| **PromptStore** | `prompt-store.ts` | `prompts`, `prompt_versions` | Prompt effectiveness tracking in SQLite | +| **AccessStore** | `access-store.ts` | `access_control` | Role-based permissions per sender | +| **SubMasterStore** | `sub-master-store.ts` | `sessions` | Sub-master session state + checkpoints | +| **WorkerBriefing** | `worker-briefing.ts` | (reads multiple tables) | Assembles per-worker context before spawn | +| **Retrieval** | `retrieval.ts` | (reads chunk + conversation tables) | Semantic + FTS5 search helpers | +| **Eviction** | `eviction.ts` | (writes multiple tables) | LRU eviction + AI-powered conversation compaction | + +### Schema Versioning + +`schema_versions` table tracks applied migrations (version INTEGER, applied_at, description). The migration runner only applies versions > MAX(version), wrapped in transactions for rollback safety. + +### FTS5 Search Sanitization + +All FTS5 `MATCH` queries (in `searchConversations()` and `searchWorkspaceChunks()`) are sanitized via `sanitizeFts5Query()` (exported from `src/memory/retrieval.ts`) before being passed to the FTS5 engine. This prevents `SqliteError: fts5: syntax error near "'"` crashes when user messages or search queries contain FTS5 special characters. + +```typescript +// src/memory/retrieval.ts +export function sanitizeFts5Query(query: string): string { + // Strips FTS5 special characters: " * ( ) { } [ ] : ^ ~ ? @ # $ % & \ | < > = ! + , ; + // Quotes each token as a literal phrase + const cleaned = query.replace(/["*(){}[\]:^~?@#$%&\\|<>=!+,;]/g, ' '); + const tokens = cleaned.split(/\s+/).filter(Boolean); + if (tokens.length === 0) return ''; + return tokens.map((t) => `"${t}"`).join(' '); +} ``` -Features: streaming via async generator, session support, prompt sanitization, graceful shutdown guard (active child processes tracked and waited for during SIGTERM/SIGINT). +If `sanitizeFts5Query()` returns an empty string (e.g. query was only special characters), the search returns `[]` immediately — no SQL is executed. + +This fix was applied in Phase 63 (OB-F38). All FTS5 callers in the codebase route through `sanitizeFts5Query()` from the shared location in `retrieval.ts`. + +### memory.md Update Mechanism + +`triggerMemoryUpdate()` in `MasterManager` (`src/master/master-manager.ts`) drives the memory.md update pipeline: + +``` +Triggers: + - Every 10 completed tasks (MEMORY_UPDATE_INTERVAL = 10) + - On shutdown (before process.exit) + +Pipeline: + 1. Call memory.getRecentMessages(20) → fetch last 20 user+master messages from SQLite + 2. Format each entry as "[YYYY-MM-DD HH:MM] Role: content" (content truncated at 300 chars) + 3. Prepend "## Recent conversation history:\n" section to the memory-update prompt + 4. Spawn a stateless --print agent to write the updated memory.md + 5. Agent writes updated notes to .openbridge/context/memory.md + +Fallbacks: + - No messages in SQLite → prompt sent without history section (graceful degradation) + - this.memory is null → same fallback (no crash) + - Agent exits non-zero → logged as WARN, not fatal + +Design rationale: + - --print mode is intentional (no TTY hang). The injected conversation history + compensates for the stateless context — the agent has enough context to write + meaningful notes without needing a persistent session. + - Prompt uses generic language ("write your updated notes to ") rather than + Claude-specific tool names — works with Claude and Codex as Master AI. +``` + +### Conversation History Commands + +The router (`src/core/router.ts`) handles built-in `/history` commands — intercepted before routing to Master AI: + +| Command | Description | +| ------------------------ | -------------------------------------------------- | +| `history` | List last 10 sessions (title, date, message count) | +| `history search ` | FTS5 search across all past conversations | +| `history ` | Show full transcript for one session | + +WebChat connector also exposes `/api/sessions` (list) and `/api/sessions/:id` (full session) REST endpoints. + +--- + +## Layer 6: MCP Integration + +MCP (Model Context Protocol) allows OpenBridge workers to call external services — Gmail, Canva, Slack, GitHub, databases, and any other MCP-compatible API — without OpenBridge needing to know anything about those services. The Claude CLI handles tool discovery and invocation; MCP servers are external processes that translate tool calls into real API requests. + +### End-to-End Flow + +``` +User (WhatsApp / Console / Telegram) + → OpenBridge Router + → Master AI (reads available MCP servers from system prompt context) + → decides: "this task needs Canva" + → includes mcpServers: [{ name: "canva", ... }] in worker TaskManifest + → AgentRunner.manifestToSpawnOptions() writes per-worker temp config + → claude --print --mcp-config /tmp/ob-mcp-.json --strict-mcp-config ... + → Claude CLI spawns Canva MCP server process + → MCP server calls Canva API + → MCP server exits when worker completes + → temp config file deleted after spawn + → worker returns result + → Master formats and sends back to channel +``` + +### Per-Worker MCP Isolation (Security) + +Each worker receives a **temporary MCP config** containing only the servers it needs, not all configured servers. This prevents cross-contamination of API keys and limits each worker's external access to its task scope. + +```typescript +// manifestToSpawnOptions() in agent-runner.ts +// When manifest.mcpServers is non-empty: +// 1. Generate /tmp/ob-mcp-.json with ONLY the requested servers +// 2. Set spawnOpts.mcpConfigPath = temp file path +// 3. Set strictMcpConfig: true (ignore global/project MCP configs) +// 4. Register cleanup: delete temp file after spawn completes +``` + +The `--strict-mcp-config` flag ensures workers cannot access MCP servers from the user's global `~/.claude/` or project-level configs — only those explicitly provisioned for the task. + +### Master Awareness + +The Master AI sees all configured MCP servers in its system prompt (injected via `MasterSystemPromptContext.mcpServers`). This allows the Master to autonomously decide when a task requires an external service and which workers should receive MCP access: + +``` +Available MCP Servers: +- canva: Design creation and editing via Canva API +- gmail: Send and read Gmail messages +- filesystem: Read and write local files via MCP protocol + +To use an external service, include `mcpServers` in the worker TaskManifest +with only the servers that worker needs. +``` + +### Claude CLI MCP Flags + +`ClaudeAdapter` (`src/core/adapters/claude-adapter.ts`) passes MCP flags when `SpawnOptions.mcpConfigPath` is set: + +```bash +claude --print \ + --mcp-config /tmp/ob-mcp-.json \ + --strict-mcp-config \ + --allowedTools "Read,mcp__canva__*" \ + "Create a banner for the Q3 campaign" +``` + +### MCP Config Format (what Claude CLI expects) + +```json +{ + "mcpServers": { + "canva": { + "command": "npx", + "args": ["-y", "@anthropic/canva-mcp-server"], + "env": { "CANVA_API_KEY": "sk-..." } + } + } +} +``` + +### OpenBridge Config Format (what users write in config.json) + +```json +{ + "mcp": { + "enabled": true, + "servers": [ + { + "name": "canva", + "command": "npx", + "args": ["-y", "@anthropic/canva-mcp-server"], + "env": { "CANVA_API_KEY": "sk-..." } + } + ], + "configPath": "~/.claude/claude_desktop_config.json" + } +} +``` + +`configPath` lets users import their existing Claude Desktop or Claude Code MCP configuration without duplicating server definitions. + +### MCP Scope + +- **Claude workers:** `--mcp-config` + `--strict-mcp-config` (implemented in `ClaudeAdapter`) +- **Codex workers:** native MCP support via `codex mcp` — when `opts.mcpConfigPath` is set, passed via `-c ` in `CodexAdapter` (wired in Phase 58, OB-1105) +- **Aider workers:** no MCP support (Aider has no MCP integration) + +### MCP Server Lifecycle + +MCP servers are **per-worker processes** spawned by the Claude CLI at invocation time and terminated when the worker completes. This matches OpenBridge's bounded-worker philosophy — no long-lived external service connections. Each worker starts fresh. + +### Global MCP Config on Startup + +On bridge startup, `src/core/config.ts` writes `.openbridge/mcp-config.json` from `V2Config.mcp`: + +- If `configPath` is set: validates + imports the external config file +- If inline `servers` are defined: transforms them to Claude CLI format and writes the file +- If both: merges (inline servers override same-name imports from `configPath`) + +`getMcpConfigPath()` returns the path to this file (or `null` when MCP is not configured), used by `MasterManager` when building Master session context. --- @@ -337,15 +859,27 @@ The config loader auto-detects the format and runs the appropriate startup flow. ### V2 Flow (with discovery + Master) ``` -1. loadConfig() → detect V2 format -2. scanForAITools() → discover claude, codex, etc. -3. new Bridge(config) → create bridge with auth, queue, router -4. registerBuiltInConnectors() → register WhatsApp, Console -5. bridge.start() → initialize connectors, health, metrics -6. new MasterManager(tool, path) → create Master with discovered tool -7. bridge.setMaster(master) → wire Master into router -8. master.startExploration() → fire-and-forget background exploration -9. Ready — waiting for messages +1. loadConfig() → detect V2 format +2. writeMcpConfig(config.mcp) → if MCP configured: merge inline servers + configPath import, + write .openbridge/mcp-config.json in Claude CLI format +3. scanForAITools() → discover claude, codex, etc. + → ScanResult { tools, master } +4. adapterRegistry.getForTool(master) + → resolve master to CLIAdapter + (ClaudeAdapter, CodexAdapter, or AiderAdapter) +5. new Bridge(config) → create bridge with auth, queue, router +6. registerBuiltInConnectors() → register Console, WebChat, WhatsApp, Telegram, Discord +7. bridge.start() → initialize connectors, health, metrics +8. new MasterManager(tool, path) → create Master with discovered tool + adapter +9. bridge.setMaster(master) → wire Master into router +10. master.startExploration() → fire-and-forget incremental exploration + ├─ ExplorationCoordinator.explore() + ├─ Load exploration-state.json (if exists) + ├─ Skip completed phases + ├─ Execute remaining passes (each calls agentRunner.spawn() → adapter.buildSpawnConfig()) + ├─ Checkpoint after each pass + └─ State transitions: idle → exploring → ready +11. Ready — waiting for messages with full project context and MCP servers available ``` ### V0 Flow (legacy — direct provider) @@ -353,7 +887,7 @@ The config loader auto-detects the format and runs the appropriate startup flow. ``` 1. loadConfig() → detect V0 format 2. new Bridge(config) → create bridge -3. registerBuiltInConnectors() → register WhatsApp, Console +3. registerBuiltInConnectors() → register Console, WebChat, WhatsApp, Telegram, Discord 4. registerBuiltInProviders() → register Claude Code provider 5. bridge.start() → initialize connectors + providers 6. Ready — messages route directly to provider @@ -361,19 +895,113 @@ The config loader auto-detects the format and runs the appropriate startup flow. --- +## Resilient Startup + +On restart, the Master AI reuses valid state and resumes incomplete exploration: + +``` +Scenario 1: Valid .openbridge/ exists + → Skip exploration, load workspace-map.json + → State: ready immediately + +Scenario 2: Incomplete exploration (exploration-state.json exists) + → Load exploration-state.json + → Resume from last completed phase + → Continue with remaining passes + → State: exploring → ready + +Scenario 3: Corrupted or missing workspace-map.json + → Delete exploration/ folder + → Start fresh 5-pass exploration + → State: exploring → ready + +Scenario 4: First run (no .openbridge/) + → Create .openbridge/ folder + → Start 5-pass exploration + → State: exploring → ready +``` + +--- + +## Graceful Shutdown + +OpenBridge handles `SIGINT` (Ctrl+C) and `SIGTERM` (process manager) with a structured shutdown sequence that ensures session state and memory are preserved before exit. + +### Shutdown Flow + +``` +SIGINT / SIGTERM received + │ + ├─ shutdownInProgress guard → if already shutting down, ignore signal (no double-shutdown) + │ + ├─ console.log('\nShutting down gracefully... please wait') ← user-facing message + │ + ├─ logger.info('Shutting down...') ← structured log + │ + ├─ Promise.race([ + │ bridge.stop(), ← clean connector shutdown + │ timeout(10_000) ← 10s hard limit + │ ]) + │ + │ bridge.stop() calls: + │ └─ MasterManager.shutdown() + │ ├─ saveMasterSessionToStore() ← FIRST: fast SQLite write (< 100ms) + │ │ critical state always persisted + │ └─ triggerMemoryUpdate() ← SECOND: slow AI spawn (10–30s) + │ may be interrupted by 10s timeout + │ + ├─ On success (shutdown completed before timeout): + │ process.exit(0) + │ + └─ On timeout (10s exceeded): + console.error('Shutdown timeout exceeded (10s) — forcing exit') + process.exit(1) +``` + +### Critical-First Ordering + +`MasterManager.shutdown()` saves session state to SQLite **before** triggering the memory update. This ordering guarantees: + +- **SQLite session state** (fast write, < 100ms) — always persisted, even if the memory update is interrupted +- **memory.md update** (slow AI spawn, 10–30s) — attempted after session state is safe, may be cut short by the 10s timeout + +A try/catch wraps `triggerMemoryUpdate()` so its failure cannot block `saveMasterSessionToStore()`. + +### shutdownInProgress Guard + +`src/index.ts` maintains a `shutdownInProgress` boolean flag. If the signal handler fires a second time while shutdown is in progress (e.g. user presses Ctrl+C twice), the second call is a no-op. This prevents concurrent shutdown races and double-logging. + +### `tsx` Force-Kill Behavior + +When running via `npm run dev` (which uses `tsx`), `tsx` may print `[tsx] Previous process hasn't exited yet. Force killing...` if it receives a second Ctrl+C or if the 10s timeout fires. This is expected behavior — see `CONTRIBUTING.md` for details on avoiding force-kills. + +--- + ## Key Design Decisions -1. **The AI does the exploring, not our code.** We don't write framework detectors or package.json parsers. We send the AI a prompt and let it figure out the project. This is simpler and more powerful. +1. **Incremental exploration, not monolithic.** The old architecture used a single giant AI call that timed out on real projects. The new 5-pass strategy breaks exploration into short, checkpointed phases that never timeout. + +2. **The AI does the exploring, not our code.** We don't write framework detectors or package.json parsers. We send the AI focused prompts and let it figure out the project. This is simpler and more powerful. + +3. **`.openbridge/` lives inside the target project.** The AI's knowledge is co-located with the code it knows. Uses a SQLite database (`openbridge.db`, shipped in v0.0.2) alongside JSON files for exploration state. JSON files remain for exploration checkpointing; `openbridge.db` stores conversations, tasks, prompts, agent activity, access control, and schema versions. + +4. **Session continuity enables multi-turn conversations.** The Master tracks sessions per sender with 30-minute TTL. First message creates a session, subsequent messages resume it. This enables natural business conversations: "which invoices are overdue?" → "send reminders to those clients". + +5. **Discovery runs once at startup.** We don't continuously scan for tools. Restart to re-discover. + +6. **V0 config stays supported.** Auto-detect config version, run the appropriate flow. No breaking changes. + +7. **AgentRunner replaced the original executor.** The `agent-runner.ts` module provides retries, model fallback, tool restrictions (`--allowedTools`), bounded execution (`--max-turns`), and disk logging — inspired by the bash scripts in `scripts/`. -2. **`.openbridge/` lives inside the target project.** The AI's knowledge is co-located with the code it knows. It has its own git repo so changes are tracked without polluting the project's git history. +8. **CLIAdapter decouples AgentRunner from per-tool CLI details.** Each tool (`claude`, `codex`, `aider`) has its own `CLIAdapter` that translates provider-neutral `SpawnOptions` into CLI-specific binary + args + env. Lossy translation is intentional — if a CLI doesn't support a feature (e.g. Codex has no `--max-turns`), the adapter silently drops it. This means AgentRunner code never needs to special-case individual tools. -3. **Discovery runs once at startup.** We don't continuously scan for tools. Restart to re-discover. +9. **Dead code is deleted, not archived.** Old modules (like `claude-code-executor.ts`) are removed once replaced. Git history preserves them if needed. -4. **V0 config stays supported.** Auto-detect config version, run the appropriate flow. No breaking changes. +10. **MCP integration is per-worker and isolated.** Each worker receives a temporary MCP config containing only the servers it needs. Combined with `--strict-mcp-config`, no worker can access more external services than its TaskManifest declares. API keys from one worker never leak to another. -5. **The executor is generalized, not rewritten.** The existing `claude-code-executor.ts` handles spawning, streaming, sanitization, sessions, and graceful shutdown. Adding `command` option was a one-line change. +11. **Master drives MCP assignment autonomously.** The Master AI sees available MCP servers in its system prompt context. It decides which workers need external service access based on the task. No hardcoded rules — the Master reasons about this the same way it reasons about tool profiles and model selection. -6. **Dead code is archived, not deleted.** Old knowledge/ and orchestrator/ modules go to `src/_archived/` — out of the compile path but preserved in git. +12. **MCP scope follows CLI capabilities.** `--mcp-config` is a Claude CLI flag. Codex has its own native MCP system (`codex mcp`), wired in Phase 58. Aider has no MCP support. Each adapter translates `mcpConfigPath` to its CLI's native mechanism (or silently drops it if unsupported). --- @@ -393,7 +1021,7 @@ src/ │ ├── common.ts ← Shared types │ ├── agent.ts ← Agent / TaskAgent types (reused) │ ├── discovery.ts ← DiscoveredTool, ScanResult schemas -│ └── master.ts ← MasterState, ExplorationSummary schemas +│ └── master.ts ← MasterState, ExplorationSummary, exploration schemas ├── core/ │ ├── bridge.ts ← Main orchestrator (setMaster + lifecycle) │ ├── router.ts ← Message routing (Master → provider fallback) @@ -402,6 +1030,15 @@ src/ │ ├── registry.ts ← Plugin registry (auto-discovery) │ ├── config.ts ← Config loader (V2 detection + V0 fallback) │ ├── config-watcher.ts ← Config hot-reload +│ ├── agent-runner.ts ← Unified CLI executor (--allowedTools, --max-turns, --model, retries) +│ ├── model-selector.ts ← Model recommendation per task type + profile +│ ├── cli-adapter.ts ← CLIAdapter interface + CLISpawnConfig + CapabilityLevel +│ ├── adapter-registry.ts ← Maps tool names to CLIAdapter instances (lazy-loads built-ins) +│ ├── adapters/ ← Built-in CLIAdapter implementations +│ │ ├── index.ts ← Re-exports all adapters +│ │ ├── claude-adapter.ts ← ClaudeAdapter: --print, --session-id, --resume, --allowedTools +│ │ ├── codex-adapter.ts ← CodexAdapter: exec, --skip-git-repo-check, --json, -o, sandbox +│ │ └── aider-adapter.ts ← AiderAdapter: --no-auto-commits, --yes, --message │ ├── health.ts ← Health check endpoint │ ├── metrics.ts ← Metrics collection │ ├── audit-logger.ts ← Audit trail @@ -409,24 +1046,44 @@ src/ │ └── logger.ts ← Pino logger ├── connectors/ │ ├── index.ts ← Connector registry -│ ├── whatsapp/ ← WhatsApp connector (V0) -│ └── console/ ← Console connector (reference impl) +│ ├── console/ ← Console connector (reference impl) +│ ├── webchat/ ← WebChat connector (browser UI) +│ ├── whatsapp/ ← WhatsApp connector +│ ├── telegram/ ← Telegram connector (grammY) +│ └── discord/ ← Discord connector (discord.js v14) ├── providers/ │ ├── index.ts ← Provider registry -│ └── claude-code/ ← Claude Code CLI provider (V0) -│ ├── claude-code-provider.ts -│ ├── claude-code-executor.ts ← Generalized executor (any CLI) -│ ├── claude-code-config.ts -│ ├── session-manager.ts -│ └── provider-error.ts +│ ├── claude-code/ ← Claude Code CLI provider (uses AgentRunner + ClaudeAdapter) +│ │ ├── claude-code-provider.ts +│ │ ├── claude-code-config.ts +│ │ ├── session-manager.ts +│ │ └── provider-error.ts +│ └── codex/ ← Codex CLI provider (uses AgentRunner + CodexAdapter) +│ ├── codex-provider.ts ← CodexProvider: processMessage(), streamMessage(), isAvailable() +│ ├── codex-config.ts ← CodexConfig Zod schema (workspacePath, timeout, model, sandbox) +│ └── session-manager.ts ← Codex session management (ephemeral → named session → resume) ├── discovery/ │ ├── index.ts ← scanForAITools() export │ ├── tool-scanner.ts ← CLI tool detection (which) │ └── vscode-scanner.ts ← VS Code extension detection └── master/ ├── index.ts ← Module exports - ├── master-manager.ts ← Master AI lifecycle + message routing - ├── dotfolder-manager.ts ← .openbridge/ CRUD + git operations - ├── exploration-prompt.ts ← System prompt for workspace exploration + ├── master-manager.ts ← Master AI lifecycle + task classification + sessions + + │ checkpointSession() / resumeSession() (5710+ LOC) + ├── master-system-prompt.ts ← Master AI system prompt builder (includes memory.md guidance) + ├── worker-registry.ts ← Active worker tracking + concurrency limits + ├── dotfolder-manager.ts ← .openbridge/ CRUD + exploration state + prompt library + + │ readMemoryFile() / writeMemoryFile() (866+ LOC) + ├── exploration-coordinator.ts ← 5-pass orchestration + checkpointing + ├── exploration-prompts.ts ← Pass-specific prompt generators + ├── exploration-prompt.ts ← Legacy monolithic exploration prompt (V0) + ├── result-parser.ts ← Robust JSON extraction with fallbacks + ├── seed-prompts.ts ← Initial prompt templates for Master AI + ├── spawn-parser.ts ← Parse worker spawn requests from Master output + ├── worker-result-formatter.ts ← Format worker results for Master + ├── workspace-change-tracker.ts ← Git-based workspace change detection + ├── prompt-evolver.ts ← Prompt effectiveness tracking + self-improvement + ├── sub-master-detector.ts ← Sub-master capability detection + ├── sub-master-manager.ts ← Sub-master session pool management └── delegation.ts ← Multi-AI task delegation ``` diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 105c1fc9..0f68dc98 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -1,6 +1,6 @@ # OpenBridge — Configuration Guide -> **Last Updated:** 2026-02-20 +> **Last Updated:** 2026-02-27 --- @@ -44,6 +44,7 @@ npx openbridge init | `audit` | `object` | No | `{}` | Audit logging settings | | `health` | `object` | No | `{}` | Health check endpoint settings | | `metrics` | `object` | No | `{}` | Metrics endpoint settings | +| `mcp` | `object` | No | — | MCP server configuration (Claude-only) | | `logLevel` | `string` | No | `info` | One of: trace, debug, info, warn, error, fatal | ### `workspacePath` @@ -73,11 +74,43 @@ Array of messaging channel configurations. At least one required. ] ``` -| Field | Type | Required | Default | Description | -| --------- | --------- | :------: | ------- | ------------------------------------ | -| `type` | `string` | Yes | — | Channel type (`whatsapp`, `console`) | -| `enabled` | `boolean` | No | `true` | Enable/disable this channel | -| `options` | `object` | No | `{}` | Channel-specific options | +| Field | Type | Required | Default | Description | +| --------- | --------- | :------: | ------- | ------------------------------------------------------------------------ | +| `type` | `string` | Yes | — | Channel type: `console`, `webchat`, `whatsapp`, `telegram`, or `discord` | +| `enabled` | `boolean` | No | `true` | Enable/disable this channel | +| `options` | `object` | No | `{}` | Channel-specific options (see per-type tables below) | + +#### Console Options + +No options required. The Console connector reads from stdin and writes to stdout. + +> **Note:** Console messages are sent as `console-user`. Add `"console-user"` to your whitelist, or leave whitelist empty (V0 only) to allow all. + +#### WebChat Options + +| Option | Type | Default | Description | +| ------ | -------- | ----------- | ----------------------------------------- | +| `port` | `number` | `3000` | TCP port the HTTP + WebSocket server uses | +| `host` | `string` | `localhost` | Hostname the server binds to | + +> **Tip:** To expose WebChat on your local network, set `"host": "0.0.0.0"`. + +#### Telegram Options + +| Option | Type | Required | Description | +| ------------- | -------- | :------: | ------------------------------------------------------- | +| `token` | `string` | Yes | Bot token from @BotFather | +| `botUsername` | `string` | No | Bot username without `@` — required for group @mentions | + +> **Setup:** Create a bot via [@BotFather](https://t.me/botfather) (`/newbot`), copy the token. Telegram whitelist entries use the sender's phone number or numeric user ID. + +#### Discord Options + +| Option | Type | Required | Description | +| ------- | -------- | :------: | ----------------------------------- | +| `token` | `string` | Yes | Bot token from the Developer Portal | + +> **Setup:** Create an application at [discord.com/developers/applications](https://discord.com/developers/applications), add a Bot, copy the token. Discord whitelist entries use numeric user IDs (e.g. `"123456789012345678"`). #### WhatsApp Options @@ -105,12 +138,14 @@ Authentication and security configuration. } ``` -| Field | Type | Default | Description | -| --------------- | ---------- | ------- | ------------------------------------------------ | -| `whitelist` | `string[]` | `[]` | Phone numbers allowed to use the bridge | -| `prefix` | `string` | `/ai` | Command prefix (messages without it are ignored) | -| `rateLimit` | `object` | `{}` | Per-user rate limiting | -| `commandFilter` | `object` | `{}` | Command allow/deny patterns | +| Field | Type | Default | Description | +| --------------- | ---------- | ------------------------- | --------------------------------------------------------------------- | +| `whitelist` | `string[]` | `[]` (V0) / required (V2) | Senders allowed to use the bridge. **V2 requires at least one entry** | +| `prefix` | `string` | `/ai` | Command prefix (messages without it are ignored) | +| `rateLimit` | `object` | `{}` | Per-user rate limiting | +| `commandFilter` | `object` | `{}` | Command allow/deny patterns | + +> **V2 whitelist requirement:** In V2 config, `auth.whitelist` must contain at least one entry. An empty array is rejected at startup with a Zod validation error. Use the sender's identifier for each connector: phone number for WhatsApp/Telegram (e.g. `"+1234567890"`), `"console-user"` for Console, `"webchat-user"` for WebChat, and numeric user ID for Discord. If you need open access during development, use V0 config format instead. #### Rate Limit @@ -130,19 +165,183 @@ Authentication and security configuration. ### `master` (optional) -Override the auto-detected Master AI settings. +Override the auto-detected Master AI settings. By default, OpenBridge scans your machine for AI tools (`claude`, `codex`, `aider`, etc.) and picks the most capable one as Master. Use this section to override that behavior. + +```json +"master": { + "tool": "codex" +} +``` + +| Field | Type | Default | Description | +| ------------------- | -------- | ------------- | ---------------------------------------------------- | +| `tool` | `string` | auto-detected | Force a specific tool as Master (exact name or path) | +| `explorationPrompt` | `string` | built-in | Custom prompt for workspace exploration _(planned)_ | +| `sessionTtlMs` | `number` | `1800000` | Session lifetime in milliseconds _(planned)_ | + +#### Discovery Override Examples + +**Force a specific tool by name:** + +```json +"master": { + "tool": "aider" +} +``` + +OpenBridge will skip auto-detection and use `aider` if it's installed. + +**Force Codex as Master:** + +```json +"master": { + "tool": "codex" +} +``` + +Codex requires `OPENAI_API_KEY` to be set in your environment — see [AI Provider Requirements](#ai-provider-requirements) below. + +**Use a specific Claude installation:** ```json "master": { - "tool": "codex", - "explorationPrompt": "Custom exploration instructions..." + "tool": "/usr/local/bin/claude" +} +``` + +Useful if you have multiple AI CLIs installed and want to pick a specific one. + +**Planned features (not yet implemented):** + +- `explorationPrompt`: Custom exploration instructions for non-code workspaces +- `sessionTtlMs`: Override session expiry time for privacy/performance tuning + +### `mcp` (optional) + +Enable MCP (Model Context Protocol) servers so workers can call external services — Gmail, Slack, Canva, databases, and anything else with an MCP server. + +> **Claude-only:** MCP via `--mcp-config` is supported for Claude workers only. Codex has native MCP support wired separately via `codex mcp`. + +```json +"mcp": { + "enabled": true, + "servers": [ + { + "name": "filesystem", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-filesystem", "/tmp"] + }, + { + "name": "gmail", + "command": "npx", + "args": ["-y", "@anthropic/gmail-mcp-server"], + "env": { "GMAIL_OAUTH_TOKEN": "ya29...." } + } + ] +} +``` + +| Field | Type | Default | Description | +| ------------ | --------- | ------- | -------------------------------------------------------------------------------- | +| `enabled` | `boolean` | `true` | Enable/disable MCP globally | +| `servers` | `array` | `[]` | Inline server definitions (see table below) | +| `configPath` | `string` | — | Path to an existing Claude Desktop / Claude Code MCP config to import (optional) | + +#### MCP Server Fields + +Each entry in `servers` defines one MCP server: + +| Field | Type | Required | Description | +| --------- | ------------------------ | :------: | ------------------------------------------------ | +| `name` | `string` | Yes | Unique name used to reference this server | +| `command` | `string` | Yes | Executable to run (e.g. `npx`, `python`, `node`) | +| `args` | `string[]` | No | Arguments passed to the command | +| `env` | `Record` | No | Environment variables injected into the process | + +#### Importing from Claude Desktop / Claude Code + +If you already have MCP servers configured in Claude Desktop or Claude Code, point `configPath` at that file to reuse the same servers without duplication: + +```json +"mcp": { + "configPath": "~/.claude/claude_desktop_config.json" +} +``` + +You can combine both — inline `servers` entries take precedence over same-named imports from `configPath`: + +```json +"mcp": { + "configPath": "~/.claude/claude_desktop_config.json", + "servers": [ + { + "name": "gmail", + "command": "npx", + "args": ["-y", "@anthropic/gmail-mcp-server"], + "env": { "GMAIL_OAUTH_TOKEN": "ya29...." } + } + ] +} +``` + +#### Security Model — Per-Worker Isolation + +OpenBridge uses **per-worker temp config files** and `--strict-mcp-config` to isolate each worker's MCP access: + +1. When the Master AI spawns a worker, it specifies exactly which MCP servers that worker needs in its `TaskManifest.mcpServers`. +2. OpenBridge writes a temporary JSON file containing **only those servers** — not the full server list. +3. The worker is launched with `--mcp-config /tmp/ob-mcp-.json --strict-mcp-config`. +4. After the worker exits, the temp file is deleted. + +This means a worker that only needs `gmail` cannot call `canva` tools — even if `canva` is configured globally. Each worker sees exactly what it needs, nothing more. + +#### Examples for Popular MCP Servers + +**Filesystem (read/write local files):** + +```json +{ + "name": "filesystem", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-filesystem", "/Users/you/projects"] +} +``` + +**Git (query repo history and diffs):** + +```json +{ + "name": "git", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-git", "--repository", "/Users/you/projects/my-app"] +} +``` + +**Gmail (send and read emails):** + +```json +{ + "name": "gmail", + "command": "npx", + "args": ["-y", "@anthropic/gmail-mcp-server"], + "env": { "GMAIL_OAUTH_TOKEN": "ya29...." } +} +``` + +**Slack (post messages, read channels):** + +```json +{ + "name": "slack", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-slack"], + "env": { "SLACK_BOT_TOKEN": "xoxb-..." } } ``` -| Field | Type | Default | Description | -| ------------------- | -------- | ------------- | ----------------------------------------- | -| `tool` | `string` | auto-detected | Force a specific tool as Master (by name) | -| `explorationPrompt` | `string` | built-in | Custom prompt for workspace exploration | +> **MCP ecosystem:** Browse available servers at [modelcontextprotocol.io](https://modelcontextprotocol.io) and the [MCP servers GitHub repo](https://github.com/modelcontextprotocol/servers). + +--- ### `queue` @@ -231,13 +430,69 @@ V0 config runs the legacy startup flow (direct provider routing, no discovery, n --- +## AI Provider Requirements + +OpenBridge auto-discovers AI tools on your machine. Each tool has its own authentication model: + +| Tool | Auth Method | Requirement | +| ---------- | ------------------------------------ | --------------------------------------------- | +| **Claude** | Local CLI auth (`claude auth login`) | None — uses your existing Claude subscription | +| **Codex** | API key via environment variable | `OPENAI_API_KEY` must be set before starting | +| **Aider** | API key via environment variable | Depends on the underlying model provider | + +### Claude + +Claude Code authenticates via your local Claude subscription. Run `claude auth login` once to set it up. No API keys or environment variables required for OpenBridge. + +### Codex + +Codex requires a valid OpenAI API key. Set it in your environment before starting OpenBridge: + +**Option 1 — Shell environment (recommended):** + +```bash +export OPENAI_API_KEY="sk-..." +npm run dev +``` + +**Option 2 — `.env` file in the OpenBridge directory:** + +```bash +# .env +OPENAI_API_KEY=sk-... +``` + +OpenBridge reads `.env` automatically on startup (via Node.js `--env-file` or dotenv if present). + +> **Note:** If `OPENAI_API_KEY` is missing, Codex workers will fail immediately with an auth error. OpenBridge logs a clear message: `"Codex requires OPENAI_API_KEY environment variable"` — no retries are wasted. + +#### Codex Sandbox Modes + +Codex workers run in a sandboxed environment that controls what actions they can take. The sandbox mode is inferred from the worker's tool profile: + +| Sandbox Mode | Codex Flag | What the Worker Can Do | +| ------------ | ---------------------- | -------------------------------------------------------------------------------------------------- | +| `read-only` | `--sandbox read-only` | Read files, run read-only commands. No writes or edits. Default when no tool profile is specified. | +| `read-write` | `--sandbox read-write` | Read and write files. Cannot run arbitrary shell commands. | +| `full-auto` | `--full-auto` | Full access — read, write, run commands. Use with caution. | + +OpenBridge maps tool profiles to sandbox modes automatically: + +- `read-only` tool profile → `read-only` sandbox +- `code-edit` tool profile → `read-write` sandbox +- `full-access` tool profile → `full-auto` sandbox +- No profile specified → `read-only` sandbox (safe default) + +--- + ## Environment Variables -| Variable | Description | Default | -| ------------- | ---------------------------------------- | ------------- | -| `CONFIG_PATH` | Path to config file | `config.json` | -| `LOG_LEVEL` | Override log level from config | from config | -| `NODE_ENV` | Environment (`development`/`production`) | `development` | +| Variable | Description | Default | +| ---------------- | -------------------------------------------------------------- | ------------- | +| `CONFIG_PATH` | Path to config file | `config.json` | +| `LOG_LEVEL` | Override log level from config | from config | +| `NODE_ENV` | Environment (`development`/`production`) | `development` | +| `OPENAI_API_KEY` | OpenAI API key — required when using Codex as Master or worker | _(none)_ | --- @@ -287,6 +542,22 @@ Load config.json "denyPatterns": ["rm -rf", "DROP TABLE", "sudo", "format c:"] } }, + "mcp": { + "enabled": true, + "servers": [ + { + "name": "filesystem", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-filesystem", "/Users/you/projects/my-saas-app"] + }, + { + "name": "slack", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-slack"], + "env": { "SLACK_BOT_TOKEN": "xoxb-..." } + } + ] + }, "queue": { "maxRetries": 3, "retryDelayMs": 1000 diff --git a/docs/CONNECTORS.md b/docs/CONNECTORS.md new file mode 100644 index 00000000..9afbd211 --- /dev/null +++ b/docs/CONNECTORS.md @@ -0,0 +1,294 @@ +# OpenBridge — Connector Testing Guide + +How to enable and test each connector. All connectors use the same `config.json` — just swap the `channels` array. + +--- + +## Console + +The simplest connector — reads from stdin, writes to stdout. No credentials needed. + +**When to use:** Local development, scripting, demos without a phone/browser. + +**Setup:** Console is enabled by default. Just run the bridge. + +```bash +npm run dev +``` + +Type a message and press Enter. Prefix with `/ai` (or whatever your `auth.prefix` is set to). + +**Sample config.json:** + +```json +{ + "workspacePath": "/absolute/path/to/your/project", + "channels": [ + { + "type": "console", + "enabled": true + } + ], + "auth": { + "whitelist": ["console-user"], + "prefix": "/ai" + } +} +``` + +> **Note:** The Console connector sends messages as `console-user`. Add that string to your whitelist, or leave whitelist empty to allow all. + +--- + +## WebChat + +A browser-based chat UI served on `localhost`. No phone or bot account needed. + +**When to use:** Demos, local testing with a polished UI, sharing with teammates on the same machine. + +**Setup:** + +1. Add `webchat` to `channels` in config.json (see sample below). +2. Run the bridge: `npm run dev` +3. Open `http://localhost:3000` in your browser. +4. Type a message and send. + +**Sample config.json:** + +```json +{ + "workspacePath": "/absolute/path/to/your/project", + "channels": [ + { + "type": "webchat", + "enabled": true, + "options": { + "port": 3000, + "host": "localhost" + } + } + ], + "auth": { + "whitelist": ["webchat-user"], + "prefix": "/ai" + } +} +``` + +**Options:** + +| Option | Default | Description | +| ------ | ----------- | ----------------------------------------- | +| `port` | `3000` | TCP port the HTTP + WebSocket server uses | +| `host` | `localhost` | Hostname the server binds to | + +> **Tip:** To expose WebChat on your local network (e.g. for phone testing), set `"host": "0.0.0.0"` and open `http://:3000`. + +--- + +## Telegram + +A Telegram bot that responds to direct messages and group `@mentions`. + +**When to use:** Mobile testing, team deployments, production use with Telegram users. + +**Setup:** + +1. Create a bot via [@BotFather](https://t.me/botfather): + - Send `/newbot` + - Choose a name (e.g. `My OpenBridge Bot`) + - Choose a username ending in `bot` (e.g. `my_openbridge_bot`) + - Copy the token (format: `123456789:ABC-DEF...`) +2. Add the token to `config.json` (see sample below). +3. Run the bridge: `npm run dev` +4. Open Telegram, find your bot, send a message. + +**Sample config.json:** + +```json +{ + "workspacePath": "/absolute/path/to/your/project", + "channels": [ + { + "type": "telegram", + "enabled": true, + "options": { + "token": "123456789:ABC-DEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnop", + "botUsername": "my_openbridge_bot" + } + } + ], + "auth": { + "whitelist": ["+1234567890"], + "prefix": "/ai" + } +} +``` + +**Options:** + +| Option | Required | Description | +| ------------- | :------: | ------------------------------------------------------- | +| `token` | Yes | Bot token from @BotFather | +| `botUsername` | No | Bot username without `@` — required for group @mentions | + +**Group chats:** Add the bot to a group and mention it: `@my_openbridge_bot /ai what's in this project?` + +**Whitelist:** Telegram messages use the sender's phone number (if available) or numeric user ID. To allow all users, set `"whitelist": []`. + +--- + +## Discord + +A Discord bot that responds to DMs and messages in guild channels. + +**When to use:** Team deployments on Discord servers, developer communities. + +**Setup:** + +1. Create a Discord application and bot: + - Go to [discord.com/developers/applications](https://discord.com/developers/applications) + - Click **New Application** → name it (e.g. `OpenBridge`) + - Go to **Bot** → click **Add Bot** → confirm + - Under **Token**, click **Reset Token** and copy it + - Under **Privileged Gateway Intents**, enable: + - **Server Members Intent** + - **Message Content Intent** + - Click **Save Changes** +2. Invite the bot to your server: + - Go to **OAuth2 → URL Generator** + - Select scopes: `bot` + - Select bot permissions: `Send Messages`, `Read Message History`, `View Channels` + - Open the generated URL and add the bot to your server +3. Add the token to `config.json` (see sample below). +4. Run the bridge: `npm run dev` +5. Send a message to the bot in a channel or DM. + +**Sample config.json:** + +```json +{ + "workspacePath": "/absolute/path/to/your/project", + "channels": [ + { + "type": "discord", + "enabled": true, + "options": { + "token": "YOUR_DISCORD_BOT_TOKEN_HERE" + } + } + ], + "auth": { + "whitelist": ["your-discord-user-id"], + "prefix": "/ai" + } +} +``` + +**Options:** + +| Option | Required | Description | +| ------- | :------: | ----------------------------------- | +| `token` | Yes | Bot token from the Developer Portal | + +**Whitelist:** Discord messages use the sender's Discord user ID (numeric string, e.g. `"123456789012345678"`). Enable Developer Mode in Discord settings to copy user IDs. To allow all users, set `"whitelist": []`. + +--- + +## WhatsApp + +Connects to WhatsApp via a QR code scan (uses the whatsapp-web.js library). + +**When to use:** Production deployments, mobile-first workflows, sharing with non-technical users. + +**Setup:** + +1. Make sure Google Chrome or Chromium is installed on your machine (required by whatsapp-web.js). +2. Add `whatsapp` to `channels` in config.json (see sample below). +3. Run the bridge: `npm run dev` +4. A QR code appears in the terminal. +5. Open WhatsApp on your phone → **Settings → Linked Devices → Link a Device** → scan the QR code. +6. Send a message from a whitelisted number. Prefix with `/ai`. + +**Sample config.json:** + +```json +{ + "workspacePath": "/absolute/path/to/your/project", + "channels": [ + { + "type": "whatsapp", + "enabled": true, + "options": { + "sessionName": "openbridge-default" + } + } + ], + "auth": { + "whitelist": ["+1234567890"], + "prefix": "/ai" + } +} +``` + +**Options:** + +| Option | Default | Description | +| ------------- | -------------------- | --------------------------------------------------- | +| `sessionName` | `openbridge-default` | Name for the WhatsApp session (used as folder name) | +| `sessionPath` | (auto) | Custom path to store session data | +| `headless` | `true` | Run Chromium headlessly (set `false` to debug) | +| `reconnect` | see below | Auto-reconnect settings | + +**Reconnect defaults:** + +```json +{ + "enabled": true, + "maxAttempts": 10, + "initialDelayMs": 2000, + "maxDelayMs": 60000, + "backoffFactor": 2 +} +``` + +**Whitelist:** Use the full international format with `+` and country code, e.g. `"+1234567890"`. + +**Session persistence:** Once linked, the session is saved to disk. On restart, OpenBridge reconnects automatically without re-scanning the QR code. + +**Troubleshooting:** + +- **QR code not appearing:** Ensure Chrome/Chromium is installed (`google-chrome --version` or `chromium --version`). +- **`ProtocolError: Execution context was destroyed`:** Transient Chromium crash during startup — OpenBridge retries automatically (up to 3 attempts with backoff). If it persists, try restarting. +- **Session expired:** Delete the `.wwebjs_auth/` folder and re-scan the QR code. + +--- + +## Running Multiple Connectors + +You can enable multiple connectors at the same time. Each operates independently — the same Master AI handles messages from all channels. + +```json +{ + "workspacePath": "/absolute/path/to/your/project", + "channels": [ + { "type": "console", "enabled": true }, + { + "type": "webchat", + "enabled": true, + "options": { "port": 3000 } + }, + { + "type": "telegram", + "enabled": true, + "options": { "token": "YOUR_TELEGRAM_TOKEN" } + } + ], + "auth": { + "whitelist": [], + "prefix": "/ai" + } +} +``` + +All connectors start in parallel on `npm run dev`. Responses are delivered back through the same connector the message came from. diff --git a/docs/CREATIVE_OUTPUT.md b/docs/CREATIVE_OUTPUT.md new file mode 100644 index 00000000..c52f2e35 --- /dev/null +++ b/docs/CREATIVE_OUTPUT.md @@ -0,0 +1,211 @@ +# Creative Output Guide + +OpenBridge supports generating visual and creative content through its skill pack system. Workers can produce diagrams, charts, web pages, presentations, generative art, and brand assets — and deliver them via messaging channels. + +--- + +## Supported Output Types + +| Skill Pack | What it produces | Output format | +| ----------------- | ------------------------------------------------------------- | ------------------------------------ | +| `diagram-maker` | Architecture diagrams, flowcharts, sequence, ER, class, Gantt | Mermaid / PlantUML / D2 text + PNG | +| `chart-generator` | Bar, line, pie, scatter, area charts from data | Self-contained HTML | +| `web-designer` | Landing pages, marketing sites, HTML email templates | Self-contained HTML | +| `slide-designer` | Presentation slides, pitch decks, training materials | Self-contained HTML (PDF-exportable) | +| `generative-art` | Algorithmic art, SVG patterns, p5.js sketches | HTML or SVG | +| `brand-assets` | Logo concepts, social media images, favicons | SVG / HTML | + +--- + +## Triggering Creative Output + +The Master AI automatically selects the right skill pack based on your request. Phrases that trigger creative mode: + +``` +/ai draw an architecture diagram of the project +/ai create a bar chart from sales.csv +/ai build a landing page for my product +/ai make a pitch deck with 5 slides +/ai generate a logo concept for "Acme Corp" +/ai create a flowchart of the authentication flow +``` + +No configuration required — the router classifies intent and selects the appropriate skill pack. + +--- + +## Rendering Pipeline + +Creative output follows this pipeline: + +``` +Worker generates content (Mermaid / SVG / HTML) + │ + ▼ +File Server serves the file at a local URL + │ + ▼ +HTML Renderer converts to PNG (optional, requires Puppeteer) + │ + ▼ +Channel delivers as image (WhatsApp / Telegram) or link (WebChat) +``` + +### When images are delivered directly + +For WhatsApp and Telegram, OpenBridge renders the output to PNG and sends it as a media message if the renderer is available. Otherwise it sends a preview URL. + +For WebChat, outputs are served as interactive HTML previews via the built-in file server. + +--- + +## Mermaid Rendering + +Mermaid diagrams are rendered using two backends, tried in order: + +1. **mermaid.ink API** — no installation required; needs network access. +2. **Puppeteer fallback** — renders locally in a headless browser; requires `puppeteer` installed. + +If both backends fail, the diagram definition is returned as text so you can paste it into [mermaid.live](https://mermaid.live) or any Markdown preview. + +**Supported Mermaid diagram types:** + +- Flowchart (`flowchart TD / LR`) +- Sequence diagram (`sequenceDiagram`) +- Entity-relationship (`erDiagram`) +- Class diagram (`classDiagram`) +- State diagram (`stateDiagram-v2`) +- Gantt chart (`gantt`) +- Pie chart (`pie`) + +--- + +## HTML-to-Image Rendering + +HTML and SVG outputs are converted to PNG/JPEG using `HTMLRenderer` (Puppeteer-based). + +**Default settings:** + +| Setting | Default | +| ------------ | -------- | +| Format | PNG | +| Viewport | 1280×720 | +| JPEG quality | 90 | +| Full page | false | + +SVG files are automatically sized to their natural `viewBox` or `width`/`height` dimensions. + +Rendered images are saved to `.openbridge/generated/` inside the workspace. + +--- + +## Prerequisites + +### No extras needed + +- **Diagrams via mermaid.ink** — works out of the box (requires internet access). +- **SVG output** — no dependencies; served directly by the file server. +- **HTML output as links** — no dependencies; served via the built-in HTTP server. + +### Optional: Puppeteer (for local PNG rendering) + +Install Puppeteer to enable local HTML-to-image and Mermaid fallback rendering: + +```bash +npm install puppeteer +``` + +Puppeteer downloads Chromium on first install (~300 MB). After installation, all rendering is local — no network access required. + +Check if Puppeteer is available: + +```typescript +import { HTMLRenderer } from './src/core/html-renderer.js'; +const available = await HTMLRenderer.isAvailable(); // true/false +``` + +### Diagram tools (optional, for text-based rendering only) + +These tools are NOT required — they are used by workers to generate diagram syntax, not to render it. + +| Tool | Install | Used for | +| ----------- | ---------------------------------------- | ---------------------- | +| Mermaid CLI | `npm install -g @mermaid-js/mermaid-cli` | Local `mmdc` rendering | +| PlantUML | `brew install plantuml` (macOS) | PlantUML rendering | +| D2 | `brew install d2` (macOS) | D2 diagram rendering | + +Workers produce diagram syntax; rendering is handled by OpenBridge's rendering pipeline. + +--- + +## Format Selection Guide + +Workers default to Mermaid for maximum portability. Use these guidelines: + +| Use case | Recommended format | +| --------------------------------------------- | ------------------ | +| Flowcharts, decision trees, process flows | Mermaid flowchart | +| API call sequences, service communication | Mermaid sequence | +| Database schemas, table relationships | Mermaid ER | +| Class hierarchies, interface implementations | Mermaid class | +| State machines, lifecycle diagrams | Mermaid state | +| Project timelines, sprint plans | Mermaid Gantt | +| System architecture, infrastructure maps | D2 | +| UML (formal notation, component diagrams) | PlantUML | +| Data visualizations (bar, line, pie, scatter) | Chart.js / D3.js | +| Web UI, landing pages, email | HTML/CSS | +| Presentations, pitch decks | Reveal.js HTML | +| Algorithmic art, patterns | p5.js / SVG | +| Logos, social media images, favicons | SVG | + +--- + +## Output Locations + +All generated files are written to: + +``` +/.openbridge/generated/ +├── render-.png ← HTML/SVG renders +├── mermaid-.png ← Mermaid renders +└── .html ← HTML outputs from workers +``` + +The file server exposes these files at: + +``` +http://localhost:/files/ +``` + +Workers write HTML and SVG files directly to the workspace or to `.openbridge/generated/`. The Master AI coordinates delivery to the channel. + +--- + +## Troubleshooting + +### "Puppeteer is not installed" + +Install it: `npm install puppeteer` + +If you don't want to install Puppeteer, mermaid.ink (for Mermaid diagrams) and direct SVG serving still work without it. + +### "mermaid.ink API unavailable" + +The mermaid.ink public API may be rate-limited or unreachable. Install Puppeteer for offline fallback rendering. + +### "Mermaid rendering failed" + +Both backends failed. Check: + +1. Network access for mermaid.ink +2. Puppeteer installation: `npm list puppeteer` + +If both are unavailable, the worker returns the diagram as a Mermaid code block — paste it into [mermaid.live](https://mermaid.live) to render manually. + +### SVG not rendering in WhatsApp + +WhatsApp does not support SVG. OpenBridge automatically converts SVG to PNG via `HTMLRenderer` before sending. This requires Puppeteer. If Puppeteer is not installed, a download link is sent instead. + +### HTML preview not accessible + +The file server must be running and reachable from your device. In development, ensure the configured port is accessible on your network. diff --git a/docs/IMPLEMENTATION-PLAN.md b/docs/IMPLEMENTATION-PLAN.md new file mode 100644 index 00000000..9bb13bbd --- /dev/null +++ b/docs/IMPLEMENTATION-PLAN.md @@ -0,0 +1,1662 @@ +# OpenBridge Business Platform — Implementation Plan + +> **Goal**: Transform OpenBridge from a developer AI bridge into a universal business AI platform. +> Every pattern below is inspired by production ERPs/CRMs and adapted for AI-first, conversation-driven usage. + +--- + +## Table of Contents + +- [Part 1: Architecture Patterns (What We Learn From)](#part-1-architecture-patterns) +- [Part 2: The DocType Engine](#part-2-the-doctype-engine) +- [Part 3: Document Intelligence Layer](#part-3-document-intelligence-layer) +- [Part 4: State Machine & Lifecycle Engine](#part-4-state-machine--lifecycle-engine) +- [Part 5: Computed Fields & Reactive Cascade](#part-5-computed-fields--reactive-cascade) +- [Part 6: Integration Hub](#part-6-integration-hub) +- [Part 7: Workflow Engine](#part-7-workflow-engine) +- [Part 8: Document Generation Pipeline](#part-8-document-generation-pipeline) +- [Part 9: Web Page & App Generation](#part-9-web-page--app-generation) +- [Part 10: Credential Security](#part-10-credential-security) +- [Part 11: Self-Improvement & Skill Learning](#part-11-self-improvement--skill-learning) +- [Part 12: Marketplace Integration](#part-12-marketplace-integration) +- [Part 13: Industry Templates](#part-13-industry-templates) +- [Part 14: Implementation Phases & Task Breakdown](#part-14-implementation-phases--task-breakdown) +- [Part 15: Tech Stack Additions](#part-15-tech-stack-additions) + +--- + +## Part 1: Architecture Patterns + +### Pattern Map: ERP/CRM → OpenBridge + +Every major feature is inspired by a proven production system. This table is the reference for all implementation decisions. + +| Business Need | Inspiration Source | How They Solve It | OpenBridge Adaptation | +| --------------------------- | ------------------------------------ | -------------------------------------------------------------------------------------- | ------------------------------------------------------------------------- | +| **Organized data** | Frappe DocType | ONE JSON definition → DB table + REST API + Web form + PDF + permissions | AI creates DocType from conversation → SQLite table + API + form | +| **Auto-numbering** | Frappe `naming_series` + `tabSeries` | Partitioned counter table, `FOR UPDATE` row lock, zero-padded | `dt_series` table in SQLite, per-prefix counters | +| **Child records** | Frappe child tables | `parent` / `parentfield` / `parenttype` triple, delete-and-reinsert on save | Invoice items, project tasks, order lines as child tables | +| **Computed fields** | Odoo `@api.depends` | Reactive dependency graph, cascading recomputation, stored vs non-stored | SQLite `GENERATED` columns + triggers for cross-table cascade | +| **Unified documents** | Odoo `account.move` | ONE model for invoices + bills + credit notes + payments, discriminated by `move_type` | Single `dt_document` concept with `doc_type` discriminator | +| **Payment matching** | Odoo reconciliation engine | Match debit/credit lines on same account, partial + full reconcile | Stripe payment → match to invoice by payment link ID | +| **Email from any doc** | Odoo `mail.thread` mixin | `_inherit = ['mail.thread']` adds messaging to any model | `[SHARE:email]` output markers already exist | +| **Auto-actions** | Odoo `base.automation` | AOP: monkey-patch `create()`/`write()` to inject trigger checks | Workflow triggers on DocType field changes | +| **Pipeline stages** | HubSpot CRM | Stages with probability weighting → revenue forecasting | Task/order pipelines with weighted metrics | +| **Activity timeline** | HubSpot contact timeline | Multi-source aggregation via associations, custom timeline events API | Knowledge Graph + `conversation_messages` aggregation | +| **Dynamic schema** | Twenty CRM | Metadata tables → runtime `ALTER TABLE` → regenerate GraphQL → cache | DocType metadata → runtime `CREATE TABLE` → auto-generate API | +| **Custom objects** | Twenty CRM | `ObjectMetadata` + `FieldMetadata` tables, per-workspace PostgreSQL schema | `doctypes` + `doctype_fields` metadata tables in SQLite | +| **Execution pipeline** | Salesforce triggers | 20-step deterministic order: before-save → validate → save → after-save → commit | Message → auth → classify → before-hooks → execute → after-hooks → commit | +| **Visual automation** | Salesforce Flows | Interpreter pattern: flow definition = program, runtime = interpreter | Workflow definition in SQLite, executed by workflow engine | +| **Formula fields** | Salesforce | Compiled expressions evaluated at query time, cross-object traversal | SQLite `GENERATED` columns + AI-evaluated complex formulas | +| **Payment gateways** | Invoice Ninja | Strategy pattern: `BasePaymentDriver` → 25+ gateway implementations | `BusinessIntegration` interface → Stripe/PayPal/Flouci adapters | +| **Recurring billing** | Invoice Ninja | Cron job + template clone → new invoice + auto-bill stored payment method | Workflow scheduler + DocType template instantiation | +| **Client portal** | Invoice Ninja | Token-based auth, scoped views (client sees only their records) | Generated HTML pages with UUID-based access | +| **PDF generation** | Frappe + Odoo | HTML template (Jinja/QWeb) → wkhtmltopdf subprocess → PDF | pdfmake (declarative JSON → PDF) + Puppeteer fallback | +| **Data flow between steps** | n8n | `{ json: data, binary: files }` structure between pipeline nodes | Worker output as structured JSON + file attachments | +| **Credential encryption** | n8n | AES-256-GCM encrypt-at-rest, decrypt-on-demand per node execution | Encrypt in SQLite, decrypt only when worker needs it | +| **Webhook lifecycle** | n8n | Register endpoint on activate, unregister on deactivate, persist in DB | Integration webhook endpoints on file-server | +| **Report generation** | Odoo QWeb | XML template with directives + data context → HTML → wkhtmltopdf → PDF | Skill pack + pdfmake/Puppeteer → PDF served via file-server | + +--- + +## Part 2: The DocType Engine + +### What It Is + +A DocType is a business entity definition that the Master AI creates when it detects the user needs to track something. ONE definition generates: database table, REST API, web form, list view, PDF template, state machine, FTS5 index, and WhatsApp commands. + +**Inspired by**: Frappe DocType (metadata-first), Twenty CRM (runtime schema), NocoDB (dual-database separation) + +### How It Works Step by Step + +``` +User says: "I need to track my invoices" + │ + ▼ +Step 1: INTENT DETECTION (Master AI, existing classification engine) + Master recognizes: user needs a new business entity + No existing DocType "Invoice" found + │ + ▼ +Step 2: SCHEMA GENERATION (Master AI spawns worker) + Worker prompt: "Design an Invoice entity with appropriate fields, + lifecycle states, and computed fields for a + [detected-industry] business" + Worker returns DocType JSON definition + │ + ▼ +Step 3: METADATA STORAGE + INSERT INTO doctypes (name, label, icon, source, ...) VALUES (...) + INSERT INTO doctype_fields (doctype_id, name, type, ...) VALUES (...) -- per field + INSERT INTO doctype_states (doctype_id, name, transitions, ...) VALUES (...) -- per state + INSERT INTO doctype_hooks (doctype_id, event, action, ...) VALUES (...) -- per hook + │ + ▼ +Step 4: TABLE CREATION (dynamic DDL) + CREATE TABLE dt_invoice ( + id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))), + invoice_number TEXT UNIQUE, + client_id TEXT, + issue_date TEXT DEFAULT (date('now')), + due_date TEXT, + subtotal REAL DEFAULT 0, + tax_rate REAL DEFAULT 19, + tax_amount REAL GENERATED ALWAYS AS (subtotal * tax_rate / 100) STORED, + total REAL GENERATED ALWAYS AS (subtotal + (subtotal * tax_rate / 100)) STORED, + status TEXT DEFAULT 'draft', + payment_link TEXT, + pdf_path TEXT, + notes TEXT, + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP, + created_by TEXT + ); + CREATE INDEX idx_dt_invoice_status ON dt_invoice(status); + CREATE INDEX idx_dt_invoice_client ON dt_invoice(client_id); + │ + ▼ +Step 5: CHILD TABLE CREATION (Frappe pattern) + CREATE TABLE dt_invoice__items ( + id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))), + parent_id TEXT NOT NULL REFERENCES dt_invoice(id) ON DELETE CASCADE, + idx INTEGER NOT NULL, -- sort order (Frappe pattern) + description TEXT NOT NULL, + quantity REAL DEFAULT 1, + unit_price REAL NOT NULL, + amount REAL GENERATED ALWAYS AS (quantity * unit_price) STORED, + UNIQUE(parent_id, idx) + ); + │ + ▼ +Step 6: SERIES TABLE (Frappe naming_series pattern) + INSERT INTO dt_series (prefix, current_value) VALUES ('INV-2026-', 0); + │ + ▼ +Step 7: FTS5 INDEX + CREATE VIRTUAL TABLE dt_invoice_fts USING fts5( + invoice_number, notes, content=dt_invoice, content_rowid=rowid + ); + -- Triggers to keep FTS5 in sync with data table + │ + ▼ +Step 8: RECOMPUTATION TRIGGERS (Odoo @api.depends pattern) + CREATE TRIGGER trg_invoice_recompute + AFTER INSERT ON dt_invoice__items + BEGIN + UPDATE dt_invoice SET + subtotal = (SELECT COALESCE(SUM(amount), 0) FROM dt_invoice__items WHERE parent_id = NEW.parent_id), + updated_at = datetime('now') + WHERE id = NEW.parent_id; + END; + -- Similar triggers for UPDATE and DELETE on items + │ + ▼ +Step 9: CONFIRMATION + Master replies: "✓ Invoice tracking is ready. + I can now create, send, and track invoices. + Try: 'Invoice Mohamed for web design TND 2000'" +``` + +### Database Schema (Metadata Tables) + +```sql +-- DocType definitions (schema registry) +CREATE TABLE doctypes ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL UNIQUE, -- "Invoice", "Customer", "Vehicle" + label_singular TEXT NOT NULL, -- "Invoice" + label_plural TEXT NOT NULL, -- "Invoices" + icon TEXT, -- "📄" + table_name TEXT NOT NULL UNIQUE, -- "dt_invoice" + source TEXT NOT NULL, -- 'ai-created', 'imported', 'integration', 'template' + template_id TEXT, -- industry template that spawned this + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP +); + +-- Field definitions per DocType +CREATE TABLE doctype_fields ( + id TEXT PRIMARY KEY, + doctype_id TEXT NOT NULL REFERENCES doctypes(id) ON DELETE CASCADE, + name TEXT NOT NULL, -- "invoice_number", "client_id" + label TEXT NOT NULL, -- "Invoice #", "Client" + field_type TEXT NOT NULL, -- 'text', 'number', 'currency', 'date', 'link', 'table', ... + required INTEGER DEFAULT 0, + default_value TEXT, -- JSON-encoded default + options TEXT, -- JSON array for select/multiselect + formula TEXT, -- Expression for computed fields + depends_on TEXT, -- Conditional visibility expression + searchable INTEGER DEFAULT 0, -- Include in FTS5 + sort_order INTEGER NOT NULL, -- Display order + link_doctype TEXT, -- For 'link' type: target DocType name + child_doctype TEXT, -- For 'table' type: child DocType name + UNIQUE(doctype_id, name) +); + +-- State machine definitions +CREATE TABLE doctype_states ( + id TEXT PRIMARY KEY, + doctype_id TEXT NOT NULL REFERENCES doctypes(id) ON DELETE CASCADE, + name TEXT NOT NULL, -- "draft", "sent", "paid", "overdue" + label TEXT NOT NULL, -- "Draft", "Sent", "Paid", "Overdue" + color TEXT DEFAULT 'gray', -- UI color hint + is_initial INTEGER DEFAULT 0, -- Starting state + is_terminal INTEGER DEFAULT 0, -- End state (no transitions out) + sort_order INTEGER NOT NULL, + UNIQUE(doctype_id, name) +); + +-- State transitions +CREATE TABLE doctype_transitions ( + id TEXT PRIMARY KEY, + doctype_id TEXT NOT NULL REFERENCES doctypes(id) ON DELETE CASCADE, + from_state TEXT NOT NULL, + to_state TEXT NOT NULL, + action_name TEXT NOT NULL, -- "send", "mark_paid", "cancel" + action_label TEXT NOT NULL, -- "Send Invoice", "Mark as Paid" + allowed_roles TEXT, -- JSON array: ["owner", "admin"] + condition TEXT, -- Expression: "total > 0" + UNIQUE(doctype_id, from_state, action_name) +); + +-- Lifecycle hooks (Odoo base.automation + Salesforce triggers) +CREATE TABLE doctype_hooks ( + id TEXT PRIMARY KEY, + doctype_id TEXT NOT NULL REFERENCES doctypes(id) ON DELETE CASCADE, + event TEXT NOT NULL, -- 'create', 'update', 'delete', 'transition:{action}' + action_type TEXT NOT NULL, -- 'generate_number', 'generate_pdf', 'send_notification', + -- 'create_payment_link', 'update_field', 'run_workflow', + -- 'call_integration', 'spawn_worker' + action_config TEXT NOT NULL, -- JSON: hook-specific configuration + sort_order INTEGER DEFAULT 0, -- Execution order (Salesforce-inspired deterministic ordering) + enabled INTEGER DEFAULT 1 +); + +-- Relations between DocTypes +CREATE TABLE doctype_relations ( + id TEXT PRIMARY KEY, + from_doctype TEXT NOT NULL REFERENCES doctypes(id) ON DELETE CASCADE, + to_doctype TEXT NOT NULL REFERENCES doctypes(id) ON DELETE CASCADE, + relation_type TEXT NOT NULL, -- 'has_many', 'belongs_to', 'many_to_many' + from_field TEXT NOT NULL, -- Field on source DocType + to_field TEXT DEFAULT 'id', -- Field on target DocType + label TEXT -- "Customer's Invoices" +); + +-- Auto-numbering (Frappe naming_series pattern) +CREATE TABLE dt_series ( + prefix TEXT PRIMARY KEY, -- "INV-2026-", "QUO-2026-", "PO-2026-" + current_value INTEGER DEFAULT 0 +); +``` + +### Auto-Numbering Implementation (Frappe naming_series) + +``` +How Frappe does it: + Table: tabSeries + ┌──────────────┬─────────┐ + │ name (prefix)│ current │ + ├──────────────┼─────────┤ + │ INV-2026- │ 42 │ + └──────────────┴─────────┘ + + On INSERT: + BEGIN IMMEDIATE; -- SQLite equivalent of FOR UPDATE + SELECT current_value FROM dt_series WHERE prefix = 'INV-2026-'; + UPDATE dt_series SET current_value = current_value + 1 WHERE prefix = 'INV-2026-'; + COMMIT; + → Result: INV-2026-00043 + +How OpenBridge does it: + Same pattern, executed inside the 'create' hook: + + Hook config: { + "type": "generate_number", + "pattern": "INV-{YYYY}-{#####}", + "field": "invoice_number" + } + + Engine parses pattern: + "INV-" + year(now) + "-" → prefix = "INV-2026-" + "#####" → 5-digit zero-padded counter + + Upsert into dt_series, increment, format → "INV-2026-00043" + Set field on the new record before INSERT +``` + +### REST API Auto-Generation + +``` +For each DocType, file-server exposes: + + GET /api/dt/:doctype → List records (with pagination, filters, search) + GET /api/dt/:doctype/:id → Get single record (with child tables) + POST /api/dt/:doctype → Create record (runs hooks: generate_number, validate) + PUT /api/dt/:doctype/:id → Update record (runs hooks: recompute, validate) + DELETE /api/dt/:doctype/:id → Soft-delete record + POST /api/dt/:doctype/:id/transition → Execute state transition (runs transition hooks) + GET /api/dt/:doctype/search?q=... → FTS5 search + +All endpoints: + - Validate fields against DocType schema (Zod generated from metadata) + - Check state machine rules for transitions + - Fire lifecycle hooks in deterministic order (Salesforce-inspired) + - Return structured JSON responses +``` + +### Web Form Auto-Generation + +``` +For each DocType, a worker generates an HTML form: + + GET /forms/:doctype/new → Create form + GET /forms/:doctype/:id/edit → Edit form + GET /forms/:doctype → List view (table with search/filter/sort) + GET /forms/:doctype/:id → Detail view + +The form is generated from field metadata: + field_type = 'text' → + field_type = 'number' → + field_type = 'currency' → with currency symbol + field_type = 'date' → + field_type = 'select' → populated from linked DocType + field_type = 'table' → Inline editable rows (add/remove) + field_type = 'longtext' → + +
+ + + + + + + + + + + + + + + + +
+ + + + +`; + +export const WEBCHAT_LOGIN_HTML = ` + + + + + OpenBridge WebChat — Sign in + + + + + + + + +`; + +export const WEBCHAT_SW_JS = `"use strict";(()=>{var a="openbridge-webchat-v1";self.addEventListener("install",function(n){n.waitUntil(caches.open(a).then(function(t){return t.addAll(["/"])})),self.skipWaiting()});self.addEventListener("activate",function(n){n.waitUntil(caches.keys().then(function(t){return Promise.all(t.filter(function(e){return e!==a}).map(function(e){return caches.delete(e)}))})),self.clients.claim()});self.addEventListener("fetch",function(n){var t=n.request;if(t.method==="GET"){var e=new URL(t.url);e.origin===self.location.origin&&(e.pathname==="/"||e.pathname==="")&&n.respondWith(caches.match(t).then(function(i){return i||fetch(t)}))}});self.addEventListener("push",function(n){var t={title:"OpenBridge",body:"New message received"};if(n.data)try{t=n.data.json()}catch{t.body=n.data.text()}n.waitUntil(self.registration.showNotification(t.title||"OpenBridge",{body:t.body,icon:"/icons/icon-192.png"}))});self.addEventListener("notificationclick",function(n){n.notification.close(),n.waitUntil(clients.matchAll({type:"window",includeUncontrolled:!0}).then(function(t){for(var e=0;e + + + + + OpenBridge WebChat + + + + + + + + + + + + +
+
+ +

OpenBridge WebChat

+ +
+ + + + + + +
+ + Connecting... +
+
+
+ +
+ + +
+ +
+
+ + +
+ + + + +
+
+
+ + + + + + + + + +
+ + + + diff --git a/src/connectors/webchat/ui/js/.gitkeep b/src/connectors/webchat/ui/js/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/src/connectors/webchat/ui/js/app.js b/src/connectors/webchat/ui/js/app.js new file mode 100644 index 00000000..b2d34a6b --- /dev/null +++ b/src/connectors/webchat/ui/js/app.js @@ -0,0 +1,1333 @@ +/** + * OpenBridge WebChat — main application entry point. + * Coordinates WebSocket, markdown rendering, and dashboard modules. + */ + +import { initWebSocket, sendMessage, isConnected } from './websocket.js'; +import { renderMarkdown } from './markdown.js'; +import { initDashboard, updateDashboard } from './dashboard.js'; +import { initSidebar, loadSessions, setOnSessionSelect, setOnNewConversation } from './sidebar.js'; +import { initAutocomplete } from './autocomplete.js'; +import { initSettings, setOnThemeChange, setOnSoundChange } from './settings.js'; +import { initDeepMode, handleDeepPhaseEvent, restoreDeepModeState, handleDeepModeStateSnapshot } from './deep-mode.js'; + +const msgs = document.getElementById('msgs'); +const form = document.getElementById('form'); +const inp = document.getElementById('inp'); +const send = document.getElementById('send'); +const dot = document.getElementById('dot'); +const connLabel = document.getElementById('connLabel'); +const statusBar = document.getElementById('status-bar'); +const statusText = document.getElementById('status-text'); +const statusTimer = document.getElementById('status-timer'); + +let timerInterval = null; +let timerStart = null; + +const _pageSessionId = + typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function' + ? crypto.randomUUID() + : Math.random().toString(36).slice(2); +let _feedbackMsgCounter = 0; + +// --- Public URL bar (shown when tunnel is active) --- + +(function initPublicUrlBar() { + const url = window.__OB_PUBLIC_URL__; + if (!url) return; + const bar = document.getElementById('public-url-bar'); + const text = document.getElementById('public-url-text'); + const btn = document.getElementById('url-copy-btn'); + if (!bar || !text || !btn) return; + text.textContent = url; + bar.classList.remove('hidden'); + bar.classList.add('visible'); + btn.addEventListener('click', function () { + navigator.clipboard.writeText(url).then( + function () { + btn.textContent = 'Copied!'; + btn.classList.add('copied'); + setTimeout(function () { + btn.textContent = 'Copy'; + btn.classList.remove('copied'); + }, 2000); + }, + function () { + // Fallback for browsers without clipboard API + const el = document.createElement('textarea'); + el.value = url; + el.style.position = 'fixed'; + el.style.opacity = '0'; + document.body.appendChild(el); + el.select(); + document.execCommand('copy'); + document.body.removeChild(el); + btn.textContent = 'Copied!'; + btn.classList.add('copied'); + setTimeout(function () { + btn.textContent = 'Copy'; + btn.classList.remove('copied'); + }, 2000); + }, + ); + }); +})(); + +// --- Share this link button --- + +(function initShareBtn() { + const btn = document.getElementById('share-btn'); + const toast = document.getElementById('share-toast'); + if (!btn || !toast) return; + + let toastTimeout = null; + + function showToast() { + if (toastTimeout) clearTimeout(toastTimeout); + toast.classList.add('visible'); + toastTimeout = setTimeout(function () { + toast.classList.remove('visible'); + toastTimeout = null; + }, 2000); + } + + btn.addEventListener('click', function () { + const url = window.location.href; + navigator.clipboard.writeText(url).then( + function () { + showToast(); + }, + function () { + // Fallback for browsers without clipboard API + const el = document.createElement('textarea'); + el.value = url; + el.style.position = 'fixed'; + el.style.opacity = '0'; + document.body.appendChild(el); + el.select(); + document.execCommand('copy'); + document.body.removeChild(el); + showToast(); + }, + ); + }); +})(); + +// --- Timestamps --- + +let tsVisible = localStorage.getItem('ob-ts') !== 'false'; + +function formatRelativeTime(date) { + const diff = Math.floor((Date.now() - date.getTime()) / 1000); + if (diff < 60) return 'just now'; + if (diff < 3600) return Math.floor(diff / 60) + 'm ago'; + if (diff < 86400) return Math.floor(diff / 3600) + 'h ago'; + return Math.floor(diff / 86400) + 'd ago'; +} + +function applyTsVisibility() { + const btn = document.getElementById('ts-toggle'); + if (btn) btn.textContent = tsVisible ? 'Hide times' : 'Show times'; + document.documentElement.setAttribute('data-ts', tsVisible ? 'show' : 'hide'); +} + +(function initTs() { + applyTsVisibility(); + const btn = document.getElementById('ts-toggle'); + if (btn) { + btn.addEventListener('click', function () { + tsVisible = !tsVisible; + localStorage.setItem('ob-ts', tsVisible ? 'true' : 'false'); + applyTsVisibility(); + }); + } + setInterval(function () { + msgs.querySelectorAll('time.bubble-ts').forEach(function (el) { + el.textContent = formatRelativeTime(new Date(el.dateTime)); + }); + }, 60000); +})(); + +// --- Theme toggle --- + +(function initTheme() { + const btn = document.getElementById('theme-toggle'); + + function applyTheme(theme) { + document.documentElement.setAttribute('data-theme', theme); + btn.textContent = theme === 'dark' ? 'Light' : 'Dark'; + localStorage.setItem('ob-theme', theme); + } + + applyTheme(localStorage.getItem('ob-theme') || 'light'); + + btn.addEventListener('click', function () { + const current = document.documentElement.getAttribute('data-theme'); + const next = current === 'dark' ? 'light' : 'dark'; + applyTheme(next); + // Keep settings panel theme select in sync + const settingsSelect = document.getElementById('settings-theme-select'); + if (settingsSelect) settingsSelect.value = next; + }); +})(); + +// --- Conversation Persistence --- + +const CONV_STORAGE_KEY = 'ob-conversation'; +const CONV_MAX_MESSAGES = 100; +let _convLog = []; +let _convPersistEnabled = true; + +function _saveConv() { + try { + localStorage.setItem(CONV_STORAGE_KEY, JSON.stringify(_convLog)); + } catch (_) { + // localStorage full or unavailable — non-critical + } +} + +function _trackMsg(content, cls, timestamp) { + if (!_convPersistEnabled) return; + _convLog.push({ content, cls, ts: (timestamp instanceof Date ? timestamp : new Date()).toISOString() }); + if (_convLog.length > CONV_MAX_MESSAGES) { + _convLog = _convLog.slice(-CONV_MAX_MESSAGES); + } + _saveConv(); +} + +function clearConversation() { + _convLog = []; + try { + localStorage.removeItem(CONV_STORAGE_KEY); + } catch (_) {} +} + +function restoreConversation() { + try { + const raw = localStorage.getItem(CONV_STORAGE_KEY); + if (!raw) return; + const saved = JSON.parse(raw); + if (!Array.isArray(saved) || saved.length === 0) return; + _convPersistEnabled = false; + _convLog = saved.slice(-CONV_MAX_MESSAGES); + for (const msg of _convLog) { + if (msg.cls === 'user' || msg.cls === 'ai') { + addBubble(msg.content, msg.cls, msg.ts ? new Date(msg.ts) : new Date()); + } + } + _convPersistEnabled = true; + } catch (_) { + _convPersistEnabled = true; + } +} + +// --- Messages --- + +function makeAvatar(cls) { + const av = document.createElement('div'); + av.className = 'avatar avatar-' + cls; + av.setAttribute('aria-hidden', 'true'); + av.textContent = cls === 'user' ? 'You' : 'AI'; + return av; +} + +function addBubble(content, cls, timestamp) { + const div = document.createElement('div'); + div.className = 'bubble ' + cls; + if (cls === 'ai') { + const html = renderMarkdown(content); + if (content.length > 500) { + const wrap = document.createElement('div'); + wrap.className = 'collapsible-wrap'; + + const inner = document.createElement('div'); + inner.className = 'collapsible-inner'; + inner.style.maxHeight = '120px'; + inner.innerHTML = html; + + const fade = document.createElement('div'); + fade.className = 'collapsible-fade'; + + const btn = document.createElement('button'); + btn.className = 'show-more-btn'; + btn.textContent = 'Show more'; + btn.setAttribute('aria-expanded', 'false'); + btn.addEventListener('click', function () { + const isCollapsed = btn.getAttribute('aria-expanded') === 'false'; + if (isCollapsed) { + inner.style.maxHeight = inner.scrollHeight + 'px'; + fade.style.display = 'none'; + btn.textContent = 'Show less'; + btn.setAttribute('aria-expanded', 'true'); + } else { + inner.style.maxHeight = '120px'; + fade.style.display = ''; + btn.textContent = 'Show more'; + btn.setAttribute('aria-expanded', 'false'); + } + }); + + wrap.appendChild(inner); + wrap.appendChild(fade); + div.appendChild(wrap); + div.appendChild(btn); + } else { + div.innerHTML = html; + } + } else { + div.textContent = content; + } + if (cls !== 'sys') { + const tsDate = timestamp instanceof Date ? timestamp : new Date(); + const ts = document.createElement('time'); + ts.className = 'bubble-ts'; + ts.dateTime = tsDate.toISOString(); + ts.title = tsDate.toLocaleString(); + ts.textContent = formatRelativeTime(tsDate); + div.appendChild(ts); + if (cls === 'ai') { + _feedbackMsgCounter++; + const feedbackRow = document.createElement('div'); + feedbackRow.className = 'feedback-row'; + const upBtn = document.createElement('button'); + upBtn.type = 'button'; + upBtn.className = 'feedback-btn'; + upBtn.setAttribute('aria-label', 'Good response'); + upBtn.dataset.rating = 'up'; + upBtn.dataset.msgIdx = String(_feedbackMsgCounter); + upBtn.textContent = '\uD83D\uDC4D'; + const downBtn = document.createElement('button'); + downBtn.type = 'button'; + downBtn.className = 'feedback-btn'; + downBtn.setAttribute('aria-label', 'Poor response'); + downBtn.dataset.rating = 'down'; + downBtn.dataset.msgIdx = String(_feedbackMsgCounter); + downBtn.textContent = '\uD83D\uDC4E'; + feedbackRow.appendChild(upBtn); + feedbackRow.appendChild(downBtn); + div.appendChild(feedbackRow); + } + const row = document.createElement('div'); + row.className = 'msg-row ' + cls; + row.appendChild(makeAvatar(cls)); + row.appendChild(div); + msgs.appendChild(row); + } else { + msgs.appendChild(div); + } + msgs.scrollTop = msgs.scrollHeight; + if (cls === 'user' || cls === 'ai') { + _trackMsg(content, cls, timestamp instanceof Date ? timestamp : new Date()); + } + return div; +} + +// --- Copy button for code blocks --- + +msgs.addEventListener('click', function (e) { + const btn = e.target.closest('.copy-btn'); + if (!btn) return; + const code = btn.dataset.code; + if (!code) return; + navigator.clipboard.writeText(code).then(function () { + btn.textContent = 'Copied!'; + btn.classList.add('copied'); + setTimeout(function () { + btn.textContent = 'Copy'; + btn.classList.remove('copied'); + }, 2000); + }); +}); + +// --- Feedback buttons --- + +const _feedbackToastEl = (function () { + const el = document.createElement('div'); + el.className = 'feedback-toast'; + el.textContent = 'Thanks!'; + document.body.appendChild(el); + return el; +})(); + +let _feedbackToastTimer = null; + +function showFeedbackToast() { + if (_feedbackToastTimer) clearTimeout(_feedbackToastTimer); + _feedbackToastEl.classList.add('visible'); + _feedbackToastTimer = setTimeout(function () { + _feedbackToastEl.classList.remove('visible'); + _feedbackToastTimer = null; + }, 2000); +} + +msgs.addEventListener('click', function (e) { + const btn = e.target.closest('.feedback-btn'); + if (!btn || btn.disabled) return; + const rating = btn.dataset.rating; + const msgIdx = btn.dataset.msgIdx; + const feedbackRow = btn.closest('.feedback-row'); + if (feedbackRow) { + feedbackRow.querySelectorAll('.feedback-btn').forEach(function (b) { + b.disabled = true; + if (b.dataset.rating === rating) { + b.classList.add(rating === 'up' ? 'active-up' : 'active-down'); + } + }); + } + showFeedbackToast(); + fetch('/api/feedback', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ session: _pageSessionId, message: msgIdx, rating: rating }), + }).catch(function () { + // Feedback failed — non-critical + }); +}); + +// --- Status bar --- + +function startTimer() { + if (timerInterval) return; + timerStart = Date.now(); + statusTimer.textContent = '0s'; + timerInterval = setInterval(function () { + const elapsed = Math.floor((Date.now() - timerStart) / 1000); + statusTimer.textContent = elapsed + 's'; + }, 1000); +} + +function stopTimer() { + if (timerInterval) { + clearInterval(timerInterval); + timerInterval = null; + } + timerStart = null; + statusTimer.textContent = ''; +} + +function showStatus(html) { + statusBar.classList.remove('hidden'); + statusText.innerHTML = html; + if (!timerInterval) startTimer(); +} + +function hideStatus() { + statusBar.classList.add('hidden'); + statusText.innerHTML = ''; + stopTimer(); +} + +// --- Progress event labels --- + +function progressLabel(event) { + if (event.type === 'classifying') { + return '\uD83D\uDD0D Analyzing request...'; + } + if (event.type === 'planning') { + return '\uD83D\uDCCB Planning subtasks...'; + } + if (event.type === 'spawning') { + const n = event.workerCount; + return ( + '\uD83D\uDCCB Breaking into ' + + n + + ' subtask' + + (n !== 1 ? 's' : '') + + '...' + ); + } + if (event.type === 'worker-progress') { + const label = event.workerName ? '\u2699\uFE0F ' + event.workerName + ': ' : '\u2699\uFE0F '; + return ( + label + + event.completed + + '/' + + event.total + + ' workers done...' + ); + } + if (event.type === 'synthesizing') { + return '\uD83D\uDCDD Preparing final response...'; + } + if (event.type === 'exploring') { + return ( + '\uD83D\uDDFA\uFE0F ' + + event.phase + + '...' + ); + } + if (event.type === 'exploring-directory') { + return ( + '\uD83D\uDCC2 Exploring directories: ' + + event.completed + + '/' + + event.total + + (event.directory ? ' (' + event.directory + ')' : '') + + '...' + ); + } + return null; +} + +// --- Connection state --- + +function setOnline(online, reconnecting) { + dot.className = 'conn-dot' + (online ? ' online' : ''); + if (online) { + connLabel.textContent = 'Connected'; + } else if (reconnecting) { + connLabel.textContent = 'Reconnecting...'; + } else { + connLabel.textContent = 'Disconnected'; + } + inp.disabled = !online; + send.disabled = !online; + const uploadBtn = document.getElementById('upload-btn'); + if (uploadBtn) uploadBtn.disabled = !online; + const micBtn = document.getElementById('mic-btn'); + if (micBtn) micBtn.disabled = !online; +} + +// --- Permission Prompt --- + +function showPermissionPrompt(data) { + var container = document.getElementById('permission-container'); + if (!container) return; + + // Only one permission prompt at a time + container.replaceChildren(); + + var timeoutSec = Math.max(1, Math.round((data.timeoutMs || 60000) / 1000)); + var remaining = timeoutSec; + var countdownInterval = null; + + function respond(approved) { + if (countdownInterval) clearInterval(countdownInterval); + container.replaceChildren(); + sendMessage({ + type: 'permission-response', + permissionId: data.permissionId, + approved: approved, + }); + } + + // Overlay + var overlay = document.createElement('div'); + overlay.className = 'permission-overlay'; + overlay.setAttribute('role', 'dialog'); + overlay.setAttribute('aria-modal', 'true'); + overlay.setAttribute('aria-label', 'Permission request'); + + // Modal + var modal = document.createElement('div'); + modal.className = 'permission-modal'; + + // Header + var header = document.createElement('div'); + header.className = 'permission-header'; + var icon = document.createElement('span'); + icon.className = 'permission-icon'; + icon.setAttribute('aria-hidden', 'true'); + icon.textContent = '\uD83D\uDD10'; + var title = document.createElement('span'); + title.className = 'permission-title'; + title.textContent = 'Permission Request'; + header.appendChild(icon); + header.appendChild(title); + + // Body + var body = document.createElement('div'); + body.className = 'permission-body'; + + var toolRow = document.createElement('div'); + toolRow.className = 'permission-tool'; + var toolName = document.createElement('span'); + toolName.className = 'permission-tool-name'; + toolName.textContent = data.toolName || 'Unknown tool'; + toolRow.appendChild(toolName); + body.appendChild(toolRow); + + if (data.detail) { + var detail = document.createElement('div'); + detail.className = 'permission-detail'; + detail.textContent = data.detail; + body.appendChild(detail); + } + + // Actions + var actions = document.createElement('div'); + actions.className = 'permission-actions'; + + var allowBtn = document.createElement('button'); + allowBtn.className = 'permission-btn permission-btn-allow'; + allowBtn.textContent = 'Allow'; + allowBtn.setAttribute('aria-label', 'Allow this action'); + allowBtn.addEventListener('click', function () { + respond(true); + }); + + var denyBtn = document.createElement('button'); + denyBtn.className = 'permission-btn permission-btn-deny'; + denyBtn.textContent = 'Deny'; + denyBtn.setAttribute('aria-label', 'Deny this action'); + denyBtn.addEventListener('click', function () { + respond(false); + }); + + actions.appendChild(allowBtn); + actions.appendChild(denyBtn); + + // Countdown + var countdown = document.createElement('div'); + countdown.className = 'permission-countdown'; + var countdownText = document.createElement('span'); + countdownText.textContent = 'Auto-deny in ' + remaining + 's'; + var countdownBar = document.createElement('div'); + countdownBar.className = 'permission-countdown-bar'; + countdownBar.style.width = '100%'; + countdown.appendChild(countdownText); + countdown.appendChild(countdownBar); + + modal.appendChild(header); + modal.appendChild(body); + modal.appendChild(actions); + modal.appendChild(countdown); + overlay.appendChild(modal); + container.appendChild(overlay); + + // Focus the Allow button for keyboard accessibility + allowBtn.focus(); + + // Keyboard handler: Escape to deny + function onKeydown(e) { + if (e.key === 'Escape') { + e.preventDefault(); + respond(false); + } + } + document.addEventListener('keydown', onKeydown); + + // Start countdown + countdownInterval = setInterval(function () { + remaining--; + if (remaining <= 0) { + document.removeEventListener('keydown', onKeydown); + respond(false); + return; + } + countdownText.textContent = 'Auto-deny in ' + remaining + 's'; + countdownBar.style.width = Math.round((remaining / timeoutSec) * 100) + '%'; + }, 1000); + + // Cleanup keyboard handler when modal is dismissed + var origRespond = respond; + respond = function (approved) { + document.removeEventListener('keydown', onKeydown); + origRespond(approved); + }; +} + +// --- WebSocket message handler --- + +function handleMessage(data) { + if (data.type === 'response') { + hideStatus(); + addBubble(data.content, 'ai', data.timestamp ? new Date(data.timestamp) : new Date()); + incrementUnread(); + showTaskNotification(data.content); + playNotificationSound(); + void loadSessions(); + } else if (data.type === 'download') { + hideStatus(); + const tsDate = data.timestamp ? new Date(data.timestamp) : new Date(); + const div = document.createElement('div'); + div.className = 'bubble ai'; + if (data.content) { + div.innerHTML = renderMarkdown(data.content) + '
'; + } + const link = document.createElement('a'); + link.href = data.url; + link.download = data.filename || 'download'; + link.className = 'download-link'; + link.textContent = '\u2B07\uFE0F Download ' + (data.filename || 'file'); + link.setAttribute('aria-label', 'Download ' + (data.filename || 'file')); + div.appendChild(link); + const ts = document.createElement('time'); + ts.className = 'bubble-ts'; + ts.dateTime = tsDate.toISOString(); + ts.title = tsDate.toLocaleString(); + ts.textContent = formatRelativeTime(tsDate); + div.appendChild(ts); + const dlRow = document.createElement('div'); + dlRow.className = 'msg-row ai'; + dlRow.appendChild(makeAvatar('ai')); + dlRow.appendChild(div); + msgs.appendChild(dlRow); + msgs.scrollTop = msgs.scrollHeight; + } else if (data.type === 'typing') { + showStatus( + '\uD83E\uDD14 Thinking...', + ); + } else if (data.type === 'progress') { + if (data.event && data.event.type === 'complete') { + hideStatus(); + } else if (data.event && data.event.type === 'worker-result') { + const icon = data.event.success ? '\u2705' : '\u274C'; + const toolLabel = data.event.tool ? ' \u00b7 ' + data.event.tool : ''; + const header = + icon + + ' **Subtask ' + + data.event.workerIndex + + '/' + + data.event.total + + '** (' + + data.event.profile + + toolLabel + + '):\n\n'; + addBubble(header + data.event.content, 'ai', new Date()); + } else if (data.event && data.event.type === 'worker-cancelled') { + addBubble( + '\uD83D\uDED1 Worker ' + + data.event.workerId + + ' was stopped by ' + + data.event.cancelledBy + + '.', + 'sys', + ); + } else if (data.event && data.event.type === 'deep-phase') { + handleDeepPhaseEvent(data.event); + } else if (data.event) { + const label = progressLabel(data.event); + if (label) showStatus(label); + } + } else if (data.type === 'deep-mode-state') { + // Server-sent canonical snapshot of active deep-phase events (sent on connection or by request) + handleDeepModeStateSnapshot(data.events); + } else if (data.type === 'permission-request') { + showPermissionPrompt(data); + } else if (data.type === 'agent-status') { + updateDashboard(data.agents); + } +} + +// --- Textarea auto-resize and character count --- + +const charCount = document.getElementById('char-count'); + +function autoResize() { + inp.style.height = 'auto'; + inp.style.height = inp.scrollHeight + 'px'; +} + +function updateCharCount() { + const len = inp.value.length; + if (len > 500) { + charCount.textContent = len.toLocaleString() + ' chars'; + charCount.classList.remove('hidden'); + } else { + charCount.classList.add('hidden'); + } +} + +inp.addEventListener('input', function () { + autoResize(); + updateCharCount(); +}); + +// --- Keyboard navigation --- + +inp.addEventListener('keydown', function (e) { + if (e.key === 'Enter' && !e.shiftKey) { + e.preventDefault(); + form.requestSubmit(); + } else if (e.key === 'Escape') { + inp.value = ''; + autoResize(); + updateCharCount(); + } +}); + +// --- Form submit --- + +form.addEventListener('submit', function (e) { + e.preventDefault(); + const text = inp.value.trim(); + const hasFiles = _selectedFiles.length > 0; + if ((!text && !hasFiles) || !isConnected()) return; + + // Snapshot and clear selected files immediately + const filesToUpload = _selectedFiles.slice(); + _selectedFiles = []; + renderFilePreviews(); + + const displayText = text || '(\uD83D\uDCCE file upload)'; + addBubble(displayText, 'user', new Date()); + inp.value = ''; + autoResize(); + updateCharCount(); + showStatus( + '\uD83E\uDD14 Thinking...', + ); + + if (filesToUpload.length === 0) { + sendMessage({ type: 'message', content: text }); + return; + } + + // Upload files first, then send message with file paths appended + Promise.all( + filesToUpload.map(function (file) { + const fd = new FormData(); + fd.append('file', file, file.name); + return fetch('/api/upload', { method: 'POST', body: fd }) + .then(function (r) { + return r.ok ? r.json() : null; + }) + .catch(function () { + return null; + }); + }), + ).then(function (results) { + var uploadedFiles = results.filter(function (r) { + return r && r.fileId; + }); + + var content = text; + if (uploadedFiles.length === 0 && !content) { + content = '[File upload failed — no files were saved]'; + } + sendMessage({ + type: 'message', + content: content || '', + files: uploadedFiles.map(function (r) { + return { + filename: r.filename, + path: r.path, + mimeType: r.mimeType, + size: r.size, + }; + }), + }); + }); +}); + +// --- File Upload --- + +let _selectedFiles = []; + +function formatFileSize(bytes) { + if (bytes < 1024) return bytes + ' B'; + if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(1) + ' KB'; + return (bytes / (1024 * 1024)).toFixed(1) + ' MB'; +} + +function renderFilePreviews() { + const preview = document.getElementById('file-preview'); + if (!preview) return; + if (_selectedFiles.length === 0) { + preview.classList.add('hidden'); + preview.replaceChildren(); + return; + } + preview.classList.remove('hidden'); + preview.replaceChildren(); + for (let i = 0; i < _selectedFiles.length; i++) { + const file = _selectedFiles[i]; + const chip = document.createElement('div'); + chip.className = 'file-chip'; + + const icon = document.createElement('span'); + icon.className = 'file-chip-icon'; + icon.setAttribute('aria-hidden', 'true'); + icon.textContent = '\uD83D\uDCC4'; + + const info = document.createElement('span'); + info.className = 'file-chip-info'; + + const name = document.createElement('span'); + name.className = 'file-chip-name'; + name.textContent = file.name; + + const meta = document.createElement('span'); + meta.className = 'file-chip-meta'; + meta.textContent = formatFileSize(file.size) + (file.type ? ' \u00b7 ' + file.type : ''); + + info.appendChild(name); + info.appendChild(meta); + + const removeBtn = document.createElement('button'); + removeBtn.type = 'button'; + removeBtn.className = 'file-chip-remove'; + removeBtn.setAttribute('aria-label', 'Remove ' + file.name); + removeBtn.textContent = '\u00d7'; + const idx = i; + removeBtn.addEventListener('click', function () { + _selectedFiles.splice(idx, 1); + renderFilePreviews(); + }); + + chip.appendChild(icon); + chip.appendChild(info); + chip.appendChild(removeBtn); + preview.appendChild(chip); + } +} + +(function initFileUpload() { + const uploadBtn = document.getElementById('upload-btn'); + const fileInput = document.getElementById('file-input'); + const chatWrap = document.querySelector('.chat-wrap'); + if (!uploadBtn || !fileInput) return; + + uploadBtn.addEventListener('click', function () { + fileInput.click(); + }); + + fileInput.addEventListener('change', function () { + const files = Array.from(fileInput.files || []); + for (const f of files) { + _selectedFiles.push(f); + } + fileInput.value = ''; + renderFilePreviews(); + }); + + if (chatWrap) { + chatWrap.addEventListener('dragover', function (e) { + e.preventDefault(); + chatWrap.classList.add('drag-over'); + }); + + chatWrap.addEventListener('dragleave', function (e) { + if (!chatWrap.contains(/** @type {Node} */ (e.relatedTarget))) { + chatWrap.classList.remove('drag-over'); + } + }); + + chatWrap.addEventListener('drop', function (e) { + e.preventDefault(); + chatWrap.classList.remove('drag-over'); + const files = Array.from(e.dataTransfer ? e.dataTransfer.files : []); + for (const f of files) { + _selectedFiles.push(f); + } + renderFilePreviews(); + }); + } +})(); + +// --- Voice Input --- + +(function initVoiceInput() { + const micBtn = document.getElementById('mic-btn'); + if (!micBtn) return; + + // Check for MediaRecorder support + if (typeof MediaRecorder === 'undefined' || !navigator.mediaDevices) { + micBtn.style.display = 'none'; + return; + } + + let mediaRecorder = null; + let audioChunks = []; + let recordingIndicator = null; + + function showRecordingIndicator() { + if (recordingIndicator) return; + const filePreview = document.getElementById('file-preview'); + if (!filePreview) return; + recordingIndicator = document.createElement('div'); + recordingIndicator.className = 'recording-indicator'; + recordingIndicator.innerHTML = 'Recording\u2026'; + filePreview.classList.remove('hidden'); + filePreview.appendChild(recordingIndicator); + } + + function hideRecordingIndicator() { + if (!recordingIndicator) return; + const filePreview = document.getElementById('file-preview'); + recordingIndicator.remove(); + recordingIndicator = null; + // Hide preview row if no file chips remain + if (filePreview && filePreview.children.length === 0) { + filePreview.classList.add('hidden'); + } + } + + function startRecording() { + audioChunks = []; + navigator.mediaDevices + .getUserMedia({ audio: true }) + .then(function (stream) { + const mimeType = MediaRecorder.isTypeSupported('audio/webm') ? 'audio/webm' : 'audio/ogg'; + mediaRecorder = new MediaRecorder(stream, { mimeType }); + + mediaRecorder.addEventListener('dataavailable', function (e) { + if (e.data && e.data.size > 0) audioChunks.push(e.data); + }); + + mediaRecorder.addEventListener('stop', function () { + // Stop all mic tracks to release the mic + stream.getTracks().forEach(function (t) { t.stop(); }); + + const blob = new Blob(audioChunks, { type: mimeType }); + audioChunks = []; + + hideRecordingIndicator(); + micBtn.classList.remove('recording'); + micBtn.title = 'Record voice message'; + micBtn.setAttribute('aria-label', 'Record voice message'); + + // Upload blob to /api/transcribe + const ext = mimeType === 'audio/webm' ? '.webm' : '.ogg'; + const fd = new FormData(); + fd.append('file', blob, 'voice' + ext); + + addBubble( + '\uD83C\uDFA4 Transcribing voice\u2026', + 'sys', + ); + + fetch('/api/transcribe', { method: 'POST', body: fd }) + .then(function (r) { + return r.ok ? r.json() : Promise.reject(r.status); + }) + .then(function (data) { + if (data && data.text) { + inp.value = data.text; + inp.dispatchEvent(new Event('input')); + inp.focus(); + // Remove the "Transcribing…" sys bubble + const lastSys = msgs.querySelector('.bubble.sys:last-of-type'); + if (lastSys && lastSys.textContent.includes('Transcribing')) { + lastSys.closest('.bubble.sys') && lastSys.remove(); + // also handle when it's the bubble itself + const sysBubbles = msgs.querySelectorAll('.bubble.sys'); + sysBubbles.forEach(function (el) { + if (el.textContent.includes('Transcribing')) el.remove(); + }); + } + } + }) + .catch(function () { + addBubble('\u26A0\uFE0F Voice transcription failed.', 'sys'); + }); + }); + + mediaRecorder.start(); + micBtn.classList.add('recording'); + micBtn.title = 'Stop recording'; + micBtn.setAttribute('aria-label', 'Stop recording'); + showRecordingIndicator(); + }) + .catch(function () { + addBubble( + '\u26A0\uFE0F Microphone access denied. Please allow microphone permissions.', + 'sys', + ); + }); + } + + function stopRecording() { + if (mediaRecorder && mediaRecorder.state !== 'inactive') { + mediaRecorder.stop(); + } + } + + micBtn.addEventListener('click', function () { + if (micBtn.classList.contains('recording')) { + stopRecording(); + } else { + startRecording(); + } + }); +})(); + +// --- Tab Title Unread Count --- + +let unreadCount = 0; +const baseTitle = 'OpenBridge'; + +function updateTabTitle() { + document.title = unreadCount > 0 ? '(' + unreadCount + ') ' + baseTitle : baseTitle; +} + +function incrementUnread() { + if (document.visibilityState === 'visible') return; + unreadCount++; + updateTabTitle(); +} + +function resetUnread() { + unreadCount = 0; + updateTabTitle(); +} + +document.addEventListener('visibilitychange', function () { + if (document.visibilityState === 'visible') { + resetUnread(); + } +}); + +// --- Browser Notifications --- + +function showTaskNotification(content) { + if (document.visibilityState === 'visible') return; + if (!('Notification' in window)) return; + if (Notification.permission !== 'granted') return; + var preview = content.length > 100 ? content.slice(0, 97) + '...' : content; + new Notification('OpenBridge', { + body: preview, + icon: '/icons/icon-192.png', + }); +} + +(function initNotifications() { + if (!('Notification' in window)) return; + if (Notification.permission === 'default') { + // Delay the permission request so it doesn't interrupt the initial page load + setTimeout(function () { + Notification.requestPermission(); + }, 3000); + } +})(); + +// --- Sound Notifications --- + +let soundMuted = localStorage.getItem('ob-sound') === 'false'; +let audioCtx = null; + +function getAudioContext() { + if (!audioCtx) { + audioCtx = new (window.AudioContext || window.webkitAudioContext)(); + } + return audioCtx; +} + +function playNotificationSound() { + if (soundMuted) return; + if (!window.AudioContext && !window.webkitAudioContext) return; + try { + const ctx = getAudioContext(); + const oscillator = ctx.createOscillator(); + const gainNode = ctx.createGain(); + oscillator.connect(gainNode); + gainNode.connect(ctx.destination); + oscillator.type = 'sine'; + oscillator.frequency.setValueAtTime(880, ctx.currentTime); + oscillator.frequency.exponentialRampToValueAtTime(660, ctx.currentTime + 0.15); + gainNode.gain.setValueAtTime(0.3, ctx.currentTime); + gainNode.gain.exponentialRampToValueAtTime(0.001, ctx.currentTime + 0.25); + oscillator.start(ctx.currentTime); + oscillator.stop(ctx.currentTime + 0.25); + } catch (_) { + // Web Audio API unavailable or blocked — non-critical + } +} + +function applySoundToggle() { + const btn = document.getElementById('sound-toggle'); + if (!btn) return; + btn.textContent = soundMuted ? '\uD83D\uDD07' : '\uD83D\uDD0A'; + btn.setAttribute('aria-label', soundMuted ? 'Unmute notifications' : 'Mute notifications'); + btn.setAttribute('aria-pressed', soundMuted ? 'true' : 'false'); +} + +(function initSoundToggle() { + applySoundToggle(); + const btn = document.getElementById('sound-toggle'); + if (!btn) return; + btn.addEventListener('click', function () { + soundMuted = !soundMuted; + localStorage.setItem('ob-sound', soundMuted ? 'false' : 'true'); + applySoundToggle(); + // Play a preview tone when unmuting so the user knows it works + if (!soundMuted) playNotificationSound(); + }); +})(); + +// --- Add to Home Screen Banner --- + +(function initPwaBanner() { + // Only show on mobile devices + const isMobile = + window.matchMedia('(max-width: 767px)').matches || + (('ontouchstart' in window || navigator.maxTouchPoints > 0) && screen.width <= 1024); + if (!isMobile) return; + + // Already running as installed PWA (standalone mode) + const isStandalone = + window.matchMedia('(display-mode: standalone)').matches || + window.navigator.standalone === true; + if (isStandalone) return; + + // User already dismissed permanently + if (localStorage.getItem('ob-pwa-dismissed') === '1') return; + + const banner = document.getElementById('pwa-banner'); + const installBtn = document.getElementById('pwa-install-btn'); + const dismissBtn = document.getElementById('pwa-dismiss-btn'); + const hint = document.getElementById('pwa-banner-hint'); + if (!banner || !installBtn || !dismissBtn) return; + + let deferredPrompt = null; + + // Detect iOS Safari (no beforeinstallprompt — must use manual instructions) + const isIos = /iphone|ipad|ipod/i.test(navigator.userAgent); + const isSafari = /safari/i.test(navigator.userAgent) && !/chrome|crios|fxios/i.test(navigator.userAgent); + + function showBanner() { + banner.classList.remove('hidden'); + } + + function hideBanner() { + banner.classList.add('hidden'); + localStorage.setItem('ob-pwa-dismissed', '1'); + } + + dismissBtn.addEventListener('click', hideBanner); + + if (isIos && isSafari) { + // iOS Safari: show manual share-sheet instructions + if (hint) hint.textContent = 'Tap Share \u238e then \u201cAdd to Home Screen\u201d'; + installBtn.style.display = 'none'; + setTimeout(showBanner, 2000); + } else { + // Chrome / Android: use the native beforeinstallprompt event + window.addEventListener('beforeinstallprompt', function (e) { + e.preventDefault(); + deferredPrompt = e; + setTimeout(showBanner, 2000); + }); + + installBtn.addEventListener('click', function () { + if (!deferredPrompt) return; + deferredPrompt.prompt(); + deferredPrompt.userChoice.then(function (choiceResult) { + if (choiceResult.outcome === 'accepted') { + localStorage.setItem('ob-pwa-dismissed', '1'); + } + deferredPrompt = null; + banner.classList.add('hidden'); + }); + }); + + // If app is installed later (appinstalled event), hide and dismiss + window.addEventListener('appinstalled', function () { + banner.classList.add('hidden'); + localStorage.setItem('ob-pwa-dismissed', '1'); + deferredPrompt = null; + }); + } +})(); + +// --- Service Worker Registration --- + +(function registerServiceWorker() { + if (!('serviceWorker' in navigator)) return; + navigator.serviceWorker.register('/sw.js').catch(function (err) { + // Registration failed — non-critical, app functions without it + if (typeof console !== 'undefined') console.warn('SW registration failed:', err); + }); +})(); + +// --- Session transcript loader --- + +/** + * Fetch a past session from /api/sessions/{id} and render it in the message area. + * Clears existing messages before rendering. + * @param {string} sessionId + */ +async function loadSessionTranscript(sessionId) { + clearConversation(); + _convPersistEnabled = false; + msgs.replaceChildren(); + addBubble('Loading conversation\u2026', 'sys'); + try { + const res = await fetch('/api/sessions/' + encodeURIComponent(sessionId)); + if (!res.ok) { + msgs.replaceChildren(); + addBubble('Failed to load conversation.', 'sys'); + return; + } + const data = await res.json(); + const messages = data.messages; + msgs.replaceChildren(); + if (!Array.isArray(messages) || messages.length === 0) { + addBubble('No messages in this conversation.', 'sys'); + return; + } + for (const msg of messages) { + const cls = msg.role === 'user' ? 'user' : msg.role === 'system' ? 'sys' : 'ai'; + const ts = msg.created_at ? new Date(msg.created_at) : new Date(); + addBubble(msg.content, cls, ts); + } + } catch (_) { + msgs.replaceChildren(); + addBubble('Failed to load conversation.', 'sys'); + } finally { + _convPersistEnabled = true; + } +} + +// --- New conversation --- + +/** + * Start a fresh conversation: clear the message area, notify the backend, + * and reload the session list so the new session appears in the sidebar. + */ +function startNewConversation() { + clearConversation(); + msgs.replaceChildren(); + addBubble('New conversation started.', 'sys'); + sendMessage({ type: 'new-session' }); + void loadSessions(); +} + +// --- Boot --- + +initAutocomplete(inp); +restoreConversation(); +initSidebar(); +setOnSessionSelect(loadSessionTranscript); +setOnNewConversation(startNewConversation); +void loadSessions(); +initDashboard(); +initDeepMode(msgs); +// Immediately restore Deep Mode stepper from sessionStorage (before WS connects) +restoreDeepModeState(); +initSettings(); +setOnThemeChange(function (theme) { + // Keep header theme-toggle button label in sync when theme changes via settings + const btn = document.getElementById('theme-toggle'); + if (btn) btn.textContent = theme === 'dark' ? 'Light' : 'Dark'; +}); +setOnSoundChange(function (enabled) { + // Apply immediately: update soundMuted and sync header button + soundMuted = !enabled; + applySoundToggle(); + // Play preview tone when unmuting so the user knows it works + if (!soundMuted) playNotificationSound(); +}); +let _wsConnectCount = 0; +initWebSocket({ + onOpen: function () { + _wsConnectCount++; + setOnline(true); + addBubble('Connected to OpenBridge', 'sys'); + // On reconnect (not first connect), restore local stepper state and request + // the server's authoritative snapshot to update any phase that changed while offline. + if (_wsConnectCount > 1) { + restoreDeepModeState(); + } + // Always request the server's canonical Deep Mode state so the stepper is + // accurate after any connection (first load or reconnect). + sendMessage({ type: 'get-deep-mode-state' }); + }, + onClose: function () { + setOnline(false, true); + hideStatus(); + addBubble('Disconnected \u2014 reconnecting...', 'sys'); + }, + onMessage: handleMessage, +}); diff --git a/src/connectors/webchat/ui/js/autocomplete.js b/src/connectors/webchat/ui/js/autocomplete.js new file mode 100644 index 00000000..d77ee2d2 --- /dev/null +++ b/src/connectors/webchat/ui/js/autocomplete.js @@ -0,0 +1,226 @@ +/** + * OpenBridge WebChat — Slash command autocomplete. + * Shows a filtered dropdown when "/" is typed at the start of the input. + * Arrow keys and Enter/Tab to navigate and select. Escape to close. + * + * Commands are fetched from GET /api/commands on first use and cached in + * module-level state so all autocomplete instances share the same list. + */ + +const FALLBACK_COMMANDS = [ + { name: '/history', description: 'Show conversation history' }, + { name: '/stop', description: 'Stop the current worker' }, + { name: '/status', description: 'Show agent status' }, + { name: '/deep', description: 'Enable deep mode for complex tasks' }, + { name: '/audit', description: 'Run a workspace audit' }, + { name: '/scope', description: 'Show or change task scope' }, + { name: '/apps', description: 'List connected apps' }, + { name: '/help', description: 'Show available commands' }, + { name: '/doctor', description: 'Run system health diagnostics' }, + { name: '/confirm', description: 'Confirm a pending action' }, + { name: '/skip', description: 'Skip a pending confirmation' }, +]; + +/** Module-level command cache — populated by fetchCommands() */ +let cachedCommands = null; +/** In-flight fetch promise — prevents duplicate requests */ +let fetchPromise = null; + +/** + * Fetch the command list from /api/commands. + * Returns the cached list on subsequent calls. + * Falls back to FALLBACK_COMMANDS on network/parse errors. + * @returns {Promise>} + */ +export async function fetchCommands() { + if (cachedCommands !== null) return cachedCommands; + if (fetchPromise !== null) return fetchPromise; + + fetchPromise = fetch('/api/commands') + .then(function (res) { + if (!res.ok) throw new Error('HTTP ' + res.status); + return res.json(); + }) + .then(function (data) { + if (Array.isArray(data) && data.length > 0) { + cachedCommands = data; + } else { + cachedCommands = FALLBACK_COMMANDS; + } + fetchPromise = null; + return cachedCommands; + }) + .catch(function () { + cachedCommands = FALLBACK_COMMANDS; + fetchPromise = null; + return cachedCommands; + }); + + return fetchPromise; +} + +/** + * Initialize slash command autocomplete for a textarea element. + * Kicks off a background fetch of /api/commands immediately so the list + * is ready before the user types. + * @param {HTMLTextAreaElement} input + */ +export function initAutocomplete(input) { + if (!input) return; + + const inpWrap = input.closest('.inp-wrap'); + if (!inpWrap) return; + + // Pre-fetch commands in the background so the list is warm before first use + void fetchCommands(); + + // Create dropdown element + const dropdown = document.createElement('ul'); + dropdown.className = 'autocomplete-dropdown'; + dropdown.setAttribute('role', 'listbox'); + dropdown.setAttribute('aria-label', 'Command suggestions'); + dropdown.id = 'autocomplete-dropdown'; + input.setAttribute('aria-autocomplete', 'list'); + input.setAttribute('aria-controls', 'autocomplete-dropdown'); + inpWrap.appendChild(dropdown); + + let activeIndex = -1; + let visible = false; + let filteredCommands = []; + + function show(commands) { + filteredCommands = commands; + activeIndex = -1; + visible = true; + + dropdown.replaceChildren(); + for (let i = 0; i < commands.length; i++) { + const cmd = commands[i]; + const li = document.createElement('li'); + li.className = 'autocomplete-item'; + li.setAttribute('role', 'option'); + li.setAttribute('aria-selected', 'false'); + li.dataset.index = String(i); + + const name = document.createElement('span'); + name.className = 'autocomplete-cmd'; + name.textContent = cmd.name; + + const desc = document.createElement('span'); + desc.className = 'autocomplete-desc'; + desc.textContent = cmd.description; + + li.appendChild(name); + li.appendChild(desc); + + // Use mousedown so it fires before the blur event on the input + li.addEventListener('mousedown', function (e) { + e.preventDefault(); + selectCommand(i); + }); + + dropdown.appendChild(li); + } + dropdown.classList.add('visible'); + input.setAttribute('aria-expanded', 'true'); + } + + function hide() { + visible = false; + activeIndex = -1; + dropdown.classList.remove('visible'); + dropdown.replaceChildren(); + input.setAttribute('aria-expanded', 'false'); + } + + function setActive(index) { + const items = dropdown.querySelectorAll('.autocomplete-item'); + items.forEach(function (el, i) { + if (i === index) { + el.classList.add('active'); + el.setAttribute('aria-selected', 'true'); + el.scrollIntoView({ block: 'nearest' }); + } else { + el.classList.remove('active'); + el.setAttribute('aria-selected', 'false'); + } + }); + activeIndex = index; + } + + function selectCommand(index) { + const cmd = filteredCommands[index]; + if (!cmd) return; + // Replace the current slash query with the command name + trailing space + input.value = cmd.name + ' '; + // Trigger input event so textarea resizes and char count updates + input.dispatchEvent(new Event('input')); + input.focus(); + hide(); + } + + /** + * Returns the current slash query if the input starts with "/" and the + * cursor hasn't moved past the first word yet (no space typed), otherwise null. + */ + function getQuery() { + const val = input.value; + if (!val.startsWith('/')) return null; + // Stop showing if the user has typed a space (command word is complete) + if (val.includes(' ')) return null; + return val; + } + + input.addEventListener('input', function () { + const query = getQuery(); + if (query === null) { + hide(); + return; + } + const lower = query.toLowerCase(); + // Use cached commands if available, fall back to FALLBACK_COMMANDS synchronously + const commands = cachedCommands !== null ? cachedCommands : FALLBACK_COMMANDS; + const matches = commands.filter(function (cmd) { + return cmd.name.startsWith(lower); + }); + if (matches.length === 0) { + hide(); + } else { + show(matches); + } + }); + + input.addEventListener('keydown', function (e) { + if (!visible) return; + + if (e.key === 'ArrowDown') { + e.preventDefault(); + const next = Math.min(activeIndex + 1, filteredCommands.length - 1); + setActive(next); + } else if (e.key === 'ArrowUp') { + e.preventDefault(); + const prev = Math.max(activeIndex - 1, 0); + setActive(prev); + } else if (e.key === 'Enter') { + if (activeIndex >= 0) { + // Prevent form submission and select the highlighted item + e.preventDefault(); + e.stopPropagation(); + selectCommand(activeIndex); + } + } else if (e.key === 'Tab') { + if (filteredCommands.length > 0) { + e.preventDefault(); + const idx = activeIndex >= 0 ? activeIndex : 0; + selectCommand(idx); + } + } else if (e.key === 'Escape') { + hide(); + } + }); + + // Hide on blur (delay allows mousedown on dropdown item to fire first) + input.addEventListener('blur', function () { + setTimeout(hide, 150); + }); +} diff --git a/src/connectors/webchat/ui/js/dashboard.js b/src/connectors/webchat/ui/js/dashboard.js new file mode 100644 index 00000000..66389a6f --- /dev/null +++ b/src/connectors/webchat/ui/js/dashboard.js @@ -0,0 +1,141 @@ +/** + * Agent dashboard module. + * Renders live Master + worker status and provides stop controls. + */ + +import { sendMessage } from './websocket.js'; + +let dashOpen = true; + +/** + * Initialize the dashboard collapse toggle and stop-all button. + * Must be called after the DOM is ready. + */ +export function initDashboard() { + const hdr = document.getElementById('dash-hdr'); + const stopAllBtn = document.getElementById('stop-all-btn'); + + hdr.addEventListener('click', function () { + dashOpen = !dashOpen; + document.getElementById('dash-body').style.display = dashOpen ? '' : 'none'; + document.getElementById('dash-icon').textContent = dashOpen ? '\u25B2' : '\u25BC'; + hdr.setAttribute('aria-expanded', String(dashOpen)); + }); + + hdr.addEventListener('keydown', function (e) { + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault(); + hdr.click(); + } + }); + + stopAllBtn.addEventListener('click', function () { + sendMessage({ type: 'stop-all' }); + }); +} + +/** + * Update the dashboard with the current list of agents. + * + * @param {Array} agents - array of ActivityRecord objects from the server + */ +export function updateDashboard(agents) { + const dash = document.getElementById('dash'); + const stopAllBtn = document.getElementById('stop-all-btn'); + + if (!agents || agents.length === 0) { + dash.classList.add('hidden'); + stopAllBtn.disabled = true; + return; + } + + dash.classList.remove('hidden'); + + let master = null; + const workers = []; + for (let i = 0; i < agents.length; i++) { + if (agents[i].type === 'master') master = agents[i]; + else workers.push(agents[i]); + } + + const masterDiv = document.getElementById('dash-master'); + masterDiv.innerHTML = master + ? '
Master: ' + + (master.model || 'unknown') + + ' \u00a0|\u00a0 ' + + master.status + + '
' + : ''; + + const workersDiv = document.getElementById('dash-workers'); + if (workers.length === 0) { + workersDiv.innerHTML = ''; + stopAllBtn.disabled = true; + } else { + stopAllBtn.disabled = false; + let h = '
Workers (' + workers.length + '):
'; + for (let j = 0; j < workers.length; j++) { + const w = workers[j]; + const pct = w.progress_pct || 0; + const sc = 's-' + (w.status || 'running'); + const elapsed = w.started_at + ? Math.floor((Date.now() - new Date(w.started_at).getTime()) / 1000) + 's' + : ''; + const wid = String(w.id); + h += + '
' + + '' + + wid.slice(0, 8) + + '' + + '' + + (w.model || '\u2014') + + '' + + '' + + (w.profile || '\u2014') + + '' + + '' + + (w.task_summary || '\u2014') + + '' + + '
' + + '' + + pct + + '%' + + '' + + elapsed + + '' + + '' + + '
'; + } + workersDiv.innerHTML = h; + + // Wire stop buttons via event delegation (avoids inline onclick) + workersDiv.querySelectorAll('[data-worker-id]').forEach(function (btn) { + btn.addEventListener('click', function () { + sendMessage({ type: 'stop-worker', workerId: btn.dataset.workerId }); + }); + }); + } + + let totalCost = 0; + for (let k = 0; k < agents.length; k++) { + totalCost += agents[k].cost_usd || 0; + } + + document.getElementById('dash-cost').innerHTML = + '
Cost: $' + + totalCost.toFixed(4) + + ' \u00a0|\u00a0 Active workers: ' + + workers.length + + '
'; + + document.getElementById('dash-lbl').textContent = + 'Agent Status (' + agents.length + ' active)'; +} diff --git a/src/connectors/webchat/ui/js/deep-mode.js b/src/connectors/webchat/ui/js/deep-mode.js new file mode 100644 index 00000000..eec34f56 --- /dev/null +++ b/src/connectors/webchat/ui/js/deep-mode.js @@ -0,0 +1,570 @@ +/** + * OpenBridge WebChat — Deep Mode stepper UI. + * Shows a horizontal progress bar with 5 phase dots when Deep Mode is active. + * Phases: Investigate, Report, Plan, Execute, Verify. + * Active phase is highlighted; completed phases show a checkmark. + * Hidden when Deep Mode is inactive. + * + * Also renders phase action buttons: + * - Proceed (green) — sends /proceed via WebSocket + * - Focus on # dropdown — sends /focus N via WebSocket (enabled after investigate/report) + * - Skip # dropdown — sends /skip N via WebSocket (enabled after plan) + * All buttons are disabled while a phase is running or no session is active. + * + * Phase transition cards are rendered into the msgs container to show + * phase progress inline in the conversation flow. + */ + +import { sendMessage } from './websocket.js'; + +const PHASES = ['investigate', 'report', 'plan', 'execute', 'verify']; + +const PHASE_LABELS = { + investigate: 'Investigate', + report: 'Report', + plan: 'Plan', + execute: 'Execute', + verify: 'Verify', +}; + +const PHASE_ICONS = { + investigate: '🔍', + report: '📋', + plan: '📝', + execute: '⚙️', + verify: '✅', +}; + +const PHASE_COLORS = { + investigate: 'blue', + report: 'purple', + plan: 'orange', + execute: 'green', + verify: 'teal', +}; + +/** Phases where /focus N is applicable */ +const FOCUS_PHASES = new Set(['investigate', 'report']); + +/** Phases where /skip N is applicable */ +const SKIP_PHASES = new Set(['plan']); + +let _bar = null; + +/** @type {HTMLElement|null} The msgs container for rendering phase cards */ +let _msgsContainer = null; + +/** @type {Map} "sessionId:phase" -> card DOM element */ +const _phaseCards = new Map(); + +/** @type {Map>} sessionId -> Set of completed phase names */ +const _completedPhases = new Map(); + +/** @type {Map} sessionId -> current phase name */ +const _currentPhases = new Map(); + +/** @type {string|null} */ +let _activeSession = null; + +/** Whether the current phase is still running (started but not yet completed/skipped) */ +let _phaseRunning = false; + +/** The last phase that completed — used to determine which action buttons to enable */ +let _lastCompletedPhase = null; + +// ── Reconnection state persistence ─────────────────────────────────────────── + +/** sessionStorage key for persisting received deep-phase events across reconnects */ +const SESSION_KEY = 'ob-deep-mode-events'; + +/** + * Flat log of all deep-phase events received (deduplicated per sessionId+phase, + * keeping the latest status). Persisted to sessionStorage for reconnect recovery. + * @type {Array<{sessionId: string, phase: string, status: string, result?: string}>} + */ +let _eventLog = []; + +/** + * True while replaying a stored event log — suppresses card rendering and + * prevents recursive sessionStorage writes. + */ +let _restoringState = false; + +function _persistEventLog() { + try { + sessionStorage.setItem(SESSION_KEY, JSON.stringify(_eventLog)); + } catch (_) { + // sessionStorage unavailable or full — non-critical + } +} + +function _loadEventLog() { + try { + const raw = sessionStorage.getItem(SESSION_KEY); + return raw ? JSON.parse(raw) : []; + } catch (_) { + return []; + } +} + +function _resetInMemoryState() { + _completedPhases.clear(); + _currentPhases.clear(); + _phaseCards.clear(); + _activeSession = null; + _phaseRunning = false; + _lastCompletedPhase = null; +} + +// Action button DOM references +let _proceedBtn = null; +let _focusSelect = null; +let _focusBtn = null; +let _skipSelect = null; +let _skipBtn = null; + +function getBar() { + if (!_bar) _bar = document.getElementById('deep-mode-bar'); + return _bar; +} + +function render() { + const bar = getBar(); + if (!bar) return; + if (!_activeSession) { + bar.classList.add('hidden'); + return; + } + const completed = _completedPhases.get(_activeSession) || new Set(); + const current = _currentPhases.get(_activeSession) || null; + bar.classList.remove('hidden'); + const dots = bar.querySelectorAll('.dm-phase-dot'); + dots.forEach(function (dot) { + const phase = dot.dataset.phase; + dot.classList.remove('dm-phase-current', 'dm-phase-done', 'dm-phase-pending'); + const icon = dot.querySelector('.dm-phase-icon'); + if (completed.has(phase)) { + dot.classList.add('dm-phase-done'); + if (icon) icon.textContent = '\u2713'; + dot.setAttribute('aria-label', (PHASE_LABELS[phase] || phase) + ' \u2014 completed'); + } else if (phase === current) { + dot.classList.add('dm-phase-current'); + if (icon) icon.textContent = '\u25CF'; + dot.setAttribute('aria-label', (PHASE_LABELS[phase] || phase) + ' \u2014 in progress'); + } else { + dot.classList.add('dm-phase-pending'); + if (icon) icon.textContent = '\u25CB'; + dot.setAttribute('aria-label', (PHASE_LABELS[phase] || phase) + ' \u2014 pending'); + } + }); +} + +/** Enable/disable action buttons based on current Deep Mode state */ +function updateActionButtons() { + if (!_proceedBtn) return; + const active = !!_activeSession; + const canAct = active && !_phaseRunning; + + _proceedBtn.disabled = !canAct; + + const canFocus = canAct && FOCUS_PHASES.has(_lastCompletedPhase); + _focusBtn.disabled = !canFocus; + _focusSelect.disabled = !canFocus; + + const canSkip = canAct && SKIP_PHASES.has(_lastCompletedPhase); + _skipBtn.disabled = !canSkip; + _skipSelect.disabled = !canSkip; +} + +/** Build a numeric ' + + '' + + '' + + '' + + ''; + list.appendChild(item); + } + list.querySelectorAll('.mcp-toggle-input').forEach(function (input) { + input.addEventListener('change', function () { + const name = input.getAttribute('data-name'); + fetch('/api/mcp/servers/' + encodeURIComponent(name) + '/toggle', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ enabled: input.checked }), + }).catch(function () {}); + }); + }); + list.querySelectorAll('.mcp-remove-btn').forEach(function (btn) { + btn.addEventListener('click', function () { + const name = btn.getAttribute('data-name'); + if (!confirm('Remove MCP server "' + name + '"?')) return; + fetch('/api/mcp/servers/' + encodeURIComponent(name), { method: 'DELETE' }) + .then(function (r) { if (r.ok) loadMcpServers(); }) + .catch(function () {}); + }); + }); + }) + .catch(function () {}); +} + +function initMcpPanel() { + const addBtn = document.getElementById('mcp-add-btn'); + const form = document.getElementById('mcp-add-form'); + const submitBtn = document.getElementById('mcp-add-submit'); + const cancelBtn = document.getElementById('mcp-add-cancel'); + if (!addBtn || !form) return; + + addBtn.addEventListener('click', function () { + form.style.display = form.style.display === 'none' ? 'block' : 'none'; + }); + if (cancelBtn) { + cancelBtn.addEventListener('click', function () { + form.style.display = 'none'; + }); + } + if (submitBtn) { + submitBtn.addEventListener('click', function () { + const nameInput = document.getElementById('mcp-new-name'); + const cmdInput = document.getElementById('mcp-new-command'); + if (!nameInput || !cmdInput) return; + const name = nameInput.value.trim(); + const command = cmdInput.value.trim(); + if (!name || !command) return; + fetch('/api/mcp/servers', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ name, command }), + }) + .then(function (r) { + if (r.ok) { + form.style.display = 'none'; + nameInput.value = ''; + cmdInput.value = ''; + loadMcpServers(); + } + }) + .catch(function () {}); + }); + } +} + +/** + * Initialize the settings panel. Must be called after DOM is ready. + */ +export function initSettings() { + _panel = document.getElementById('settings-panel'); + _overlay = document.getElementById('settings-overlay'); + const gearBtn = document.getElementById('settings-btn'); + const closeBtn = _panel && _panel.querySelector('.settings-close-btn'); + + if (!_panel || !_overlay || !gearBtn) return; + + // Gear button opens panel + gearBtn.addEventListener('click', function () { + if (isOpen()) { + closeSettings(); + } else { + loadDiscoveredTools(); + loadMcpServers(); + openSettings(); + } + }); + + // Close button + if (closeBtn) { + closeBtn.addEventListener('click', closeSettings); + } + + // Overlay click closes panel + _overlay.addEventListener('click', closeSettings); + + // Escape key closes panel + document.addEventListener('keydown', function (e) { + if (e.key === 'Escape' && isOpen()) { + closeSettings(); + } + }); + + // Init sub-components + initToolSelector(); + initExecutionProfile(); + initNotifications(); + initThemeSelector(); + initMcpPanel(); +} + +/** + * Returns the currently saved execution profile. + * @returns {'fast'|'thorough'|'manual'} + */ +export function getExecutionProfile() { + return localStorage.getItem('ob-exec-profile') || 'thorough'; +} + +/** + * Returns the currently saved preferred AI tool name. + * @returns {string|null} + */ +export function getPreferredTool() { + return localStorage.getItem('ob-preferred-tool') || null; +} diff --git a/src/connectors/webchat/ui/js/sidebar.js b/src/connectors/webchat/ui/js/sidebar.js new file mode 100644 index 00000000..5b26d7da --- /dev/null +++ b/src/connectors/webchat/ui/js/sidebar.js @@ -0,0 +1,407 @@ +/** + * OpenBridge WebChat — Sidebar component. + * Slide-out panel (left). Hidden on mobile, toggleable via hamburger. + * Desktop: optionally always visible (300px), persisted to localStorage. + */ + +const BREAKPOINT_DESKTOP = 768; + +let _open = false; +let _sidebar = null; +let _overlay = null; +let _toggle = null; +let _currentSessionId = null; +let _onSessionSelect = null; +let _onNewConversation = null; + +/** + * Register a callback invoked when the user clicks a session card. + * @param {function(string): void} fn + */ +export function setOnSessionSelect(fn) { + _onSessionSelect = fn; +} + +/** + * Register a callback invoked when the user clicks "New conversation". + * @param {function(): void} fn + */ +export function setOnNewConversation(fn) { + _onNewConversation = fn; +} + +function isDesktop() { + return window.innerWidth >= BREAKPOINT_DESKTOP; +} + +export function isSidebarOpen() { + return _open; +} + +function openSidebar() { + _open = true; + _sidebar.classList.add('open'); + if (!isDesktop()) { + _overlay.classList.add('visible'); + _overlay.removeAttribute('aria-hidden'); + } + _toggle.setAttribute('aria-expanded', 'true'); + _toggle.setAttribute('aria-label', 'Close sidebar'); + _sidebar.setAttribute('aria-hidden', 'false'); +} + +function closeSidebar() { + _open = false; + _sidebar.classList.remove('open'); + _overlay.classList.remove('visible'); + _overlay.setAttribute('aria-hidden', 'true'); + _toggle.setAttribute('aria-expanded', 'false'); + _toggle.setAttribute('aria-label', 'Open sidebar'); + _sidebar.setAttribute('aria-hidden', 'true'); +} + +export function toggleSidebar() { + if (_open) { + closeSidebar(); + if (isDesktop()) { + localStorage.setItem('ob-sidebar-open', 'false'); + } + } else { + openSidebar(); + if (isDesktop()) { + localStorage.setItem('ob-sidebar-open', 'true'); + } + } +} + +// --------------------------------------------------------------------------- +// Session list +// --------------------------------------------------------------------------- + +/** + * Format a timestamp string as a short relative date for sidebar cards. + * @param {string} isoString + * @returns {string} + */ +function formatSessionDate(isoString) { + if (!isoString) return ''; + const date = new Date(isoString); + const diff = Math.floor((Date.now() - date.getTime()) / 1000); + if (diff < 60) return 'just now'; + if (diff < 3600) return Math.floor(diff / 60) + 'm ago'; + if (diff < 86400) return Math.floor(diff / 3600) + 'h ago'; + if (diff < 86400 * 7) return Math.floor(diff / 86400) + 'd ago'; + return date.toLocaleDateString(undefined, { month: 'short', day: 'numeric' }); +} + +/** + * Build a session card element. + * @param {{ session_id: string, title: string|null, last_message_at: string, message_count: number }} session + * @param {boolean} isActive + * @returns {HTMLElement} + */ +function buildSessionCard(session, isActive) { + const item = document.createElement('div'); + item.className = 'sidebar-session-item' + (isActive ? ' active' : ''); + item.setAttribute('role', 'listitem'); + item.setAttribute('tabindex', '0'); + item.dataset.sessionId = session.session_id; + + const titleEl = document.createElement('div'); + titleEl.className = 'sidebar-session-title'; + titleEl.textContent = session.title || 'Conversation'; + + const metaEl = document.createElement('div'); + metaEl.className = 'sidebar-session-meta'; + + const dateSpan = document.createElement('span'); + dateSpan.textContent = formatSessionDate(session.last_message_at); + + const countSpan = document.createElement('span'); + const count = session.message_count || 0; + countSpan.textContent = count + (count === 1 ? ' msg' : ' msgs'); + + metaEl.appendChild(dateSpan); + metaEl.appendChild(countSpan); + + item.appendChild(titleEl); + item.appendChild(metaEl); + + return item; +} + +/** + * Fetch /api/sessions and render the list into #sidebar-sessions. + * The most recent session (first in list) is highlighted as the current session. + * Pass an explicit sessionId to override which item is highlighted. + * + * @param {string|null} [activeSessionId] + */ +export async function loadSessions(activeSessionId) { + const container = document.getElementById('sidebar-sessions'); + if (!container) return; + + let sessions; + try { + const res = await fetch('/api/sessions?limit=50'); + if (!res.ok) return; + sessions = await res.json(); + } catch (_) { + return; + } + + if (!Array.isArray(sessions) || sessions.length === 0) { + container.innerHTML = '
No conversations yet.
'; + return; + } + + // Default: highlight the most recent session (first in list) + const effectiveId = activeSessionId != null ? activeSessionId : sessions[0].session_id; + _currentSessionId = effectiveId; + + const frag = document.createDocumentFragment(); + for (const session of sessions) { + const card = buildSessionCard(session, session.session_id === effectiveId); + frag.appendChild(card); + } + container.replaceChildren(frag); +} + +// --------------------------------------------------------------------------- +// Search +// --------------------------------------------------------------------------- + +/** + * Escape a string for safe insertion as HTML text content. + * @param {string} str + * @returns {string} + */ +function escapeHtml(str) { + return str + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} + +/** + * Extract a short snippet from `content` centred around the first query term. + * @param {string} content + * @param {string} query + * @param {number} [maxLen=120] + * @returns {string} + */ +function truncateSnippet(content, query, maxLen) { + if (!content) return ''; + maxLen = maxLen || 120; + const terms = query.trim().split(/\s+/).filter(Boolean); + let pos = -1; + for (let i = 0; i < terms.length; i++) { + const idx = content.toLowerCase().indexOf(terms[i].toLowerCase()); + if (idx !== -1) { pos = idx; break; } + } + if (pos === -1) { + return content.slice(0, maxLen) + (content.length > maxLen ? '\u2026' : ''); + } + const start = Math.max(0, pos - 30); + const end = Math.min(content.length, start + maxLen); + const snippet = content.slice(start, end); + return (start > 0 ? '\u2026' : '') + snippet + (end < content.length ? '\u2026' : ''); +} + +/** + * HTML-escape `text` then wrap each query term occurrence in a ``. + * @param {string} text + * @param {string} query + * @returns {string} HTML string safe for innerHTML + */ +function highlightTerms(text, query) { + if (!text) return ''; + const escaped = escapeHtml(text); + const terms = query.trim().split(/\s+/).filter(Boolean); + if (terms.length === 0) return escaped; + const pattern = terms + .map(function (t) { return escapeHtml(t).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); }) + .join('|'); + const re = new RegExp('(' + pattern + ')', 'gi'); + return escaped.replace(re, '$1'); +} + +/** + * Build a search result card element. + * @param {{ session_id: string, role: string, content: string, created_at: string }} result + * @param {string} query + * @returns {HTMLElement} + */ +function buildSearchResultCard(result, query) { + const item = document.createElement('div'); + item.className = 'sidebar-session-item sidebar-search-result'; + item.setAttribute('role', 'listitem'); + item.setAttribute('tabindex', '0'); + item.dataset.sessionId = result.session_id; + + const snippet = truncateSnippet(result.content, query); + const highlighted = highlightTerms(snippet, query); + + const snippetEl = document.createElement('div'); + snippetEl.className = 'sidebar-search-snippet'; + snippetEl.innerHTML = highlighted; + + const metaEl = document.createElement('div'); + metaEl.className = 'sidebar-session-meta'; + + const roleSpan = document.createElement('span'); + roleSpan.textContent = result.role === 'user' ? 'You' : 'AI'; + + const dateSpan = document.createElement('span'); + dateSpan.textContent = formatSessionDate(result.created_at); + + metaEl.appendChild(roleSpan); + metaEl.appendChild(dateSpan); + + item.appendChild(snippetEl); + item.appendChild(metaEl); + + return item; +} + +/** + * Fetch /api/sessions/search?q={query} and render matching messages. + * @param {string} query + */ +async function runSearch(query) { + const container = document.getElementById('sidebar-sessions'); + if (!container) return; + + container.innerHTML = '
Searching\u2026
'; + + let results; + try { + const res = await fetch('/api/sessions/search?q=' + encodeURIComponent(query) + '&limit=20'); + if (!res.ok) { + container.innerHTML = '
Search failed.
'; + return; + } + results = await res.json(); + } catch (_) { + container.innerHTML = '
Search failed.
'; + return; + } + + if (!Array.isArray(results) || results.length === 0) { + container.innerHTML = '
No results for \u201c' + escapeHtml(query) + '\u201d.
'; + return; + } + + const frag = document.createDocumentFragment(); + for (const result of results) { + frag.appendChild(buildSearchResultCard(result, query)); + } + container.replaceChildren(frag); +} + +// --------------------------------------------------------------------------- + +export function initSidebar() { + _sidebar = document.getElementById('sidebar'); + _overlay = document.getElementById('sidebar-overlay'); + _toggle = document.getElementById('sidebar-toggle'); + if (!_sidebar || !_overlay || !_toggle) return; + + _toggle.addEventListener('click', toggleSidebar); + + // New conversation button + const newConvBtn = document.getElementById('new-conversation-btn'); + if (newConvBtn) { + newConvBtn.addEventListener('click', function () { + if (_onNewConversation) { + _onNewConversation(); + } + if (!isDesktop()) { + closeSidebar(); + } + }); + } + + // Overlay click closes sidebar on mobile + _overlay.addEventListener('click', function () { + closeSidebar(); + }); + + // Escape closes sidebar on mobile + document.addEventListener('keydown', function (e) { + if (e.key === 'Escape' && _open && !isDesktop()) { + closeSidebar(); + } + }); + + // On resize: update overlay visibility (no overlay on desktop) + window.addEventListener('resize', function () { + if (_open) { + if (isDesktop()) { + _overlay.classList.remove('visible'); + _overlay.setAttribute('aria-hidden', 'true'); + } else { + _overlay.classList.add('visible'); + _overlay.removeAttribute('aria-hidden'); + } + } + }); + + // Event delegation for session card clicks + const sessionsContainer = document.getElementById('sidebar-sessions'); + if (sessionsContainer) { + sessionsContainer.addEventListener('click', function (e) { + const item = e.target.closest('.sidebar-session-item'); + if (!item) return; + const sessionId = item.dataset.sessionId; + if (!sessionId) return; + // Update active highlight + sessionsContainer.querySelectorAll('.sidebar-session-item').forEach(function (el) { + el.classList.toggle('active', el === item); + }); + _currentSessionId = sessionId; + // Close sidebar on mobile after selection + if (!isDesktop()) { + closeSidebar(); + } + if (_onSessionSelect) { + _onSessionSelect(sessionId); + } + }); + + // Keyboard: Enter or Space activates a focused card + sessionsContainer.addEventListener('keydown', function (e) { + if (e.key !== 'Enter' && e.key !== ' ') return; + const item = e.target.closest('.sidebar-session-item'); + if (!item) return; + e.preventDefault(); + item.click(); + }); + } + + // Search input — debounced 300ms + const searchInput = document.getElementById('sidebar-search-input'); + let _searchTimer = null; + if (searchInput) { + searchInput.addEventListener('input', function () { + clearTimeout(_searchTimer); + const query = searchInput.value.trim(); + if (!query) { + void loadSessions(_currentSessionId); + return; + } + _searchTimer = setTimeout(function () { + void runSearch(query); + }, 300); + }); + } + + // Initial state: open on desktop unless user explicitly closed it + if (isDesktop()) { + const saved = localStorage.getItem('ob-sidebar-open'); + if (saved !== 'false') { + openSidebar(); + } + } +} diff --git a/src/connectors/webchat/ui/js/sw.js b/src/connectors/webchat/ui/js/sw.js new file mode 100644 index 00000000..d9568a34 --- /dev/null +++ b/src/connectors/webchat/ui/js/sw.js @@ -0,0 +1,82 @@ +/** + * OpenBridge WebChat — Service Worker + * Caches the HTML/CSS/JS shell for offline use and handles push notifications. + */ + +var CACHE_NAME = 'openbridge-webchat-v1'; + +// Install: pre-cache the app shell HTML so it loads when offline +self.addEventListener('install', function (event) { + event.waitUntil( + caches.open(CACHE_NAME).then(function (cache) { + return cache.addAll(['/']); + }), + ); + self.skipWaiting(); +}); + +// Activate: remove stale caches from previous versions +self.addEventListener('activate', function (event) { + event.waitUntil( + caches.keys().then(function (keys) { + return Promise.all( + keys + .filter(function (k) { + return k !== CACHE_NAME; + }) + .map(function (k) { + return caches.delete(k); + }), + ); + }), + ); + self.clients.claim(); +}); + +// Fetch: cache-first for the main HTML shell so the UI loads offline +self.addEventListener('fetch', function (event) { + var req = event.request; + if (req.method !== 'GET') return; + var url = new URL(req.url); + if (url.origin !== self.location.origin) return; + // Only intercept the root HTML shell + if (url.pathname === '/' || url.pathname === '') { + event.respondWith( + caches.match(req).then(function (cached) { + return cached || fetch(req); + }), + ); + } +}); + +// Push: display a browser notification when a push event is received +self.addEventListener('push', function (event) { + var data = { title: 'OpenBridge', body: 'New message received' }; + if (event.data) { + try { + data = event.data.json(); + } catch (_) { + data.body = event.data.text(); + } + } + event.waitUntil( + self.registration.showNotification(data.title || 'OpenBridge', { + body: data.body, + icon: '/icons/icon-192.png', + }), + ); +}); + +// Notification click: focus the existing WebChat tab or open a new one +self.addEventListener('notificationclick', function (event) { + event.notification.close(); + event.waitUntil( + clients.matchAll({ type: 'window', includeUncontrolled: true }).then(function (list) { + for (var i = 0; i < list.length; i++) { + var client = list[i]; + if ('focus' in client) return client.focus(); + } + return clients.openWindow('/'); + }), + ); +}); diff --git a/src/connectors/webchat/ui/js/websocket.js b/src/connectors/webchat/ui/js/websocket.js new file mode 100644 index 00000000..a6f54735 --- /dev/null +++ b/src/connectors/webchat/ui/js/websocket.js @@ -0,0 +1,70 @@ +/** + * WebSocket connection module. + * Handles connection lifecycle, auto-reconnection, and message sending. + */ + +/** @type {WebSocket|null} */ +let ws = null; + +/** + * Initialize the WebSocket connection with auto-reconnect. + * + * @param {{ onOpen: Function, onClose: Function, onMessage: Function }} handlers + */ +export function initWebSocket(handlers) { + function connect() { + // Pass auth token from page URL to WebSocket upgrade request + var proto = location.protocol === 'https:' ? 'wss:' : 'ws:'; + var wsUrl = proto + '//' + location.host; + var params = new URLSearchParams(location.search); + var token = params.get('token'); + if (token) wsUrl += '?token=' + encodeURIComponent(token); + ws = new WebSocket(wsUrl); + + ws.onopen = function () { + handlers.onOpen(); + }; + + ws.onclose = function () { + ws = null; + handlers.onClose(); + setTimeout(connect, 2000); + }; + + ws.onmessage = function (e) { + try { + const data = JSON.parse(e.data); + handlers.onMessage(data); + } catch (_) { + // ignore malformed messages + } + }; + + ws.onerror = function () { + // onclose will fire next and trigger reconnect + }; + } + + connect(); +} + +/** + * Send a JSON message over the WebSocket. + * No-ops if the connection is not open. + * + * @param {Object} data + */ +export function sendMessage(data) { + if (ws && ws.readyState === WebSocket.OPEN) { + ws.send(JSON.stringify(data)); + } +} + +/** + * Returns true if the WebSocket is currently open. + * + * @returns {boolean} + */ +export function isConnected() { + return ws !== null && ws.readyState === WebSocket.OPEN; +} diff --git a/src/connectors/webchat/ui/login.html b/src/connectors/webchat/ui/login.html new file mode 100644 index 00000000..9f16b1b1 --- /dev/null +++ b/src/connectors/webchat/ui/login.html @@ -0,0 +1,202 @@ + + + + + + OpenBridge WebChat — Sign in + + + + + + + + diff --git a/src/connectors/webchat/webchat-auth.ts b/src/connectors/webchat/webchat-auth.ts new file mode 100644 index 00000000..c4092c5c --- /dev/null +++ b/src/connectors/webchat/webchat-auth.ts @@ -0,0 +1,59 @@ +import { randomBytes } from 'node:crypto'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import bcrypt from 'bcryptjs'; +import { createLogger } from '../../core/logger.js'; + +const logger = createLogger('webchat-auth'); + +/** + * Reads the existing WebChat auth token from disk, or generates and persists a new one. + * + * Token is stored at /.openbridge/webchat-token (64-char hex string). + * File permissions are set to 0o600 (owner read/write only). + * + * @param baseDir - Directory under which `.openbridge/webchat-token` is stored. + * Defaults to `process.cwd()` if not provided. + */ +export function getOrCreateAuthToken(baseDir: string = process.cwd()): string { + const tokenDir = join(baseDir, '.openbridge'); + const tokenPath = join(tokenDir, 'webchat-token'); + + if (existsSync(tokenPath)) { + const existing = readFileSync(tokenPath, 'utf8').trim(); + if (existing.length > 0) { + logger.debug({ tokenPath }, 'Loaded existing WebChat auth token'); + return existing; + } + } + + const token = randomBytes(32).toString('hex'); + mkdirSync(tokenDir, { recursive: true }); + writeFileSync(tokenPath, token, { mode: 0o600, encoding: 'utf8' }); + logger.info({ tokenPath }, 'Generated new WebChat auth token'); + return token; +} + +/** bcrypt cost factor for password hashing */ +const BCRYPT_ROUNDS = 10; + +/** + * Hash a plain-text password with bcrypt. + * + * @param password - The plain-text password to hash. + * @returns A bcrypt hash string. + */ +export async function hashPassword(password: string): Promise { + return bcrypt.hash(password, BCRYPT_ROUNDS); +} + +/** + * Verify a submitted plain-text password against a stored bcrypt hash. + * + * @param submitted - The password submitted by the user. + * @param hash - The bcrypt hash stored on disk. + * @returns `true` if the password matches, `false` otherwise. + */ +export async function verifyPassword(submitted: string, hash: string): Promise { + return bcrypt.compare(submitted, hash); +} diff --git a/src/connectors/webchat/webchat-config.ts b/src/connectors/webchat/webchat-config.ts new file mode 100644 index 00000000..a04ffe56 --- /dev/null +++ b/src/connectors/webchat/webchat-config.ts @@ -0,0 +1,25 @@ +import { z } from 'zod/v3'; + +/** + * Schema for validating the PUT /api/webchat/settings request body. + */ +export const WebchatSettingsPutSchema = z.object({ + profile: z.enum(['fast', 'thorough', 'manual']), +}); + +export type WebchatSettingsPut = z.infer; + +export const WebChatConfigSchema = z.object({ + /** TCP port the HTTP + WebSocket server listens on */ + port: z.number().int().positive().default(3000), + /** Hostname the server binds to — defaults to 0.0.0.0 for LAN access. Use localhost to restrict to local machine only */ + host: z.string().default('0.0.0.0'), + /** + * Optional password for WebChat access. + * When set, token-based auth is replaced by a password login screen. + * The value is hashed with bcrypt before being stored or compared. + */ + password: z.string().min(1).optional(), +}); + +export type WebChatConfig = z.infer; diff --git a/src/connectors/webchat/webchat-connector.ts b/src/connectors/webchat/webchat-connector.ts new file mode 100644 index 00000000..b4f32009 --- /dev/null +++ b/src/connectors/webchat/webchat-connector.ts @@ -0,0 +1,1595 @@ +import { randomUUID } from 'node:crypto'; +import { mkdir, unlink, writeFile } from 'node:fs/promises'; +import type { IncomingMessage, ServerResponse } from 'node:http'; +import { networkInterfaces, tmpdir } from 'node:os'; +import { extname, join } from 'node:path'; +import type { Connector, ConnectorEvents } from '../../types/connector.js'; +import type { InboundMessage, OutboundMessage, ProgressEvent } from '../../types/message.js'; +import { WebChatConfigSchema, WebchatSettingsPutSchema } from './webchat-config.js'; +import type { WebChatConfig } from './webchat-config.js'; +import { createLogger } from '../../core/logger.js'; +import { getQrCode } from '../../core/qr-store.js'; +import type { ActivityRecord } from '../../memory/activity-store.js'; +import type { AccessControlEntry } from '../../memory/access-store.js'; +import type { MemoryManager } from '../../memory/index.js'; +import type { DiscoveredTool } from '../../types/discovery.js'; +import type { McpRegistry } from '../../core/mcp-registry.js'; +import type { MCPServer } from '../../types/config.js'; +import { WEBCHAT_HTML, WEBCHAT_LOGIN_HTML, WEBCHAT_SW_JS } from './ui-bundle.js'; +import { getOrCreateAuthToken, hashPassword, verifyPassword } from './webchat-auth.js'; +import { transcribeAudio, TRANSCRIPTION_FALLBACK_MESSAGE } from '../../core/voice-transcriber.js'; +import { processDocument } from '../../intelligence/document-processor.js'; +import { stat } from 'node:fs/promises'; + +/** Name of the HTTP-only session cookie set after successful token validation */ +const SESSION_COOKIE_NAME = 'ob_session'; +/** Session lifetime: 24 hours in milliseconds */ +const SESSION_TTL_MS = 24 * 60 * 60 * 1000; + +/** Per-IP login rate limiting — max failures before a 30-min block */ +const LOGIN_MAX_FAILURES = 5; +/** Sliding window for counting failures: 15 minutes */ +const LOGIN_WINDOW_MS = 15 * 60 * 1000; +/** Block duration after exceeding failure threshold: 30 minutes */ +const LOGIN_BLOCK_MS = 30 * 60 * 1000; + +interface IpRateEntry { + failures: number; + windowStart: number; + blockedUntil?: number; +} + +const logger = createLogger('webchat'); + +type EventListeners = { + [E in keyof ConnectorEvents]: ConnectorEvents[E][]; +}; + +/** Minimal WS client interface — avoids importing ws types at module level */ +interface WsClient { + readyState: number; + send(data: string): void; + ping(): void; + on(event: 'message', listener: (data: Buffer | string) => void): void; + on(event: 'close', listener: () => void): void; + on(event: 'error', listener: (err: Error) => void): void; +} + +/** Minimal WebSocketServer interface */ +interface WssServer { + on(event: 'connection', listener: (socket: WsClient) => void): void; + on(event: 'close', listener: () => void): void; + close(callback?: () => void): void; +} + +/** WebSocket OPEN state constant */ +const WS_OPEN = 1; + +/** Maximum upload size: 10 MB */ +const UPLOAD_MAX_BYTES = 10 * 1024 * 1024; + +/** + * Parse the first file part from a multipart/form-data body. + * Returns null if no file part is found. + */ +function parseMultipartFile( + body: Buffer, + boundary: string, +): { filename: string; mimeType: string; data: Buffer } | null { + const delimiter = Buffer.from(`--${boundary}`); + const crlfcrlf = Buffer.from('\r\n\r\n'); + + // Find the first boundary + let pos = body.indexOf(delimiter); + if (pos === -1) return null; + pos += delimiter.length; + + // Skip CRLF after opening boundary + if (body[pos] === 0x0d && body[pos + 1] === 0x0a) pos += 2; + + // Find end of headers + const headersEnd = body.indexOf(crlfcrlf, pos); + if (headersEnd === -1) return null; + + const headersStr = body.subarray(pos, headersEnd).toString('utf8'); + pos = headersEnd + 4; // skip \r\n\r\n + + // Parse headers + let filename = 'upload'; + let mimeType = 'application/octet-stream'; + for (const line of headersStr.split('\r\n')) { + const lower = line.toLowerCase(); + if (lower.startsWith('content-disposition:')) { + const m = line.match(/filename="([^"]+)"/i); + if (m) filename = m[1]!; + } else if (lower.startsWith('content-type:')) { + const ct = line.slice('content-type:'.length).trim(); + if (ct) mimeType = ct; + } + } + + // Find end boundary (CRLF + -- + boundary) + const endDelimiter = Buffer.from(`\r\n--${boundary}`); + const dataEnd = body.indexOf(endDelimiter, pos); + const data = dataEnd !== -1 ? body.subarray(pos, dataEnd) : body.subarray(pos); + return { filename, mimeType, data }; +} + +/** + * WebChat connector — serves a minimal HTML chat UI on localhost:3000 + * and exchanges messages via WebSocket. + * + * Uses Node.js built-in `http` module + the `ws` package. + * No auth required for localhost connections. + * + * Usage in config.json: + * ```json + * { + * "channels": [{ "type": "webchat", "options": { "port": 3000 } }] + * } + * ``` + */ +export class WebChatConnector implements Connector { + readonly name = 'webchat'; + readonly supportsFileAttachments = true as const; + private config: WebChatConfig; + private connected = false; + private httpServer: { close(cb?: (err?: Error) => void): void } | null = null; + private wss: WssServer | null = null; + private clients = new Set(); + private messageCounter = 0; + private readonly pendingDownloads = new Map< + string, + { data: Buffer; mimeType: string; filename?: string; timer: ReturnType } + >(); + private readonly listeners: EventListeners = { + message: [], + ready: [], + auth: [], + error: [], + disconnected: [], + }; + private memory: MemoryManager | null = null; + private discoveredTools: DiscoveredTool[] = []; + private mcpRegistry: McpRegistry | null = null; + private authToken: string | null = null; + private storeDir: string = process.cwd(); + /** bcrypt hash of the configured password, or null when token auth is active */ + private passwordHash: string | null = null; + /** Tunnel URL override for QR code — set before initialize() for tunnel-preferred QR */ + private tunnelUrl: string | null = null; + /** In-memory session store: sessionId → expiry timestamp (ms since epoch) */ + private readonly sessions: Map = new Map(); + /** Periodic cleanup interval handle for expired session eviction */ + private sessionCleanupInterval: ReturnType | null = null; + /** Per-IP login failure tracker for rate limiting */ + private readonly loginRateLimiter: Map = new Map(); + /** Server-side execution profile preference — synced from client settings panel */ + private webchatSettings: { profile: 'fast' | 'thorough' | 'manual' } = { profile: 'thorough' }; + + /** + * Per-session deep-phase event cache for reconnect state replay. + * Keyed by sessionId; values are the latest event per phase (deduplicated). + * Cleared when a session aborts or finishes (with a short delay). + */ + private readonly _deepPhaseCache = new Map< + string, + Array> + >(); + + constructor(options: Record) { + this.config = WebChatConfigSchema.parse(options); + } + + /** Wire the SQLite memory manager — enables the /api/sessions REST endpoint. */ + setMemory(memory: MemoryManager): void { + this.memory = memory; + } + + /** Wire discovered AI tools — enables the /api/discovery REST endpoint. */ + setDiscoveryResult(tools: DiscoveredTool[]): void { + this.discoveredTools = tools; + } + + /** Wire the MCP registry — enables the /api/mcp/servers REST endpoints. */ + setMcpRegistry(registry: McpRegistry): void { + this.mcpRegistry = registry; + } + + /** + * Set the workspace path used for token persistence. + * Must be called before initialize() to take effect. + * Defaults to process.cwd() if not set. + */ + setWorkspacePath(workspacePath: string): void { + this.storeDir = workspacePath; + } + + /** Returns the auth token, or null if initialize() has not been called yet or password mode is active. */ + getAuthToken(): string | null { + return this.authToken; + } + + /** Returns true when password-based auth is active (webchat.password was configured). */ + isPasswordMode(): boolean { + return this.passwordHash !== null; + } + + /** Returns the full URL to access WebChat with the auth token appended, or null before initialize(). */ + getWebChatAccessUrl(): string | null { + if (!this.authToken) return null; + const host = this.config.host === '0.0.0.0' ? 'localhost' : this.config.host; + return `http://${host}:${this.config.port}/?token=${this.authToken}`; + } + + /** + * Returns the tunnel (public) URL with auth token appended when a tunnel is active, + * or null if no tunnel URL has been set via setTunnelUrl(). + */ + getPublicUrl(): string | null { + if (!this.tunnelUrl) return null; + const token = this.authToken; + const sep = this.tunnelUrl.includes('?') ? '&' : '?'; + return token ? `${this.tunnelUrl}${sep}token=${token}` : this.tunnelUrl; + } + + /** + * Set a tunnel public URL so that initialize() uses it as the QR code target + * instead of the LAN IP. Must be called before initialize() to take effect. + * The token is automatically appended when the QR is generated. + */ + setTunnelUrl(url: string): void { + this.tunnelUrl = url; + } + + /** + * Returns the best URL for QR code scanning (tunnel > first LAN IP > localhost). + * The URL includes the auth token query parameter when token auth is active. + * Returns null if initialize() has not been called yet (no auth token). + */ + getLanAccessUrl(): string | null { + const token = this.authToken; + // Tunnel URL is preferred when set + if (this.tunnelUrl) { + const sep = this.tunnelUrl.includes('?') ? '&' : '?'; + return token ? `${this.tunnelUrl}${sep}token=${token}` : this.tunnelUrl; + } + // Otherwise use the first detected LAN IP + const lanIps = this.getLanIps(); + if (lanIps.length > 0) { + const suffix = token ? `/?token=${token}` : '/'; + return `http://${lanIps[0]!}:${this.config.port}${suffix}`; + } + // Fall back to localhost URL + return this.getWebChatAccessUrl(); + } + + /** + * Extract a bearer token from the request — checks (in order): + * 1. `?token=` query parameter + * 2. `Authorization: Bearer ` header + */ + private extractToken(url: string, req: IncomingMessage): string | null { + try { + const parsed = new URL(url, 'http://localhost'); + const tokenFromQuery = parsed.searchParams.get('token'); + if (tokenFromQuery) return tokenFromQuery; + } catch { + // malformed URL — fall through + } + const authHeader = req.headers?.['authorization']; + if (typeof authHeader === 'string' && authHeader.startsWith('Bearer ')) { + return authHeader.slice(7); + } + return null; + } + + /** Extract the session ID from the `ob_session` cookie, or null if absent. */ + private extractSessionId(req: IncomingMessage): string | null { + const cookieHeader = req.headers?.['cookie']; + if (!cookieHeader) return null; + for (const part of cookieHeader.split(';')) { + const trimmed = part.trim(); + const eqIdx = trimmed.indexOf('='); + if (eqIdx === -1) continue; + const name = trimmed.slice(0, eqIdx).trim(); + if (name === SESSION_COOKIE_NAME) { + return trimmed.slice(eqIdx + 1).trim(); + } + } + return null; + } + + /** Return true if the session exists and has not expired. Evicts on expiry. */ + private isValidSession(sessionId: string): boolean { + const expiry = this.sessions.get(sessionId); + if (expiry === undefined) return false; + if (Date.now() > expiry) { + this.sessions.delete(sessionId); + return false; + } + return true; + } + + /** Remove all expired sessions from the in-memory store. */ + private evictExpiredSessions(): void { + const now = Date.now(); + for (const [id, expiry] of this.sessions) { + if (now > expiry) this.sessions.delete(id); + } + } + + /** Extract the best-effort client IP from an incoming HTTP request. */ + private extractClientIp(req: IncomingMessage): string { + const forwarded = req.headers?.['x-forwarded-for']; + if (typeof forwarded === 'string' && forwarded.length > 0) { + return forwarded.split(',')[0]!.trim(); + } + return req.socket?.remoteAddress ?? 'unknown'; + } + + /** + * Check whether the given IP is currently rate-limited. + * Returns true if the IP should be blocked (429). + */ + private isLoginBlocked(ip: string): boolean { + const entry = this.loginRateLimiter.get(ip); + if (!entry) return false; + const now = Date.now(); + if (entry.blockedUntil !== undefined && now < entry.blockedUntil) { + return true; + } + // Reset if outside the sliding window + if (now - entry.windowStart > LOGIN_WINDOW_MS) { + this.loginRateLimiter.delete(ip); + return false; + } + return false; + } + + /** + * Record a login failure for the given IP. + * Blocks the IP for LOGIN_BLOCK_MS after LOGIN_MAX_FAILURES within LOGIN_WINDOW_MS. + */ + private recordLoginFailure(ip: string): void { + const now = Date.now(); + const entry = this.loginRateLimiter.get(ip); + if (!entry || now - entry.windowStart > LOGIN_WINDOW_MS) { + // Start a fresh window + this.loginRateLimiter.set(ip, { failures: 1, windowStart: now }); + return; + } + entry.failures += 1; + if (entry.failures >= LOGIN_MAX_FAILURES) { + entry.blockedUntil = now + LOGIN_BLOCK_MS; + logger.warn( + { ip, failures: entry.failures }, + 'WebChat login rate limit exceeded — IP blocked', + ); + } + } + + /** Reset the login failure counter for an IP on successful authentication. */ + private resetLoginFailures(ip: string): void { + this.loginRateLimiter.delete(ip); + } + + /** + * Ensure the 'webchat-user' entry exists in the access-control store. + * Called after a successful authentication event (token or password). + * Creates the entry with a default 'viewer' role if none exists. + * Preserves any existing entry (admin-customised roles are not overwritten). + */ + private ensureWebchatAccessEntry(): void { + if (!this.memory) return; + void this.memory.getAccess('webchat-user', 'webchat').then((existing) => { + if (!existing) { + const entry: AccessControlEntry = { + user_id: 'webchat-user', + channel: 'webchat', + role: 'viewer', + active: true, + }; + void this.memory!.setAccess(entry).then(() => { + logger.debug('WebChat: created access-store entry for webchat-user'); + }); + } + }); + } + + /** + * Create a new session and set the `ob_session` HTTP-only cookie on the response. + * Called when a valid bearer token is presented on an HTTP request. + */ + private startSession(res: ServerResponse): void { + this.evictExpiredSessions(); + const sessionId = randomUUID(); + const expiry = Date.now() + SESSION_TTL_MS; + this.sessions.set(sessionId, expiry); + const maxAge = Math.floor(SESSION_TTL_MS / 1000); + res.setHeader( + 'Set-Cookie', + `${SESSION_COOKIE_NAME}=${sessionId}; HttpOnly; SameSite=Strict; Max-Age=${maxAge}; Path=/`, + ); + } + + /** + * Check whether the incoming HTTP request carries a valid token or an active + * session cookie. Returns `{ ok, tokenProvided }`: + * - `ok` — whether the request is authorised + * - `tokenProvided` — true when the token itself (not a cookie) was used, + * so the caller can issue a new session cookie + * + * In password mode (`passwordHash !== null`), only valid session cookies are + * accepted — token auth is disabled so that the login screen is the sole + * entry point. + */ + private isAuthenticated( + url: string, + req: IncomingMessage, + ): { ok: boolean; tokenProvided: boolean } { + // Password mode: only session cookies grant access + if (this.passwordHash !== null) { + const sessionId = this.extractSessionId(req); + if (sessionId !== null && this.isValidSession(sessionId)) { + return { ok: true, tokenProvided: false }; + } + return { ok: false, tokenProvided: false }; + } + + // Token mode (default) + if (this.authToken === null) { + // Token not yet initialised — deny access + return { ok: false, tokenProvided: false }; + } + const token = this.extractToken(url, req); + if (token !== null && token === this.authToken) { + return { ok: true, tokenProvided: true }; + } + const sessionId = this.extractSessionId(req); + if (sessionId !== null && this.isValidSession(sessionId)) { + return { ok: true, tokenProvided: false }; + } + return { ok: false, tokenProvided: false }; + } + + async initialize(): Promise { + // Password mode: hash the configured password; skip token generation + if (this.config.password) { + this.passwordHash = await hashPassword(this.config.password); + logger.info('WebChat running in password-auth mode'); + } else { + // Token mode: generate or load persisted auth token + this.authToken = getOrCreateAuthToken(this.storeDir); + } + + const http = await import('node:http'); + + const WsServer = (await import('ws')).WebSocketServer as unknown as new (opts: { + server: unknown; + }) => WssServer; + + const server = http.createServer((req: IncomingMessage, res: ServerResponse) => { + const url = req.url ?? '/'; + + // ── Password login endpoint (public — no auth guard) ─────────────────── + if (url === '/api/webchat/login' && req.method === 'POST') { + if (this.passwordHash === null) { + // Not in password mode — endpoint not available + res.writeHead(404, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Password auth is not enabled' })); + return; + } + + const clientIp = this.extractClientIp(req); + + // Per-IP rate limit check + if (this.isLoginBlocked(clientIp)) { + res.writeHead(429, { 'Content-Type': 'application/json', 'Retry-After': '1800' }); + res.end(JSON.stringify({ error: 'Too many failed login attempts. Try again later.' })); + logger.warn({ ip: clientIp }, 'WebChat login blocked — rate limit active'); + return; + } + + let body = ''; + req.on('data', (chunk: Buffer) => { + body += chunk.toString(); + if (body.length > 1024) { + req.destroy(); + } + }); + req.on('end', () => { + void (async (): Promise => { + let parsed: { password?: string }; + try { + parsed = JSON.parse(body) as { password?: string }; + } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON body' })); + return; + } + if (typeof parsed.password !== 'string' || parsed.password.length === 0) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Missing password field' })); + return; + } + const valid = await verifyPassword(parsed.password, this.passwordHash!); + if (!valid) { + this.recordLoginFailure(clientIp); + res.writeHead(401, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid password' })); + logger.warn({ ip: clientIp }, 'WebChat login failed — invalid password'); + return; + } + this.resetLoginFailures(clientIp); + this.startSession(res); + this.ensureWebchatAccessEntry(); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + logger.info('WebChat login successful — session created'); + })(); + }); + return; + } + // ────────────────────────────────────────────────────────────────────── + + // ── PWA manifest (public — no auth required) ──────────────────────────── + if (url === '/manifest.json') { + const manifest = { + name: 'OpenBridge WebChat', + short_name: 'OpenBridge', + description: 'AI Bridge WebChat — connect to your AI assistant', + start_url: '/', + display: 'standalone', + theme_color: '#1a73e8', + background_color: '#f0f2f5', + icons: [ + { src: '/icons/icon-192.png', sizes: '192x192', type: 'image/png' }, + { src: '/icons/icon-512.png', sizes: '512x512', type: 'image/png' }, + ], + }; + res.writeHead(200, { 'Content-Type': 'application/manifest+json' }); + res.end(JSON.stringify(manifest)); + return; + } + // ───────────────────────────────────────────────────────────────────────── + + // ── Service Worker (public — no auth required, must be at root scope) ─── + if (url === '/sw.js') { + res.writeHead(200, { + 'Content-Type': 'application/javascript; charset=utf-8', + 'Service-Worker-Allowed': '/', + 'Cache-Control': 'no-cache', + }); + res.end(WEBCHAT_SW_JS); + return; + } + // ───────────────────────────────────────────────────────────────────────── + + // ── Auth guard ───────────────────────────────────────────────────────── + const auth = this.isAuthenticated(url, req); + if (!auth.ok) { + // In password mode, serve the login screen for page requests instead + // of returning a bare 401 — only API and non-GET requests get 401. + const isPageRequest = + req.method === 'GET' && + this.passwordHash !== null && + (url === '/' || url === '' || url.startsWith('/?')); + if (isPageRequest) { + res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); + res.end(WEBCHAT_LOGIN_HTML); + } else { + res.writeHead(401, { 'Content-Type': 'text/plain' }); + res.end('Unauthorized'); + } + return; + } + // When the client authenticates with the bearer token, issue a session + // cookie so subsequent requests (without the token in the URL) are also + // allowed through. + if (auth.tokenProvided) { + this.startSession(res); + this.ensureWebchatAccessEntry(); + } + // ────────────────────────────────────────────────────────────────────── + + // QR code endpoint — serves a scannable QR page in headless mode + if (url === '/qr') { + const qrData = getQrCode(); + if (!qrData) { + res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); + res.end( + 'WhatsApp QR' + + '' + + '

Waiting for QR code...

This page will auto-refresh.

', + ); + return; + } + const html = + 'Scan WhatsApp QR' + + '' + + '' + + '

Scan with WhatsApp

' + + '

Open WhatsApp → Linked Devices → Link a Device

' + + '
' + + ''; + res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); + res.end(html); + return; + } + + const match = url.match(/^\/download\/([0-9a-f-]+)$/i); + if (match) { + const fileId = match[1]!; + const entry = this.pendingDownloads.get(fileId); + if (!entry) { + res.writeHead(404, { 'Content-Type': 'text/plain' }); + res.end('Not found'); + return; + } + const filename = entry.filename ?? 'download'; + res.writeHead(200, { + 'Content-Type': entry.mimeType, + 'Content-Disposition': `attachment; filename="${filename}"`, + 'Content-Length': entry.data.length, + }); + res.end(entry.data); + return; + } + + // /api/upload — multipart file upload (POST) + if (url === '/api/upload' && req.method === 'POST') { + void (async (): Promise => { + const contentType = req.headers['content-type'] ?? ''; + const boundaryMatch = contentType.match(/boundary=([^\s;]+)/); + if (!boundaryMatch) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Missing multipart boundary' })); + return; + } + + // Reject by Content-Length before reading the body + const declaredLength = parseInt(req.headers['content-length'] ?? '0', 10); + if (Number.isFinite(declaredLength) && declaredLength > UPLOAD_MAX_BYTES) { + res.writeHead(413, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'File too large (max 10MB)' })); + return; + } + + // Collect body, enforcing the size limit + const chunks: Buffer[] = []; + let totalSize = 0; + let tooLarge = false; + await new Promise((resolve, reject) => { + req.on('data', (chunk: Buffer) => { + totalSize += chunk.length; + if (totalSize > UPLOAD_MAX_BYTES) { + tooLarge = true; + req.destroy(); + resolve(); + return; + } + chunks.push(chunk); + }); + req.on('end', resolve); + req.on('error', reject); + }); + + if (tooLarge) { + res.writeHead(413, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'File too large (max 10MB)' })); + return; + } + + const body = Buffer.concat(chunks); + const file = parseMultipartFile(body, boundaryMatch[1]!); + if (!file) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'No file found in upload' })); + return; + } + + // Store file in /.openbridge/uploads/ + const uploadsDir = join(this.storeDir, '.openbridge', 'uploads'); + await mkdir(uploadsDir, { recursive: true }); + + const fileId = randomUUID(); + const ext = extname(file.filename); + const storedName = `${fileId}${ext}`; + const filePath = join(uploadsDir, storedName); + await writeFile(filePath, file.data); + + logger.info( + { fileId, filename: file.filename, size: file.data.length }, + 'WebChat: file uploaded', + ); + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + fileId, + filename: file.filename, + size: file.data.length, + mimeType: file.mimeType, + path: filePath, + }), + ); + })(); + return; + } + + // /api/transcribe — audio file → transcribed text (POST) + if (url === '/api/transcribe' && req.method === 'POST') { + void (async (): Promise => { + const contentType = req.headers['content-type'] ?? ''; + const boundaryMatch = contentType.match(/boundary=([^\s;]+)/); + if (!boundaryMatch) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Missing multipart boundary' })); + return; + } + + const declaredLength = parseInt(req.headers['content-length'] ?? '0', 10); + if (Number.isFinite(declaredLength) && declaredLength > UPLOAD_MAX_BYTES) { + res.writeHead(413, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Audio file too large (max 10MB)' })); + return; + } + + const chunks: Buffer[] = []; + let totalSize = 0; + let tooLarge = false; + await new Promise((resolve, reject) => { + req.on('data', (chunk: Buffer) => { + totalSize += chunk.length; + if (totalSize > UPLOAD_MAX_BYTES) { + tooLarge = true; + req.destroy(); + resolve(); + return; + } + chunks.push(chunk); + }); + req.on('end', resolve); + req.on('error', reject); + }); + + if (tooLarge) { + res.writeHead(413, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Audio file too large (max 10MB)' })); + return; + } + + const body = Buffer.concat(chunks); + const file = parseMultipartFile(body, boundaryMatch[1]!); + if (!file) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'No audio file found in upload' })); + return; + } + + const audioId = randomUUID(); + const ext = extname(file.filename) || '.webm'; + const audioPath = join(tmpdir(), `ob-voice-${audioId}${ext}`); + await writeFile(audioPath, file.data); + + try { + const result = await transcribeAudio(audioPath); + if (!result) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ text: TRANSCRIPTION_FALLBACK_MESSAGE, backend: 'none' })); + return; + } + logger.info( + { backend: result.backend, durationMs: result.durationMs }, + 'WebChat: voice transcription complete', + ); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ text: result.text, backend: result.backend })); + } finally { + await unlink(audioPath).catch(() => {}); + } + })(); + return; + } + + // /api/sessions — JSON list of sessions for the WebChat history view + if (url === '/api/sessions' || url.startsWith('/api/sessions?')) { + void (async (): Promise => { + if (!this.memory) { + res.writeHead(503, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Memory not available' })); + return; + } + try { + const parsed = new URL(url, 'http://localhost'); + const limitParam = parseInt(parsed.searchParams.get('limit') ?? '20', 10); + const offsetParam = parseInt(parsed.searchParams.get('offset') ?? '0', 10); + const limit = Number.isFinite(limitParam) ? Math.min(100, Math.max(1, limitParam)) : 20; + const offset = Number.isFinite(offsetParam) ? Math.max(0, offsetParam) : 0; + const sessions = await this.memory.listSessions(limit, offset); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(sessions)); + } catch (err) { + logger.error({ err }, 'GET /api/sessions failed'); + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Internal server error' })); + } + })(); + return; + } + + // /api/sessions/search — FTS5 full-text search across conversations + if (url === '/api/sessions/search' || url.startsWith('/api/sessions/search?')) { + void (async (): Promise => { + if (!this.memory) { + res.writeHead(503, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Memory not available' })); + return; + } + try { + const parsed = new URL(url, 'http://localhost'); + const query = parsed.searchParams.get('q') ?? ''; + const limitParam = parseInt(parsed.searchParams.get('limit') ?? '20', 10); + const limit = Number.isFinite(limitParam) ? Math.min(50, Math.max(1, limitParam)) : 20; + if (!query.trim()) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify([])); + return; + } + const results = await this.memory.searchConversations(query, limit); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(results)); + } catch (err) { + logger.error({ err }, 'GET /api/sessions/search failed'); + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Internal server error' })); + } + })(); + return; + } + + // /api/sessions/:id — full conversation JSON for one session + const sessionMatch = url.match(/^\/api\/sessions\/([^/?#]+)(?:\?.*)?$/); + if (sessionMatch) { + const sessionId = decodeURIComponent(sessionMatch[1]!); + void (async (): Promise => { + if (!this.memory) { + res.writeHead(503, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Memory not available' })); + return; + } + try { + const parsed = new URL(url, 'http://localhost'); + const limitParam = parseInt(parsed.searchParams.get('limit') ?? '100', 10); + const limit = Number.isFinite(limitParam) + ? Math.min(500, Math.max(1, limitParam)) + : 100; + const messages = await this.memory.getSessionHistory(sessionId, limit); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ session_id: sessionId, messages })); + } catch (err) { + logger.error({ err }, 'GET /api/sessions/:id failed'); + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Internal server error' })); + } + })(); + return; + } + + // /api/feedback — record user thumbs-up/down on an AI response (POST) + if (url === '/api/feedback' && req.method === 'POST') { + let body = ''; + req.on('data', (chunk: Buffer) => { + body += chunk.toString(); + if (body.length > 4096) req.destroy(); + }); + req.on('end', () => { + let parsed: { session?: unknown; message?: unknown; rating?: unknown }; + try { + parsed = JSON.parse(body) as { session?: unknown; message?: unknown; rating?: unknown }; + } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON body' })); + return; + } + const rating = typeof parsed.rating === 'string' ? parsed.rating : null; + if (rating !== 'up' && rating !== 'down') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'rating must be "up" or "down"' })); + return; + } + const success = rating === 'up'; + logger.info( + { session: parsed.session, message: parsed.message, rating }, + 'WebChat: user feedback received', + ); + if (this.memory) { + void this.memory + .recordPromptOutcome('webchat-response-quality', success) + .catch(() => {}); + } + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + }); + return; + } + + // /api/discovery — list discovered AI tools for settings panel (GET) + if (url === '/api/discovery' && req.method === 'GET') { + const tools = this.discoveredTools + .filter((t) => t.available) + .map((t) => ({ name: t.name, version: t.version })); + // Only cache when tools have been wired — avoid caching empty results + // that would stick in the browser for 5 minutes during startup race. + const cacheControl = tools.length > 0 ? 'public, max-age=300' : 'no-cache, no-store'; + res.writeHead(200, { + 'Content-Type': 'application/json', + 'Cache-Control': cacheControl, + }); + res.end(JSON.stringify({ tools })); + return; + } + + // /api/mcp/servers — GET list of MCP servers + if (url === '/api/mcp/servers' && req.method === 'GET') { + if (!this.mcpRegistry) { + res.writeHead(503, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'MCP registry not available' })); + return; + } + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ servers: this.mcpRegistry.listServers() })); + return; + } + + // /api/mcp/servers — POST add a new MCP server + if (url === '/api/mcp/servers' && req.method === 'POST') { + if (!this.mcpRegistry) { + res.writeHead(503, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'MCP registry not available' })); + return; + } + let body = ''; + req.on('data', (chunk: Buffer) => { + body += chunk.toString(); + if (body.length > 4096) req.destroy(); + }); + req.on('end', () => { + let parsed: unknown; + try { + parsed = JSON.parse(body); + } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON body' })); + return; + } + const server = parsed as MCPServer; + if (typeof server.name !== 'string' || typeof server.command !== 'string') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'name and command are required' })); + return; + } + try { + this.mcpRegistry!.addServer(server); + res.writeHead(201, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + } catch (err) { + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: (err as Error).message })); + } + }); + return; + } + + // /api/mcp/servers/:name — DELETE remove an MCP server + const mcpDeleteMatch = url.match(/^\/api\/mcp\/servers\/([^/?#]+)$/); + if (mcpDeleteMatch && req.method === 'DELETE') { + if (!this.mcpRegistry) { + res.writeHead(503, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'MCP registry not available' })); + return; + } + const name = decodeURIComponent(mcpDeleteMatch[1]!); + try { + this.mcpRegistry.removeServer(name); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + } catch (err) { + res.writeHead(404, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: (err as Error).message })); + } + return; + } + + // /api/mcp/servers/:name/toggle — PUT enable/disable an MCP server + const mcpToggleMatch = url.match(/^\/api\/mcp\/servers\/([^/?#]+)\/toggle$/); + if (mcpToggleMatch && req.method === 'PUT') { + if (!this.mcpRegistry) { + res.writeHead(503, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'MCP registry not available' })); + return; + } + const name = decodeURIComponent(mcpToggleMatch[1]!); + let body = ''; + req.on('data', (chunk: Buffer) => { + body += chunk.toString(); + if (body.length > 256) req.destroy(); + }); + req.on('end', () => { + let parsed: unknown; + try { + parsed = JSON.parse(body); + } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON body' })); + return; + } + const { enabled } = parsed as { enabled?: boolean }; + if (typeof enabled !== 'boolean') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'enabled (boolean) is required' })); + return; + } + try { + this.mcpRegistry!.toggleServer(name, enabled); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + } catch (err) { + res.writeHead(404, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: (err as Error).message })); + } + }); + return; + } + + // /api/webchat/settings — GET current session settings + if (url === '/api/webchat/settings' && req.method === 'GET') { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(this.webchatSettings)); + return; + } + + // /api/webchat/settings — PUT update session settings + if (url === '/api/webchat/settings' && req.method === 'PUT') { + let body = ''; + req.on('data', (chunk: Buffer) => { + body += chunk.toString(); + if (body.length > 1024) req.destroy(); + }); + req.on('end', () => { + let parsed: unknown; + try { + parsed = JSON.parse(body); + } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON body' })); + return; + } + const validated = WebchatSettingsPutSchema.safeParse(parsed); + if (!validated.success) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + const msg = validated.error.errors[0]?.message ?? 'Invalid request body'; + res.end(JSON.stringify({ error: msg })); + return; + } + const { profile } = validated.data; + this.webchatSettings.profile = profile; + logger.debug({ profile }, 'WebChat: execution profile updated'); + // Optional: persist to access-store (fire-and-forget, non-fatal on failure) + if (this.memory) { + void (async (): Promise => { + try { + const existing = await this.memory!.getAccess('webchat-user', 'webchat'); + await this.memory!.setAccess({ + user_id: 'webchat-user', + channel: 'webchat', + role: existing?.role ?? 'viewer', + executionProfile: profile, + }); + } catch (err) { + logger.debug({ err }, 'WebChat: settings access-store persist failed (non-fatal)'); + } + })(); + } + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true, profile: this.webchatSettings.profile })); + }); + return; + } + + // /api/commands — list available slash commands for autocomplete (GET) + if (url === '/api/commands' && req.method === 'GET') { + const commands = [ + { name: '/history', description: 'Show conversation history' }, + { name: '/stop', description: 'Stop the current worker' }, + { name: '/status', description: 'Show agent status' }, + { name: '/deep', description: 'Enable deep mode for complex tasks' }, + { name: '/audit', description: 'Run a workspace audit' }, + { name: '/scope', description: 'Show or change task scope' }, + { name: '/apps', description: 'List connected apps' }, + { name: '/help', description: 'Show available commands' }, + { name: '/doctor', description: 'Run system health diagnostics' }, + { name: '/confirm', description: 'Confirm a pending action' }, + { name: '/skip', description: 'Skip a pending confirmation' }, + ]; + res.writeHead(200, { + 'Content-Type': 'application/json', + 'Cache-Control': 'public, max-age=300', + }); + res.end(JSON.stringify(commands)); + return; + } + + res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); + res.end(this.buildHtmlPage()); + }); + + this.httpServer = server; + + // ── WebSocket upgrade auth guard ────────────────────────────────────────── + // Validate the auth token before the ws library accepts the upgrade. + // Registered before WebSocketServer so this listener runs first. + server.on('upgrade', (req, socket) => { + const url = req.url ?? '/'; + const auth = this.isAuthenticated(url, req); + if (!auth.ok) { + socket.write( + 'HTTP/1.1 401 Unauthorized\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\nUnauthorized', + ); + socket.destroy(); + logger.warn({ url }, 'WebSocket upgrade rejected — invalid or missing auth token'); + } + }); + // ───────────────────────────────────────────────────────────────────────── + + const wss = new WsServer({ server }); + this.wss = wss; + + // Keep WebSocket connections alive during long-running tasks (workers can take 10+ minutes) + const PING_INTERVAL_MS = 30_000; + const pingTimer = setInterval(() => { + for (const client of this.clients) { + if (client.readyState === WS_OPEN) { + client.ping(); + } + } + }, PING_INTERVAL_MS); + wss.on('close', () => clearInterval(pingTimer)); + + wss.on('connection', (socket: WsClient) => { + this.clients.add(socket); + + // Per-socket sender ID — rotated on each "new-session" message so that + // the router and Master AI treat subsequent messages as a fresh conversation. + let socketSender = 'webchat-user'; + + // Send any active Deep Mode session state to the newly connected client so + // the stepper is immediately correct even after a page refresh or reconnect. + const cachedEvents: Array> = []; + for (const sessionEvents of this._deepPhaseCache.values()) { + cachedEvents.push(...sessionEvents); + } + if (cachedEvents.length > 0 && socket.readyState === WS_OPEN) { + socket.send(JSON.stringify({ type: 'deep-mode-state', events: cachedEvents })); + } + + socket.on('message', (raw: Buffer | string) => { + let payload: { type: string; content?: string; workerId?: string }; + try { + payload = JSON.parse(raw.toString()) as { + type: string; + content?: string; + workerId?: string; + }; + } catch { + return; + } + + if ( + payload.type === 'permission-response' && + typeof (payload as Record)['approved'] === 'boolean' + ) { + // Permission response from WebChat UI — route as YES/NO text message + const approved = (payload as Record)['approved'] as boolean; + this.messageCounter++; + const message: InboundMessage = { + id: `webchat-${this.messageCounter.toString()}`, + source: 'webchat', + sender: socketSender, + rawContent: approved ? 'YES' : 'NO', + content: approved ? 'YES' : 'NO', + timestamp: new Date(), + }; + this.emit('message', message); + } else if (payload.type === 'message' && typeof payload.content === 'string') { + this.messageCounter++; + + // Build attachments from file metadata sent by the client + const files = Array.isArray((payload as Record)['files']) + ? ((payload as Record)['files'] as Array<{ + filename?: string; + path?: string; + mimeType?: string; + size?: number; + }>) + : []; + + const buildMessage = async (): Promise => { + let attachments: InboundMessage['attachments']; + let processedDoc: InboundMessage['processedDocument']; + + if (files.length > 0) { + attachments = []; + for (const f of files) { + if (!f.path) continue; + const mime = f.mimeType ?? 'application/octet-stream'; + const type: 'image' | 'document' | 'audio' | 'video' = mime.startsWith('image/') + ? 'image' + : mime.startsWith('audio/') + ? 'audio' + : mime.startsWith('video/') + ? 'video' + : 'document'; + let sizeBytes = f.size ?? 0; + if (!sizeBytes) { + try { + sizeBytes = (await stat(f.path)).size; + } catch { + /* ignore */ + } + } + attachments.push({ + type, + filePath: f.path, + mimeType: mime, + filename: f.filename, + sizeBytes, + }); + } + + // Process the first attachment for document intelligence + if (attachments.length > 0) { + try { + processedDoc = await processDocument(attachments[0]!.filePath); + logger.debug( + { filePath: attachments[0]!.filePath, docType: processedDoc.docType }, + 'WebChat file processed', + ); + } catch (err) { + logger.warn( + { err, filePath: attachments[0]!.filePath }, + 'Document processing failed for WebChat file', + ); + } + } + } + + const content = (payload.content as string) || (attachments?.length ? '[File]' : ''); + const message: InboundMessage = { + id: `webchat-${this.messageCounter.toString()}`, + source: 'webchat', + sender: socketSender, + rawContent: content, + content, + timestamp: new Date(), + attachments, + }; + if (processedDoc !== undefined) { + message.processedDocument = processedDoc; + } + this.emit('message', message); + }; + + buildMessage().catch((err: Error) => { + logger.warn({ err }, 'Failed to process WebChat message with attachments'); + // Fallback: emit message without attachments + const message: InboundMessage = { + id: `webchat-${this.messageCounter.toString()}`, + source: 'webchat', + sender: socketSender, + rawContent: payload.content as string, + content: payload.content as string, + timestamp: new Date(), + }; + this.emit('message', message); + }); + } else if (payload.type === 'stop-worker' && typeof payload.workerId === 'string') { + this.messageCounter++; + const content = `stop ${payload.workerId}`; + const message: InboundMessage = { + id: `webchat-${this.messageCounter.toString()}`, + source: 'webchat', + sender: socketSender, + rawContent: content, + content, + timestamp: new Date(), + }; + this.emit('message', message); + } else if (payload.type === 'stop-all') { + this.messageCounter++; + const message: InboundMessage = { + id: `webchat-${this.messageCounter.toString()}`, + source: 'webchat', + sender: socketSender, + rawContent: 'stop all', + content: 'stop all', + timestamp: new Date(), + }; + this.emit('message', message); + } else if (payload.type === 'new-session') { + // Rotate the per-socket sender so the Master AI starts a fresh conversation. + socketSender = `webchat-user-${randomUUID()}`; + logger.debug({ sender: socketSender }, 'WebChat new session started'); + } else if (payload.type === 'get-deep-mode-state') { + // Client requests current Deep Mode snapshot (e.g. after reconnect) + const events: Array> = []; + for (const sessionEvents of this._deepPhaseCache.values()) { + events.push(...sessionEvents); + } + if (socket.readyState === WS_OPEN) { + socket.send(JSON.stringify({ type: 'deep-mode-state', events })); + } + } + }); + + socket.on('close', () => { + this.clients.delete(socket); + }); + + socket.on('error', (err: Error) => { + this.clients.delete(socket); + logger.warn({ err }, 'WebChat client error'); + }); + }); + + // Start periodic cleanup of expired sessions (every 5 minutes) + const SESSION_CLEANUP_INTERVAL_MS = 5 * 60 * 1000; + this.sessionCleanupInterval = setInterval(() => { + this.evictExpiredSessions(); + }, SESSION_CLEANUP_INTERVAL_MS); + + await new Promise((resolve) => { + server.listen(this.config.port, this.config.host, () => { + this.connected = true; + logger.info({ port: this.config.port, host: this.config.host }, 'WebChat connector ready'); + + // Display LAN access URLs when binding to all interfaces + if (this.config.host === '0.0.0.0') { + const lanIps = this.getLanIps(); + if (lanIps.length > 0) { + for (const ip of lanIps) { + const suffix = this.authToken ? `/?token=${this.authToken}` : '/'; + const lanUrl = `http://${ip}:${this.config.port}${suffix}`; + console.log(` WebChat LAN URL: ${lanUrl}`); + logger.info({ url: lanUrl }, 'WebChat LAN URL'); + } + } else { + logger.warn('WebChat: no LAN interfaces detected'); + } + } + + // Display public (tunnel) URL when a tunnel is active + const publicUrl = this.getPublicUrl(); + if (publicUrl) { + console.log(` WebChat Public URL: ${publicUrl}`); + logger.info({ url: publicUrl }, 'WebChat Public URL (tunnel active)'); + } + + // Display QR code for phone scanning — prefer tunnel URL if set, else first LAN URL + const qrUrl = this.getLanAccessUrl(); + if (qrUrl) { + console.log(' Scan to open WebChat on your phone:'); + import('qrcode-terminal') + .then((qrcodeTerminal) => { + const mod = qrcodeTerminal.default ?? qrcodeTerminal; + mod.generate(qrUrl, { small: true }); + }) + .catch(() => { + // qrcode-terminal unavailable — URL printed above is sufficient + }); + } + + this.emit('ready'); + resolve(); + }); + }); + } + + sendMessage(message: OutboundMessage): Promise { + if (!this.connected) { + return Promise.reject(new Error('WebChat connector is not connected')); + } + + let payload: string; + if (message.metadata?.['permissionRequest'] === true) { + // Permission request — send structured message for WebChat UI modal + payload = JSON.stringify({ + type: 'permission-request', + toolName: message.metadata['toolName'] ?? 'Unknown', + detail: message.metadata['detail'] ?? '', + timeoutMs: message.metadata['timeoutMs'] ?? 60000, + permissionId: randomUUID(), + timestamp: new Date().toISOString(), + }); + } else if (message.media) { + const fileId = randomUUID(); + const { data, mimeType, filename } = message.media; + const timer = setTimeout( + () => { + this.pendingDownloads.delete(fileId); + }, + 60 * 60 * 1000, + ); // 1 hour + this.pendingDownloads.set(fileId, { data, mimeType, filename, timer }); + payload = JSON.stringify({ + type: 'download', + content: message.content, + fileId, + filename: filename ?? 'download', + url: `/download/${fileId}`, + mimeType, + timestamp: new Date().toISOString(), + }); + } else { + payload = JSON.stringify({ + type: 'response', + content: message.content, + timestamp: new Date().toISOString(), + }); + } + + for (const client of this.clients) { + if (client.readyState === WS_OPEN) { + client.send(payload); + } + } + return Promise.resolve(); + } + + sendTypingIndicator(_chatId: string): Promise { + if (!this.connected) return Promise.resolve(); + const payload = JSON.stringify({ type: 'typing' }); + for (const client of this.clients) { + if (client.readyState === WS_OPEN) { + client.send(payload); + } + } + return Promise.resolve(); + } + + sendProgress(event: ProgressEvent, _chatId: string): Promise { + if (!this.connected) return Promise.resolve(); + + // Cache deep-phase events so newly connected clients can restore stepper state + if (event.type === 'deep-phase') { + const sessionEvents = this._deepPhaseCache.get(event.sessionId) ?? []; + const existingIdx = sessionEvents.findIndex((e) => e.phase === event.phase); + if (existingIdx >= 0) { + sessionEvents[existingIdx] = event; + } else { + sessionEvents.push(event); + } + this._deepPhaseCache.set(event.sessionId, sessionEvents); + + // Remove session from cache when it is fully done + if ( + event.status === 'aborted' || + (event.phase === 'verify' && (event.status === 'completed' || event.status === 'skipped')) + ) { + // Delay removal so a client that reconnects moments after completion still gets the state + setTimeout(() => { + this._deepPhaseCache.delete(event.sessionId); + }, 30_000); + } + } + + const payload = JSON.stringify({ type: 'progress', event }); + for (const client of this.clients) { + if (client.readyState === WS_OPEN) { + client.send(payload); + } + } + return Promise.resolve(); + } + + /** Broadcast current agent activity to all connected WebSocket clients. */ + broadcastAgentStatus(agents: ActivityRecord[]): void { + if (!this.connected || this.clients.size === 0) return; + const payload = JSON.stringify({ + type: 'agent-status', + agents, + timestamp: new Date().toISOString(), + }); + for (const client of this.clients) { + if (client.readyState === WS_OPEN) { + client.send(payload); + } + } + } + + on(event: E, listener: ConnectorEvents[E]): void { + this.listeners[event].push(listener); + } + + async shutdown(): Promise { + this.connected = false; + this.clients.clear(); + + if (this.sessionCleanupInterval !== null) { + clearInterval(this.sessionCleanupInterval); + this.sessionCleanupInterval = null; + } + + for (const entry of this.pendingDownloads.values()) { + clearTimeout(entry.timer); + } + this.pendingDownloads.clear(); + + if (this.wss) { + await new Promise((resolve) => { + this.wss!.close(() => resolve()); + }); + this.wss = null; + } + + if (this.httpServer) { + await new Promise((resolve, reject) => { + this.httpServer!.close((err?: Error) => { + if (err) reject(err); + else resolve(); + }); + }); + this.httpServer = null; + } + + logger.info('WebChat connector shut down'); + } + + isConnected(): boolean { + return this.connected; + } + + /** + * Build the main HTML page, injecting the public tunnel URL when active. + * Replaces the `window.__OB_PUBLIC_URL__ = null;` placeholder in the bundle + * with the actual tunnel URL (including auth token) so the frontend can + * display it in the header copy button. + */ + private buildHtmlPage(): string { + const publicUrl = this.getPublicUrl(); + if (!publicUrl) return WEBCHAT_HTML; + return WEBCHAT_HTML.replace( + 'window.__OB_PUBLIC_URL__ = null;', + `window.__OB_PUBLIC_URL__ = ${JSON.stringify(publicUrl)};`, + ); + } + + /** Returns a list of non-internal IPv4 addresses for LAN URL display. */ + private getLanIps(): string[] { + const nets = networkInterfaces(); + const results: string[] = []; + for (const ifaces of Object.values(nets)) { + if (!ifaces) continue; + for (const net of ifaces) { + if (net.family === 'IPv4' && !net.internal) { + results.push(net.address); + } + } + } + return results; + } + + private emit( + event: E, + ...args: Parameters + ): void { + for (const listener of this.listeners[event]) { + (listener as (...a: Parameters) => void)(...args); + } + } +} diff --git a/src/connectors/whatsapp/whatsapp-config.ts b/src/connectors/whatsapp/whatsapp-config.ts index 0f444f1f..926e5b8a 100644 --- a/src/connectors/whatsapp/whatsapp-config.ts +++ b/src/connectors/whatsapp/whatsapp-config.ts @@ -1,8 +1,9 @@ -import { z } from 'zod'; +import { z } from 'zod/v3'; export const WhatsAppConfigSchema = z.object({ sessionName: z.string().default('openbridge-default'), sessionPath: z.string().optional(), + headless: z.boolean().default(true), reconnect: z .object({ enabled: z.boolean().default(true), diff --git a/src/connectors/whatsapp/whatsapp-connector.ts b/src/connectors/whatsapp/whatsapp-connector.ts index 40b2371e..6d155305 100644 --- a/src/connectors/whatsapp/whatsapp-connector.ts +++ b/src/connectors/whatsapp/whatsapp-connector.ts @@ -1,13 +1,68 @@ import type { Connector, ConnectorEvents } from '../../types/connector.js'; -import type { OutboundMessage } from '../../types/message.js'; +import type { InboundMessage, OutboundMessage, ProgressEvent } from '../../types/message.js'; +import { processDocument } from '../../intelligence/document-processor.js'; +import { setQrCode } from '../../core/qr-store.js'; import { WhatsAppConfigSchema } from './whatsapp-config.js'; import type { WhatsAppConfig } from './whatsapp-config.js'; import { parseWhatsAppMessage, splitForWhatsApp } from './whatsapp-message.js'; import { formatMarkdownForWhatsApp } from './whatsapp-formatter.js'; import { createLogger } from '../../core/logger.js'; +import { transcribeAudio, TRANSCRIPTION_FALLBACK_MESSAGE } from '../../core/voice-transcriber.js'; +import type { MediaManager, SaveMediaResult } from '../../core/media-manager.js'; +import { isPackagedMode } from '../../cli/utils.js'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { existsSync } from 'node:fs'; +import { unlink, readlink, writeFile, readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import type { MessageMedia } from 'whatsapp-web.js'; + +const execFileAsync = promisify(execFile); const logger = createLogger('whatsapp'); +/** Common Chrome/Chromium install paths per OS, checked when PUPPETEER_EXECUTABLE_PATH is not set. */ +const CHROMIUM_PATHS: Record<'macos' | 'windows' | 'linux', string[]> = { + macos: [ + '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', + '/Applications/Chromium.app/Contents/MacOS/Chromium', + ], + windows: [ + 'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe', + 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe', + ], + linux: [ + '/usr/bin/chromium-browser', + '/usr/bin/chromium', + '/usr/bin/google-chrome', + '/usr/bin/google-chrome-stable', + ], +}; + +/** + * Resolves a Chrome/Chromium executable path for Puppeteer. + * Priority: PUPPETEER_EXECUTABLE_PATH env var → common OS install paths → null. + * When in packaged mode (pkg binary), Puppeteer cannot use its bundled Chromium, + * so the host system must have Chrome/Chromium installed. + */ +function resolveChromiumPath(): string | null { + const envPath = process.env['PUPPETEER_EXECUTABLE_PATH']; + if (envPath) return envPath; + + const platform = process.platform; + const os: 'macos' | 'windows' | 'linux' = + platform === 'darwin' ? 'macos' : platform === 'win32' ? 'windows' : 'linux'; + + for (const candidate of CHROMIUM_PATHS[os]) { + if (existsSync(candidate)) { + return candidate; + } + } + + return null; +} + type EventListeners = { [E in keyof ConnectorEvents]: ConnectorEvents[E][]; }; @@ -16,22 +71,53 @@ interface WAChat { sendStateTyping: () => Promise; } +interface WAMediaData { + data: string; // base64-encoded media + mimetype: string; + filename?: string; // original filename (present for documents) +} + +interface WAMessage { + id: { id: string }; + from: string; + body: string; + timestamp: number; + hasMedia?: boolean; + type?: string; + downloadMedia?: () => Promise; +} + +interface WASendOptions { + caption?: string; + sendMediaAsDocument?: boolean; + sendAudioAsVoice?: boolean; +} + interface WAClient { on: (event: string, handler: (...args: never[]) => void) => void; initialize: () => Promise; - sendMessage: (to: string, content: string) => Promise; + sendMessage: ( + to: string, + content: string | MessageMedia, + options?: WASendOptions, + ) => Promise; getChatById: (chatId: string) => Promise; destroy: () => Promise; } export class WhatsAppConnector implements Connector { readonly name = 'whatsapp'; + readonly supportsFileAttachments = true as const; private config: WhatsAppConfig; private connected = false; private client: WAClient | null = null; private reconnectAttempt = 0; private reconnectTimer: ReturnType | null = null; + private progressSentCleanupInterval: ReturnType | null = null; private shuttingDown = false; + private mediaManager: MediaManager | null = null; + /** Tracks chat IDs that have received a progress status message (to avoid repeat sends). */ + private readonly progressSent = new Set(); private readonly listeners: EventListeners = { message: [], ready: [], @@ -40,10 +126,24 @@ export class WhatsAppConnector implements Connector { disconnected: [], }; + /** Media types that can be downloaded as file attachments (excludes PTT voice). */ + private static readonly DOWNLOADABLE_TYPES = new Set([ + 'image', + 'document', + 'video', + 'audio', + 'sticker', + ]); + constructor(options: Record) { this.config = WhatsAppConfigSchema.parse(options); } + /** Wire a MediaManager for saving incoming media attachments to disk. */ + setMediaManager(manager: MediaManager): void { + this.mediaManager = manager; + } + async initialize(): Promise { logger.info( { @@ -53,6 +153,14 @@ export class WhatsAppConnector implements Connector { 'Initializing WhatsApp connector — session will persist across restarts', ); this.shuttingDown = false; + // Clear progressSent every 10 minutes — entries are ephemeral and accumulate over time. + this.progressSentCleanupInterval = setInterval( + () => { + this.progressSent.clear(); + logger.debug('Cleared progressSent Set (periodic cleanup)'); + }, + 10 * 60 * 1000, + ); await this.createAndStartClient(); } @@ -70,22 +178,62 @@ export class WhatsAppConnector implements Connector { localAuthOptions.dataPath = this.config.sessionPath; } + // Remove stale SingletonLock — left behind when Chromium crashes or the process is killed. + // Without this, Puppeteer hangs trying to connect to a dead browser. + await this.removeStaleLock(); + + // Resolve the Chrome/Chromium executable. In packaged mode (pkg binary), Puppeteer's + // bundled Chromium is not accessible, so we must find an existing system installation. + const chromiumPath = resolveChromiumPath(); + if (chromiumPath) { + process.env['PUPPETEER_EXECUTABLE_PATH'] = chromiumPath; + logger.info({ path: chromiumPath }, 'Using system Chrome/Chromium for WhatsApp connector'); + } else if (isPackagedMode()) { + logger.warn( + 'Chrome or Chromium not found on this system. ' + + 'Install Google Chrome (https://www.google.com/chrome) or set the ' + + 'PUPPETEER_EXECUTABLE_PATH environment variable to enable WhatsApp.', + ); + } + this.client = new Client({ authStrategy: new LocalAuth(localAuthOptions), + // Use local cache to avoid remote fetch failures (GitHub URL can be unreachable) + webVersionCache: { + type: 'local', + }, + puppeteer: { + ...(chromiumPath !== null && { executablePath: chromiumPath }), + headless: this.config.headless, + protocolTimeout: 300_000, // 5 min — WhatsApp Web can be slow to load + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-gpu', + '--disable-dev-shm-usage', + '--disable-extensions', + ], + }, }) as unknown as WAClient; this.client.on('qr', (qr: string) => { - logger.info('QR code received — scan with WhatsApp'); - // Render QR code to terminal so the user can scan it - import('qrcode-terminal') - .then((qrcodeTerminal) => { - const mod = qrcodeTerminal.default ?? qrcodeTerminal; - mod.generate(qr, { small: true }); - }) - .catch(() => { - // Fallback: print raw QR string if qrcode-terminal is not available - logger.info({ qr }, 'Install qrcode-terminal to display QR in terminal'); - }); + setQrCode(qr); + if (process.env['OPENBRIDGE_HEADLESS'] === 'true') { + // Headless mode: skip terminal display, log QR as JSON and serve via WebChat /qr endpoint + logger.info({ qr }, 'QR code received — scan at http://localhost:3000/qr'); + } else { + // Interactive mode: render QR to terminal + logger.info('QR code received — scan with WhatsApp'); + import('qrcode-terminal') + .then((qrcodeTerminal) => { + const mod = qrcodeTerminal.default ?? qrcodeTerminal; + mod.generate(qr, { small: true }); + }) + .catch(() => { + // Fallback: print raw QR string if qrcode-terminal is not available + logger.info({ qr }, 'Install qrcode-terminal to display QR in terminal'); + }); + } this.emit('auth', qr); }); @@ -108,13 +256,9 @@ export class WhatsAppConnector implements Connector { this.emit('ready'); }); - this.client.on( - 'message', - (msg: { id: { id: string }; from: string; body: string; timestamp: number }) => { - const parsed = parseWhatsAppMessage(msg.id.id, msg.from, msg.body, msg.timestamp); - this.emit('message', parsed); - }, - ); + this.client.on('message', (msg: WAMessage) => { + void this.handleIncomingMessage(msg); + }); this.client.on('disconnected', (reason: string) => { this.connected = false; @@ -123,7 +267,311 @@ export class WhatsAppConnector implements Connector { this.scheduleReconnect(); }); - await this.client.initialize(); + // Catch Puppeteer ProtocolError / browser crashes that don't trigger 'disconnected'. + // Log the phase (pre-ready vs post-ready) to help diagnose where the error occurs. + this.client.on('error', (err: Error) => { + const phase = this.connected ? 'post-ready' : 'pre-ready'; + const isProtocolError = + err.message.includes('ProtocolError') || + err.message.includes('Execution context was destroyed'); + if (isProtocolError) { + logger.error( + { err: err.message, phase }, + 'WhatsApp ProtocolError — Chromium context destroyed', + ); + } else { + logger.error({ err: err.message, phase }, 'WhatsApp client error'); + } + if (this.connected) { + this.connected = false; + this.emit('error', err); + this.scheduleReconnect(); + } + }); + + // Retry initialize() up to 3 times with exponential backoff. + // ProtocolError: Execution context was destroyed can occur transiently during startup. + const MAX_INIT_ATTEMPTS = 3; + const { initialDelayMs, backoffFactor, maxDelayMs } = this.config.reconnect; + let lastInitError: Error | null = null; + for (let attempt = 1; attempt <= MAX_INIT_ATTEMPTS; attempt++) { + try { + if (attempt > 1) { + logger.info( + { attempt, maxAttempts: MAX_INIT_ATTEMPTS }, + 'Retrying client.initialize() after previous failure', + ); + } else { + logger.info('Launching Chromium and loading WhatsApp Web...'); + } + await this.client.initialize(); + logger.info('WhatsApp client initialized successfully'); + return; + } catch (err: unknown) { + lastInitError = err instanceof Error ? err : new Error(String(err)); + logger.warn( + { attempt, maxAttempts: MAX_INIT_ATTEMPTS, err: lastInitError.message }, + 'client.initialize() failed', + ); + if (attempt < MAX_INIT_ATTEMPTS) { + const backoffMs = Math.min( + initialDelayMs * Math.pow(backoffFactor, attempt - 1), + maxDelayMs, + ); + await new Promise((resolve) => setTimeout(resolve, backoffMs)); + } + } + } + throw lastInitError ?? new Error('client.initialize() failed after all retries'); + } + + /** + * Remove stale SingletonLock from the Chromium profile directory. + * This lock is a symlink like `Mac-`. If the PID is no longer running, + * the lock is stale and will prevent Puppeteer from launching. + */ + private async removeStaleLock(): Promise { + const dataPath = this.config.sessionPath ?? '.wwebjs_auth'; + const lockPath = join(dataPath, `session-${this.config.sessionName}`, 'SingletonLock'); + + try { + const target = await readlink(lockPath); + // Target format: "Mac-" or "-" + const pidMatch = target.match(/-(\d+)$/); + if (!pidMatch?.[1]) return; + + const pid = parseInt(pidMatch[1], 10); + try { + // signal 0 = check if process exists without sending a signal + process.kill(pid, 0); + // Process is alive — lock is valid, don't remove + logger.debug({ pid }, 'SingletonLock held by running process'); + } catch { + // Process doesn't exist — lock is stale + await unlink(lockPath); + logger.info({ pid }, 'Removed stale SingletonLock from previous Chromium crash'); + } + } catch { + // Lock doesn't exist or can't be read — nothing to do + } + } + + private async handleIncomingMessage(msg: WAMessage): Promise { + if (msg.hasMedia) { + void this.sendTypingIndicator(msg.from); + } + + if (msg.hasMedia && msg.type === 'ptt') { + const transcription = await this.transcribeVoiceMessage(msg); + const content = transcription ?? TRANSCRIPTION_FALLBACK_MESSAGE; + const parsed = parseWhatsAppMessage(msg.id.id, msg.from, content, msg.timestamp); + this.emit('message', parsed); + return; + } + + // Download non-PTT media (image, document, video, audio) + const downloadResult = + msg.hasMedia && msg.type && WhatsAppConnector.DOWNLOADABLE_TYPES.has(msg.type) + ? await this.downloadIncomingMedia(msg) + : null; + + let content = msg.body; + let attachments: InboundMessage['attachments']; + let processedDoc: InboundMessage['processedDocument']; + + if (downloadResult && !downloadResult.failed) { + attachments = [ + { + type: downloadResult.type, + filePath: downloadResult.result.filePath, + mimeType: downloadResult.mimeType, + ...(downloadResult.filename !== undefined && { filename: downloadResult.filename }), + sizeBytes: downloadResult.result.sizeBytes, + }, + ]; + // Use caption (msg.body) as text; fall back to type label if no caption + if (!content) { + const fallbackMap: Record = { + image: '[Image]', + document: '[Document]', + video: '[Video]', + audio: '[Audio]', + }; + content = fallbackMap[downloadResult.type] ?? `[${downloadResult.type}]`; + } + logger.debug( + { filePath: downloadResult.result.filePath, type: downloadResult.type }, + 'Incoming media downloaded', + ); + + // Process document to extract structured content for Master AI + try { + processedDoc = await processDocument(downloadResult.result.filePath); + logger.debug( + { filePath: downloadResult.result.filePath, docType: processedDoc.docType }, + 'Document processed', + ); + } catch (err) { + logger.warn( + { err, filePath: downloadResult.result.filePath }, + 'Document processing failed', + ); + } + } else if (downloadResult?.failed) { + // Download was attempted but failed — prepend failure notice to any caption text + const failureText = `[Media attachment failed to download — ${downloadResult.type}]`; + content = content ? `${failureText}\n${content}` : failureText; + } + + const parsed = parseWhatsAppMessage(msg.id.id, msg.from, content, msg.timestamp, attachments); + if (processedDoc !== undefined) { + parsed.processedDocument = processedDoc; + } + this.emit('message', parsed); + } + + /** + * Download an incoming non-PTT media message and save it via MediaManager. + * Returns: + * - null if not applicable (no MediaManager configured, or no media data) + * - { failed: false, ... } on success + * - { failed: true, type } when download was attempted but threw an error + */ + private async downloadIncomingMedia(msg: WAMessage): Promise< + | { + failed: false; + result: SaveMediaResult; + mimeType: string; + filename?: string; + type: 'image' | 'document' | 'audio' | 'video'; + } + | { failed: true; type: string } + | null + > { + if (!this.mediaManager) return null; + + // Stickers are .webp images — map to 'image' attachment type + const attachmentType: 'image' | 'document' | 'audio' | 'video' = + msg.type === 'sticker' ? 'image' : (msg.type as 'image' | 'document' | 'audio' | 'video'); + + try { + const media = await msg.downloadMedia?.(); + if (!media?.data) return null; + + const buffer = Buffer.from(media.data, 'base64'); + const result = await this.mediaManager.saveMedia(buffer, media.mimetype, media.filename); + + return { + failed: false, + result, + mimeType: media.mimetype, + filename: media.filename, + type: attachmentType, + }; + } catch (err) { + logger.warn({ err, type: msg.type }, 'Failed to download incoming media'); + return { failed: true, type: msg.type ?? 'unknown' }; + } + } + + private async transcribeVoiceMessage(msg: WAMessage): Promise { + try { + const media = await msg.downloadMedia?.(); + if (!media?.data) return null; + + const tmpPath = join(tmpdir(), `wa-voice-${Date.now()}.ogg`); + await writeFile(tmpPath, Buffer.from(media.data, 'base64')); + try { + const result = await transcribeAudio(tmpPath); + if (result) { + logger.debug( + { backend: result.backend, durationMs: result.durationMs }, + 'Voice transcription complete', + ); + } + return result?.text ?? null; + } finally { + await unlink(tmpPath).catch(() => {}); + } + } catch (err) { + logger.warn({ err }, 'Voice message transcription failed'); + return null; + } + } + + private async findTtsTool(): Promise<{ + bin: string; + ext: string; + mimeType: string; + argsFor: (text: string, outPath: string) => string[]; + } | null> { + // macOS: 'say' command → AIFF output + try { + const { stdout } = await execFileAsync('which', ['say']); + if (stdout.trim()) { + return { + bin: stdout.trim(), + ext: 'aiff', + mimeType: 'audio/aiff', + argsFor: (text, outPath) => ['-o', outPath, text], + }; + } + } catch { + // not found + } + // Linux: 'espeak' command → WAV output + try { + const { stdout } = await execFileAsync('which', ['espeak']); + if (stdout.trim()) { + return { + bin: stdout.trim(), + ext: 'wav', + mimeType: 'audio/wav', + argsFor: (text, outPath) => ['-w', outPath, text], + }; + } + } catch { + // not found + } + return null; + } + + async sendVoiceReply(chatId: string, text: string): Promise { + if (!this.client || !this.connected) { + throw new Error('WhatsApp connector is not connected'); + } + + const ttsTool = await this.findTtsTool(); + if (!ttsTool) { + logger.warn({ chatId }, 'No TTS tool found — falling back to text for voice reply'); + const formatted = formatMarkdownForWhatsApp(text); + const chunks = splitForWhatsApp(formatted); + for (const chunk of chunks) { + await this.client.sendMessage(chatId, chunk); + } + return; + } + + const tmpPath = join(tmpdir(), `wa-tts-${Date.now()}.${ttsTool.ext}`); + try { + await execFileAsync(ttsTool.bin, ttsTool.argsFor(text, tmpPath)); + const audioData = await readFile(tmpPath); + const base64 = audioData.toString('base64'); + const WAWebJS = await import('whatsapp-web.js'); + const { MessageMedia: MessageMediaClass } = WAWebJS; + const media = new MessageMediaClass(ttsTool.mimeType, base64, null); + await this.client.sendMessage(chatId, media, { sendAudioAsVoice: true }); + logger.debug({ chatId }, 'Voice reply sent via TTS'); + } catch (err) { + logger.warn({ err, chatId }, 'TTS generation failed — falling back to text reply'); + const formatted = formatMarkdownForWhatsApp(text); + const chunks = splitForWhatsApp(formatted); + for (const chunk of chunks) { + await this.client.sendMessage(chatId, chunk); + } + } finally { + await unlink(tmpPath).catch(() => {}); + } } private scheduleReconnect(): void { @@ -134,6 +582,12 @@ export class WhatsAppConnector implements Connector { return; } + // Guard against double-scheduling (e.g. both 'error' and 'disconnected' fire together) + if (this.reconnectTimer !== null) { + logger.debug('Reconnect already scheduled — skipping duplicate'); + return; + } + if (maxAttempts > 0 && this.reconnectAttempt >= maxAttempts) { logger.error({ maxAttempts }, 'WhatsApp reconnect: max attempts reached, giving up'); this.emit('error', new Error('WhatsApp reconnect failed: max attempts reached')); @@ -151,24 +605,27 @@ export class WhatsAppConnector implements Connector { 'WhatsApp scheduling reconnect', ); - this.reconnectTimer = setTimeout(() => { + // eslint-disable-next-line @typescript-eslint/no-misused-promises + this.reconnectTimer = setTimeout(async () => { this.reconnectTimer = null; if (this.shuttingDown) return; logger.info({ attempt: this.reconnectAttempt }, 'WhatsApp attempting reconnect'); if (this.client) { - this.client.destroy().catch((err: unknown) => { + await this.client.destroy().catch((err: unknown) => { logger.warn({ err }, 'Error destroying old WhatsApp client before reconnect'); }); this.client = null; } - this.createAndStartClient().catch((err: unknown) => { + try { + await this.createAndStartClient(); + } catch (err: unknown) { logger.error({ err }, 'WhatsApp reconnect failed'); this.emit('error', err instanceof Error ? err : new Error(String(err))); this.scheduleReconnect(); - }); + } }, delay); } @@ -177,6 +634,24 @@ export class WhatsAppConnector implements Connector { throw new Error('WhatsApp connector is not connected'); } + if (message.media) { + const WAWebJS = await import('whatsapp-web.js'); + const { MessageMedia: MessageMediaClass } = WAWebJS; + const base64 = message.media.data.toString('base64'); + const media = new MessageMediaClass( + message.media.mimeType, + base64, + message.media.filename ?? null, + ); + const caption = message.content || message.media.filename; + const options: WASendOptions = {}; + if (caption) options.caption = caption; + if (message.media.type === 'document') options.sendMediaAsDocument = true; + await this.client.sendMessage(message.recipient, media, options); + logger.debug({ recipient: message.recipient, type: message.media.type }, 'Media sent'); + return; + } + const formatted = formatMarkdownForWhatsApp(message.content); const chunks = splitForWhatsApp(formatted); for (const chunk of chunks) { @@ -186,6 +661,24 @@ export class WhatsAppConnector implements Connector { logger.debug({ recipient: message.recipient, chunks: chunks.length }, 'Message sent'); } + async sendProactive(recipient: string, content: string): Promise { + if (!this.client || !this.connected) { + throw new Error('WhatsApp connector is not connected'); + } + + // Normalize to WhatsApp chat ID format: digits only + @c.us + const digits = recipient.replace(/\D/g, ''); + const chatId = digits.includes('@') ? recipient : `${digits}@c.us`; + + const formatted = formatMarkdownForWhatsApp(content); + const chunks = splitForWhatsApp(formatted); + for (const chunk of chunks) { + await this.client.sendMessage(chatId, chunk); + } + + logger.debug({ recipient: chatId, chunks: chunks.length }, 'Proactive message sent'); + } + async sendTypingIndicator(chatId: string): Promise { if (!this.client || !this.connected) { return; // Best-effort — silently skip if not connected @@ -200,18 +693,73 @@ export class WhatsAppConnector implements Connector { } } + async sendProgress(event: ProgressEvent, chatId: string): Promise { + if (!this.client || !this.connected) return; + + if (event.type === 'complete') { + this.progressSent.delete(chatId); + return; + } + + // Send spawning status once per conversation + if (event.type === 'spawning' && !this.progressSent.has(chatId)) { + const n = event.workerCount; + const text = `🔄 Breaking into ${n.toString()} subtask${n !== 1 ? 's' : ''}...`; + try { + await this.client.sendMessage(chatId, text); + this.progressSent.add(chatId); + } catch (err: unknown) { + logger.debug({ chatId, err }, 'Failed to send WhatsApp progress message'); + } + } + + // Send each worker result as it completes + if (event.type === 'worker-result') { + const icon = event.success ? '✅' : '❌'; + const header = `${icon} Subtask ${event.workerIndex}/${event.total} (${event.profile}):`; + const maxLen = 4000; + const body = + event.content.length > maxLen + ? event.content.slice(0, maxLen) + '\n...(truncated)' + : event.content; + try { + await this.client.sendMessage(chatId, `${header}\n${body}`); + } catch (err: unknown) { + logger.debug({ chatId, err }, 'Failed to send WhatsApp worker result'); + } + } + + // Notify all connected chats when a worker is cancelled (OB-883) + if (event.type === 'worker-cancelled') { + try { + await this.client.sendMessage( + chatId, + `🛑 Worker ${event.workerId} was stopped by ${event.cancelledBy}.`, + ); + } catch (err: unknown) { + logger.debug({ chatId, err }, 'Failed to send WhatsApp worker-cancelled notification'); + } + } + } + on(event: E, listener: ConnectorEvents[E]): void { this.listeners[event].push(listener); } async shutdown(): Promise { - this.shuttingDown = true; - - if (this.reconnectTimer !== null) { + if (this.reconnectTimer) { clearTimeout(this.reconnectTimer); this.reconnectTimer = null; } + if (this.progressSentCleanupInterval) { + clearInterval(this.progressSentCleanupInterval); + this.progressSentCleanupInterval = null; + } + + this.shuttingDown = true; + this.progressSent.clear(); + if (this.client) { await this.client.destroy(); this.connected = false; diff --git a/src/connectors/whatsapp/whatsapp-message.ts b/src/connectors/whatsapp/whatsapp-message.ts index b575ecd8..1a6b5940 100644 --- a/src/connectors/whatsapp/whatsapp-message.ts +++ b/src/connectors/whatsapp/whatsapp-message.ts @@ -1,7 +1,8 @@ import type { InboundMessage } from '../../types/message.js'; +import { splitMessage, PLATFORM_MAX_LENGTH } from '../message-splitter.js'; /** Maximum message length for WhatsApp responses */ -export const WHATSAPP_MAX_LENGTH = 4096; +export const WHATSAPP_MAX_LENGTH = PLATFORM_MAX_LENGTH.whatsapp; /** Parse a raw WhatsApp message into an InboundMessage */ export function parseWhatsAppMessage( @@ -9,6 +10,7 @@ export function parseWhatsAppMessage( sender: string, body: string, timestamp: number, + attachments?: InboundMessage['attachments'], ): InboundMessage { return { id, @@ -17,58 +19,14 @@ export function parseWhatsAppMessage( rawContent: body, content: body, // Will be cleaned by AuthService timestamp: new Date(timestamp * 1000), + ...(attachments !== undefined && { attachments }), }; } /** * Split a response into WhatsApp-safe chunks (≤ WHATSAPP_MAX_LENGTH each). - * - * Splitting strategy (in order of preference): - * 1. If content fits in a single message, return it as-is. - * 2. Split on the last double-newline (paragraph break) within the limit. - * 3. Split on the last single newline within the limit. - * 4. Split on the last space (word boundary) within the limit. - * 5. Hard-split at the limit as a last resort. - * - * Each chunk (except the last) gets a "[n/total]" suffix so the user - * knows the message continues. + * Delegates to the shared splitMessage() utility. */ export function splitForWhatsApp(content: string): string[] { - if (content.length <= WHATSAPP_MAX_LENGTH) { - return [content]; - } - - const chunks: string[] = []; - let remaining = content; - - while (remaining.length > 0) { - if (remaining.length <= WHATSAPP_MAX_LENGTH) { - chunks.push(remaining); - break; - } - - // Reserve space for the part indicator suffix (e.g. "\n\n[3/10]") - const reservedForSuffix = 12; - const maxChunk = WHATSAPP_MAX_LENGTH - reservedForSuffix; - - const slice = remaining.slice(0, maxChunk); - - // Try split points from best to worst - let splitAt = slice.lastIndexOf('\n\n'); - if (splitAt <= 0) splitAt = slice.lastIndexOf('\n'); - if (splitAt <= 0) splitAt = slice.lastIndexOf(' '); - if (splitAt <= 0) splitAt = maxChunk; - - chunks.push(remaining.slice(0, splitAt)); - remaining = remaining.slice(splitAt).trimStart(); - } - - // If only one chunk after splitting (edge case), return as-is - if (chunks.length === 1) { - return chunks; - } - - // Add part indicators - const total = chunks.length; - return chunks.map((chunk, i) => `${chunk}\n\n[${i + 1}/${total}]`); + return splitMessage(content, WHATSAPP_MAX_LENGTH); } diff --git a/src/core/adapter-registry.ts b/src/core/adapter-registry.ts new file mode 100644 index 00000000..b9db45dd --- /dev/null +++ b/src/core/adapter-registry.ts @@ -0,0 +1,147 @@ +/** + * Adapter Registry — Maps provider names to CLIAdapter instances. + * + * Resolves discovered tools to their corresponding CLI adapters. + * Built-in adapters are lazy-loaded on first access; custom adapters + * can be registered via register() and take priority over built-ins. + */ + +import type { CLIAdapter, CapabilityLevel } from './cli-adapter.js'; +import type { DiscoveredTool } from '../types/discovery.js'; +import type { ConsentMode } from '../memory/access-store.js'; +import { ClaudeAdapter } from './adapters/claude-adapter.js'; +import { CodexAdapter } from './adapters/codex-adapter.js'; +import { AiderAdapter } from './adapters/aider-adapter.js'; +import { ClaudeSDKAdapter } from './adapters/claude-sdk.js'; +import { createLogger } from './logger.js'; + +const logger = createLogger('adapter-registry'); + +/** + * Trust level controlling which adapter is selected for Claude tools. + * + * - `auto` — pre-approved `--allowedTools` list; uses the CLI adapter (no prompts). + * - `edit` — auto-approve reads/edits, relay Bash/Write to user; uses the SDK adapter. + * - `ask` — relay every tool call to the user for approval; uses the SDK adapter. + */ +export type TrustLevel = 'auto' | 'edit' | 'ask'; + +/** + * Map a stored ConsentMode value to the corresponding TrustLevel for adapter selection. + * + * - `auto-approve-all` → `auto` (CLI adapter, no per-tool prompts) + * - `auto-approve-up-to-edit` → `edit` (SDK adapter, prompt only for Bash/Write) + * - `always-ask` → `ask` (SDK adapter, prompt for every tool call) + * - `auto-approve-read` → `ask` (SDK adapter, conservative fallback) + */ +export function consentModeToTrustLevel(mode: ConsentMode): TrustLevel { + if (mode === 'auto-approve-all') return 'auto'; + if (mode === 'auto-approve-up-to-edit') return 'edit'; + return 'ask'; +} + +/** Built-in adapter factories keyed by tool name */ +const BUILT_IN_ADAPTERS: Record CLIAdapter> = { + claude: () => new ClaudeAdapter(), + 'claude-sdk': () => new ClaudeSDKAdapter(), + codex: () => new CodexAdapter(), + aider: () => new AiderAdapter(), +}; + +export class AdapterRegistry { + private adapters = new Map(); + + /** Register a CLIAdapter for a tool name */ + register(name: string, adapter: CLIAdapter): void { + this.adapters.set(name, adapter); + logger.debug({ name }, 'Registered CLI adapter'); + } + + /** Get the adapter for a tool name, creating from built-ins if needed */ + get(name: string): CLIAdapter | undefined { + let adapter = this.adapters.get(name); + if (!adapter) { + const factory = BUILT_IN_ADAPTERS[name]; + if (factory) { + adapter = factory(); + this.adapters.set(name, adapter); + } + } + return adapter; + } + + /** Get the adapter for a discovered tool */ + getForTool(tool: DiscoveredTool): CLIAdapter | undefined { + return this.get(tool.name); + } + + /** + * Get the adapter for a tool name, choosing between CLI and SDK adapters + * based on the user's trust level. + * + * - `auto` → CLI adapter (pre-approved --allowedTools, no prompts). + * - `edit` → SDK adapter (auto-approve reads/edits, relay Bash/Write to user). + * - `ask` → SDK adapter (relay every tool call to the user for approval). + * + * Non-Claude tools are unaffected — `trustLevel` is ignored and the + * standard adapter for that tool name is returned. + */ + getForTrustLevel(toolName: string, trustLevel: TrustLevel): CLIAdapter | undefined { + if (toolName === 'claude') { + const adapterName = trustLevel === 'auto' ? 'claude' : 'claude-sdk'; + return this.get(adapterName); + } + return this.get(toolName); + } + + /** Check if an adapter exists (registered or built-in) for a tool name */ + has(name: string): boolean { + return this.adapters.has(name) || name in BUILT_IN_ADAPTERS; + } + + /** + * Validate whether an adapter natively enforces a capability profile. + * + * Returns `{ supported: true }` if the adapter enforces the profile via + * native CLI mechanisms (e.g. Claude's --allowedTools named tool lists). + * Returns `{ supported: false }` if the adapter emulates the profile via + * sandbox modes or system-prompt constraints instead (e.g. Codex, Aider). + * + * Callers can use this to decide whether additional system-prompt constraints + * are needed or whether to accept the adapter's emulated enforcement. + * This does NOT change spawn behavior — adapters always handle tool profiles + * in buildSpawnConfig(), using whatever mechanism they support. + */ + resolveProfileForAdapter(adapterName: string, profile: CapabilityLevel): { supported: boolean } { + const adapter = this.get(adapterName); + if (!adapter) { + logger.debug( + { adapterName, profile }, + 'resolveProfileForAdapter: no adapter found — profile support unknown', + ); + return { supported: false }; + } + + const profiles = adapter.supportedProfiles?.(); + if (!profiles || !profiles.includes(profile)) { + logger.debug( + { adapterName, profile }, + 'Adapter does not natively enforce profile via tool lists — ' + + 'profile emulated via sandbox mode or system-prompt constraints', + ); + return { supported: false }; + } + + return { supported: true }; + } +} + +/** Create an AdapterRegistry pre-loaded with the Claude CLI and SDK adapters */ +export function createAdapterRegistry(): AdapterRegistry { + const registry = new AdapterRegistry(); + // Claude CLI adapter — default for trust=auto + registry.register('claude', new ClaudeAdapter()); + // Claude SDK adapter — used for trust=edit or trust=ask (interactive approval) + registry.register('claude-sdk', new ClaudeSDKAdapter()); + return registry; +} diff --git a/src/core/adapters/aider-adapter.ts b/src/core/adapters/aider-adapter.ts new file mode 100644 index 00000000..0b05f792 --- /dev/null +++ b/src/core/adapters/aider-adapter.ts @@ -0,0 +1,146 @@ +/** + * Aider CLI Adapter + * + * Translates provider-neutral SpawnOptions into Aider CLI arguments. + * + * Aider CLI flags: + * --model Model to use (gpt-4o, o1, claude-3-sonnet, etc.) + * --message Non-interactive message (runs and exits) + * --yes Auto-confirm all changes + * --no-auto-commits Don't auto-commit changes (for read-only tasks) + * + * Feature mapping (lossy — Aider doesn't support these): + * --max-turns → dropped (aider manages its own loop) + * --max-budget-usd → dropped + * --session-id → dropped (aider uses git for state) + * --append-system-prompt → prepended to message text + * --allowedTools → mapped to --no-auto-commits for read-only + */ + +import type { CLIAdapter, CLISpawnConfig, CapabilityLevel } from '../cli-adapter.js'; +import type { SpawnOptions } from '../agent-runner.js'; +import { sanitizePrompt } from '../agent-runner.js'; +import { createLogger } from '../logger.js'; +import { sanitizeEnv } from '../env-sanitizer.js'; +import { SecurityConfigSchema } from '../../types/config.js'; +import type { SecurityConfig } from '../../types/config.js'; + +const logger = createLogger('aider-adapter'); + +const DEFAULT_SECURITY_CONFIG: SecurityConfig = SecurityConfigSchema.parse({}); + +export class AiderAdapter implements CLIAdapter { + readonly name = 'aider'; + private readonly securityConfig: SecurityConfig; + + constructor(securityConfig?: SecurityConfig) { + this.securityConfig = securityConfig ?? DEFAULT_SECURITY_CONFIG; + } + + buildSpawnConfig(opts: SpawnOptions): CLISpawnConfig { + const args: string[] = []; + + if (opts.model) { + args.push('--model', opts.model); + } + + // --yes auto-confirms changes (non-interactive mode) + args.push('--yes'); + + // Map read-only tools to --no-auto-commits (best-effort access control) + if (opts.allowedTools) { + const hasEditWrite = opts.allowedTools.some((t) => t === 'Edit' || t === 'Write'); + if (!hasEditWrite) { + args.push('--no-auto-commits'); + } + } + + // systemPrompt: prepend to message (aider has no --append-system-prompt) + let message = sanitizePrompt( + opts.prompt, + this.getPromptBudget(opts.model).maxPromptChars, + 'worker', + ); + if (opts.systemPrompt) { + message = opts.systemPrompt + '\n\n' + message; + } + + args.push('--message', message); + + // Log dropped options at debug level + if (opts.maxTurns) { + logger.debug({ maxTurns: opts.maxTurns }, 'aider: --max-turns not supported, ignoring'); + } + if (opts.maxBudgetUsd) { + logger.debug( + { maxBudgetUsd: opts.maxBudgetUsd }, + 'aider: --max-budget-usd not supported, ignoring', + ); + } + if (opts.resumeSessionId || opts.sessionId) { + logger.debug('aider: sessions not supported, ignoring session options'); + } + + return { + binary: 'aider', + args, + env: this.cleanEnv({ ...process.env }), + }; + } + + cleanEnv(env: Record): Record { + // Remove Claude env vars in case both Claude and Aider are installed + const cleaned = { ...env }; + for (const key of Object.keys(cleaned)) { + if ( + key === 'CLAUDECODE' || + key.startsWith('CLAUDE_CODE_') || + key.startsWith('CLAUDE_AGENT_SDK_') + ) { + delete cleaned[key]; + } + } + return sanitizeEnv(cleaned, this.securityConfig); + } + + mapCapabilityLevel(_level: CapabilityLevel): string[] | undefined { + // Aider doesn't use tool lists — it manages its own file access. + return undefined; + } + + isValidModel(model: string): boolean { + // Aider uses litellm which supports a very wide range of models. + // Accept anything non-empty — aider will validate at runtime. + return model.length > 0; + } + + supportedProfiles(): readonly CapabilityLevel[] { + // Aider manages its own file access via git integration — no --allowedTools support. + // Access control is applied via --no-auto-commits for read-only tasks. + return []; + } + + getPromptBudget(model?: string): { maxPromptChars: number; maxSystemPromptChars: number } { + // Aider prepends systemPrompt to the --message text in buildSpawnConfig(), so both + // fields share the same underlying `--message` argument. There is no separate + // system-prompt channel in the Aider CLI. + // + // Aider uses litellm and supports a wide range of models. Budgets are set per model family: + // - gpt-4.1: ~1M token context window (~4M chars) → 400K combined budget + // - o3 / o4-mini: 200K token context window (~800K chars) → 200K combined budget + // - Default: 100K chars (~25K tokens) — safe for smaller / unknown models + // + // Both fields are set to the same value to signal that they share a single pool + // (system + user prompt merged into one --message string). PromptAssembler should + // treat these as a combined budget when targeting AiderAdapter. + let combined: number; + if (model && /gpt-4\.1/i.test(model)) { + combined = 400_000; + } else if (model && /\bo3\b|\bo4-mini\b/i.test(model)) { + combined = 200_000; + } else { + combined = 100_000; + } + return { maxPromptChars: combined, maxSystemPromptChars: combined }; + } +} diff --git a/src/core/adapters/claude-adapter.ts b/src/core/adapters/claude-adapter.ts new file mode 100644 index 00000000..338b8e2f --- /dev/null +++ b/src/core/adapters/claude-adapter.ts @@ -0,0 +1,163 @@ +/** + * Claude CLI Adapter + * + * Translates provider-neutral SpawnOptions into Claude Code CLI arguments. + * Extracted from the hardcoded logic in agent-runner.ts buildArgs() and execOnce(). + * + * Claude Code CLI flags: + * --print Single-turn stateless mode (no session) + * --model Model to use (haiku, sonnet, opus, or full ID) + * --max-turns Max agentic turns + * --allowedTools Tool names (variadic, repeated per tool) + * --append-system-prompt Append to system prompt + * --max-budget-usd Maximum spend in USD + * --mcp-config Path to MCP server config JSON (per-worker isolation) + * --strict-mcp-config Ignore global/project MCP configs (use only --mcp-config) + * --resume Resume existing session + * --session-id Start new session with specific ID + */ + +import type { CLIAdapter, CLISpawnConfig, CapabilityLevel } from '../cli-adapter.js'; +import type { SpawnOptions } from '../agent-runner.js'; +import { sanitizePrompt, MODEL_ALIASES, DEFAULT_MAX_TURNS_TASK } from '../agent-runner.js'; +import type { ModelAlias } from '../agent-runner.js'; +import { createLogger } from '../logger.js'; +import { sanitizeEnv } from '../env-sanitizer.js'; +import { SecurityConfigSchema } from '../../types/config.js'; +import type { SecurityConfig } from '../../types/config.js'; +import { getClaudePromptBudget } from './claude-budget.js'; + +const logger = createLogger('claude-adapter'); + +const DEFAULT_SECURITY_CONFIG: SecurityConfig = SecurityConfigSchema.parse({}); + +export class ClaudeAdapter implements CLIAdapter { + readonly name = 'claude'; + private readonly securityConfig: SecurityConfig; + + constructor(securityConfig?: SecurityConfig) { + this.securityConfig = securityConfig ?? DEFAULT_SECURITY_CONFIG; + } + + buildSpawnConfig(opts: SpawnOptions): CLISpawnConfig { + const args: string[] = []; + + // Depth limiting: --print (single-turn) vs --session-id/--resume (multi-turn) + if (opts.resumeSessionId) { + args.push('--resume', opts.resumeSessionId); + } else if (opts.sessionId) { + args.push('--session-id', opts.sessionId); + } else { + args.push('--print'); + } + + if (opts.model) { + if (!this.isValidModel(opts.model)) { + logger.warn( + { model: opts.model }, + 'Unrecognized model — passing through to CLI, which may reject it', + ); + } + args.push('--model', opts.model); + } + + const maxTurns = opts.maxTurns ?? DEFAULT_MAX_TURNS_TASK; + args.push('--max-turns', String(maxTurns)); + + if (opts.systemPrompt) { + args.push('--append-system-prompt', opts.systemPrompt); + } + + if (opts.maxBudgetUsd !== undefined && opts.maxBudgetUsd > 0) { + args.push('--max-budget-usd', String(opts.maxBudgetUsd)); + } + + if (opts.mcpConfigPath) { + args.push('--mcp-config', opts.mcpConfigPath); + } + + if (opts.strictMcpConfig) { + args.push('--strict-mcp-config'); + } + + // Place the prompt BEFORE --allowedTools. Commander.js parses the first + // positional argument as the prompt. --allowedTools is variadic () + // and would consume a trailing prompt as a tool name. + args.push( + sanitizePrompt(opts.prompt, this.getPromptBudget(opts.model).maxPromptChars, 'worker'), + ); + + if (opts.allowedTools && opts.allowedTools.length > 0) { + for (const tool of opts.allowedTools) { + args.push('--allowedTools', tool); + } + } + + // NOTE: Workspace boundary is enforced via spawn({ cwd: workspacePath }) in agent-runner.ts, + // not via CLI flags. Claude CLI does not support --cwd. + + return { + binary: 'claude', + args, + env: this.cleanEnv({ ...process.env }, opts.securityConfig?.trustLevel), + }; + } + + cleanEnv( + env: Record, + trustLevel?: string, + ): Record { + const cleaned = { ...env }; + for (const key of Object.keys(cleaned)) { + if ( + key === 'CLAUDECODE' || + key.startsWith('CLAUDE_CODE_') || + key.startsWith('CLAUDE_AGENT_SDK_') + ) { + delete cleaned[key]; + } + } + + // In trusted mode, remove HOME to prevent agents from reading ~/.ssh, ~/.bashrc, etc. + if (trustLevel === 'trusted') { + delete cleaned['HOME']; + } + + return sanitizeEnv(cleaned, this.securityConfig); + } + + mapCapabilityLevel(level: CapabilityLevel): string[] | undefined { + switch (level) { + case 'read-only': + return ['Read', 'Glob', 'Grep']; + case 'code-edit': + return [ + 'Read', + 'Edit', + 'Write', + 'Glob', + 'Grep', + 'Bash(git:*)', + 'Bash(npm:*)', + 'Bash(npx:*)', + ]; + case 'full-access': + return ['Read', 'Edit', 'Write', 'Glob', 'Grep', 'Bash(*)']; + } + } + + isValidModel(model: string): boolean { + if (MODEL_ALIASES.includes(model as ModelAlias)) return true; + // Full model IDs follow the pattern: claude-- + return /^claude-[a-z0-9]+-[a-z0-9._-]+$/.test(model); + } + + supportedProfiles(): readonly CapabilityLevel[] { + // Claude enforces all profiles natively via --allowedTools named tool lists. + return ['read-only', 'code-edit', 'full-access']; + } + + getPromptBudget(model?: string): { maxPromptChars: number; maxSystemPromptChars: number } { + return getClaudePromptBudget(model); + } +} diff --git a/src/core/adapters/claude-budget.ts b/src/core/adapters/claude-budget.ts new file mode 100644 index 00000000..2edd660b --- /dev/null +++ b/src/core/adapters/claude-budget.ts @@ -0,0 +1,41 @@ +/** + * Shared prompt budget helper for Claude adapters. + * + * Used by both ClaudeAdapter (CLI-based) and ClaudeSDKAdapter (SDK-based) to + * return identical model-aware prompt budgets. Centralised here to eliminate + * duplication between the two adapters. + * + * Official context windows: + * Tier 1 — Opus 4.6 (claude-opus-4-6): 1M tokens (~3.4M chars), max output 128k tokens + * Sonnet 4.6 (claude-sonnet-4-6): 1M tokens (~3.4M chars), max output 64k tokens + * → maxPromptChars: 128_000, maxSystemPromptChars: 800_000 + * Tier 2 — Haiku 4.5 (claude-haiku-4-5-20251001): 200k tokens (~680k chars), max output 64k tokens + * → maxPromptChars: 32_768, maxSystemPromptChars: 180_000 + * Tier 3 — Unrecognized / unspecified: Sonnet-class default + * → maxPromptChars: 128_000, maxSystemPromptChars: 800_000 + */ + +export function getClaudePromptBudget(model?: string): { + maxPromptChars: number; + maxSystemPromptChars: number; +} { + const isOpus46 = + model != null && + (/opus.*4[.-]6/i.test(model) || model === 'claude-opus-4-6' || model === 'opus'); + const isSonnet46 = + model != null && + (/sonnet.*4[.-]6/i.test(model) || model === 'claude-sonnet-4-6' || model === 'sonnet'); + + if (isOpus46 || isSonnet46) { + return { maxPromptChars: 128_000, maxSystemPromptChars: 800_000 }; + } + + // Haiku workers always specify model: "haiku" explicitly — keep their conservative limits. + if (model != null && /haiku/i.test(model)) { + return { maxPromptChars: 32_768, maxSystemPromptChars: 180_000 }; + } + + // Undefined, empty, or unrecognized model — default to Sonnet-class budget. + // Haiku workers always set model explicitly; unspecified workers run on Sonnet-class hardware. + return { maxPromptChars: 128_000, maxSystemPromptChars: 800_000 }; +} diff --git a/src/core/adapters/claude-sdk.ts b/src/core/adapters/claude-sdk.ts new file mode 100644 index 00000000..2f33fee3 --- /dev/null +++ b/src/core/adapters/claude-sdk.ts @@ -0,0 +1,377 @@ +/* eslint-disable @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-redundant-type-constituents */ +/* All rules above are disabled because @anthropic-ai/claude-agent-sdk is an optional peer dependency + that is not always installed. Type resolution fails at lint time but works at runtime. */ + +/** + * Claude Agent SDK Adapter + * + * Uses `query()` from `@anthropic-ai/claude-agent-sdk` instead of + * `child_process.spawn('claude', ...)`. The `canUseTool` callback provides + * per-tool-call control: + * + * - Non-interactive mode: auto-approve tools in the allowed list + * (mirrors `--allowedTools` behavior from the CLI adapter). + * - Interactive mode: delegate to a permission relay callback (OB-1498) + * that routes approval requests through messaging channels. + * + * This adapter implements `CLIAdapter` for compatibility with the adapter + * registry, but its primary execution path is `executeQuery()` — not + * `buildSpawnConfig()`. AgentRunner detects SDK adapters via `isSDKAdapter()` + * and uses `executeQuery()` instead of spawning a child process. + */ + +import type { CLIAdapter, CLISpawnConfig, CapabilityLevel } from '../cli-adapter.js'; +import type { SpawnOptions } from '../agent-runner.js'; +import { sanitizePrompt, MODEL_ALIASES } from '../agent-runner.js'; +import type { ModelAlias } from '../agent-runner.js'; +import { createLogger } from '../logger.js'; +import { sanitizeEnv } from '../env-sanitizer.js'; +import { SecurityConfigSchema } from '../../types/config.js'; +import type { SecurityConfig } from '../../types/config.js'; +import { getClaudePromptBudget } from './claude-budget.js'; +import type { + CanUseTool, + Options as SDKOptions, + PermissionResult, + Query, + SDKMessage, + SDKResultMessage, +} from '@anthropic-ai/claude-agent-sdk'; +import { query } from '@anthropic-ai/claude-agent-sdk'; + +const logger = createLogger('claude-sdk-adapter'); + +const DEFAULT_SECURITY_CONFIG: SecurityConfig = SecurityConfigSchema.parse({}); + +/** + * Callback signature for interactive permission relay. + * When set, tool calls not in the allowed list are relayed to the user + * through the messaging channel for approval. + */ +export type PermissionRelayFn = (params: { + toolName: string; + input: Record; + userId: string; + channel: string; +}) => Promise; + +/** + * Options for executing a query via the SDK adapter. + */ +export interface SDKExecuteOptions { + /** SpawnOptions from AgentRunner */ + spawnOptions: SpawnOptions; + /** User ID for permission relay */ + userId?: string; + /** Channel name for permission relay */ + channel?: string; + /** Optional permission relay function for interactive mode */ + permissionRelay?: PermissionRelayFn; + /** AbortController for cancellation */ + abortController?: AbortController; + /** Callback for streaming messages */ + onMessage?: (message: SDKMessage) => void; +} + +/** + * Result from an SDK query execution. + */ +export interface SDKExecuteResult { + /** The text result from the agent */ + stdout: string; + /** Duration in milliseconds */ + durationMs: number; + /** Number of agentic turns used */ + numTurns: number; + /** Total cost in USD */ + costUsd: number; + /** Session ID from the query */ + sessionId: string; + /** Whether the query ended due to an error */ + isError: boolean; + /** Error subtype if applicable */ + errorSubtype?: string; +} + +export class ClaudeSDKAdapter implements CLIAdapter { + readonly name = 'claude-sdk'; + private readonly securityConfig: SecurityConfig; + + constructor(securityConfig?: SecurityConfig) { + this.securityConfig = securityConfig ?? DEFAULT_SECURITY_CONFIG; + } + + /** + * Build a CLISpawnConfig for compatibility with the CLIAdapter interface. + * + * This adapter does NOT use child_process.spawn() — callers should use + * `executeQuery()` instead. This method exists only to satisfy the interface. + * AgentRunner detects SDK adapters via `isSDKAdapter()` and routes accordingly. + */ + buildSpawnConfig(opts: SpawnOptions): CLISpawnConfig { + logger.warn('buildSpawnConfig() called on SDK adapter — use executeQuery() instead'); + // Return a no-op config that cannot actually spawn + return { + binary: '__claude_sdk__', + args: [ + sanitizePrompt(opts.prompt, this.getPromptBudget(opts.model).maxPromptChars, 'worker'), + ], + env: this.cleanEnv({ ...process.env }), + }; + } + + cleanEnv(env: Record): Record { + const cleaned = { ...env }; + for (const key of Object.keys(cleaned)) { + if ( + key === 'CLAUDECODE' || + key.startsWith('CLAUDE_CODE_') || + key.startsWith('CLAUDE_AGENT_SDK_') + ) { + delete cleaned[key]; + } + } + return sanitizeEnv(cleaned, this.securityConfig); + } + + mapCapabilityLevel(level: CapabilityLevel): string[] | undefined { + switch (level) { + case 'read-only': + return ['Read', 'Glob', 'Grep']; + case 'code-edit': + return [ + 'Read', + 'Edit', + 'Write', + 'Glob', + 'Grep', + 'Bash(git:*)', + 'Bash(npm:*)', + 'Bash(npx:*)', + ]; + case 'full-access': + return ['Read', 'Edit', 'Write', 'Glob', 'Grep', 'Bash(*)']; + } + } + + isValidModel(model: string): boolean { + if (MODEL_ALIASES.includes(model as ModelAlias)) return true; + return /^claude-[a-z0-9]+-[a-z0-9._-]+$/.test(model); + } + + supportedProfiles(): readonly CapabilityLevel[] { + return ['read-only', 'code-edit', 'full-access']; + } + + getPromptBudget(model?: string): { maxPromptChars: number; maxSystemPromptChars: number } { + return getClaudePromptBudget(model); + } + + /** + * Type guard: identifies this as an SDK-based adapter. + * AgentRunner uses this to choose `executeQuery()` over `child_process.spawn()`. + */ + isSDKAdapter(): boolean { + return true; + } + + /** + * Build a `canUseTool` callback that implements the permission model. + * + * - Tools in `allowedTools` are auto-approved (mirrors --allowedTools). + * - If a `permissionRelay` is provided, non-allowed tools are relayed + * to the user through the messaging channel for approval. + * - Without a `permissionRelay`, non-allowed tools are denied. + */ + buildCanUseTool( + allowedTools: string[] | undefined, + permissionRelay?: PermissionRelayFn, + userId?: string, + channel?: string, + ): CanUseTool { + const allowed = new Set(allowedTools ?? []); + + return async ( + toolName: string, + input: Record, + _options, + ): Promise => { + // Auto-approve tools in the allowed list + if (allowed.has(toolName)) { + return { behavior: 'allow' }; + } + + // Check for wildcard patterns (e.g. Bash(*) allows any Bash tool) + for (const pattern of allowed) { + if (pattern.endsWith('(*)') && toolName.startsWith(pattern.slice(0, -3))) { + return { behavior: 'allow' }; + } + // Handle patterns like Bash(git:*) — allow Bash if the command starts with 'git' + const wildcardMatch = pattern.match(/^(\w+)\(([^)]+):\*\)$/); + if (wildcardMatch) { + const [, baseTool, prefix] = wildcardMatch; + if (toolName === baseTool && prefix) { + const command = typeof input['command'] === 'string' ? input['command'].trim() : ''; + if (command.startsWith(prefix)) { + return { behavior: 'allow' }; + } + } + } + } + + // Interactive mode: relay to user for approval + if (permissionRelay && userId && channel) { + logger.info({ toolName, userId, channel }, 'Relaying tool permission to user'); + const approved = await permissionRelay({ + toolName, + input, + userId, + channel, + }); + return approved + ? { behavior: 'allow', updatedInput: input } + : { behavior: 'deny', message: 'User denied via messaging channel' }; + } + + // No relay — deny by default + logger.info({ toolName }, 'Tool not in allowed list and no permission relay — denying'); + return { + behavior: 'deny', + message: `Tool "${toolName}" is not in the allowed list`, + }; + }; + } + + /** + * Build SDK query options from SpawnOptions. + */ + buildQueryOptions(opts: SDKExecuteOptions): { prompt: string; options: SDKOptions } { + const { spawnOptions, permissionRelay, userId, channel, abortController } = opts; + const budget = this.getPromptBudget(spawnOptions.model); + const prompt = sanitizePrompt(spawnOptions.prompt, budget.maxPromptChars, 'worker'); + + const sdkOptions: SDKOptions = { + cwd: spawnOptions.workspacePath, + env: this.cleanEnv({ ...process.env }), + persistSession: false, + }; + + if (spawnOptions.model) { + sdkOptions.model = spawnOptions.model; + } + + if (spawnOptions.maxTurns) { + sdkOptions.maxTurns = spawnOptions.maxTurns; + } + + if (spawnOptions.maxBudgetUsd !== undefined && spawnOptions.maxBudgetUsd > 0) { + sdkOptions.maxBudgetUsd = spawnOptions.maxBudgetUsd; + } + + if (spawnOptions.systemPrompt) { + sdkOptions.systemPrompt = { + type: 'preset', + preset: 'claude_code', + append: spawnOptions.systemPrompt, + }; + } + + if (spawnOptions.resumeSessionId) { + sdkOptions.resume = spawnOptions.resumeSessionId; + sdkOptions.persistSession = true; + } else if (spawnOptions.sessionId) { + sdkOptions.sessionId = spawnOptions.sessionId; + sdkOptions.persistSession = true; + } + + if (abortController) { + sdkOptions.abortController = abortController; + } + + // Set up tool control via canUseTool callback + if (spawnOptions.allowedTools && spawnOptions.allowedTools.length > 0) { + sdkOptions.allowedTools = spawnOptions.allowedTools; + sdkOptions.canUseTool = this.buildCanUseTool( + spawnOptions.allowedTools, + permissionRelay, + userId, + channel, + ); + // Use dontAsk so only canUseTool controls approval + sdkOptions.permissionMode = 'dontAsk'; + } + + return { prompt, options: sdkOptions }; + } + + /** + * Execute a query using the Claude Agent SDK. + * + * This is the primary execution path for the SDK adapter, replacing + * `child_process.spawn('claude', ...)` with a programmatic `query()` call. + */ + async executeQuery(opts: SDKExecuteOptions): Promise { + const startTime = Date.now(); + const { prompt, options } = this.buildQueryOptions(opts); + + logger.info( + { + model: opts.spawnOptions.model, + maxTurns: opts.spawnOptions.maxTurns, + hasPermissionRelay: !!opts.permissionRelay, + cwd: opts.spawnOptions.workspacePath, + }, + 'Executing SDK query', + ); + + let resultMessage: SDKResultMessage | undefined; + + const queryIterator: Query = query({ prompt, options }); + + try { + for await (const message of queryIterator) { + // Forward streaming messages to the caller + if (opts.onMessage) { + opts.onMessage(message); + } + + // Capture the final result + if (message.type === 'result') { + resultMessage = message; + } + } + } catch (error) { + const durationMs = Date.now() - startTime; + logger.error({ error, durationMs }, 'SDK query failed'); + throw error; + } + + const durationMs = Date.now() - startTime; + + if (!resultMessage) { + logger.warn({ durationMs }, 'SDK query completed without a result message'); + return { + stdout: '', + durationMs, + numTurns: 0, + costUsd: 0, + sessionId: '', + isError: true, + errorSubtype: 'no_result', + }; + } + + const isSuccess = resultMessage.subtype === 'success'; + const stdout = isSuccess && 'result' in resultMessage ? resultMessage.result : ''; + + return { + stdout, + durationMs, + numTurns: resultMessage.num_turns, + costUsd: resultMessage.total_cost_usd, + sessionId: resultMessage.session_id, + isError: resultMessage.is_error, + errorSubtype: isSuccess ? undefined : resultMessage.subtype, + }; + } +} diff --git a/src/core/adapters/codex-adapter.ts b/src/core/adapters/codex-adapter.ts new file mode 100644 index 00000000..ab447eb4 --- /dev/null +++ b/src/core/adapters/codex-adapter.ts @@ -0,0 +1,419 @@ +/** + * Codex CLI Adapter + * + * Translates provider-neutral SpawnOptions into OpenAI Codex CLI arguments. + * + * Codex uses the `exec` subcommand for non-interactive execution: + * codex exec [OPTIONS] + * + * Codex exec flags: + * -m, --model Model to use (gpt-5.2-codex is default; o3/o4-mini require API key auth) + * -s, --sandbox read-only | workspace-write | danger-full-access + * --full-auto Auto-approve all actions (convenience flag) + * --skip-git-repo-check Skip git repo trust check (required for non-git workspaces) + * --json Output JSONL events to stdout (structured output) + * -o, --output-last-message Write final answer to file (reliable capture) + * -C, --cd Working directory + * --ephemeral No session persistence + * -c, --config Config file for MCP server definitions (MCP passthrough) + * Positional argument (after exec) + * + * Feature mapping (lossy — Codex doesn't support these Claude CLI features): + * --max-turns → dropped (codex runs to completion) + * --max-budget-usd → dropped + * --session-id → new named session (omits --ephemeral so Codex saves state) + * --resume-session → `codex exec resume --last` (or explicit session ID) + * --append-system-prompt → prepended to prompt text + * --allowedTools → NOT passed to Codex (unsupported); mapped to --sandbox for access + * control + system prompt constraints for behavioral guidance. + * Codex uses shell-level sandbox modes (read-only, workspace-write, + * danger-full-access) rather than named tool lists. + */ + +import { readFileSync, unlinkSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { randomBytes } from 'node:crypto'; +import type { CLIAdapter, CLISpawnConfig, CapabilityLevel } from '../cli-adapter.js'; +import type { SpawnOptions } from '../agent-runner.js'; +import { sanitizePrompt } from '../agent-runner.js'; +import { createLogger } from '../logger.js'; +import { sanitizeEnv } from '../env-sanitizer.js'; +import { SecurityConfigSchema } from '../../types/config.js'; +import type { SecurityConfig } from '../../types/config.js'; + +const logger = createLogger('codex-adapter'); + +const DEFAULT_SECURITY_CONFIG: SecurityConfig = SecurityConfigSchema.parse({}); + +/** + * Extract the final message content from Codex `--json` JSONL output. + * + * Codex with `--json` emits one JSON object per line to stdout: + * {"type":"message","content":"Hello, world!"} + * {"type":"tool_call","name":"bash","input":"..."} + * {"type":"tool_result","output":"..."} + * {"type":"message","content":"Final answer"} + * + * We find the last `type === "message"` event and return its `content` field. + * Falls back to raw stdout if no parseable message event is found. + */ +export function parseCodexJsonlOutput(stdout: string): string { + const lines = stdout.split('\n').filter(Boolean); + let lastContent: string | undefined; + + for (const line of lines) { + try { + const event = JSON.parse(line) as Record; + if (event['type'] === 'message' && typeof event['content'] === 'string') { + lastContent = event['content']; + } + } catch { + // Not valid JSON — skip (could be mixed terminal output or partial lines) + } + } + + return lastContent ?? stdout; +} + +/** + * Extract human-readable text from a single Codex `--json` streaming JSONL chunk. + * + * Codex with `--json` emits one JSON event per line during streaming. This function + * parses a single line and extracts visible text content, returning null for events + * that are not user-visible (thread.started, item.started without output, etc.). + * + * Handled event types: + * - `type: "message"` with `content` string → returns content + * - `type: "item.completed"` + `item.type: "command_execution"` → formatted output + * - `type: "item.completed"` + `item.type: "reasoning"` → reasoning text (if not hidden) + * - All other event types → null (not user-visible) + * + * @param chunk A single line from Codex `--json` streaming output + * @returns Human-readable text, or null if the event has no user-visible content + */ +export function parseCodexStreamChunk(chunk: string): string | null { + const line = chunk.trim(); + if (!line) return null; + + let event: Record; + try { + event = JSON.parse(line) as Record; + } catch { + // Not valid JSON — could be a partial line or mixed terminal output + return null; + } + + const type = event['type']; + + // type: "message" — final or intermediate message with string content + if (type === 'message' && typeof event['content'] === 'string') { + return event['content']; + } + + // type: "item.completed" — completed reasoning step or command execution + if (type === 'item.completed') { + const item = event['item']; + if (!item || typeof item !== 'object') return null; + const itemObj = item as Record; + const itemType = itemObj['type']; + + // Reasoning step — extract text (skip if hidden or empty) + if (itemType === 'reasoning') { + const text = itemObj['text']; + if (typeof text === 'string' && text.trim()) { + return `[thinking] ${text.trim()}`; + } + return null; + } + + // Command execution output + if (itemType === 'command_execution') { + const output = itemObj['output']; + if (typeof output === 'string' && output.trim()) { + return `[cmd] ${output.trim()}`; + } + return null; + } + } + + // All other event types (thread.started, item.started, etc.) — not user-visible + return null; +} + +/** Map capability levels to codex sandbox modes */ +const CAPABILITY_TO_SANDBOX: Record = { + 'read-only': 'read-only', + 'code-edit': 'workspace-write', + 'full-access': 'danger-full-access', +}; + +/** + * System prompt constraints injected when tool restrictions are specified via allowedTools. + * + * Codex CLI does NOT support --allowedTools (Claude-style named tool restriction lists). + * Instead, we rely on two complementary mechanisms: + * 1. --sandbox — shell-level access control (blocks filesystem writes in read-only) + * 2. System prompt — behavioral guidance telling Codex how to approach the task + * + * Without explicit guidance, Codex workers on read-only tasks tend to resort to complex + * inline Python scripts via `/bin/zsh -lc "python -c '...'"` with deeply nested shell + * escaping, exhausting their turn budget without reading any files. The constraint text + * below steers Codex toward simple, direct commands that work within the sandbox. + */ +const SANDBOX_CONSTRAINTS: Record = { + 'read-only': + 'IMPORTANT: For this task, only READ files. Do NOT create, modify, or delete any files.\n' + + 'Do NOT run complex bash scripts, inline Python (-c "..."), or deeply nested shell escaping.\n' + + 'Use simple, direct shell commands: cat, head, tail, grep, find, ls.\n' + + 'Example: to read a file use `cat /path/to/file`. To search use `grep -r "pattern" /dir`.\n' + + 'Keep every command short and direct — avoid multi-line shell gymnastics.', + 'workspace-write': + 'For this task, you can read and write files.\n' + + 'Prefer direct file operations (cat, head, tail, grep, sed, echo, tee) over complex scripts.\n' + + 'Keep shell commands simple and avoid deeply nested shell escaping.', +}; + +export class CodexAdapter implements CLIAdapter { + readonly name = 'codex'; + private readonly securityConfig: SecurityConfig; + + constructor(securityConfig?: SecurityConfig) { + this.securityConfig = securityConfig ?? DEFAULT_SECURITY_CONFIG; + } + + buildSpawnConfig(opts: SpawnOptions): CLISpawnConfig { + // Codex CLI supports multiple auth methods: + // 1. `codex login` — OAuth via ChatGPT (zero API key, like Claude Code CLI) + // 2. OPENAI_API_KEY env var — direct API key auth + // We don't check for either here — Codex handles auth internally and gives + // a clear error if the user isn't authenticated. + + // Start with `exec` subcommand for non-interactive mode + const args: string[] = ['exec', '--skip-git-repo-check']; + + if (opts.model) { + args.push('--model', opts.model); + } + + // Map allowedTools → sandbox mode via heuristic. + // Codex does not support --allowedTools (Claude-style named tool lists). Instead, we: + // 1. Infer a --sandbox mode from the tool list for shell-level access control + // 2. Inject a system prompt constraint to guide behavioral compliance (see below) + const sandboxMode = this.inferSandboxMode(opts.allowedTools); + if (opts.allowedTools && opts.allowedTools.length > 0) { + logger.debug( + { allowedTools: opts.allowedTools, sandboxMode }, + 'codex: --allowedTools not supported — mapped to --sandbox mode + system prompt constraints', + ); + } + if (sandboxMode) { + if (sandboxMode === 'danger-full-access') { + // Use --full-auto for full access (enables auto-approve + full sandbox) + args.push('--full-auto'); + } else { + args.push('--sandbox', sandboxMode); + } + } + + // Session management: + // - resumeSessionId set → resume last session via `exec resume --last` (or explicit ID) + // - sessionId set → new named session start (no --ephemeral so Codex saves state) + // - neither set → ephemeral (no session persistence, current worker behavior) + if (opts.resumeSessionId) { + // Insert `resume --last` (or explicit session ID) after `exec`. + // codex exec resume --last [OPTIONS] + args.splice( + 1, + 0, + 'resume', + opts.resumeSessionId === '__last__' ? '--last' : opts.resumeSessionId, + ); + } else if (!opts.sessionId) { + // Worker spawn path — stateless, no session saved. + args.push('--ephemeral'); + } + // When sessionId is set (new provider session), omit --ephemeral so Codex saves state. + + // MCP passthrough: Codex supports MCP natively via `codex mcp add`. + // When a config path is provided, pass it via the -c flag so Codex can + // load MCP server definitions without requiring global pre-configuration. + if (opts.mcpConfigPath) { + logger.debug({ mcpConfigPath: opts.mcpConfigPath }, 'codex: passing MCP config via -c flag'); + args.push('-c', opts.mcpConfigPath); + } + + // Enable JSONL structured output — each event is a JSON object on its own line. + // AgentRunner applies parseOutput() to extract the final message content. + args.push('--json'); + + // -o / --output-last-message: Codex's recommended way to capture the final answer reliably. + // Generates a unique temp file; parseOutput reads it after process exit and cleans up. + const tempFile = join(tmpdir(), `ob-codex-${Date.now()}-${randomBytes(4).toString('hex')}.txt`); + args.push('-o', tempFile); + + // Build the combined prompt with system-level guidance prepended. + // Codex has no --append-system-prompt flag, so all guidance is prepended to the prompt. + // + // Order (first → last in the text): + // 1. Sandbox constraint — behavioral guidance when tool restrictions were specified + // 2. User system prompt — caller-supplied context or instructions + // 3. Task prompt — the actual task description + let prompt = sanitizePrompt( + opts.prompt, + this.getPromptBudget(opts.model).maxPromptChars, + 'worker', + ); + const systemParts: string[] = []; + + // Inject behavioral constraint based on sandbox mode — always applied so Codex + // receives guidance appropriate for its access level. In particular, read-only + // workers always get the file-read-only instruction, preventing shell gymnastics + // even when no explicit allowedTools list was passed. + const constraint = SANDBOX_CONSTRAINTS[sandboxMode]; + if (constraint) { + systemParts.push(constraint); + } + + if (opts.systemPrompt) { + systemParts.push(opts.systemPrompt); + } + + if (systemParts.length > 0) { + prompt = systemParts.join('\n\n') + '\n\n' + prompt; + } + + // Prompt is positional for codex exec + args.push(prompt); + + // Log dropped options at debug level + if (opts.maxTurns) { + logger.debug({ maxTurns: opts.maxTurns }, 'codex: --max-turns not supported, ignoring'); + } + if (opts.maxBudgetUsd) { + logger.debug( + { maxBudgetUsd: opts.maxBudgetUsd }, + 'codex: --max-budget-usd not supported, ignoring', + ); + } + return { + binary: 'codex', + args, + env: this.cleanEnv({ ...process.env }), + parseStreamChunk: parseCodexStreamChunk, + parseOutput: (stdout: string): string => { + // Prefer the -o temp file — Codex writes the final answer there reliably. + // Falls back to --json JSONL parsing if the file is missing or unreadable + // (older Codex versions, unsupported flag, or process crash before write). + try { + const content = readFileSync(tempFile, 'utf-8').trim(); + if (content) { + logger.debug( + { tempFile, contentLength: content.length }, + 'codex: tempfile output read successfully — using as primary source', + ); + try { + unlinkSync(tempFile); + logger.debug({ tempFile }, 'codex: tempfile cleaned up'); + } catch { + // Best-effort cleanup — not critical if it fails + } + return content; + } + // File exists but is empty — Codex may not have written the final answer + logger.warn( + { tempFile }, + 'codex: tempfile exists but is empty — falling back to JSONL stdout parsing', + ); + } catch { + // Temp file not found or unreadable — normal if -o flag is unsupported + logger.debug( + { tempFile }, + 'codex: tempfile not found — falling back to JSONL stdout parsing', + ); + } + return parseCodexJsonlOutput(stdout); + }, + }; + } + + cleanEnv(env: Record): Record { + // Remove Claude env vars in case both Claude and Codex are installed + const cleaned = { ...env }; + for (const key of Object.keys(cleaned)) { + if ( + key === 'CLAUDECODE' || + key.startsWith('CLAUDE_CODE_') || + key.startsWith('CLAUDE_AGENT_SDK_') + ) { + delete cleaned[key]; + } + } + return sanitizeEnv(cleaned, this.securityConfig); + } + + mapCapabilityLevel(_level: CapabilityLevel): string[] | undefined { + // Codex doesn't use tool lists — it uses sandbox modes. + // Return undefined; the sandbox mode is set in buildSpawnConfig via inferSandboxMode. + return undefined; + } + + supportedProfiles(): readonly CapabilityLevel[] { + // Codex uses --sandbox modes + system-prompt constraints, not --allowedTools named + // tool lists. No profiles are natively enforced via named tool restrictions. + // The adapter emulates profile restrictions via sandbox mode and injected constraints. + return []; + } + + isValidModel(model: string): boolean { + // ChatGPT-account auth only supports gpt-5.2-codex (the default in v0.104.0). + // o3, o4-mini, codex-mini are rejected with "not supported when using Codex with a ChatGPT account". + // API-key auth may support more models — accept OpenAI-style IDs for forward compat. + const codexModels = [ + 'gpt-5.2-codex', // default — works with both ChatGPT auth and API key + ]; + if (codexModels.includes(model)) return true; + // Accept OpenAI-style model IDs for forward compatibility (API-key users) + return /^(gpt-|o[0-9]|codex)/.test(model); + } + + getPromptBudget(_model?: string): { maxPromptChars: number; maxSystemPromptChars: number } { + // Codex merges systemPrompt INTO the prompt positional argument — there is no + // separate system-prompt channel (no --append-system-prompt flag). Both system + // context and task description are concatenated into a single string before + // being passed to `codex exec` as one positional arg. + // + // Because of this merger the two fields share the same underlying budget. + // We return a conservative combined limit based on OpenAI model context windows: + // - gpt-5.3-codex / gpt-5.2-codex: ~400K token context window (~1.6M chars) + // - We use 400K chars total as a conservative combined budget to leave + // ample room for tool call outputs and model response tokens. + // + // Both fields are set to the same value to signal that they share a single pool + // rather than having independent channels. The PromptAssembler should treat these + // as a combined budget when targeting CodexAdapter. + const combined = 400_000; + return { maxPromptChars: combined, maxSystemPromptChars: combined }; + } + + /** + * Infer codex sandbox mode from Claude-style allowedTools list. + * + * Heuristic: + * - Bash(*) present → danger-full-access (unrestricted) + * - Edit or Write present → workspace-write (file modifications) + * - Otherwise (incl. empty/undefined) → read-only (safe default) + */ + private inferSandboxMode(allowedTools?: string[]): string { + if (!allowedTools || allowedTools.length === 0) return 'read-only'; + + const hasUnrestrictedBash = allowedTools.some((t) => t === 'Bash(*)'); + const hasEditWrite = allowedTools.some((t) => t === 'Edit' || t === 'Write'); + + if (hasUnrestrictedBash) return 'danger-full-access'; + if (hasEditWrite) return 'workspace-write'; + return 'read-only'; + } +} + +export { CAPABILITY_TO_SANDBOX }; diff --git a/src/core/adapters/index.ts b/src/core/adapters/index.ts new file mode 100644 index 00000000..3faff3a7 --- /dev/null +++ b/src/core/adapters/index.ts @@ -0,0 +1,5 @@ +export { ClaudeAdapter } from './claude-adapter.js'; +export { ClaudeSDKAdapter } from './claude-sdk.js'; +export { CodexAdapter } from './codex-adapter.js'; +export { AiderAdapter } from './aider-adapter.js'; +export type { CLIAdapter, CLISpawnConfig, CapabilityLevel } from '../cli-adapter.js'; diff --git a/src/core/agent-runner.ts b/src/core/agent-runner.ts new file mode 100644 index 00000000..091caa5d --- /dev/null +++ b/src/core/agent-runner.ts @@ -0,0 +1,2547 @@ +import { spawn as nodeSpawn } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { mkdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, isAbsolute, resolve, sep } from 'node:path'; +import { createLogger } from './logger.js'; +import { DockerSandbox } from './docker-sandbox.js'; +import { sanitizeEnv } from './env-sanitizer.js'; +import { BUILT_IN_PROFILES } from '../types/agent.js'; +import type { TaskManifest, ToolProfile } from '../types/agent.js'; +import type { ModelRegistry } from './model-registry.js'; +import type { CLIAdapter, CLISpawnConfig } from './cli-adapter.js'; +import { ClaudeAdapter } from './adapters/claude-adapter.js'; +import { getClaudePromptBudget } from './adapters/claude-budget.js'; +import type { SandboxConfig, SecurityConfig, WorkspaceTrustLevel } from '../types/config.js'; +import { classifyError, isMaxTurnsExhausted, isRateLimitError } from './error-classifier.js'; +import { checkProfileCostSpike, estimateCostUsd, getProfileCostCap } from './cost-manager.js'; +import type { MetricsCollector } from './metrics.js'; +export type { ErrorCategory } from './error-classifier.js'; +export { + MAX_TURNS_PATTERNS, + classifyError, + isMaxTurnsExhausted, + isRateLimitError, +} from './error-classifier.js'; +export type { CostEstimate } from './cost-manager.js'; +export { + PROFILE_COST_CAPS, + getProfileCostCap, + checkProfileCostSpike, + getProfileCostAverages, + resetProfileCostAverages, + estimateCostUsd, + estimateCost, +} from './cost-manager.js'; + +const logger = createLogger('agent-runner'); + +/** Seconds of execution budget allocated per agent turn for timeout derivation. */ +const SECS_PER_TURN = 30; + +/** + * Returns model-aware max prompt length. + * Opus 4.6 / Sonnet 4.6: 128_000 chars (1M token context window). + * Haiku 4.5 and all others: 32_768 chars (conservative default). + */ +export function getMaxPromptLength(model?: string): number { + return getClaudePromptBudget(model).maxPromptChars; +} + +/** Module-level metrics collector — set once by the host (e.g. Bridge) via setAgentRunnerMetrics(). */ +let _promptMetrics: MetricsCollector | null = null; + +/** + * Wire a MetricsCollector into the agent-runner module so that + * `truncatePrompt` can emit prompt-size metrics without requiring + * the collector to be threaded through every call site. + */ +export function setAgentRunnerMetrics(collector: MetricsCollector | null): void { + _promptMetrics = collector; +} + +/** + * Default max-turns limits to prevent runaway agents. + * Without --max-turns, Claude can make unlimited tool calls until + * the process timeout kills it (OB-F14: exit code 143 / SIGTERM). + */ + +/** Max turns for exploration tasks (file listing, classification) — fast, bounded */ +export const DEFAULT_MAX_TURNS_EXPLORATION = 15; + +/** Max turns for user-facing tasks (implementation, reasoning) — more room to work */ +export const DEFAULT_MAX_TURNS_TASK = 25; + +/** + * Accepted model short names for the --model flag. + * @deprecated Use ModelRegistry with capability tiers ('fast', 'balanced', 'powerful') instead. + * Kept for backward compatibility — these are the Claude-specific aliases. + */ +export const MODEL_ALIASES = ['haiku', 'sonnet', 'opus'] as const; +export type ModelAlias = (typeof MODEL_ALIASES)[number]; + +/** + * Model fallback chain: opus → sonnet → haiku. + * @deprecated Use ModelRegistry.getFallback() for provider-agnostic fallback. + * If the preferred model is unavailable or rate-limited, the runner + * falls back to the next model in the chain before retrying. + */ +export const MODEL_FALLBACK_CHAIN: Record = { + opus: 'sonnet', + sonnet: 'haiku', + haiku: undefined, // no further fallback +}; + +/** + * Default maximum number of lint/test fix iterations before escalating to Master. + * Configurable via `worker.maxFixIterations` in config (OB-1791). + */ +export const DEFAULT_MAX_FIX_ITERATIONS = 3; + +/** + * Patterns in worker stdout that indicate a lint/test fix iteration. + * Each pattern match signals one fix attempt cycle (run checks → fix errors). + * Matched case-insensitively against accumulated stdout. + */ +export const FIX_ITERATION_PATTERNS: RegExp[] = [ + /\bnpm run lint\b/i, + /\bnpm run typecheck\b/i, + /\bvite(?:st)?\s+run\b/i, + /\bRunning\s+(?:lint|test|typecheck)\s+fix/i, + /\bAttempting\s+to\s+fix\b/i, + /\bFix\s+attempt\s+\d+\b/i, + /\bApplying\s+(?:lint|type)\s+fix/i, + /\bRe-running\s+(?:lint|tests?|typecheck)\b/i, +]; + +/** + * Count the number of fix iteration signals in worker stdout. + * Each pattern match counts as one fix attempt. + * Used to enforce the maxFixIterations cap. + */ +export function countFixIterations(stdout: string): number { + let count = 0; + for (const pattern of FIX_ITERATION_PATTERNS) { + const globalPattern = new RegExp( + pattern.source, + pattern.flags.includes('g') ? pattern.flags : pattern.flags + 'g', + ); + const matches = stdout.match(globalPattern); + if (matches) count += matches.length; + } + return count; +} + +/** + * Patterns that identify unresolved error lines in worker stdout. + * Used by extractRemainingErrors() to surface errors that the worker + * failed to fix before hitting the iteration cap (OB-1790). + */ +const REMAINING_ERROR_PATTERNS: RegExp[] = [ + /\berror\s+TS\d+:/i, // TypeScript: error TS2345: + /\btype\s+error\b/i, // TypeScript: Type error + /^\d+\s+error/i, // "3 errors" summary line + /\berror\b[^:]*:\s+.{10,}/i, // generic "error: some message" + /[✕✗×]/, // test failure symbols + /\bfailing\b/i, // "2 failing" + /\bFAIL\b/, // Jest/Vitest FAIL + /\bAssertionError\b/i, // assertion failure + /Expected.*Received/is, // Jest/Vitest assertion diff + /^\s*●\s+.{10,}/, // Jest bullet error +]; + +/** + * Extract unresolved error lines from worker stdout. + * + * Scans the last 3 000 characters of output (most recent activity) for + * lint, TypeScript, and test failure patterns. Returns up to 10 distinct + * error lines that likely represent issues the worker could not fix before + * hitting the iteration cap. + * + * Used to build the error-details section of the [FIX CAP REACHED] report + * injected into the Master session (OB-1790). + */ +export function extractRemainingErrors(stdout: string): string[] { + // Focus on the tail — most recent output is the most relevant + const tail = stdout.length > 3_000 ? stdout.slice(-3_000) : stdout; + const lines = tail + .split('\n') + .map((l) => l.trim()) + .filter(Boolean); + + const seen = new Set(); + const errors: string[] = []; + + for (const line of lines) { + if (errors.length >= 10) break; + if (REMAINING_ERROR_PATTERNS.some((re) => re.test(line))) { + const truncated = line.slice(0, 200); + if (!seen.has(truncated)) { + seen.add(truncated); + errors.push(truncated); + } + } + } + + return errors; +} + +/** + * Patterns in streaming stdout that indicate a new agent turn has started. + * Matched against each chunk to extract the current turn number. + * Claude CLI emits these markers at the beginning of each agentic turn. + */ +const TURN_INDICATOR_PATTERNS: RegExp[] = [ + /\bturn\s+(\d+)\b/i, // "Turn 1", "Turn 2", "agentic turn 3" + /"turn":\s*(\d+)/, // JSON: "turn": 1 + /\((\d+)\s+agentic\s+turn/i, // "(3 agentic turns used)" + /step\s+(\d+)\s+of\s+\d+/i, // "Step 1 of 25" +]; + +/** + * Result of parsing a turn indicator from a streaming stdout chunk. + */ +export interface TurnIndicator { + /** Number of agentic turns used so far */ + turnsUsed: number; + /** The last action text extracted from the chunk, if detectable */ + lastAction?: string; +} + +/** + * Parse a turn indicator from a chunk of Claude CLI streaming stdout. + * + * Returns a `TurnIndicator` if a turn marker is found in the chunk, + * or `null` if the chunk contains no recognizable turn information. + * Used by workers streaming real-time progress via `execOnceStreaming()`. + */ +export function parseTurnIndicator(chunk: string): TurnIndicator | null { + for (const pattern of TURN_INDICATOR_PATTERNS) { + const match = chunk.match(pattern); + if (match?.[1]) { + const turnsUsed = parseInt(match[1], 10); + if (!isNaN(turnsUsed) && turnsUsed > 0) { + // Extract a short lastAction hint from the first non-empty line of the chunk + const firstLine = chunk + .split('\n') + .map((l) => l.trim()) + .find((l) => l.length > 0); + return { turnsUsed, lastAction: firstLine }; + } + } + } + return null; +} + +/** + * Get the next model in the fallback chain for a given model. + * If a ModelRegistry is provided, uses tier-aware fallback (provider-agnostic). + * Otherwise falls back to the hardcoded Claude chain. + */ +export function getNextFallbackModel( + currentModel: string, + registry?: ModelRegistry, +): string | undefined { + if (registry) { + return registry.getFallback(currentModel); + } + return MODEL_FALLBACK_CHAIN[currentModel] ?? (currentModel === 'haiku' ? undefined : 'sonnet'); +} + +/** + * Validate a model string. + * If a ModelRegistry is provided, checks against registered models (provider-agnostic). + * Otherwise falls back to Claude-specific validation. + */ +export function isValidModel(model: string, registry?: ModelRegistry): boolean { + if (registry) { + return registry.isValid(model); + } + if (MODEL_ALIASES.includes(model as ModelAlias)) return true; + // Full model IDs follow the pattern: claude-- + return /^claude-[a-z0-9]+-[a-z0-9._-]+$/.test(model); +} + +/** + * Tool group constants for --allowedTools. + * Used instead of --dangerously-skip-permissions to give agents + * only the tools they need for a given task type. + */ + +/** Read-only tools — safe for exploration and information gathering */ +export const TOOLS_READ_ONLY = ['Read', 'Glob', 'Grep'] as const; + +/** Data query tools — read-only data exploration with query commands (no file modifications) */ +export const TOOLS_DATA_QUERY = [ + 'Read', + 'Glob', + 'Grep', + 'Bash(sqlite3:*)', + 'Bash(python3:*)', + 'Bash(node:*)', + 'Bash(jq:*)', + 'Bash(awk:*)', + 'Bash(head:*)', + 'Bash(tail:*)', + 'Bash(wc:*)', + 'Bash(sort:*)', + 'Bash(uniq:*)', + 'Bash(cut:*)', +] as const; + +/** Code editing tools — for implementation tasks that modify files */ +export const TOOLS_CODE_EDIT = [ + 'Read', + 'Edit', + 'Write', + 'Glob', + 'Grep', + 'Bash(git:*)', + 'Bash(npm:*)', + 'Bash(npx:*)', + 'Bash(rm:*)', + 'Bash(mv:*)', + 'Bash(cp:*)', + 'Bash(mkdir:*)', +] as const; + +/** Full access tools — unrestricted (use sparingly) */ +export const TOOLS_FULL = ['Read', 'Edit', 'Write', 'Glob', 'Grep', 'Bash(*)'] as const; + +/** File management tools — for moving, copying, or deleting files/directories within the workspace */ +export const TOOLS_FILE_MANAGEMENT = [ + 'Read', + 'Glob', + 'Grep', + 'Write', + 'Edit', + 'Bash(rm:*)', + 'Bash(mv:*)', + 'Bash(cp:*)', + 'Bash(mkdir:*)', + 'Bash(chmod:*)', + 'Bash(git:*)', +] as const; + +/** Code audit tools — read files and run test/lint/typecheck commands, no file modifications */ +export const TOOLS_CODE_AUDIT = [ + 'Read', + 'Glob', + 'Grep', + 'Bash(npm:test)', + 'Bash(npm:run:lint)', + 'Bash(npm:run:typecheck)', + 'Bash(npx:vitest:*)', + 'Bash(npx:eslint:*)', + 'Bash(npx:tsc:*)', + 'Bash(npm:run:test:*)', + 'Bash(pytest:*)', + 'Bash(cargo:test)', +] as const; + +/** + * Resolve a profile name to its tool list. + * Checks custom profiles first (if provided), then falls back to built-in profiles. + * Returns undefined if the profile name is not recognized in either source. + */ +export function resolveProfile( + profileName: string, + customProfiles?: Record, + trustLevel?: WorkspaceTrustLevel, +): string[] | undefined { + if (trustLevel === 'trusted') return [...TOOLS_FULL]; + if (trustLevel === 'sandbox') return [...TOOLS_READ_ONLY]; + if (customProfiles) { + const custom = customProfiles[profileName]; + if (custom) return custom.tools; + } + const profile = BUILT_IN_PROFILES[profileName as keyof typeof BUILT_IN_PROFILES]; + return profile?.tools; +} + +/** + * Resolve a profile name to its tool list using the built-in TOOLS_* constants. + * For profiles not covered by the constants, falls back to resolveProfile(). + * Used when profile: 'code-audit' (or other built-in names) is requested via + * SPAWN markers or config — returns the correct tool list for --allowedTools flags. + */ +export function resolveTools( + profileName: string, + customProfiles?: Record, + trustLevel?: WorkspaceTrustLevel, +): string[] | undefined { + switch (profileName) { + case 'read-only': + return [...TOOLS_READ_ONLY]; + case 'data-query': + return [...TOOLS_DATA_QUERY]; + case 'code-edit': + return [...TOOLS_CODE_EDIT]; + case 'file-management': + return [...TOOLS_FILE_MANAGEMENT]; + case 'full-access': + return [...TOOLS_FULL]; + case 'code-audit': + return [...TOOLS_CODE_AUDIT]; + default: + return resolveProfile(profileName, customProfiles, trustLevel); + } +} + +/** + * Check whether a target path is within the given workspace root. + * + * Both paths are resolved to absolute form before comparison so that relative + * paths and `..` components are handled correctly. + * + * Returns `false` when the resolved target is NOT a descendant of `workspacePath`, + * which signals that a destructive operation (`rm`, `mv`) would escape the workspace. + * + * Exported so callers and unit tests can validate paths independently (OB-1494). + */ +export function isPathWithinWorkspace(targetPath: string, workspacePath: string): boolean { + const resolvedTarget = isAbsolute(targetPath) + ? resolve(targetPath) + : resolve(workspacePath, targetPath); + const resolvedWorkspace = resolve(workspacePath); + // Append sep so '/workspace-extra' does not falsely match '/workspace' + const workspaceWithSep = resolvedWorkspace.endsWith(sep) + ? resolvedWorkspace + : resolvedWorkspace + sep; + return resolvedTarget === resolvedWorkspace || resolvedTarget.startsWith(workspaceWithSep); +} + +/** + * Command patterns used to extract paths from worker stdout. + * 'destructive' entries (rm, mv) are logged at ERROR level. + * 'boundary' entries (cat, cp, curl, etc.) are read-only escapes — logged at WARN. + */ +const BOUNDARY_CMD_PATTERNS: Array<{ + cmd: string; + re: RegExp; + severity: 'destructive' | 'boundary'; +}> = [ + { cmd: 'rm', re: /\brm\s+(?:-[a-zA-Z]+\s+)*([^\s;|&><"']+)/g, severity: 'destructive' }, + { cmd: 'mv', re: /\bmv\s+(?:-[a-zA-Z]+\s+)*([^\s;|&><"']+)/g, severity: 'destructive' }, + { cmd: 'cat', re: /\bcat\s+(?:-[a-zA-Z]+\s+)*([^\s;|&><"']+)/g, severity: 'boundary' }, + { cmd: 'cp', re: /\bcp\s+(?:-[a-zA-Z]+\s+)*([^\s;|&><"']+)/g, severity: 'boundary' }, + { cmd: 'scp', re: /\bscp\s+(?:-[a-zA-Z]+\s+)*([^\s;|&><"']+)/g, severity: 'boundary' }, + { cmd: 'curl', re: /\bcurl\s+[^;|&\n]*?-o\s+([^\s;|&><"']+)/g, severity: 'boundary' }, + { cmd: 'wget', re: /\bwget\s+[^;|&\n]*?-O\s+([^\s;|&><"']+)/g, severity: 'boundary' }, + { cmd: 'rsync', re: /\brsync\s+(?:-[a-zA-Z]+\s+)*([^\s;|&><"']+)/g, severity: 'boundary' }, + { cmd: 'ln', re: /\bln\s+(?:-[a-zA-Z]+\s+)*([^\s;|&><"']+)/g, severity: 'boundary' }, +]; + +/** + * Scan worker stdout for shell commands whose target paths fall outside the configured + * `workspacePath`. Covers destructive commands (`rm`, `mv`) and boundary-escaping + * read/copy/transfer commands (`cat`, `cp`, `scp`, `curl`, `wget`, `rsync`, `ln`). + * + * Returns an array of violations with a `severity` field: + * - `'destructive'`: rm/mv targeting paths outside workspace (logged at ERROR) + * - `'boundary'`: read/copy commands accessing paths outside workspace (logged at WARN) + * + * This is a best-effort text scan — it cannot replace a real shell parser but catches + * the common cases where absolute paths outside the workspace are referenced. + * + * Exported for unit testing (OB-1494). + */ +export function scanDestructiveCommandViolations( + stdout: string, + workspacePath: string, +): Array<{ command: string; path: string; severity: 'destructive' | 'boundary' }> { + const violations: Array<{ command: string; path: string; severity: 'destructive' | 'boundary' }> = + []; + + for (const { cmd, re, severity } of BOUNDARY_CMD_PATTERNS) { + // Re-create with global flag to reset lastIndex on each call + const globalRe = new RegExp(re.source, re.flags.includes('g') ? re.flags : re.flags + 'g'); + let match: RegExpExecArray | null; + while ((match = globalRe.exec(stdout)) !== null) { + const targetPath = match[1]; + if (targetPath && !isPathWithinWorkspace(targetPath, workspacePath)) { + violations.push({ command: cmd, path: targetPath, severity }); + } + } + } + + return violations; +} + +/** + * Result returned by manifestToSpawnOptions(). + * Contains the resolved SpawnOptions and a cleanup function that deletes + * any temporary files created for per-worker MCP isolation. + */ +export interface ManifestSpawnResult { + /** Resolved spawn options ready for AgentRunner.spawn() */ + spawnOptions: SpawnOptions; + /** + * Async cleanup function — call after spawn completes (success or failure). + * Deletes the per-worker MCP temp file if one was created; no-op otherwise. + */ + cleanup: () => Promise; +} + +/** + * Convert a TaskManifest into SpawnOptions. + * + * Resolution rules: + * - If `allowedTools` is provided explicitly, it takes priority over `profile` + * - If only `profile` is provided, resolve it via custom profiles then built-in + * - If neither is provided, no tools restriction is applied + * - All other fields map directly to SpawnOptions equivalents + * + * Per-worker MCP isolation: + * - When `manifest.mcpServers` is non-empty, a temporary JSON file is written + * containing only those servers (not all globally configured servers). + * - `spawnOptions.mcpConfigPath` is set to the temp file path and + * `strictMcpConfig` is set to `true` so the worker cannot access any + * globally configured MCP servers. + * - Call `cleanup()` after the spawn completes to delete the temp file. + */ +export async function manifestToSpawnOptions( + manifest: TaskManifest, + customProfiles?: Record, +): Promise { + let allowedTools: string[] | undefined = manifest.allowedTools; + + if (!allowedTools && manifest.profile) { + const resolved = resolveProfile(manifest.profile, customProfiles); + if (resolved) { + allowedTools = resolved; + } else { + logger.warn( + { profile: manifest.profile }, + 'Unknown profile name — no tools restriction applied', + ); + } + } + + const baseOptions: SpawnOptions = { + prompt: manifest.prompt, + workspacePath: manifest.workspacePath, + model: manifest.model, + allowedTools, + maxTurns: manifest.maxTurns, + timeout: manifest.timeout, + retries: manifest.retries, + retryDelay: manifest.retryDelay, + maxBudgetUsd: manifest.maxBudgetUsd, + maxCostUsd: manifest.maxCostUsd, + profile: manifest.profile, + }; + + // No MCP servers requested — return immediately with a no-op cleanup + if (!manifest.mcpServers || manifest.mcpServers.length === 0) { + return { + spawnOptions: baseOptions, + cleanup: async (): Promise => { + /* no-op */ + }, + }; + } + + // Per-worker MCP isolation: write a temp file with ONLY the requested servers. + // This ensures each worker sees only the MCP servers it needs, not all + // globally configured servers (security: least-privilege per worker). + const tempFilePath = `${tmpdir()}/ob-mcp-${randomUUID()}.json`; + + const mcpServersConfig: Record< + string, + { command: string; args?: string[]; env?: Record } + > = {}; + for (const server of manifest.mcpServers) { + mcpServersConfig[server.name] = { + command: server.command, + ...(server.args !== undefined ? { args: server.args } : {}), + ...(server.env !== undefined ? { env: server.env } : {}), + }; + } + + await writeFile(tempFilePath, JSON.stringify({ mcpServers: mcpServersConfig }, null, 2), 'utf-8'); + + logger.debug( + { tempFilePath, serverCount: manifest.mcpServers.length }, + 'Wrote per-worker MCP config — worker sees only its requested servers', + ); + + return { + spawnOptions: { + ...baseOptions, + mcpConfigPath: tempFilePath, + strictMcpConfig: true, + }, + cleanup: async (): Promise => { + try { + await rm(tempFilePath, { force: true }); + logger.debug({ tempFilePath }, 'Cleaned up per-worker MCP temp file'); + } catch (err) { + logger.warn({ tempFilePath, err }, 'Failed to clean up MCP temp file'); + } + }, + }; +} + +/** + * Apply graduated size checks and hard truncation to a prompt. + * + * - Logs WARN when the prompt exceeds 80 % of `maxLength` so callers can + * trigger early compaction or investigate prompt bloat. + * - Logs WARN (with a "lost" byte count) when the prompt is actually + * truncated (> 100 % of `maxLength`). + * - `context` identifies the call site in the log so operators can tell + * whether bloat is coming from exploration, message-processing, or a + * worker spawn. + */ +export function truncatePrompt( + prompt: string, + maxLength: number = getMaxPromptLength(), + context: string = 'unknown', +): string { + const warnThreshold = Math.floor(maxLength * 0.8); + + if (prompt.length > maxLength) { + const bytesLost = prompt.length - maxLength; + const percentLost = Math.round((bytesLost / prompt.length) * 100); + logger.warn( + { + context, + originalChars: prompt.length, + maxLength, + bytesLost, + percentLost, + }, + `[${context}] Prompt truncated: ${bytesLost} chars lost (${percentLost}% of content, limit ${maxLength})`, + ); + _promptMetrics?.recordPromptSize(prompt.length, maxLength, percentLost); + return prompt.slice(0, maxLength); + } + + if (prompt.length > warnThreshold) { + const pct = Math.round((prompt.length / maxLength) * 100); + logger.warn( + { + context, + promptChars: prompt.length, + maxLength, + usagePct: pct, + }, + `[${context}] Prompt at ${pct}% of limit (${prompt.length}/${maxLength} chars) — consider early compaction`, + ); + } + + _promptMetrics?.recordPromptSize(prompt.length, maxLength, 0); + return prompt; +} + +/** + * Sanitize a user-supplied prompt before passing it to the CLI. + * + * Removes null bytes and ASCII control characters (except tab, newline, and + * carriage return). Delegates size checking and truncation to `truncatePrompt`. + * spawn() is used without shell: true, so shell metacharacters are already safe. + * + * Pass the adapter-aware budget via `maxLength` so Master prompts use the + * correct provider limit instead of the hardcoded 32 K default. + * Pass `context` to identify the call site in truncation warnings. + */ +export function sanitizePrompt( + prompt: string, + maxLength: number = getMaxPromptLength(), + context: string = 'unknown', +): string { + // eslint-disable-next-line no-control-regex + const cleaned = prompt.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, ''); + return truncatePrompt(cleaned, maxLength, context); +} + +/** Options accepted by AgentRunner.spawn() */ +export interface SpawnOptions { + /** The prompt to send to the AI agent */ + prompt: string; + /** Working directory for the agent */ + workspacePath: string; + /** Model to use: 'haiku', 'sonnet', 'opus', or a full model ID */ + model?: string; + /** List of tools the agent is allowed to use (passed as --allowedTools) */ + allowedTools?: string[]; + /** Maximum number of agentic turns before the agent stops */ + maxTurns?: number; + /** Timeout in milliseconds for each individual attempt */ + timeout?: number; + /** Number of retry attempts on non-zero exit codes (default: 3) */ + retries?: number; + /** Delay in milliseconds between retry attempts (default: 10000) */ + retryDelay?: number; + /** Path to write the full log output */ + logFile?: string; + /** Resume an existing session */ + resumeSessionId?: string; + /** Start a new conversation with a specific session ID */ + sessionId?: string; + /** System prompt to append to the default Claude system prompt */ + systemPrompt?: string; + /** Maximum spend in USD for this agent run (passed as --max-budget-usd) */ + maxBudgetUsd?: number; + /** + * Path to an MCP config JSON file to pass to the agent CLI. + * For Claude: passed as `--mcp-config ` so the worker can use MCP tools. + * For Codex: passed via the `-c` flag when set. + */ + mcpConfigPath?: string; + /** + * When true, passes `--strict-mcp-config` to the Claude CLI, isolating the worker + * from any globally configured MCP servers (e.g. ~/.claude/claude_desktop_config.json). + * Only the servers listed in `mcpConfigPath` will be available to the worker. + */ + strictMcpConfig?: boolean; + /** + * Sandbox configuration for this worker. + * When `sandbox.mode` is `'docker'`, the worker is spawned inside a Docker container + * using the `openbridge-worker:latest` image with workspace volume mounts, + * resource limits, and env var sanitization applied. + * Default: `{ mode: 'none' }` — run as a regular child process. + */ + sandbox?: SandboxConfig; + /** + * Security configuration for env var sanitization inside the Docker sandbox. + * When `sandbox.mode` is `'docker'`, the sanitizer strips env vars matching + * `envDenyPatterns` (unless overridden by `envAllowPatterns`) before passing + * them to the container via `-e` flags. + * If omitted, the default deny/allow patterns from Phase 85 are used. + */ + securityConfig?: SecurityConfig; + /** + * Tool profile name used for per-profile cost cap enforcement (OB-F101). + * When set, the runner checks accumulated cost against PROFILE_COST_CAPS[profile] + * during streaming and logs a WARNING + aborts if the cap is exceeded. + * Populated automatically by manifestToSpawnOptions() from TaskManifest.profile. + */ + profile?: string; + /** + * Per-profile cost cap overrides in USD. + * Merged on top of PROFILE_COST_CAPS defaults — caller-supplied values win. + * Example: `{ 'read-only': 0.25 }` to tighten the default $0.50 cap. + */ + workerCostCaps?: Record; + /** + * Per-worker cost cap in USD (OB-1521). + * When cumulative reported cost exceeds this value during streaming, + * the process is killed with SIGTERM and the result is marked costCapped: true. + * Takes precedence over profile-based caps from workerCostCaps/PROFILE_COST_CAPS. + * Defaults set by worker-orchestrator.ts: read-only=0.05, code-edit=0.10, full-access=0.15. + */ + maxCostUsd?: number; + /** + * Maximum number of lint/test fix iterations before escalating to Master (OB-1789). + * Each time the worker runs lint/test commands and then attempts a fix, that counts + * as one iteration. When the cap is reached, the run is aborted with + * `fixCapReached: true` in the result so Master can decide the next action. + * Defaults to DEFAULT_MAX_FIX_ITERATIONS (3). Set to 0 to disable the cap. + */ + maxFixIterations?: number; + /** + * When true, `buildMasterSpawnOptions()` skips injecting the workspace context summary + * into the system prompt. Use when the prompt already contains the workspace map + * (e.g. incremental exploration prompts that embed `JSON.stringify(currentMap)`). + */ + skipWorkspaceContext?: boolean; +} + +/** + * Handle returned by AgentRunner.spawnWithHandle(). + * Provides the result promise, the PID of the initial child process, + * and an abort function that sends SIGTERM → 5s grace → SIGKILL. + */ +export interface SpawnHandle { + /** Promise that resolves to the final AgentResult (after retries if any) */ + promise: Promise; + /** PID of the initial child process (-1 if the OS did not assign one) */ + pid: number; + /** Abort the currently-running execution (SIGTERM → 5s grace period → SIGKILL) */ + abort: () => void; +} + +/** Result returned from AgentRunner.spawn() */ +export interface AgentResult { + stdout: string; + stderr: string; + exitCode: number; + durationMs: number; + retryCount: number; + /** The model that was requested (undefined = CLI default) */ + model?: string; + /** Models that were tried and fell back from due to rate limits, in order */ + modelFallbacks?: string[]; + /** Estimated cost in USD for this agent run */ + costUsd?: number; + /** + * True when the Claude CLI exited with code 0 but stdout contains a + * max-turns indicator, meaning the worker ran out of its turn budget + * before completing the task. The result is incomplete. + */ + turnsExhausted?: boolean; + /** Last agentic turn count reported during streaming (undefined if not tracked) */ + turnsUsed?: number; + /** Max turns limit that was configured for this run (undefined if not set) */ + maxTurns?: number; + /** + * Completion status of the agent run. + * - 'completed': agent finished within its turn budget + * - 'partial': agent hit the max-turns limit before finishing (turnsExhausted: true) + * - 'fix-cap-reached': agent hit the fix iteration cap (fixCapReached: true) + * - 'cost-capped': agent was killed because cumulative cost exceeded maxCostUsd (costCapped: true) + */ + status: 'completed' | 'partial' | 'fix-cap-reached' | 'cost-capped'; + /** + * Number of lint/test fix iterations detected in worker output (OB-1789). + * Populated when maxFixIterations is set. Undefined if fix cap tracking is disabled. + */ + fixIterationsUsed?: number; + /** + * True when the worker hit the fix iteration cap without resolving all errors (OB-1789). + * Master should inspect the partial output and decide whether to retry, escalate, + * or accept the partial result. + */ + fixCapReached?: boolean; + /** + * True when the worker was killed because cumulative cost exceeded maxCostUsd (OB-1522). + * The result contains partial output collected before the cap was hit. + */ + costCapped?: boolean; +} + +/** Record of a single execution attempt (used for aggregated error reporting) */ +export interface AttemptRecord { + attempt: number; + exitCode: number; + stderr: string; +} + +/** + * Error thrown when all retry attempts are exhausted. + * Contains aggregated details from every attempt so callers can inspect + * what went wrong across the full retry sequence. + */ +export class AgentExhaustedError extends Error { + readonly attempts: AttemptRecord[]; + readonly lastExitCode: number; + readonly totalAttempts: number; + readonly durationMs: number; + + constructor(attempts: AttemptRecord[], durationMs: number) { + const total = attempts.length; + const lastExit = attempts[total - 1]?.exitCode ?? 1; + const summary = attempts + .map( + (a) => + ` attempt ${a.attempt}: exit ${a.exitCode}` + + (a.stderr ? ` — ${a.stderr.slice(0, 200)}` : ''), + ) + .join('\n'); + super(`Agent failed after ${total} attempt(s) (last exit code ${lastExit}):\n${summary}`); + this.name = 'AgentExhaustedError'; + this.attempts = attempts; + this.lastExitCode = lastExit; + this.totalAttempts = total; + this.durationMs = durationMs; + } +} + +/** + * Build the CLI argument array from spawn options. + * @deprecated Use CLIAdapter.buildSpawnConfig() instead for provider-agnostic arg building. + * Kept for backward compatibility — produces Claude-specific args. + */ +export function buildArgs(opts: SpawnOptions): string[] { + const args: string[] = []; + + // Depth limiting: --print (single-turn, no session) and --session-id/--resume + // (multi-turn, persistent) are mutually exclusive. + // Workers use --print (enforces they can't spawn other workers). + // Master uses --session-id/--resume (enables persistent multi-turn behavior). + if (opts.resumeSessionId) { + args.push('--resume', opts.resumeSessionId); + } else if (opts.sessionId) { + args.push('--session-id', opts.sessionId); + } else { + // No session — use --print for single-turn, stateless execution + args.push('--print'); + } + + if (opts.model) { + if (!isValidModel(opts.model)) { + logger.warn( + { model: opts.model }, + 'Unrecognized model — passing through to CLI, which may reject it', + ); + } + args.push('--model', opts.model); + } + + const maxTurns = opts.maxTurns ?? DEFAULT_MAX_TURNS_TASK; + args.push('--max-turns', String(maxTurns)); + + if (opts.systemPrompt) { + args.push('--append-system-prompt', opts.systemPrompt); + } + + if (opts.maxBudgetUsd !== undefined && opts.maxBudgetUsd > 0) { + args.push('--max-budget-usd', String(opts.maxBudgetUsd)); + } + + // Place the prompt BEFORE --allowedTools. Commander.js parses the first + // positional argument as the prompt. --allowedTools is variadic () + // and would consume a trailing prompt as a tool name when no other option + // follows it (e.g. --append-system-prompt). + args.push(sanitizePrompt(opts.prompt, getMaxPromptLength(opts.model), 'worker')); + + if (opts.allowedTools && opts.allowedTools.length > 0) { + for (const tool of opts.allowedTools) { + args.push('--allowedTools', tool); + } + } + + return args; +} + +/** + * Grace period in milliseconds between SIGTERM and SIGKILL. + * When a worker times out, we send SIGTERM first, wait this long, + * then send SIGKILL if the process hasn't exited. + */ +const SIGTERM_GRACE_PERIOD_MS = 5000; + +/** Handle returned by execOnce() — exposes the result promise, process PID, and a kill function. */ +interface ExecOnceHandle { + promise: Promise<{ stdout: string; stderr: string; exitCode: number }>; + pid: number; + kill: () => void; +} + +/** Execute a single agent attempt. Returns stdout, stderr, exitCode. */ +function execOnce(config: CLISpawnConfig, workspacePath: string, timeout?: number): ExecOnceHandle { + const child = nodeSpawn(config.binary, config.args, { + cwd: workspacePath, + env: config.env, + stdio: [config.stdin ?? 'ignore', 'pipe', 'pipe'], + }); + + logger.debug( + { pid: child.pid, binary: config.binary, argCount: config.args.length }, + 'Spawned child process', + ); + + let stdout = ''; + let stderr = ''; + let timedOut = false; + let killed = false; + let authAborted = false; + let timeoutTimer: NodeJS.Timeout | undefined; + let gracePeriodTimer: NodeJS.Timeout | undefined; + + const promise = new Promise<{ stdout: string; stderr: string; exitCode: number }>( + (resolve, reject) => { + // Manual timeout handling with SIGTERM → SIGKILL progression + if (timeout && timeout > 0) { + timeoutTimer = setTimeout(() => { + if (killed) return; + timedOut = true; + logger.warn( + { timeout, pid: child.pid }, + 'Worker timeout exceeded — sending SIGTERM (5s grace period)', + ); + + // Send SIGTERM for graceful shutdown + const terminated = child.kill('SIGTERM'); + + if (!terminated) { + logger.warn({ pid: child.pid }, 'Failed to send SIGTERM to worker'); + // Resolve immediately if kill failed + resolve({ + stdout, + stderr: stderr + '\nTimeout: failed to terminate process', + exitCode: 143, + }); + return; + } + + // Set up grace period timer for SIGKILL + gracePeriodTimer = setTimeout(() => { + if (killed) return; + killed = true; + logger.warn({ timeout, pid: child.pid }, 'Grace period expired — sending SIGKILL'); + child.kill('SIGKILL'); + }, SIGTERM_GRACE_PERIOD_MS); + }, timeout); + } + + child.stdout!.on('data', (data: Buffer) => { + const chunk = data.toString(); + stdout += chunk; + logger.debug( + { pid: child.pid, chunkLen: chunk.length, totalLen: stdout.length }, + 'stdout data received', + ); + }); + + child.stderr!.on('data', (data: Buffer) => { + const chunk = data.toString(); + stderr += chunk; + + // Early detection for interactive OAuth/authentication URLs + const OAUTH_PATTERNS = [ + /https:\/\/.*authorize\?/i, + /https:\/\/.*login\?/i, + /https:\/\/.*oauth/i, + /Waiting for authorization/i, + /Open the following URL/i, + ]; + + if (OAUTH_PATTERNS.some((p) => p.test(chunk))) { + if (!authAborted && !killed) { + logger.warn({ pid: child.pid }, 'Worker attempting interactive auth — aborting early'); + authAborted = true; + child.kill('SIGTERM'); + } + } + }); + + child.on('close', (code, signal) => { + // Clear both timers + if (timeoutTimer) clearTimeout(timeoutTimer); + if (gracePeriodTimer) clearTimeout(gracePeriodTimer); + + logger.debug( + { pid: child.pid, code, signal, stdoutLen: stdout.length, stderrLen: stderr.length }, + 'Child process closed', + ); + + // Apply adapter-specific output parsing (e.g. Codex --json JSONL extraction) + let parsedStdout = stdout; + if (config.parseOutput) { + try { + parsedStdout = config.parseOutput(stdout); + } catch (parseErr) { + logger.warn({ parseErr }, 'parseOutput threw — falling back to raw stdout'); + } + } + + if (timedOut) { + // Process was terminated due to timeout + const exitCode = signal === 'SIGTERM' ? 143 : signal === 'SIGKILL' ? 137 : (code ?? 1); + resolve({ + stdout: parsedStdout, + stderr: + stderr + + `\nTimeout: process terminated after ${timeout}ms (signal: ${signal ?? 'none'})`, + exitCode, + }); + } else if (authAborted && signal === 'SIGTERM') { + // Process was terminated due to detected interactive auth attempt + const exitCode = 143; + resolve({ + stdout: parsedStdout, + stderr: stderr + '\nAuth-required: worker attempted interactive authentication', + exitCode, + }); + } else { + resolve({ stdout: parsedStdout, stderr, exitCode: code ?? 1 }); + } + }); + + child.on('error', (error) => { + if (timeoutTimer) clearTimeout(timeoutTimer); + if (gracePeriodTimer) clearTimeout(gracePeriodTimer); + reject(error); + }); + }, + ); + + return { + promise, + pid: child.pid ?? -1, + kill: (): void => { + // Guard: process already exited — skip kill, clear any lingering timers + if (child.exitCode !== null) { + clearTimeout(gracePeriodTimer); + gracePeriodTimer = undefined; + logger.debug( + { pid: child.pid, exitCode: child.exitCode }, + 'kill() called on already-exited process — skipping', + ); + return; + } + + killed = true; + // Clear both timers atomically to prevent any pending timeout/grace-period from firing + clearTimeout(timeoutTimer); + clearTimeout(gracePeriodTimer); + timeoutTimer = undefined; + gracePeriodTimer = undefined; + + // Graceful shutdown with SIGTERM + const terminated = child.kill('SIGTERM'); + + if (terminated) { + // Set up grace period for SIGKILL + gracePeriodTimer = setTimeout(() => { + child.kill('SIGKILL'); + }, SIGTERM_GRACE_PERIOD_MS); + } + }, + }; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Write a log file with a header and full stdout/stderr output. + * Creates the parent directory if it doesn't exist. + */ +async function writeLogFile( + logFile: string, + opts: SpawnOptions, + result: AgentResult, +): Promise { + const header = [ + `# Agent Run Log`, + `# Timestamp: ${new Date().toISOString()}`, + `# Model: ${opts.model ?? 'default'}`, + `# Tools: ${opts.allowedTools?.join(', ') ?? 'none specified'}`, + `# Max Turns: ${opts.maxTurns ?? DEFAULT_MAX_TURNS_TASK}`, + `# Prompt Length: ${opts.prompt.length}`, + `# Exit Code: ${result.exitCode}`, + `# Duration: ${result.durationMs}ms`, + `# Retries: ${result.retryCount}`, + '', + '--- STDOUT ---', + result.stdout, + '', + '--- STDERR ---', + result.stderr, + ].join('\n'); + + await mkdir(dirname(logFile), { recursive: true }); + await writeFile(logFile, header, 'utf-8'); +} + +/** Execute a single agent attempt in streaming mode. Yields stdout chunks. */ +function execOnceStreaming( + config: CLISpawnConfig, + workspacePath: string, + timeout?: number, +): { + chunks: AsyncGenerator; + abort: () => void; + pid: number; +} { + const child = nodeSpawn(config.binary, config.args, { + cwd: workspacePath, + // Don't use Node's built-in timeout — we handle it manually for graceful cleanup + env: config.env, + stdio: [config.stdin ?? 'ignore', 'pipe', 'pipe'], + }); + + let stderr = ''; + let timedOut = false; + let timeoutTimer: NodeJS.Timeout | undefined; + let gracePeriodTimer: NodeJS.Timeout | undefined; + + // Manual timeout handling with SIGTERM → SIGKILL progression + if (timeout && timeout > 0) { + timeoutTimer = setTimeout(() => { + timedOut = true; + logger.warn( + { timeout, pid: child.pid }, + 'Worker streaming timeout exceeded — sending SIGTERM (5s grace period)', + ); + + // Send SIGTERM for graceful shutdown + const terminated = child.kill('SIGTERM'); + + if (!terminated) { + logger.warn({ pid: child.pid }, 'Failed to send SIGTERM to streaming worker'); + return; + } + + // Set up grace period timer for SIGKILL + gracePeriodTimer = setTimeout(() => { + logger.warn( + { timeout, pid: child.pid }, + 'Grace period expired — sending SIGKILL to streaming worker', + ); + child.kill('SIGKILL'); + }, SIGTERM_GRACE_PERIOD_MS); + }, timeout); + } + + child.stderr!.on('data', (data: Buffer) => { + stderr += data.toString(); + }); + + const chunkQueue: string[] = []; + let done = false; + let exitCode = 1; + let _exitSignal: string | null = null; + let spawnError: Error | undefined; + + let notify: (() => void) | undefined; + function waitForData(): Promise { + return new Promise((resolve) => { + notify = resolve; + }); + } + + child.stdout!.on('data', (data: Buffer) => { + chunkQueue.push(data.toString()); + notify?.(); + }); + + child.on('close', (code, signal) => { + // Clear both timers + if (timeoutTimer) clearTimeout(timeoutTimer); + if (gracePeriodTimer) clearTimeout(gracePeriodTimer); + + exitCode = code ?? 1; + _exitSignal = signal; + + if (timedOut) { + // Process was terminated due to timeout + exitCode = signal === 'SIGTERM' ? 143 : signal === 'SIGKILL' ? 137 : (code ?? 1); + stderr += `\nTimeout: process terminated after ${timeout}ms (signal: ${signal ?? 'none'})`; + } + + done = true; + notify?.(); + }); + + child.on('error', (error) => { + if (timeoutTimer) clearTimeout(timeoutTimer); + if (gracePeriodTimer) clearTimeout(gracePeriodTimer); + + logger.error({ error }, 'Agent streaming error'); + spawnError = error; + done = true; + notify?.(); + }); + + async function* generate(): AsyncGenerator { + while (!done || chunkQueue.length > 0) { + if (chunkQueue.length > 0) { + yield chunkQueue.shift()!; + } else if (!done) { + await waitForData(); + } + } + + if (spawnError) { + throw spawnError; + } + + return { exitCode, stderr }; + } + + return { + chunks: generate(), + pid: child.pid ?? -1, + abort: (): void => { + // Clear timers if abort is called manually + if (timeoutTimer) clearTimeout(timeoutTimer); + if (gracePeriodTimer) clearTimeout(gracePeriodTimer); + + // Graceful shutdown with SIGTERM + const terminated = child.kill('SIGTERM'); + + if (terminated) { + // Set up grace period for SIGKILL + gracePeriodTimer = setTimeout(() => { + child.kill('SIGKILL'); + }, SIGTERM_GRACE_PERIOD_MS); + } + }, + }; +} + +export class AgentRunner { + private readonly adapter: CLIAdapter; + + constructor(adapter?: CLIAdapter) { + this.adapter = adapter ?? new ClaudeAdapter(); + } + + /** + * Execute a single agent attempt inside a Docker sandbox container. + * + * Creates, starts, execs, stops, and removes a container for each call. + * The container is always removed in the `finally` block — even on error. + * + * Returns the same shape as the `execOnce()` promise so it can be used as + * a drop-in replacement inside the `spawn()` retry loop. + */ + private async _execOnceDocker( + config: CLISpawnConfig, + workspacePath: string, + sandboxConfig: SandboxConfig, + timeout?: number, + maxTurns?: number, + securityConfig?: SecurityConfig, + mcpConfigPath?: string, + ): Promise<{ stdout: string; stderr: string; exitCode: number }> { + const dockerSandbox = new DockerSandbox(); + + // Apply env sanitizer (Phase 85) to strip secrets before passing to the container. + // sanitizeEnv() expects Record; Docker only accepts strings, + // so we filter undefined values out in the same pass. + const rawEnv: Record = config.env; + const sanitized = securityConfig ? sanitizeEnv(rawEnv, securityConfig) : rawEnv; + const env: Record = {}; + for (const [key, value] of Object.entries(sanitized)) { + if (value !== undefined) { + env[key] = value; + } + } + + const rawCount = Object.keys(rawEnv).filter((k) => rawEnv[k] !== undefined).length; + const sanitizedCount = Object.keys(env).length; + if (sanitizedCount < rawCount) { + logger.debug( + { stripped: rawCount - sanitizedCount, kept: sanitizedCount }, + 'Docker sandbox: stripped secret env vars via sanitizer', + ); + } + + const containerName = `ob-worker-${randomUUID().slice(0, 8)}`; + const mounts = DockerSandbox.buildWorkspaceMounts({ workspacePath }); + + // MCP config isolation (OB-1556): if a per-worker MCP config file was written + // on the host, mount it read-only into the container and rewrite the CLI args + // so the agent references the container path instead of the host path. + const CONTAINER_MCP_CONFIG_PATH = '/tmp/ob-mcp-config.json'; + let containerArgs = config.args; + if (mcpConfigPath) { + mounts.push({ + host: mcpConfigPath, + container: CONTAINER_MCP_CONFIG_PATH, + readOnly: true, + }); + // Replace the host path with the container path in the arg list. + containerArgs = config.args.map((arg) => + arg === mcpConfigPath ? CONTAINER_MCP_CONFIG_PATH : arg, + ); + logger.debug( + { mcpConfigPath, containerPath: CONTAINER_MCP_CONFIG_PATH }, + 'Mounting MCP config into Docker container', + ); + } + + // Compute exec timeout: use explicit timeout when provided, otherwise derive + // from maxTurns (30 seconds per turn) so long-running workers are eventually + // force-killed by the exec call. Fall back to 5 minutes if neither is set. + const effectiveTimeout = timeout ?? (maxTurns ? maxTurns * SECS_PER_TURN * 1_000 : 300_000); + + logger.debug( + { containerName, binary: config.binary, argCount: config.args.length, effectiveTimeout }, + 'Spawning agent inside Docker container', + ); + + let containerId: string | undefined; + try { + containerId = await dockerSandbox.createContainer({ + image: 'openbridge-worker:latest', + name: containerName, + mounts, + env, + network: sandboxConfig.network, + memoryMB: sandboxConfig.memoryMB, + cpus: sandboxConfig.cpus, + workdir: '/workspace', + }); + dockerSandbox.trackContainer(containerId); + + await dockerSandbox.startContainer(containerId); + + const result = await dockerSandbox.exec(containerId, [config.binary, ...containerArgs], { + cwd: '/workspace', + timeout: effectiveTimeout, + }); + + logger.info({ containerName, exitCode: result.exitCode }, 'Docker agent exec completed'); + + return result; + } finally { + if (containerId) { + // Untrack before cleanup so the crash-exit handler skips this container. + dockerSandbox.untrackContainer(containerId); + try { + await dockerSandbox.stopContainer(containerId, 5); + } catch { + // Container may have already exited — ignore stop errors + } + try { + await dockerSandbox.removeContainer(containerId, true); + } catch (err) { + logger.warn( + { containerName, err }, + 'Failed to remove Docker container after worker exit', + ); + } + } + } + } + + /** + * Spawn an AI CLI agent with the given options. + * + * Uses the CLIAdapter to build provider-specific CLI args, executes the + * child process, and retries on non-zero exit codes up to `retries` times + * with `retryDelay` between attempts. + */ + async spawn(opts: SpawnOptions): Promise { + const retries = opts.retries ?? 3; + const retryDelay = opts.retryDelay ?? 10_000; + let currentModel = opts.model; + let currentConfig = this.adapter.buildSpawnConfig(opts); + const startTime = Date.now(); + const modelFallbacks: string[] = []; + // Derive a bounded timeout when SPAWN markers omit one — prevents workers + // from running unbounded (only caught by the 10-30min watchdog otherwise). + const effectiveTimeout = + opts.timeout ?? (opts.maxTurns ? opts.maxTurns * SECS_PER_TURN * 1_000 : 300_000); + + logger.debug( + { + workspacePath: opts.workspacePath, + model: opts.model, + maxTurns: opts.maxTurns, + allowedTools: opts.allowedTools, + timeout: opts.timeout, + retries, + sessionId: opts.resumeSessionId ?? opts.sessionId, + }, + 'Spawning agent', + ); + + let lastResult: { stdout: string; stderr: string; exitCode: number } | undefined; + let attempt = 0; + const attemptRecords: AttemptRecord[] = []; + + for (attempt = 0; attempt <= retries; attempt++) { + if (attempt > 0) { + logger.warn( + { attempt, maxRetries: retries, delay: retryDelay }, + 'Retrying agent after non-zero exit', + ); + await sleep(retryDelay); + } + + try { + if (opts.sandbox?.mode === 'docker') { + // OB-1558: Check Docker availability before attempting sandbox spawn. + // If the daemon is unavailable, fall back to direct spawn. + // Note: DockerHealthMonitor logs WARN on Docker unavailability state transitions. + // This DEBUG log provides per-spawn visibility without creating log noise. (OB-1666) + const dockerSandboxCheck = new DockerSandbox(); + const dockerAvailable = await dockerSandboxCheck.isAvailable(); + if (!dockerAvailable) { + logger.debug( + { attempt, workspacePath: opts.workspacePath }, + 'Docker unavailable — falling back to direct (unsandboxed) spawn', + ); + const { promise: execPromise } = execOnce( + currentConfig, + opts.workspacePath, + effectiveTimeout, + ); + lastResult = await execPromise; + } else { + lastResult = await this._execOnceDocker( + currentConfig, + opts.workspacePath, + opts.sandbox, + opts.timeout, + opts.maxTurns, + opts.securityConfig, + opts.mcpConfigPath, + ); + } + } else { + const { promise: execPromise } = execOnce( + currentConfig, + opts.workspacePath, + effectiveTimeout, + ); + lastResult = await execPromise; + } + } catch (error) { + logger.error({ error, attempt }, 'Agent spawn error'); + attemptRecords.push({ + attempt, + exitCode: -1, + stderr: error instanceof Error ? error.message : String(error), + }); + if (attempt < retries) { + continue; + } + throw new AgentExhaustedError(attemptRecords, Date.now() - startTime); + } + + if (lastResult.exitCode === 0) { + break; + } + + logger.warn( + { exitCode: lastResult.exitCode, attempt, stderr: lastResult.stderr.slice(0, 500) }, + 'Agent exited with non-zero code', + ); + + attemptRecords.push({ + attempt, + exitCode: lastResult.exitCode, + stderr: lastResult.stderr, + }); + + // Skip remaining retries if the worker timed out — it will time out again + if (classifyError(lastResult.stderr, lastResult.exitCode) === 'timeout') { + logger.warn( + { exitCode: lastResult.exitCode, attempt, model: currentModel }, + 'Timeout exit — skipping remaining retries (retrying would timeout again)', + ); + break; + } + + // Check for rate-limit / model unavailability — fall back to next model + if (currentModel && isRateLimitError(lastResult.stderr) && attempt < retries) { + const nextModel = getNextFallbackModel(currentModel); + if (nextModel) { + logger.warn( + { from: currentModel, to: nextModel, attempt }, + 'Model rate-limited — falling back to next model in chain', + ); + modelFallbacks.push(currentModel); + currentModel = nextModel; + currentConfig = this.adapter.buildSpawnConfig({ ...opts, model: currentModel }); + } + } + } + + const durationMs = Date.now() - startTime; + const retryCount = Math.min(attempt, retries); + + // If we exited the loop without a success, throw aggregated error + if (!lastResult || lastResult.exitCode !== 0) { + throw new AgentExhaustedError(attemptRecords, durationMs); + } + + const turnsExhausted = isMaxTurnsExhausted(lastResult.stdout); + + // Workspace safety scan for file-management profile (OB-1494, OB-1587). + // Detect commands in worker output that targeted paths outside the workspace. + if (opts.profile === 'file-management') { + const violations = scanDestructiveCommandViolations(lastResult.stdout, opts.workspacePath); + for (const { command, path: violatingPath, severity } of violations) { + if (severity === 'destructive') { + logger.error( + { command, path: violatingPath, workspacePath: opts.workspacePath }, + `file-management worker used '${command}' on path outside workspace — destructive boundary violation`, + ); + } else { + logger.warn( + { command, path: violatingPath, workspacePath: opts.workspacePath }, + `file-management worker used '${command}' on path outside workspace — boundary escape detected`, + ); + } + } + } + + const costUsd = estimateCostUsd(currentModel, Buffer.byteLength(lastResult.stdout, 'utf8')); + + // Post-hoc cost cap warning for non-streaming path (OB-F101). + // Cannot abort after completion — log warning so callers can diagnose spikes. + const costCap = getProfileCostCap( + opts.profile, + opts.workerCostCaps, + opts.securityConfig?.trustLevel, + ); + if (costCap !== undefined && costUsd > costCap) { + logger.warn( + { cost: costUsd, cap: costCap, profile: opts.profile }, + `Worker cost cap exceeded: ${costUsd.toFixed(4)} > ${costCap} for profile ${opts.profile ?? 'unknown'}`, + ); + } + + // 10x average spike detection (OB-1673) + checkProfileCostSpike(opts.profile, costUsd); + + // Fix iteration cap check (OB-1789). + // Count how many lint/test fix attempts appear in the output. + // If the count reaches maxFixIterations, mark the result accordingly. + const maxFixIterations = opts.maxFixIterations ?? DEFAULT_MAX_FIX_ITERATIONS; + const fixIterationsUsed = + maxFixIterations > 0 ? countFixIterations(lastResult.stdout) : undefined; + const fixCapReached = fixIterationsUsed !== undefined && fixIterationsUsed >= maxFixIterations; + + let resultStatus: AgentResult['status']; + if (fixCapReached) { + resultStatus = 'fix-cap-reached'; + } else if (turnsExhausted) { + resultStatus = 'partial'; + } else { + resultStatus = 'completed'; + } + + const result: AgentResult = { + stdout: lastResult.stdout, + stderr: lastResult.stderr, + exitCode: lastResult.exitCode, + durationMs, + retryCount, + model: currentModel, + modelFallbacks: modelFallbacks.length > 0 ? modelFallbacks : undefined, + costUsd, + turnsExhausted: turnsExhausted || undefined, + maxTurns: opts.maxTurns, + status: resultStatus, + fixIterationsUsed, + fixCapReached: fixCapReached || undefined, + }; + + if (fixCapReached) { + logger.warn( + { + fixIterationsUsed, + maxFixIterations, + model: currentModel ?? 'default', + }, + 'Worker hit fix iteration cap — escalating to Master', + ); + } else if (turnsExhausted) { + logger.warn( + { model: currentModel ?? 'default', maxTurns: opts.maxTurns ?? DEFAULT_MAX_TURNS_TASK }, + 'Agent exited with code 0 but max-turns was exhausted — result may be incomplete', + ); + } + + logger.info( + { + exitCode: result.exitCode, + durationMs: result.durationMs, + model: result.model ?? 'default', + retryCount: result.retryCount, + modelFallbacks: result.modelFallbacks, + costUsd: result.costUsd, + turnsExhausted: result.turnsExhausted, + status: result.status, + fixIterationsUsed: result.fixIterationsUsed, + fixCapReached: result.fixCapReached, + }, + 'Agent completed', + ); + + if (opts.logFile) { + try { + await writeLogFile(opts.logFile, opts, result); + logger.debug({ logFile: opts.logFile }, 'Agent log written to disk'); + } catch (logError) { + logger.warn({ logFile: opts.logFile, error: logError }, 'Failed to write agent log'); + } + } + + return result; + } + + /** + * Spawn an AI CLI agent and immediately return a handle with the child PID + * and an abort function, without waiting for the run to complete. + * + * The returned `promise` resolves to the same `AgentResult` that `spawn()` + * would produce (including retries and model fallback). The `abort()` function + * always targets the *currently running* child process — it updates across + * retry boundaries so a late abort still kills an in-progress retry. + * + * Use this instead of `spawn()` when the caller needs to: + * - record the worker PID in a registry before the run finishes + * - cancel a long-running worker in response to a user "stop" command + * + * Keep `spawn()` unchanged for backward compatibility. + */ + spawnWithHandle(opts: SpawnOptions): SpawnHandle { + const retries = opts.retries ?? 3; + const retryDelay = opts.retryDelay ?? 10_000; + let currentModel = opts.model; + let currentConfig = this.adapter.buildSpawnConfig(opts); + const startTime = Date.now(); + const modelFallbacks: string[] = []; + // Derive a bounded timeout when SPAWN markers omit one — prevents workers + // from running unbounded (only caught by the 10-30min watchdog otherwise). + const effectiveTimeout = + opts.timeout ?? (opts.maxTurns ? opts.maxTurns * SECS_PER_TURN * 1_000 : 300_000); + + // Mutable reference to the kill function of the currently-running process. + // Updated on each retry so abort() always terminates the live child. + let currentKill: (() => void) | undefined; + + const abort = (): void => { + currentKill?.(); + }; + + // Launch the first execution immediately — this lets us capture its PID + // synchronously before the async retry loop begins. + const firstHandle = execOnce(currentConfig, opts.workspacePath, effectiveTimeout); + currentKill = firstHandle.kill; + const initialPid = firstHandle.pid; + + logger.debug( + { + workspacePath: opts.workspacePath, + model: opts.model, + maxTurns: opts.maxTurns, + allowedTools: opts.allowedTools, + timeout: opts.timeout, + retries, + pid: initialPid, + sessionId: opts.resumeSessionId ?? opts.sessionId, + }, + 'Spawning agent with handle', + ); + + const promise = (async (): Promise => { + const attemptRecords: AttemptRecord[] = []; + let lastResult: { stdout: string; stderr: string; exitCode: number } | undefined; + let attempt = 0; + + // The first iteration uses the already-started handle; subsequent + // iterations create new execOnce() calls for each retry. + let currentHandle: ExecOnceHandle = firstHandle; + + for (attempt = 0; attempt <= retries; attempt++) { + if (attempt > 0) { + logger.warn( + { attempt, maxRetries: retries, delay: retryDelay }, + 'Retrying agent after non-zero exit', + ); + await sleep(retryDelay); + currentHandle = execOnce(currentConfig, opts.workspacePath, effectiveTimeout); + currentKill = currentHandle.kill; + } + + try { + lastResult = await currentHandle.promise; + } catch (error) { + logger.error({ error, attempt }, 'Agent spawn error'); + attemptRecords.push({ + attempt, + exitCode: -1, + stderr: error instanceof Error ? error.message : String(error), + }); + if (attempt < retries) { + continue; + } + throw new AgentExhaustedError(attemptRecords, Date.now() - startTime); + } + + if (lastResult.exitCode === 0) { + break; + } + + logger.warn( + { exitCode: lastResult.exitCode, attempt, stderr: lastResult.stderr.slice(0, 500) }, + 'Agent exited with non-zero code', + ); + + attemptRecords.push({ + attempt, + exitCode: lastResult.exitCode, + stderr: lastResult.stderr, + }); + + // Skip remaining retries if the worker timed out — it will time out again + if (classifyError(lastResult.stderr, lastResult.exitCode) === 'timeout') { + logger.warn( + { exitCode: lastResult.exitCode, attempt, model: currentModel }, + 'Timeout exit — skipping remaining retries (retrying would timeout again)', + ); + break; + } + + // Rate-limit / model unavailability — fall back to next model + if (currentModel && isRateLimitError(lastResult.stderr) && attempt < retries) { + const nextModel = getNextFallbackModel(currentModel); + if (nextModel) { + logger.warn( + { from: currentModel, to: nextModel, attempt }, + 'Model rate-limited — falling back to next model in chain', + ); + modelFallbacks.push(currentModel); + currentModel = nextModel; + currentConfig = this.adapter.buildSpawnConfig({ ...opts, model: currentModel }); + } + } + } + + const durationMs = Date.now() - startTime; + const retryCount = Math.min(attempt, retries); + + if (!lastResult || lastResult.exitCode !== 0) { + throw new AgentExhaustedError(attemptRecords, durationMs); + } + + const turnsExhausted = isMaxTurnsExhausted(lastResult.stdout); + + // Workspace safety scan for file-management profile (OB-1494, OB-1587). + if (opts.profile === 'file-management') { + const violations = scanDestructiveCommandViolations(lastResult.stdout, opts.workspacePath); + for (const { command, path: violatingPath, severity } of violations) { + if (severity === 'destructive') { + logger.error( + { command, path: violatingPath, workspacePath: opts.workspacePath }, + `file-management worker used '${command}' on path outside workspace — destructive boundary violation`, + ); + } else { + logger.warn( + { command, path: violatingPath, workspacePath: opts.workspacePath }, + `file-management worker used '${command}' on path outside workspace — boundary escape detected`, + ); + } + } + } + + const costUsdHandle = estimateCostUsd( + currentModel, + Buffer.byteLength(lastResult.stdout, 'utf8'), + ); + + // Post-hoc cost cap warning for non-streaming path (OB-F101). + const costCapHandle = getProfileCostCap( + opts.profile, + opts.workerCostCaps, + opts.securityConfig?.trustLevel, + ); + if (costCapHandle !== undefined && costUsdHandle > costCapHandle) { + logger.warn( + { cost: costUsdHandle, cap: costCapHandle, profile: opts.profile }, + `Worker cost cap exceeded: ${costUsdHandle.toFixed(4)} > ${costCapHandle} for profile ${opts.profile ?? 'unknown'}`, + ); + } + + // 10x average spike detection (OB-1673) + checkProfileCostSpike(opts.profile, costUsdHandle); + + // Fix iteration cap check (OB-1789). + const maxFixIterationsHandle = opts.maxFixIterations ?? DEFAULT_MAX_FIX_ITERATIONS; + const fixIterationsUsedHandle = + maxFixIterationsHandle > 0 ? countFixIterations(lastResult.stdout) : undefined; + const fixCapReachedHandle = + fixIterationsUsedHandle !== undefined && fixIterationsUsedHandle >= maxFixIterationsHandle; + + let handleStatus: AgentResult['status']; + if (fixCapReachedHandle) { + handleStatus = 'fix-cap-reached'; + } else if (turnsExhausted) { + handleStatus = 'partial'; + } else { + handleStatus = 'completed'; + } + + const result: AgentResult = { + stdout: lastResult.stdout, + stderr: lastResult.stderr, + exitCode: lastResult.exitCode, + durationMs, + retryCount, + model: currentModel, + modelFallbacks: modelFallbacks.length > 0 ? modelFallbacks : undefined, + costUsd: costUsdHandle, + turnsExhausted: turnsExhausted || undefined, + maxTurns: opts.maxTurns, + status: handleStatus, + fixIterationsUsed: fixIterationsUsedHandle, + fixCapReached: fixCapReachedHandle || undefined, + }; + + if (fixCapReachedHandle) { + logger.warn( + { + fixIterationsUsed: fixIterationsUsedHandle, + maxFixIterations: maxFixIterationsHandle, + model: currentModel ?? 'default', + }, + 'Worker hit fix iteration cap — escalating to Master', + ); + } else if (turnsExhausted) { + logger.warn( + { model: currentModel ?? 'default', maxTurns: opts.maxTurns ?? DEFAULT_MAX_TURNS_TASK }, + 'Agent exited with code 0 but max-turns was exhausted — result may be incomplete', + ); + } + + logger.info( + { + exitCode: result.exitCode, + durationMs: result.durationMs, + model: result.model ?? 'default', + retryCount: result.retryCount, + modelFallbacks: result.modelFallbacks, + costUsd: result.costUsd, + turnsExhausted: result.turnsExhausted, + fixIterationsUsed: result.fixIterationsUsed, + fixCapReached: result.fixCapReached, + }, + 'Agent completed', + ); + + if (opts.logFile) { + try { + await writeLogFile(opts.logFile, opts, result); + logger.debug({ logFile: opts.logFile }, 'Agent log written to disk'); + } catch (logError) { + logger.warn({ logFile: opts.logFile, error: logError }, 'Failed to write agent log'); + } + } + + return result; + })(); + + return { promise, pid: initialPid, abort }; + } + + /** + * Spawn an AI CLI agent with real-time streaming progress via turn-indicator parsing. + * + * Identical to spawnWithHandle() — returns pid, abort, and a result promise — + * but uses execOnceStreaming() internally so that stdout chunks are inspected + * as they arrive. Whenever parseTurnIndicator() detects a new agent turn in a + * chunk, the optional onProgress callback is invoked with the TurnIndicator. + * + * Use this instead of spawnWithHandle() when the caller needs real-time turn + * visibility (e.g. to broadcast worker-turn-progress events to connectors). + */ + spawnWithStreamingHandle( + opts: SpawnOptions, + onProgress?: (indicator: TurnIndicator) => void, + ): SpawnHandle { + const retries = opts.retries ?? 3; + const retryDelay = opts.retryDelay ?? 10_000; + let currentModel = opts.model; + let currentConfig = this.adapter.buildSpawnConfig(opts); + const startTime = Date.now(); + const modelFallbacks: string[] = []; + + // Mutable reference to the abort function of the currently-running stream. + // Updated on each retry so abort() always terminates the live child. + let currentAbort: (() => void) | undefined; + const abort = (): void => { + currentAbort?.(); + }; + + // Launch the first execution immediately — this lets us capture its PID + // synchronously before the async retry loop begins. + const firstStreaming = execOnceStreaming(currentConfig, opts.workspacePath, opts.timeout); + currentAbort = firstStreaming.abort; + const initialPid = firstStreaming.pid; + + logger.debug( + { + workspacePath: opts.workspacePath, + model: opts.model, + maxTurns: opts.maxTurns, + allowedTools: opts.allowedTools, + timeout: opts.timeout, + retries, + pid: initialPid, + sessionId: opts.resumeSessionId ?? opts.sessionId, + }, + 'Spawning agent with streaming handle', + ); + + const promise = (async (): Promise => { + const attemptRecords: AttemptRecord[] = []; + let attempt = 0; + let lastTurnsUsed = 0; + + // The first iteration uses the already-started streaming handle. + let currentStreaming: ReturnType = firstStreaming; + + for (attempt = 0; attempt <= retries; attempt++) { + if (attempt > 0) { + logger.warn( + { attempt, maxRetries: retries, delay: retryDelay }, + 'Retrying streaming agent after non-zero exit', + ); + await sleep(retryDelay); + currentStreaming = execOnceStreaming(currentConfig, opts.workspacePath, opts.timeout); + currentAbort = currentStreaming.abort; + } + + let stdout = ''; + let streamResult: { exitCode: number; stderr: string } | undefined; + let spawnError: Error | undefined; + let costCapExceeded = false; + let costCapMessage = ''; + let perWorkerCostCapped = false; + let perWorkerCostAtCap = 0; + + try { + // Drain all chunks — accumulate stdout and report turn progress + let iterResult = await currentStreaming.chunks.next(); + while (!iterResult.done) { + const chunk = iterResult.value; + stdout += chunk; + + // Per-worker maxCostUsd check (OB-1522 / OB-F195). + // Kill the process and return partial output if cumulative cost exceeds the cap. + if (opts.maxCostUsd !== undefined) { + const currentCostUsd = estimateCostUsd( + currentModel, + Buffer.byteLength(stdout, 'utf8'), + ); + if (currentCostUsd > opts.maxCostUsd) { + logger.warn( + { cost: currentCostUsd, cap: opts.maxCostUsd, profile: opts.profile }, + `Per-worker cost cap exceeded: $${currentCostUsd.toFixed(4)} > $${opts.maxCostUsd} — killing worker`, + ); + currentAbort?.(); + perWorkerCostCapped = true; + perWorkerCostAtCap = currentCostUsd; + break; + } + } + + // Per-profile cost cap check (OB-F101). + // Estimate cost from accumulated output; abort early if cap exceeded. + const costCap = getProfileCostCap( + opts.profile, + opts.workerCostCaps, + opts.securityConfig?.trustLevel, + ); + if (costCap !== undefined) { + const currentCostUsd = estimateCostUsd( + currentModel, + Buffer.byteLength(stdout, 'utf8'), + ); + if (currentCostUsd > costCap) { + costCapMessage = `Worker cost cap exceeded: ${currentCostUsd.toFixed(4)} > ${costCap} for profile ${opts.profile ?? 'unknown'}`; + logger.warn( + { cost: currentCostUsd, cap: costCap, profile: opts.profile }, + costCapMessage, + ); + currentAbort?.(); + costCapExceeded = true; + break; + } + } + + if (onProgress) { + if (currentConfig.parseStreamChunk) { + // Adapter has incremental stream parser (e.g. Codex --json JSONL). + // Split chunk by lines — each line may be a separate structured event. + // Transform readable events to synthetic TurnIndicators so users see + // real-time progress instead of raw JSON. + for (const line of chunk.split('\n')) { + if (!line.trim()) continue; + const parsedText = currentConfig.parseStreamChunk(line); + if (parsedText !== null) { + lastTurnsUsed++; + onProgress({ turnsUsed: lastTurnsUsed, lastAction: parsedText }); + } + } + } else { + // Claude path — use turn indicator parsing for Claude-specific patterns. + const indicator = parseTurnIndicator(chunk); + if (indicator) { + lastTurnsUsed = indicator.turnsUsed; + onProgress(indicator); + } + } + } + iterResult = await currentStreaming.chunks.next(); + } + if (!costCapExceeded && iterResult.done) { + streamResult = iterResult.value; + } + } catch (error) { + logger.error({ error, attempt }, 'Agent streaming handle error'); + spawnError = error instanceof Error ? error : new Error(String(error)); + } + + // Per-worker cost cap exceeded (OB-1522) — return partial output, no retry + if (perWorkerCostCapped) { + const durationMs = Date.now() - startTime; + let parsedStdout = stdout; + if (currentConfig.parseOutput) { + try { + parsedStdout = currentConfig.parseOutput(stdout); + } catch { + /* fall back to raw */ + } + } + const result: AgentResult = { + stdout: parsedStdout, + stderr: `Cost capped: $${perWorkerCostAtCap.toFixed(4)} exceeded maxCostUsd $${opts.maxCostUsd}`, + exitCode: 1, + durationMs, + retryCount: attempt, + model: currentModel, + modelFallbacks: modelFallbacks.length > 0 ? modelFallbacks : undefined, + costUsd: perWorkerCostAtCap, + turnsUsed: lastTurnsUsed > 0 ? lastTurnsUsed : undefined, + maxTurns: opts.maxTurns, + status: 'cost-capped', + costCapped: true, + }; + if (opts.logFile) { + try { + await writeLogFile(opts.logFile, opts, result); + } catch { + /* ignore */ + } + } + return result; + } + + // Cost cap exceeded — abort immediately without retrying + if (costCapExceeded) { + attemptRecords.push({ attempt, exitCode: 1, stderr: costCapMessage }); + throw new AgentExhaustedError(attemptRecords, Date.now() - startTime); + } + + if (spawnError) { + attemptRecords.push({ + attempt, + exitCode: -1, + stderr: spawnError.message, + }); + if (attempt < retries) continue; + throw new AgentExhaustedError(attemptRecords, Date.now() - startTime); + } + + if (streamResult!.exitCode === 0) { + const durationMs = Date.now() - startTime; + const retryCount = attempt; + const turnsExhausted = isMaxTurnsExhausted(stdout); + + // Apply adapter-specific output parsing (e.g. Codex --json JSONL extraction) + let parsedStdout = stdout; + if (currentConfig.parseOutput) { + try { + parsedStdout = currentConfig.parseOutput(stdout); + } catch (parseErr) { + logger.warn({ parseErr }, 'parseOutput threw — falling back to raw stdout'); + } + } + + const streamCostUsd = estimateCostUsd(currentModel, Buffer.byteLength(stdout, 'utf8')); + + // 10x average spike detection (OB-1673) + checkProfileCostSpike(opts.profile, streamCostUsd); + + // Fix iteration cap check (OB-1789). + const maxFixIterationsStream = opts.maxFixIterations ?? DEFAULT_MAX_FIX_ITERATIONS; + const fixIterationsUsedStream = + maxFixIterationsStream > 0 ? countFixIterations(stdout) : undefined; + const fixCapReachedStream = + fixIterationsUsedStream !== undefined && + fixIterationsUsedStream >= maxFixIterationsStream; + + let streamStatus: AgentResult['status']; + if (fixCapReachedStream) { + streamStatus = 'fix-cap-reached'; + } else if (turnsExhausted) { + streamStatus = 'partial'; + } else { + streamStatus = 'completed'; + } + + const result: AgentResult = { + stdout: parsedStdout, + stderr: streamResult!.stderr, + exitCode: 0, + durationMs, + retryCount, + model: currentModel, + modelFallbacks: modelFallbacks.length > 0 ? modelFallbacks : undefined, + costUsd: streamCostUsd, + turnsExhausted: turnsExhausted || undefined, + turnsUsed: lastTurnsUsed > 0 ? lastTurnsUsed : undefined, + maxTurns: opts.maxTurns, + status: streamStatus, + fixIterationsUsed: fixIterationsUsedStream, + fixCapReached: fixCapReachedStream || undefined, + }; + + if (fixCapReachedStream) { + logger.warn( + { + fixIterationsUsed: fixIterationsUsedStream, + maxFixIterations: maxFixIterationsStream, + model: currentModel ?? 'default', + }, + 'Streaming worker hit fix iteration cap — escalating to Master', + ); + } else if (turnsExhausted) { + logger.warn( + { + model: currentModel ?? 'default', + maxTurns: opts.maxTurns ?? DEFAULT_MAX_TURNS_TASK, + }, + 'Streaming agent exited with code 0 but max-turns was exhausted — result may be incomplete', + ); + } + + logger.info( + { + exitCode: 0, + durationMs, + model: currentModel ?? 'default', + retryCount, + modelFallbacks: result.modelFallbacks, + costUsd: result.costUsd, + turnsExhausted: result.turnsExhausted, + fixIterationsUsed: result.fixIterationsUsed, + fixCapReached: result.fixCapReached, + }, + 'Streaming agent completed', + ); + + if (opts.logFile) { + try { + await writeLogFile(opts.logFile, opts, result); + logger.debug({ logFile: opts.logFile }, 'Streaming agent log written to disk'); + } catch (logError) { + logger.warn({ logFile: opts.logFile, error: logError }, 'Failed to write agent log'); + } + } + + return result; + } + + // Non-zero exit — record and possibly retry + logger.warn( + { + exitCode: streamResult!.exitCode, + attempt, + stderr: streamResult!.stderr.slice(0, 500), + }, + 'Streaming agent exited with non-zero code', + ); + + attemptRecords.push({ + attempt, + exitCode: streamResult!.exitCode, + stderr: streamResult!.stderr, + }); + + // Skip remaining retries if the worker timed out — it will time out again + if (classifyError(streamResult!.stderr, streamResult!.exitCode) === 'timeout') { + logger.warn( + { exitCode: streamResult!.exitCode, attempt, model: currentModel }, + 'Timeout exit — skipping remaining retries (retrying would timeout again)', + ); + break; + } + + // Rate-limit / model unavailability — fall back to next model + if (currentModel && isRateLimitError(streamResult!.stderr) && attempt < retries) { + const nextModel = getNextFallbackModel(currentModel); + if (nextModel) { + logger.warn( + { from: currentModel, to: nextModel, attempt }, + 'Model rate-limited — falling back to next model in chain', + ); + modelFallbacks.push(currentModel); + currentModel = nextModel; + currentConfig = this.adapter.buildSpawnConfig({ ...opts, model: currentModel }); + } + } + } + + // All retries exhausted + throw new AgentExhaustedError(attemptRecords, Date.now() - startTime); + })(); + + return { promise, pid: initialPid, abort }; + } + + /** + * Stream an AI CLI agent, yielding stdout chunks as they arrive. + * + * Supports all the same options as spawn() — allowedTools, maxTurns, + * model, retries, disk logging. On non-zero exit codes, retries the + * entire execution (previous chunks are discarded for that attempt). + * + * The generator's return value is an AgentResult with the accumulated + * stdout from the successful attempt. + */ + async *stream(opts: SpawnOptions): AsyncGenerator { + const retries = opts.retries ?? 3; + const retryDelay = opts.retryDelay ?? 10_000; + let currentModel = opts.model; + let currentConfig = this.adapter.buildSpawnConfig(opts); + const startTime = Date.now(); + const modelFallbacks: string[] = []; + + logger.debug( + { + workspacePath: opts.workspacePath, + model: opts.model, + maxTurns: opts.maxTurns, + allowedTools: opts.allowedTools, + timeout: opts.timeout, + retries, + sessionId: opts.resumeSessionId ?? opts.sessionId, + }, + 'Streaming agent', + ); + + const attemptRecords: AttemptRecord[] = []; + let attempt = 0; + + for (attempt = 0; attempt <= retries; attempt++) { + if (attempt > 0) { + logger.warn( + { attempt, maxRetries: retries, delay: retryDelay }, + 'Retrying stream after non-zero exit', + ); + await sleep(retryDelay); + } + + let stdout = ''; + let streamResult: { exitCode: number; stderr: string } | undefined; + let spawnError: Error | undefined; + let streamCostCapped = false; + let streamCostAtCap = 0; + + try { + const { chunks, abort: streamAbort } = execOnceStreaming( + currentConfig, + opts.workspacePath, + opts.timeout, + ); + + // Drain all chunks — yield each one and accumulate stdout + let iterResult = await chunks.next(); + while (!iterResult.done) { + const chunk = iterResult.value; + stdout += chunk; + + // Per-worker maxCostUsd check (OB-1522 / OB-F195). + if (opts.maxCostUsd !== undefined) { + const currentCostUsd = estimateCostUsd(currentModel, Buffer.byteLength(stdout, 'utf8')); + if (currentCostUsd > opts.maxCostUsd) { + logger.warn( + { cost: currentCostUsd, cap: opts.maxCostUsd, profile: opts.profile }, + `Per-worker cost cap exceeded: $${currentCostUsd.toFixed(4)} > $${opts.maxCostUsd} — killing worker`, + ); + streamAbort(); + streamCostCapped = true; + streamCostAtCap = currentCostUsd; + break; + } + } + + yield chunk; + iterResult = await chunks.next(); + } + + if (!streamCostCapped && iterResult.done) { + streamResult = iterResult.value; + } + } catch (error) { + logger.error({ error, attempt }, 'Agent stream error'); + spawnError = error instanceof Error ? error : new Error(String(error)); + } + + // Per-worker cost cap hit — return partial output, no retry (OB-1522) + if (streamCostCapped) { + const durationMs = Date.now() - startTime; + let parsedStdout = stdout; + if (currentConfig.parseOutput) { + try { + parsedStdout = currentConfig.parseOutput(stdout); + } catch { + /* fall back to raw */ + } + } + const result: AgentResult = { + stdout: parsedStdout, + stderr: `Cost capped: $${streamCostAtCap.toFixed(4)} exceeded maxCostUsd $${opts.maxCostUsd}`, + exitCode: 1, + durationMs, + retryCount: attempt, + model: currentModel, + modelFallbacks: modelFallbacks.length > 0 ? modelFallbacks : undefined, + costUsd: streamCostAtCap, + maxTurns: opts.maxTurns, + status: 'cost-capped', + costCapped: true, + }; + if (opts.logFile) { + try { + await writeLogFile(opts.logFile, opts, result); + } catch { + /* ignore */ + } + } + return result; + } + + if (spawnError) { + attemptRecords.push({ + attempt, + exitCode: -1, + stderr: spawnError.message, + }); + if (attempt < retries) { + continue; + } + throw new AgentExhaustedError(attemptRecords, Date.now() - startTime); + } + + if (streamResult!.exitCode === 0) { + const durationMs = Date.now() - startTime; + const retryCount = attempt; + const turnsExhausted = isMaxTurnsExhausted(stdout); + + // Apply adapter-specific output parsing (e.g. Codex --json JSONL extraction) + let parsedStdout = stdout; + if (currentConfig.parseOutput) { + try { + parsedStdout = currentConfig.parseOutput(stdout); + } catch (parseErr) { + logger.warn({ parseErr }, 'parseOutput threw — falling back to raw stdout'); + } + } + + const result: AgentResult = { + stdout: parsedStdout, + stderr: streamResult!.stderr, + exitCode: 0, + durationMs, + retryCount, + model: currentModel, + modelFallbacks: modelFallbacks.length > 0 ? modelFallbacks : undefined, + costUsd: estimateCostUsd(currentModel, Buffer.byteLength(stdout, 'utf8')), + turnsExhausted: turnsExhausted || undefined, + maxTurns: opts.maxTurns, + status: turnsExhausted ? 'partial' : 'completed', + }; + + if (turnsExhausted) { + logger.warn( + { + model: currentModel ?? 'default', + maxTurns: opts.maxTurns ?? DEFAULT_MAX_TURNS_TASK, + }, + 'Stream exited with code 0 but max-turns was exhausted — result may be incomplete', + ); + } + + logger.info( + { + exitCode: 0, + durationMs, + model: currentModel ?? 'default', + retryCount, + modelFallbacks: result.modelFallbacks, + costUsd: result.costUsd, + turnsExhausted: result.turnsExhausted, + }, + 'Stream completed', + ); + + if (opts.logFile) { + try { + await writeLogFile(opts.logFile, opts, result); + logger.debug({ logFile: opts.logFile }, 'Stream log written to disk'); + } catch (logError) { + logger.warn({ logFile: opts.logFile, error: logError }, 'Failed to write stream log'); + } + } + + return result; + } + + // Non-zero exit — record and possibly retry + logger.warn( + { + exitCode: streamResult!.exitCode, + attempt, + stderr: streamResult!.stderr.slice(0, 500), + }, + 'Stream exited with non-zero code', + ); + + attemptRecords.push({ + attempt, + exitCode: streamResult!.exitCode, + stderr: streamResult!.stderr, + }); + + // Skip remaining retries if the worker timed out — it will time out again + if (classifyError(streamResult!.stderr, streamResult!.exitCode) === 'timeout') { + logger.warn( + { exitCode: streamResult!.exitCode, attempt, model: currentModel }, + 'Timeout exit — skipping remaining retries (retrying would timeout again)', + ); + break; + } + + // Check for rate-limit / model unavailability — fall back to next model + if (currentModel && isRateLimitError(streamResult!.stderr) && attempt < retries) { + const nextModel = getNextFallbackModel(currentModel); + if (nextModel) { + logger.warn( + { from: currentModel, to: nextModel, attempt }, + 'Model rate-limited — falling back to next model in chain', + ); + modelFallbacks.push(currentModel); + currentModel = nextModel; + currentConfig = this.adapter.buildSpawnConfig({ ...opts, model: currentModel }); + } + } + } + + // All retries exhausted + throw new AgentExhaustedError(attemptRecords, Date.now() - startTime); + } + + /** + * Spawn an AI CLI agent from a TaskManifest. + * + * Converts the manifest into SpawnOptions, resolving the `profile` field + * into tool lists via custom profiles then built-in profiles. + * If both `profile` and explicit `allowedTools` are provided, explicit wins. + */ + async spawnFromManifest( + manifest: TaskManifest, + customProfiles?: Record, + ): Promise { + const { spawnOptions, cleanup } = await manifestToSpawnOptions(manifest, customProfiles); + try { + return await this.spawn(spawnOptions); + } finally { + await cleanup(); + } + } + + /** + * Stream an AI CLI agent from a TaskManifest. + * + * Same as spawnFromManifest but yields stdout chunks as they arrive. + * Resolves `profile` to tools the same way as spawnFromManifest. + */ + async *streamFromManifest( + manifest: TaskManifest, + customProfiles?: Record, + ): AsyncGenerator { + const { spawnOptions, cleanup } = await manifestToSpawnOptions(manifest, customProfiles); + try { + return yield* this.stream(spawnOptions); + } finally { + await cleanup(); + } + } +} diff --git a/src/core/app-server.ts b/src/core/app-server.ts new file mode 100644 index 00000000..089658cc --- /dev/null +++ b/src/core/app-server.ts @@ -0,0 +1,388 @@ +import { createServer } from 'node:net'; +import { spawn, type ChildProcess } from 'node:child_process'; +import { randomUUID, randomBytes } from 'node:crypto'; +import { access, readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { createLogger } from './logger.js'; +import { type TunnelManager } from './tunnel-manager.js'; +import { type InteractionRelay } from './interaction-relay.js'; + +const logger = createLogger('app-server'); + +export type AppStatus = 'running' | 'stopped'; +export type AppScaffoldType = 'npm' | 'static' | 'node'; + +export interface AppScaffold { + type: AppScaffoldType; + command: string; + args: string[]; +} + +export interface AppInstance { + id: string; + port: number; + url: string; + publicUrl: string | null; + status: AppStatus; + startedAt: string; + /** Authentication token for the InteractionRelay WebSocket connection. Set when an InteractionRelay is configured. */ + relayToken?: string; +} + +interface AppServerOptions { + baseUrl?: string; + portStart?: number; + portEnd?: number; + idleTimeoutMs?: number; + /** Maximum number of apps that can run concurrently. Default: 5 */ + maxConcurrent?: number; + /** Memory limit per app process in megabytes. Applied as --max-old-space-size for Node apps. Default: 256 */ + maxMemoryMB?: number; + /** + * Factory that creates a fresh TunnelManager for each app that starts. + * If provided, a tunnel is created for each app port and the public URL + * is stored in AppInstance.publicUrl. The tunnel is stopped when the app stops. + */ + tunnelFactory?: () => TunnelManager; + /** + * InteractionRelay to register per-app tokens with. + * When provided, startApp() generates a unique auth token per app, registers it + * with the relay, and returns it as AppInstance.relayToken. The token is + * unregistered when the app stops. + */ + relay?: InteractionRelay; +} + +const DEFAULT_BASE_URL = 'http://localhost'; +const DEFAULT_PORT_START = 3100; +const DEFAULT_PORT_END = 3199; +const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000; +const DEFAULT_MAX_CONCURRENT = 5; +const DEFAULT_MAX_MEMORY_MB = 256; +const HEALTH_CHECK_TIMEOUT_MS = 20_000; +const HEALTH_CHECK_INTERVAL_MS = 500; +const HEALTH_REQUEST_TIMEOUT_MS = 2_000; + +interface AppRuntime { + instance: AppInstance; + process: ChildProcess; + appPath: string; + scaffold: AppScaffold; + lastRequestAt: number; + idleTimer: ReturnType | null; + tunnelManager: TunnelManager | null; +} + +export class AppServer { + private readonly apps = new Map(); + private readonly baseUrl: string; + private readonly idleTimeoutMs: number; + private readonly portStart: number; + private readonly portEnd: number; + private readonly usedPorts = new Set(); + private readonly tunnelFactory: (() => TunnelManager) | null; + private readonly relay: InteractionRelay | null; + readonly maxConcurrent: number; + readonly maxMemoryMB: number; + + constructor(options: AppServerOptions = {}) { + this.baseUrl = options.baseUrl ?? DEFAULT_BASE_URL; + this.portStart = options.portStart ?? DEFAULT_PORT_START; + this.portEnd = options.portEnd ?? DEFAULT_PORT_END; + this.idleTimeoutMs = options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS; + this.maxConcurrent = options.maxConcurrent ?? DEFAULT_MAX_CONCURRENT; + this.maxMemoryMB = options.maxMemoryMB ?? DEFAULT_MAX_MEMORY_MB; + this.tunnelFactory = options.tunnelFactory ?? null; + this.relay = options.relay ?? null; + } + + /** + * Scan the port range and mark any already-bound ports as in use. + * Call this once on startup before the first startApp(). + */ + async scanUsedPorts(): Promise { + const checks: Promise[] = []; + for (let port = this.portStart; port <= this.portEnd; port++) { + checks.push( + isPortInUse(port).then((inUse) => { + if (inUse) { + this.usedPorts.add(port); + } + }), + ); + } + await Promise.all(checks); + logger.info( + { portStart: this.portStart, portEnd: this.portEnd, inUse: this.usedPorts.size }, + 'Port scan complete', + ); + } + + async detectAppScaffold(appPath: string): Promise { + const packageJsonPath = path.join(appPath, 'package.json'); + if (await this.pathExists(packageJsonPath)) { + try { + const raw = await readFile(packageJsonPath, 'utf-8'); + const data = JSON.parse(raw) as { scripts?: Record }; + if (data.scripts?.['start']) { + return { type: 'npm', command: 'npm', args: ['start'] }; + } + } catch (error) { + logger.warn({ err: error, appPath }, 'Failed to parse package.json for app scaffold'); + } + } + + const serverPath = path.join(appPath, 'server.js'); + if (await this.pathExists(serverPath)) { + return { type: 'node', command: 'node', args: ['server.js'] }; + } + + const indexPath = path.join(appPath, 'index.html'); + if (await this.pathExists(indexPath)) { + return { type: 'static', command: 'npx', args: ['-y', 'serve', '.'] }; + } + + return null; + } + + async startApp(appPath: string): Promise { + if (this.apps.size >= this.maxConcurrent) { + throw new Error( + `Maximum concurrent apps reached (${this.maxConcurrent}). Stop an existing app before starting a new one.`, + ); + } + + const scaffold = await this.detectAppScaffold(appPath); + if (!scaffold) { + throw new Error(`No app scaffold detected at path: ${appPath}`); + } + + const id = randomUUID(); + const port = this.allocatePort(); + const url = `${this.baseUrl}:${port}`; + const relayToken = this.relay ? randomBytes(32).toString('hex') : undefined; + const instance: AppInstance = { + id, + port, + url, + publicUrl: null, + status: 'running', + startedAt: new Date().toISOString(), + relayToken, + }; + + if (this.relay && relayToken) { + this.relay.registerApp(id, relayToken); + } + + const child = this.spawnAppProcess(scaffold, appPath, port); + const runtime: AppRuntime = { + instance, + process: child, + appPath, + scaffold, + lastRequestAt: Date.now(), + idleTimer: null, + tunnelManager: null, + }; + + this.apps.set(id, runtime); + + child.on('exit', (code, signal) => { + const active = this.apps.get(id); + if (!active) return; + logger.warn( + { id, port, appPath, code, signal }, + 'App process exited unexpectedly; stopping app', + ); + this.stopApp(id); + }); + + try { + await this.waitForHealthy(url); + } catch (error) { + this.stopApp(id); + throw error; + } + + if (this.tunnelFactory) { + const tm = this.tunnelFactory(); + try { + const publicUrl = await tm.start(port); + instance.publicUrl = publicUrl; + runtime.tunnelManager = tm; + logger.info({ id, port, publicUrl }, 'App tunnel created'); + } catch (err) { + logger.warn( + { id, port, err }, + 'Failed to create tunnel for app — continuing without tunnel', + ); + } + } + + this.scheduleIdleTimeout(id); + logger.info({ id, port, appPath, publicUrl: instance.publicUrl }, 'App started'); + return instance; + } + + stopApp(appId: string): void { + const runtime = this.apps.get(appId); + if (!runtime) return; + + runtime.instance.status = 'stopped'; + if (runtime.idleTimer) { + clearTimeout(runtime.idleTimer); + runtime.idleTimer = null; + } + + if (!runtime.process.killed) { + runtime.process.kill('SIGTERM'); + } + + if (runtime.tunnelManager) { + runtime.tunnelManager.stop(); + runtime.tunnelManager = null; + logger.info({ id: appId, port: runtime.instance.port }, 'App tunnel stopped'); + } + + if (this.relay) { + this.relay.unregisterApp(appId); + } + + this.usedPorts.delete(runtime.instance.port); + this.apps.delete(appId); + logger.info({ id: appId, port: runtime.instance.port }, 'App stopped'); + } + + stopAll(): void { + const ids = Array.from(this.apps.keys()); + for (const id of ids) { + this.stopApp(id); + } + logger.info({ count: ids.length }, 'All apps stopped'); + } + + listApps(): AppInstance[] { + return Array.from(this.apps.values(), (runtime) => runtime.instance); + } + + getApp(appId: string): AppInstance | null { + return this.apps.get(appId)?.instance ?? null; + } + + recordAppRequest(appId: string): void { + const runtime = this.apps.get(appId); + if (!runtime) return; + runtime.lastRequestAt = Date.now(); + this.scheduleIdleTimeout(appId); + } + + private allocatePort(): number { + for (let port = this.portStart; port <= this.portEnd; port++) { + if (!this.usedPorts.has(port)) { + this.usedPorts.add(port); + return port; + } + } + throw new Error( + `No available ports in range ${this.portStart}–${this.portEnd}. All ${this.portEnd - this.portStart + 1} ports in use.`, + ); + } + + private async pathExists(filePath: string): Promise { + try { + await access(filePath); + return true; + } catch { + return false; + } + } + + private spawnAppProcess(scaffold: AppScaffold, appPath: string, port: number): ChildProcess { + const env: NodeJS.ProcessEnv = { ...process.env, PORT: String(port), HOST: '0.0.0.0' }; + const args = [...scaffold.args]; + + if (scaffold.type === 'static') { + if (!args.includes('-l') && !args.includes('--listen')) { + args.push('-l', String(port)); + } + } + + // Apply memory limit to Node.js processes via --max-old-space-size + if (scaffold.type === 'node' || scaffold.type === 'npm') { + env['NODE_OPTIONS'] = `--max-old-space-size=${String(this.maxMemoryMB)}`; + } + + logger.info( + { appPath, port, command: scaffold.command, args, maxMemoryMB: this.maxMemoryMB }, + 'Spawning app process', + ); + + return spawn(scaffold.command, args, { + cwd: appPath, + env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + } + + private async waitForHealthy(url: string): Promise { + const deadline = Date.now() + HEALTH_CHECK_TIMEOUT_MS; + let lastError: string | null = null; + + while (Date.now() < deadline) { + try { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), HEALTH_REQUEST_TIMEOUT_MS); + const response = await fetch(url, { signal: controller.signal }); + clearTimeout(timeout); + if (response.ok) { + return; + } + } catch (error) { + lastError = error instanceof Error ? error.message : String(error); + } + + await new Promise((resolve) => setTimeout(resolve, HEALTH_CHECK_INTERVAL_MS)); + } + + throw new Error(`App health check failed for ${url}` + (lastError ? ` (${lastError})` : '')); + } + + private scheduleIdleTimeout(appId: string): void { + const runtime = this.apps.get(appId); + if (!runtime) return; + + if (runtime.idleTimer) { + clearTimeout(runtime.idleTimer); + } + + runtime.idleTimer = setTimeout(() => { + const latest = this.apps.get(appId); + if (!latest) return; + const idleFor = Date.now() - latest.lastRequestAt; + if (idleFor >= this.idleTimeoutMs) { + logger.info({ id: appId, port: latest.instance.port }, 'App idle timeout reached'); + this.stopApp(appId); + } else { + this.scheduleIdleTimeout(appId); + } + }, this.idleTimeoutMs); + } +} + +/** + * Check whether a TCP port is already in use by attempting to bind to it. + * Returns true if the port is in use (bind fails with EADDRINUSE). + */ +function isPortInUse(port: number): Promise { + return new Promise((resolve) => { + const server = createServer(); + server.once('error', (err: NodeJS.ErrnoException) => { + resolve(err.code === 'EADDRINUSE'); + }); + server.once('listening', () => { + server.close(() => resolve(false)); + }); + server.listen(port, '127.0.0.1'); + }); +} diff --git a/src/core/audit-logger.ts b/src/core/audit-logger.ts index 8d82f9d3..5627936e 100644 --- a/src/core/audit-logger.ts +++ b/src/core/audit-logger.ts @@ -1,13 +1,36 @@ -import { appendFile, mkdir } from 'node:fs/promises'; -import { dirname } from 'node:path'; +import { appendFile, mkdir, readdir, stat, unlink, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; import type { AuditConfig } from '../types/config.js'; import type { InboundMessage, OutboundMessage } from '../types/message.js'; +import type { MemoryManager, AuditRecord } from '../memory/index.js'; import { createLogger } from './logger.js'; const logger = createLogger('audit'); export type AuditEventType = 'inbound' | 'outbound' | 'auth_denied' | 'rate_limited' | 'error'; +/** Execution trace written to .openbridge/audit/ for each worker spawn. */ +export interface WorkerSpawnTrace { + /** Activity / task ID for this spawn */ + taskId: string; + /** Tool profile used (read-only, code-audit, code-edit, full-access, master) */ + profile: string; + /** Allowed tools list passed to the worker */ + tools: string[]; + /** Wall-clock duration in milliseconds */ + durationMs: number; + /** Estimated cost in USD (undefined if unknown) */ + costUsd?: number; + /** Number of files modified (heuristic count from worker output) */ + filesModified: number; + /** Worker result status */ + result: 'success' | 'failed' | 'timeout' | 'error'; + /** Model used by the worker */ + model?: string; + /** Short task description */ + taskSummary?: string; +} + export interface AuditEntry { timestamp: string; event: AuditEventType; @@ -20,10 +43,14 @@ export interface AuditEntry { metadata?: Record; } +/** How long to retain worker spawn trace files in .openbridge/audit/ (30 days). */ +const AUDIT_FILE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; + export class AuditLogger { private readonly enabled: boolean; private readonly logPath: string; - private dirEnsured = false; + private memory: MemoryManager | null = null; + private workspacePath: string | null = null; constructor(config: AuditConfig) { this.enabled = config.enabled; @@ -34,6 +61,95 @@ export class AuditLogger { } } + /** Attach a MemoryManager for SQLite persistence. */ + setMemory(memory: MemoryManager): void { + this.memory = memory; + } + + /** Set the workspace path — enables JSON trace files in .openbridge/audit/. */ + setWorkspacePath(workspacePath: string): void { + this.workspacePath = workspacePath; + } + + /** + * Write an execution trace JSON file to .openbridge/audit/ for a completed worker spawn. + * Filename: audit-{timestamp}-{taskId}.json + * Also triggers cleanup of files older than 30 days. + */ + async logWorkerSpawn(trace: WorkerSpawnTrace): Promise { + if (!this.workspacePath) return; + + const auditDir = join(this.workspacePath, '.openbridge', 'audit'); + try { + await mkdir(auditDir, { recursive: true }); + } catch (err) { + logger.error({ err, auditDir }, 'Failed to create .openbridge/audit directory'); + return; + } + + // Build a filesystem-safe timestamp: 2026-03-02T12:00:00.000Z → 20260302T120000Z + const ts = new Date() + .toISOString() + .replace(/[-:]/g, '') + .replace(/\.\d+Z$/, 'Z'); + const safeName = trace.taskId.replace(/[^a-zA-Z0-9_-]/g, '_').slice(0, 40); + const filename = `audit-${ts}-${safeName}.json`; + const filePath = join(auditDir, filename); + + const record = { + timestamp: new Date().toISOString(), + taskId: trace.taskId, + profile: trace.profile, + tools: trace.tools, + durationMs: trace.durationMs, + costUsd: trace.costUsd ?? null, + filesModified: trace.filesModified, + result: trace.result, + model: trace.model ?? null, + taskSummary: trace.taskSummary ?? null, + }; + + try { + await writeFile(filePath, JSON.stringify(record, null, 2), 'utf-8'); + logger.debug({ filePath, taskId: trace.taskId }, 'Worker spawn trace written'); + } catch (err) { + logger.error({ err, filePath }, 'Failed to write worker spawn trace'); + return; + } + + // Best-effort cleanup — do not await or propagate errors + this.cleanupOldAuditFiles(auditDir).catch((err) => { + logger.warn({ err, auditDir }, 'Audit file cleanup failed'); + }); + } + + /** + * Remove audit trace files older than AUDIT_FILE_RETENTION_MS from the given directory. + */ + private async cleanupOldAuditFiles(auditDir: string): Promise { + const cutoff = Date.now() - AUDIT_FILE_RETENTION_MS; + let entries: string[]; + try { + entries = await readdir(auditDir); + } catch { + return; // directory may not exist yet + } + + for (const entry of entries) { + if (!entry.startsWith('audit-') || !entry.endsWith('.json')) continue; + const filePath = join(auditDir, entry); + try { + const info = await stat(filePath); + if (info.mtimeMs < cutoff) { + await unlink(filePath); + logger.debug({ filePath }, 'Removed expired audit trace file'); + } + } catch { + // ignore per-file errors + } + } + } + async logInbound(message: InboundMessage): Promise { await this.write({ timestamp: new Date().toISOString(), @@ -83,15 +199,35 @@ export class AuditLogger { private async write(entry: AuditEntry): Promise { if (!this.enabled) return; - try { - if (!this.dirEnsured) { - await mkdir(dirname(this.logPath), { recursive: true }); - this.dirEnsured = true; - } + // Secondary sink: Pino structured log (always, for real-time console visibility) + logger.info({ audit: entry }, `audit:${entry.event}`); + // Primary sink: JSONL flat file + try { + await mkdir(dirname(this.logPath), { recursive: true }); await appendFile(this.logPath, JSON.stringify(entry) + '\n', 'utf-8'); } catch (err) { logger.error({ err, entry }, 'Failed to write audit log entry'); } + + // Secondary sink: SQLite (when memory is attached) + if (this.memory) { + try { + const record: AuditRecord = { + timestamp: entry.timestamp, + event: entry.event, + message_id: entry.messageId ?? null, + sender: entry.sender ?? null, + source: entry.source ?? null, + recipient: entry.recipient ?? null, + content_length: entry.contentLength ?? null, + error: entry.error ?? null, + metadata: entry.metadata ? JSON.stringify(entry.metadata) : null, + }; + await this.memory.insertAuditEntry(record); + } catch (err) { + logger.error({ err, entry }, 'Failed to write audit entry to SQLite'); + } + } } } diff --git a/src/core/auth.ts b/src/core/auth.ts index 7737ad59..c9d2e652 100644 --- a/src/core/auth.ts +++ b/src/core/auth.ts @@ -1,4 +1,13 @@ +import { randomInt } from 'crypto'; +import type Database from 'better-sqlite3'; import type { AuthConfig, CommandFilterConfig } from '../types/config.js'; +import { + getAccess, + setAccess, + resetDailyCosts, + incrementDailyCost as storeIncrementDailyCost, +} from '../memory/access-store.js'; +import type { AccessRole } from '../memory/access-store.js'; import { createLogger } from './logger.js'; const logger = createLogger('auth'); @@ -8,28 +17,122 @@ export interface CommandFilterResult { reason?: string; } +// --------------------------------------------------------------------------- +// Role-based default allowed actions +// null = no restriction (all actions allowed) +// --------------------------------------------------------------------------- + +const ROLE_ALLOWED_ACTIONS: Record = { + owner: null, + admin: null, + developer: ['read', 'edit', 'test', 'chat'], + viewer: ['read', 'chat'], + custom: null, // governed by entry.allowed_actions +}; + +// Keywords used to classify a message into an action category. +// Order matters — stop > deploy > edit > test > chat (most → least restrictive). +// 'chat' is the default for conversational messages with no strong action keywords. +const STOP_RE = /\bstop\b/i; +const DEPLOY_RE = /\b(deploy|release|publish|push to|ship|launch|stage)\b/i; +const EDIT_RE = + /\b(edit|modify|change|update|fix|refactor|add|create|write|implement|delete|remove|replace|rename|install|uninstall)\b/i; +const TEST_RE = /\b(test|run|execute|build|compile|lint)\b/i; + +function classifyMessageAction(content: string): string { + if (STOP_RE.test(content)) return 'stop'; + if (DEPLOY_RE.test(content)) return 'deploy'; + if (EDIT_RE.test(content)) return 'edit'; + if (TEST_RE.test(content)) return 'test'; + return 'chat'; +} + +export interface PendingPairing { + senderId: string; + channel: string; + requestedAt: Date; + attempts: number; +} + +const PAIRING_TTL_MS = 5 * 60 * 1000; // 5 minutes +const PAIRING_CLEANUP_INTERVAL_MS = 60 * 1000; // 60 seconds +const PAIRING_RATE_LIMIT_MAX = 3; // max requests per sender per hour +const PAIRING_RATE_LIMIT_WINDOW_MS = 60 * 60 * 1000; // 1 hour + export class AuthService { private whitelist: Set; private prefix: string; private allowPatterns: RegExp[]; private denyPatterns: RegExp[]; private denyMessage: string; + private db: Database.Database | null = null; + private defaultRole: string; + private channelRoles: Record; + private pairingEnabled: boolean; + private pendingPairings: Map = new Map(); + private expiryTimer: ReturnType | null = null; + private pairingRateLimits: Map = new Map(); /** * Normalize a phone number to digits-only for comparison. * Handles: "+33629539495", "33629539495@c.us", "33629539495" */ - private static normalizeNumber(input: string): string { - return input.replace('@c.us', '').replace(/\D/g, ''); + /** + * Normalize any sender identifier — phone numbers get digit-only treatment, + * non-numeric identifiers (webchat-user, console-user, usernames) are lowercased as-is. + */ + private static normalizeId(input: string): string { + const withoutSuffix = input.replace(/@c\.us$/, ''); + const withoutPrefix = withoutSuffix.replace(/^\+/, ''); + // If it's all digits, treat as phone number + if (/^\d+$/.test(withoutPrefix)) { + return withoutPrefix; + } + // Non-numeric identifier — keep as-is (lowercased) + return input.toLowerCase(); + } + + /** + * Build whitelist with per-entry diagnostics. + * Returns the normalized Set and a list of dropped entries with reasons. + */ + private static buildWhitelistDetailed(numbers: string[]): { + whitelist: Set; + dropped: Array<{ entry: string; reason: string }>; + } { + const whitelist = new Set(); + const dropped: Array<{ entry: string; reason: string }> = []; + + for (const n of numbers) { + const trimmed = n.trim(); + if (!trimmed) { + dropped.push({ entry: n, reason: 'empty entry' }); + continue; + } + + const normalized = AuthService.normalizeId(trimmed); + if (whitelist.has(normalized)) { + dropped.push({ entry: n, reason: 'duplicate' }); + continue; + } + + whitelist.add(normalized); + } + + return { whitelist, dropped }; } private static buildWhitelist(numbers: string[]): Set { - return new Set(numbers.map((n) => AuthService.normalizeNumber(n))); + return AuthService.buildWhitelistDetailed(numbers).whitelist; } constructor(config: AuthConfig) { - this.whitelist = AuthService.buildWhitelist(config.whitelist); + const detailed = AuthService.buildWhitelistDetailed(config.whitelist); + this.whitelist = detailed.whitelist; this.prefix = config.prefix; + this.defaultRole = config.defaultRole ?? 'owner'; + this.channelRoles = config.channelRoles ?? {}; + this.pairingEnabled = config.pairingEnabled ?? true; const filter: CommandFilterConfig = config.commandFilter ?? { allowPatterns: [], @@ -44,20 +147,243 @@ export class AuthService { logger.info( { whitelistedNumbers: this.whitelist.size, + rawEntries: config.whitelist.length, + droppedEntries: detailed.dropped.length, prefix: this.prefix, allowPatterns: filter.allowPatterns.length, denyPatterns: filter.denyPatterns.length, }, 'Auth service initialized', ); + + for (const { entry, reason } of detailed.dropped) { + if (reason === 'duplicate') { + logger.warn({ entry }, `Duplicate whitelist entry ignored: ${entry}`); + } else { + logger.warn({ entry, reason }, `Dropped whitelist entry '${entry}': ${reason}`); + } + } + + if (this.whitelist.size === 0) { + logger.warn( + 'Auth whitelist is empty — ALL senders are authorized. To restrict access, add phone numbers to auth.whitelist in config.json.', + ); + } + + this.startExpiryTimer(); + } + + /** Attach a SQLite database for access_control enforcement. */ + setDatabase(db: Database.Database): void { + this.db = db; + logger.info('AuthService: access_control database attached'); + + // Sync existing access_control entries with the configured defaultRole. + // This fixes stale entries created under the old schema default ('viewer') + // that don't match the config's defaultRole (typically 'owner'). + this.syncDefaultRoles(db); + } + + /** + * Update any whitelisted user whose DB role still reflects the old schema + * default ('viewer') but whose config defaultRole is different (e.g. 'owner'). + * Only touches entries that have no explicit allowed_actions override — those + * are assumed to be auto-created entries that should track the config default. + */ + private syncDefaultRoles(db: Database.Database): void { + if (this.defaultRole === 'viewer') return; // nothing to sync + try { + const result = db + .prepare( + `UPDATE access_control + SET role = ?, updated_at = datetime('now') + WHERE role = 'viewer' + AND (allowed_actions IS NULL OR allowed_actions = '[]' OR allowed_actions = '')`, + ) + .run(this.defaultRole); + if (result.changes > 0) { + logger.info( + { updatedCount: result.changes, newRole: this.defaultRole }, + 'AuthService: synced stale viewer roles to config defaultRole', + ); + } + } catch (err) { + logger.warn({ err }, 'AuthService: syncDefaultRoles failed — continuing'); + } } /** Check if a sender is allowed to use the bridge */ - isAuthorized(sender: string): boolean { + isAuthorized(sender: string, channel?: string): boolean { if (this.whitelist.size === 0) { return true; // No whitelist = open access } - return this.whitelist.has(AuthService.normalizeNumber(sender)); + const normalizedSender = AuthService.normalizeId(sender); + if (this.whitelist.has(normalizedSender)) { + return true; // Whitelisted — always authorized + } + // Prefix match: "webchat-user" in whitelist matches "webchat-user-" + for (const entry of this.whitelist) { + if (normalizedSender.startsWith(entry + '-') || normalizedSender.startsWith(entry + '_')) { + return true; + } + } + // Pairing is additive: a user approved via pairing has an active access_control entry. + // Check the DB when pairing is enabled so paired users coexist with the whitelist. + if (this.pairingEnabled && this.db && channel) { + try { + const entry = getAccess(this.db, sender, channel); + if (entry && entry.active) { + return true; + } + } catch (err) { + logger.warn( + { err, sender, channel }, + 'AuthService: getAccess check in isAuthorized failed', + ); + } + } + return false; + } + + /** + * Check access control for a whitelisted user. + * + * Enforces three rules (in order): + * 1. Entry must be active. + * 2. Daily cost budget must not be exceeded. + * 3. The classified action must be allowed for the user's role (or custom list). + * 4. If scopes are set and the message contains explicit file paths, those paths + * must fall within at least one allowed scope. + * + * When no DB is attached or no entry exists, defaults to 'owner' (backward-compatible). + * Before checking the budget, stale daily-cost resets are applied automatically. + */ + checkAccessControl( + userId: string, + channel: string, + messageContent?: string, + ): CommandFilterResult { + if (!this.db) return { allowed: true }; + + // Run any pending midnight resets before checking the budget. + try { + resetDailyCosts(this.db); + } catch (err) { + logger.warn({ err }, 'AuthService: resetDailyCosts failed — skipping'); + } + + let entry; + try { + entry = getAccess(this.db, userId, channel); + } catch (err) { + logger.warn({ err, userId, channel }, 'AuthService: getAccess failed — defaulting to allow'); + return { allowed: true }; + } + + // No entry → default to owner (backward-compatible with pre-ACL whitelist behaviour). + if (!entry) return { allowed: true }; + + // 1. Active check + if (!entry.active) { + logger.warn({ userId, channel }, 'Access denied — account inactive'); + return { allowed: false, reason: 'Your access has been revoked.' }; + } + + // 2. Daily budget check + if (entry.max_cost_per_day_usd != null && entry.daily_cost_used != null) { + if (entry.daily_cost_used >= entry.max_cost_per_day_usd) { + logger.warn( + { + userId, + channel, + used: entry.daily_cost_used, + limit: entry.max_cost_per_day_usd, + }, + 'Access denied — daily cost limit exceeded', + ); + return { + allowed: false, + reason: `Daily usage limit ($${entry.max_cost_per_day_usd.toFixed(2)}) reached. Try again tomorrow.`, + }; + } + } + + // 3. Action check (only when message content is available) + if (messageContent) { + const action = classifyMessageAction(messageContent); + + // blocked_actions always take precedence (explicit deny list) + if (entry.blocked_actions && entry.blocked_actions.includes(action)) { + logger.warn({ userId, channel, action }, 'Access denied — action in blocked list'); + return { + allowed: false, + reason: `Action "${action}" is blocked for your account (role: ${entry.role}).`, + }; + } + + // Determine the effective allowed-action set: + // • If the entry has an explicit allowed_actions list, use that. + // • Otherwise fall back to the role default. + // • null means "no restriction" (owner / admin). + const effectiveAllowed = + entry.allowed_actions && entry.allowed_actions.length > 0 + ? entry.allowed_actions + : (ROLE_ALLOWED_ACTIONS[entry.role] ?? null); + + if (effectiveAllowed !== null && !effectiveAllowed.includes(action)) { + logger.warn( + { userId, channel, role: entry.role, action, effectiveAllowed }, + 'Access denied — action not in allowed list for role', + ); + const allowedList = effectiveAllowed.join(', '); + return { + allowed: false, + reason: `Action "${action}" is not permitted for your role (${entry.role}). Your role allows: ${allowedList}.`, + }; + } + + // 4. Scope check — only enforced when explicit file paths appear in the message. + if (entry.scopes && entry.scopes.length > 0) { + // A simple heuristic: paths contain a '/' and end with a known extension. + const FILE_PATH_RE = /(?:^|\s)((?:\.{0,2}\/)?[\w./\\-]+\.[\w]{1,6})(?:\s|$)/g; + let match: RegExpExecArray | null; + const detectedPaths: string[] = []; + while ((match = FILE_PATH_RE.exec(messageContent)) !== null) { + if (match[1]) detectedPaths.push(match[1]); + } + + if (detectedPaths.length > 0) { + const outOfScope = detectedPaths.filter( + (fp) => !entry.scopes!.some((scope) => fp.startsWith(scope)), + ); + if (outOfScope.length > 0) { + logger.warn( + { userId, channel, outOfScope, scopes: entry.scopes }, + 'Access denied — file path outside allowed scopes', + ); + return { + allowed: false, + reason: 'One or more referenced file paths are outside your allowed scope.', + }; + } + } + } + } + + return { allowed: true }; + } + + /** + * Increment daily_cost_used for the given user+channel. + * No-op when no DB is attached or no entry exists for the pair. + */ + incrementDailyCost(userId: string, channel: string, costUsd: number): void { + if (!this.db || costUsd <= 0) return; + try { + storeIncrementDailyCost(this.db, userId, channel, costUsd); + } catch (err) { + logger.warn({ err, userId, channel, costUsd }, 'AuthService: incrementDailyCost failed'); + } } /** Check if a message starts with the configured prefix */ @@ -98,8 +424,12 @@ export class AuthService { /** Hot-reload auth config without restarting */ updateConfig(config: AuthConfig): void { - this.whitelist = AuthService.buildWhitelist(config.whitelist); + const detailed = AuthService.buildWhitelistDetailed(config.whitelist); + this.whitelist = detailed.whitelist; this.prefix = config.prefix; + this.defaultRole = config.defaultRole ?? 'owner'; + this.channelRoles = config.channelRoles ?? {}; + this.pairingEnabled = config.pairingEnabled ?? true; const filter: CommandFilterConfig = config.commandFilter ?? { allowPatterns: [], @@ -114,15 +444,210 @@ export class AuthService { logger.info( { whitelistedNumbers: this.whitelist.size, + droppedEntries: detailed.dropped.length, prefix: this.prefix, allowPatterns: filter.allowPatterns.length, denyPatterns: filter.denyPatterns.length, }, 'Auth service config reloaded', ); + + for (const { entry, reason } of detailed.dropped) { + if (reason === 'duplicate') { + logger.warn({ entry }, `Duplicate whitelist entry ignored: ${entry}`); + } else { + logger.warn({ entry, reason }, `Dropped whitelist entry '${entry}': ${reason}`); + } + } + } + + /** + * Returns the role to assign for a given channel when auto-creating an access_control entry. + * Checks channelRoles[channel] first, then falls back to defaultRole. + */ + getRoleForChannel(channel: string): string { + return this.channelRoles[channel] ?? this.defaultRole; + } + + /** + * Auto-create an access_control entry with the correct role for this user+channel + * if none exists. No-op when no DB is attached or an entry already exists. + * Called on the first authorized command to eliminate the "no entry = owner fallback" + * ambiguity in checkAccessControl. + */ + ensureAccessEntry(userId: string, channel: string): void { + if (!this.db) return; + try { + const existing = getAccess(this.db, userId, channel); + if (existing) return; + const role = this.getRoleForChannel(channel) as AccessRole; + setAccess(this.db, { user_id: userId, channel, role, active: true }); + logger.info({ userId, channel, role }, 'AuthService: auto-created access_control entry'); + } catch (err) { + logger.warn({ err, userId, channel }, 'AuthService: ensureAccessEntry failed — continuing'); + } + } + + /** + * Evict all pending pairings whose requestedAt is older than PAIRING_TTL_MS. + * Also removes expired entries from the DB if attached. + */ + evictExpiredPairings(): void { + const now = Date.now(); + for (const [code, pairing] of this.pendingPairings) { + if (now - pairing.requestedAt.getTime() >= PAIRING_TTL_MS) { + this.pendingPairings.delete(code); + logger.info({ code, senderId: pairing.senderId }, 'Pairing code expired — removed'); + if (this.db) { + try { + this.db.prepare(`DELETE FROM pending_pairings WHERE code = ?`).run(code); + } catch (err) { + logger.warn({ err, code }, 'AuthService: failed to remove expired pairing from DB'); + } + } + } + } + } + + /** Start the background timer that evicts expired pairings every 60 seconds. */ + private startExpiryTimer(): void { + this.expiryTimer = setInterval(() => { + this.evictExpiredPairings(); + }, PAIRING_CLEANUP_INTERVAL_MS); + // Allow the Node.js process to exit even if the timer is still running. + if (this.expiryTimer.unref) { + this.expiryTimer.unref(); + } + } + + /** Stop the background expiry timer. Call this when shutting down. */ + stopExpiryTimer(): void { + if (this.expiryTimer !== null) { + clearInterval(this.expiryTimer); + this.expiryTimer = null; + } } get commandPrefix(): string { return this.prefix; } + + get isPairingEnabled(): boolean { + return this.pairingEnabled; + } + + /** + * Generate a cryptographically secure 6-digit pairing code. + * Returns a zero-padded string in the range "100000"–"999999". + */ + static generatePairingCode(): string { + return randomInt(100000, 999999).toString(); + } + + /** Store a new pending pairing for an unknown sender. */ + storePairing(code: string, senderId: string, channel: string): void { + const requestedAt = new Date(); + this.pendingPairings.set(code, { + senderId, + channel, + requestedAt, + attempts: 0, + }); + if (this.db) { + try { + this.db + .prepare( + `INSERT OR REPLACE INTO pending_pairings (code, sender_id, channel, requested_at, attempts) + VALUES (?, ?, ?, ?, 0)`, + ) + .run(code, senderId, channel, requestedAt.toISOString()); + } catch (err) { + logger.warn( + { err, code, senderId, channel }, + 'AuthService: failed to persist pending pairing to DB', + ); + } + } + logger.info({ code, senderId, channel }, 'Pending pairing stored'); + } + + /** Retrieve a pending pairing by code, or undefined if not found. */ + getPairing(code: string): PendingPairing | undefined { + return this.pendingPairings.get(code); + } + + /** Remove a pending pairing (after approval or expiry). */ + removePairing(code: string): void { + this.pendingPairings.delete(code); + if (this.db) { + try { + this.db.prepare(`DELETE FROM pending_pairings WHERE code = ?`).run(code); + } catch (err) { + logger.warn({ err, code }, 'AuthService: failed to remove pending pairing from DB'); + } + } + } + + /** Increment the attempt counter for a pairing code. */ + incrementPairingAttempts(code: string): void { + const pairing = this.pendingPairings.get(code); + if (pairing) { + pairing.attempts += 1; + } + if (this.db) { + try { + this.db + .prepare(`UPDATE pending_pairings SET attempts = attempts + 1 WHERE code = ?`) + .run(code); + } catch (err) { + logger.warn({ err, code }, 'AuthService: failed to increment pairing attempts in DB'); + } + } + } + + /** Return all pending pairings (for inspection/testing). */ + getPendingPairings(): ReadonlyMap { + return this.pendingPairings; + } + + /** + * Check whether a sender is within the pairing rate limit. + * Returns true if the request is allowed, false if the limit is exceeded. + * Prunes timestamps older than PAIRING_RATE_LIMIT_WINDOW_MS on each call. + */ + checkPairingRateLimit(senderId: string): boolean { + const now = Date.now(); + const cutoff = now - PAIRING_RATE_LIMIT_WINDOW_MS; + const timestamps = (this.pairingRateLimits.get(senderId) ?? []).filter( + (ts) => ts.getTime() >= cutoff, + ); + if (timestamps.length >= PAIRING_RATE_LIMIT_MAX) { + this.pairingRateLimits.set(senderId, timestamps); + return false; + } + timestamps.push(new Date(now)); + this.pairingRateLimits.set(senderId, timestamps); + return true; + } + + /** + * Initiate a pairing for an unknown sender. + * Generates a 6-digit code, stores the pending pairing, and returns the + * message to send back to the unknown sender. + * Returns null if pairing is disabled or the sender has exceeded the rate limit. + */ + initiatePairing(senderId: string, channel: string): string | null { + if (!this.pairingEnabled) { + logger.info({ senderId, channel }, 'Pairing disabled — request ignored'); + return null; + } + if (!this.checkPairingRateLimit(senderId)) { + logger.warn({ senderId, channel }, 'Pairing rate limit exceeded — request denied'); + return null; + } + const code = AuthService.generatePairingCode(); + this.storePairing(code, senderId, channel); + logger.info({ senderId, channel, code }, 'Pairing initiated for unknown sender'); + return `To connect, ask the admin to approve code: ${code} (expires in 5 minutes)`; + } } diff --git a/src/core/bridge.ts b/src/core/bridge.ts index f578c845..1b526a18 100644 --- a/src/core/bridge.ts +++ b/src/core/bridge.ts @@ -1,25 +1,69 @@ -import type { AppConfig } from '../types/config.js'; -import type { InboundMessage } from '../types/message.js'; +import * as fs from 'node:fs/promises'; +import { readFileSync } from 'node:fs'; +import { execSync } from 'node:child_process'; +import path from 'node:path'; +import type { AppConfig, EmailConfig, MCPServer, SecurityConfig } from '../types/config.js'; +import { V2ConfigSchema, ENV_DENY_PATTERNS, getEffectiveSandboxMode } from '../types/config.js'; +import type { DiscoveredTool } from '../types/discovery.js'; +import { warnAboutExposedSecrets } from './env-sanitizer.js'; +import { TunnelManager } from './tunnel-manager.js'; +// Side-effect imports: each adapter auto-registers with TunnelManager on load +import './cloudflared-adapter.js'; +import './ngrok-adapter.js'; +import type { InboundMessage, OutboundMessage } from '../types/message.js'; import type { Connector } from '../types/connector.js'; import type { AIProvider } from '../types/provider.js'; import type { MasterManager } from '../master/master-manager.js'; +import { DotFolderManager } from '../master/dotfolder-manager.js'; +import { KnowledgeRetriever } from './knowledge-retriever.js'; +import { MemoryManager } from '../memory/index.js'; +import { createMediaManager } from './media-manager.js'; +import type { MediaManager } from './media-manager.js'; +import type { McpRegistry } from './mcp-registry.js'; import { AuthService } from './auth.js'; import { AuditLogger } from './audit-logger.js'; import { ConfigWatcher } from './config-watcher.js'; +import { FileServer } from './file-server.js'; import { HealthServer } from './health.js'; import type { HealthStatus, ComponentStatus } from './health.js'; import { MessageQueue } from './queue.js'; import { MetricsCollector, MetricsServer } from './metrics.js'; +import { setAgentRunnerMetrics } from './agent-runner.js'; import { PluginRegistry } from './registry.js'; import { RateLimiter } from './rate-limiter.js'; -import { Router } from './router.js'; +import { Router, classifyMessagePriority } from './router.js'; import { AgentOrchestrator } from './agent-orchestrator.js'; +import type { AppServer } from './app-server.js'; +import type { InteractionRelay } from './interaction-relay.js'; +import { SecretScanner } from './secret-scanner.js'; +import type { SecretMatch } from './secret-scanner.js'; +import { DockerSandbox, DockerHealthMonitor, cleanupSandboxContainers } from './docker-sandbox.js'; +import { IntegrationHub } from '../integrations/hub.js'; import { createLogger } from './logger.js'; const logger = createLogger('bridge'); +/** Maximum inbound message length — matches sanitizePrompt's cap in agent-runner.ts */ +const MAX_INBOUND_LENGTH = 32_768; + export interface BridgeOptions { configPath?: string; + /** Max ms to wait for queue drain on shutdown before proceeding. Default: 30 000 */ + drainTimeoutMs?: number; + /** Absolute path to the target workspace — when provided, MemoryManager is created for SQLite persistence */ + workspacePath?: string; + /** MCP server registry — when provided, exposed via getMcpRegistry() and wired to connectors */ + mcpRegistry?: McpRegistry; + /** Security config — controls which env vars are stripped from worker processes */ + securityConfig?: SecurityConfig; + /** Tunnel tool name (e.g. 'cloudflared', 'ngrok') — when set, starts a tunnel on the file server port during start() */ + tunnelTool?: string; + /** Glob patterns for files to include — only these visible to the AI (workspace.include from V2 config) */ + workspaceInclude?: readonly string[]; + /** Glob patterns for files to exclude — hidden from the AI (workspace.exclude from V2 config) */ + workspaceExclude?: readonly string[]; + /** Discovered AI tools — when provided, exposed via GET /api/discovery in WebChat */ + discoveredTools?: DiscoveredTool[]; } export class Bridge { @@ -36,24 +80,72 @@ export class Bridge { private readonly router: Router; private readonly orchestrator: AgentOrchestrator; private master: MasterManager | null = null; + private memory: MemoryManager | null = null; + private mcpRegistry: McpRegistry | null = null; + private readonly securityConfig: SecurityConfig | undefined; + private fileServer: FileServer | null = null; + private appServer: AppServer | null = null; + private interactionRelay: InteractionRelay | null = null; + private tunnelManager: TunnelManager | null = null; + private tunnelPublicUrl: string | null = null; + private readonly workspacePath: string | undefined; private readonly connectors: Connector[] = []; private readonly providers: AIProvider[] = []; private readonly startedAt: number = Date.now(); private readonly configPath?: string; + private stopped = false; + private readonly drainTimeoutMs: number; + private agentStatusInterval: ReturnType | null = null; + private evictionInterval: ReturnType | null = null; + private lastMessageAt: string | null = null; + private tunnelExitHandler: (() => void) | null = null; + private tunnelSigintHandler: (() => void) | null = null; + private dockerHealthMonitor: DockerHealthMonitor | null = null; + private readonly integrationHub: IntegrationHub; + private readonly detectedSecrets: SecretMatch[] = []; + private readonly sessionExcludePatterns: string[] = []; + private readonly workspaceInclude: readonly string[]; + private readonly workspaceExclude: readonly string[]; + private readonly discoveredTools: DiscoveredTool[]; constructor(config: AppConfig, options?: BridgeOptions) { this.config = config; this.configPath = options?.configPath; + this.drainTimeoutMs = options?.drainTimeoutMs ?? 30_000; + this.workspaceInclude = options?.workspaceInclude ?? []; + this.workspaceExclude = options?.workspaceExclude ?? []; + this.discoveredTools = options?.discoveredTools ?? []; this.auth = new AuthService(config.auth); this.auditLogger = new AuditLogger(config.audit); this.healthServer = new HealthServer(config.health); this.metrics = new MetricsCollector(); + setAgentRunnerMetrics(this.metrics); this.metricsServer = new MetricsServer(config.metrics); this.rateLimiter = new RateLimiter(config.auth.rateLimit); this.queue = new MessageQueue(config.queue, this.metrics); this.registry = new PluginRegistry(); this.router = new Router(config.defaultProvider, config.router, this.auditLogger, this.metrics); this.orchestrator = new AgentOrchestrator(config.defaultProvider); + + if (options?.workspacePath) { + this.workspacePath = options.workspacePath; + const dbPath = path.join(options.workspacePath, '.openbridge', 'openbridge.db'); + this.memory = new MemoryManager(dbPath); + this.fileServer = new FileServer(options.workspacePath); + this.router.setWorkspacePath(options.workspacePath); + this.auditLogger.setWorkspacePath(options.workspacePath); + } + + if (options?.mcpRegistry) { + this.mcpRegistry = options.mcpRegistry; + } + + if (options?.tunnelTool) { + this.tunnelManager = new TunnelManager(options.tunnelTool); + } + + this.securityConfig = options?.securityConfig; + this.integrationHub = new IntegrationHub(); } /** Register built-in and external plugins before starting */ @@ -61,20 +153,361 @@ export class Bridge { return this.registry; } + /** Returns the names of all successfully initialized connectors */ + getActiveConnectorNames(): string[] { + return this.connectors.map((c) => c.name); + } + + /** Returns the port the file server is listening on, or null if not running */ + getFileServerPort(): number | null { + if (!this.fileServer) return null; + const match = this.fileServer.baseUrl.match(/:(\d+)$/); + return match ? parseInt(match[1]!, 10) : null; + } + + /** Returns the tunnel public URL if a tunnel is active, or null if not running */ + getTunnelUrl(): string | null { + return this.tunnelPublicUrl; + } + + /** + * Ensure a tunnel is running and return its public URL. + * Called on-demand by the output-marker-processor for remote channel file delivery. + * + * Logic: + * 1. If tunnelManager already has a public URL, return it immediately. + * 2. If tunnelManager exists but hasn't started, start it on the file server port. + * 3. If no tunnel tool is configured, attempt to auto-detect `cloudflared` via `which`. + * If found, create a TunnelManager, start it, store the public URL, and notify Master. + * 4. If no tunnel tool is available, return null. + */ + async ensureTunnel(): Promise { + // 1. Already active — return immediately + if (this.tunnelManager?.isActive()) { + return this.tunnelPublicUrl; + } + + const fileServerPort = this.getFileServerPort(); + if (fileServerPort === null) { + return null; + } + + // 2. TunnelManager exists but not yet started — start it + if (this.tunnelManager) { + this.registerTunnelShutdownHandlers(); + try { + this.tunnelPublicUrl = await this.tunnelManager.start(fileServerPort); + logger.info( + { url: this.tunnelPublicUrl }, + 'Auto-tunnel started for remote channel file delivery', + ); + this.master?.setTunnelUrl(this.tunnelPublicUrl); + return this.tunnelPublicUrl; + } catch (error) { + logger.warn({ err: error }, 'ensureTunnel: existing TunnelManager failed to start'); + return null; + } + } + + // 3. No tunnel configured — attempt to auto-detect cloudflared + try { + const whichCmd = process.platform === 'win32' ? 'where' : 'which'; + execSync(`${whichCmd} cloudflared`, { stdio: 'pipe' }); + } catch { + // cloudflared not found + return null; + } + + // cloudflared is available — create and start a new TunnelManager + this.tunnelManager = new TunnelManager('cloudflared'); + this.registerTunnelShutdownHandlers(); + try { + this.tunnelPublicUrl = await this.tunnelManager.start(fileServerPort); + logger.info( + { url: this.tunnelPublicUrl }, + 'Auto-tunnel started for remote channel file delivery', + ); + this.master?.setTunnelUrl(this.tunnelPublicUrl); + return this.tunnelPublicUrl; + } catch (error) { + logger.warn({ err: error }, 'ensureTunnel: auto-detected cloudflared failed to start'); + this.tunnelManager = null; + return null; + } + } + + /** Returns WebChat access URL (with token) and the raw token if a webchat connector is active */ + getWebChatInfo(): { url: string; token: string } | null { + for (const connector of this.connectors) { + if (connector.name !== 'webchat') continue; + const c = connector as { + getAuthToken?: () => string | null; + getWebChatAccessUrl?: () => string | null; + }; + const token = typeof c.getAuthToken === 'function' ? c.getAuthToken() : null; + const url = typeof c.getWebChatAccessUrl === 'function' ? c.getWebChatAccessUrl() : null; + if (token && url) return { url, token }; + } + return null; + } + + private registerTunnelShutdownHandlers(): void { + if (!this.tunnelManager || this.tunnelExitHandler || this.tunnelSigintHandler) { + return; + } + + this.tunnelExitHandler = (): void => { + this.tunnelManager?.stop(); + this.tunnelPublicUrl = null; + }; + this.tunnelSigintHandler = (): void => { + this.tunnelManager?.stop(); + this.tunnelPublicUrl = null; + }; + process.once('exit', this.tunnelExitHandler); + process.on('SIGINT', this.tunnelSigintHandler); + } + + private clearTunnelShutdownHandlers(): void { + if (this.tunnelExitHandler) { + process.removeListener('exit', this.tunnelExitHandler); + this.tunnelExitHandler = null; + } + if (this.tunnelSigintHandler) { + process.removeListener('SIGINT', this.tunnelSigintHandler); + this.tunnelSigintHandler = null; + } + } + + /** Returns the MemoryManager instance (null if no workspacePath was provided or init failed) */ + getMemory(): MemoryManager | null { + return this.memory; + } + + /** Returns the McpRegistry instance (null if no mcpRegistry was provided) */ + getMcpRegistry(): McpRegistry | null { + return this.mcpRegistry; + } + + /** Returns the IntegrationHub instance */ + getIntegrationHub(): IntegrationHub { + return this.integrationHub; + } + + /** + * Returns glob patterns for files auto-excluded this session after startup secret scanning. + * Each entry is a relative path from the workspace root (e.g. "service-account-prod.json"). + * Pass alongside `config.workspace.exclude` when calling isFileVisible(). + */ + getSessionExcludePatterns(): readonly string[] { + return this.sessionExcludePatterns; + } + + /** + * Returns the list of sensitive files detected during startup scanning. + * Useful for the /scope command (OB-1472) to report secrets with severity. + */ + getDetectedSecrets(): readonly SecretMatch[] { + return this.detectedSecrets; + } + /** Set the Master AI — must be called before start() to enable Master routing */ setMaster(master: MasterManager): void { this.master = master; logger.info('Master AI set on Bridge'); } + /** Set the AppServer — enables graceful app cleanup on shutdown and APP marker handling in Router */ + setAppServer(appServer: AppServer): void { + this.appServer = appServer; + this.router.setAppServer(appServer); + logger.info('AppServer set on Bridge and Router'); + } + + /** Set the InteractionRelay — routes app messages to Master via Router */ + setInteractionRelay(relay: InteractionRelay): void { + this.interactionRelay = relay; + this.router.setInteractionRelay(relay); + logger.info('InteractionRelay set on Bridge and Router'); + } + + /** Set the email config — enables [SHARE:email] marker support in the router */ + setEmailConfig(config: EmailConfig): void { + this.router.setEmailConfig(config); + logger.info('Email config set on Router'); + } + + /** + * Register MCP servers with the health check endpoint. + * Call after bridge.start() with servers from V2Config.mcp.servers. + * The /health response will include an `mcp` section reporting whether + * each server's command is available on PATH. + */ + setMcpServers(servers: MCPServer[]): void { + this.healthServer.setMcpServers(servers); + logger.info({ count: servers.length }, 'MCP servers registered for health checks'); + } + /** Start the bridge: initialize all connectors and providers, begin processing */ async start(): Promise { logger.info('Starting OpenBridge...'); + // Scan process.env for secret patterns and warn operators about variables that will be stripped + const denyPatterns = this.securityConfig?.envDenyPatterns ?? [...ENV_DENY_PATTERNS]; + warnAboutExposedSecrets(process.env, denyPatterns); + + // Scan workspace root for sensitive files — non-fatal, logs warnings and builds session exclude list + if (this.workspacePath) { + await this.runSecretScan(this.workspacePath); + } + + // Resolve effective sandbox mode — auto-detects Docker/bubblewrap for trusted mode (OB-1589) + if (this.securityConfig) { + const effectiveMode = getEffectiveSandboxMode(this.securityConfig); + if (effectiveMode !== this.securityConfig.sandbox.mode) { + logger.info( + { from: this.securityConfig.sandbox.mode, to: effectiveMode }, + 'Sandbox mode resolved', + ); + this.securityConfig.sandbox.mode = effectiveMode; + } + } + + // Docker startup health check + periodic recheck (OB-1557) + if (this.securityConfig?.sandbox?.mode === 'docker') { + const dockerSandbox = new DockerSandbox(); + this.dockerHealthMonitor = new DockerHealthMonitor(dockerSandbox); + await this.dockerHealthMonitor.start(); + + // Clean up dangling containers only when daemon is reachable (OB-1554) + if (this.dockerHealthMonitor.isDockerAvailable()) { + dockerSandbox.cleanupDanglingContainers().catch((err: unknown) => { + logger.warn({ err }, 'Docker startup cleanup failed — continuing'); + }); + } + } + + // Initialize memory system (SQLite) — non-fatal: DotFolderManager is the fallback + if (this.memory) { + try { + await this.memory.init(); + await this.memory.migrate(); + logger.info('MemoryManager initialized and migrated'); + + // Wire SQLite audit persistence into AuditLogger + this.auditLogger.setMemory(this.memory); + } catch (error) { + logger.error( + { err: error }, + 'MemoryManager initialization failed — continuing with DotFolderManager fallback', + ); + this.memory = null; + } + } + + // Startup sweep: mark stale 'running' agent_activity records as 'abandoned' (OB-1518) + // Any record still 'running' after 10 minutes is an orphan from a prior crash. + if (this.memory) { + try { + const db = this.memory.getDb(); + if (db) { + const tenMinutesAgo = new Date(Date.now() - 10 * 60 * 1000).toISOString(); + const now = new Date().toISOString(); + const result = db + .prepare( + `UPDATE agent_activity + SET status = 'abandoned', completed_at = ?, updated_at = ? + WHERE status = 'running' AND started_at < ?`, + ) + .run(now, now, tenMinutesAgo); + if (result.changes > 0) { + logger.warn( + { count: result.changes }, + 'Startup sweep: marked stale running agents as abandoned (likely orphans from a prior crash)', + ); + } + } + } catch (err) { + logger.warn({ err }, 'Startup activity sweep failed — continuing'); + } + } + + // Clean up legacy .openbridge/ artifacts that were migrated to SQLite + if (this.memory && this.workspacePath) { + await this.cleanLegacyDotFolderArtifacts(this.workspacePath, this.memory); + } + + // Schedule periodic DB eviction — run once on startup, then every 24 hours + if (this.memory) { + const runEviction = (): void => { + if (!this.memory) return; + logger.info('Running scheduled DB eviction'); + void this.memory + .evictOldData() + .then(() => { + logger.info('DB eviction complete'); + }) + .catch((err: unknown) => { + logger.error({ err }, 'DB eviction failed'); + }); + }; + runEviction(); + this.evictionInterval = setInterval(runEviction, 24 * 60 * 60 * 1000); + } + + // Wire auth service into router for SEND marker whitelist enforcement + this.router.setAuth(this.auth); + + // Wire security config into router for high-risk spawn confirmation + if (this.securityConfig) { + this.router.setSecurityConfig(this.securityConfig); + } + + // Attach the SQLite DB to the auth service for access_control enforcement + if (this.memory) { + const db = this.memory.getDb(); + if (db) this.auth.setDatabase(db); + } + + // Wire memory into router for "status" command support + if (this.memory) { + this.router.setMemory(this.memory); + } + + // Wire message queue into router for queue-depth display in "status" command + this.router.setQueue(this.queue); + + // Wire workspace visibility state into router for /scope command (OB-1472) + this.router.setVisibilityState( + this.detectedSecrets, + this.sessionExcludePatterns, + this.workspaceInclude, + this.workspaceExclude, + ); + + // Wire IntegrationHub into Router and MasterManager + this.router.setIntegrationHub(this.integrationHub); + + // Wire auto-tunnel into Router for on-demand remote channel APP delivery (OB-1633) + this.router.setEnsureTunnel(() => this.ensureTunnel()); + + if (this.master) { + this.master.setIntegrationHub(this.integrationHub); + } + if (this.master) { // V2 flow: Master AI handles all routing — skip provider initialization this.router.setMaster(this.master); + this.master.setRouter(this.router); logger.info('Master AI wired into router (V2 mode — providers skipped)'); + + // Wire KnowledgeRetriever after MemoryManager and DotFolderManager are ready (OB-1344) + if (this.memory && this.workspacePath) { + const dotFolder = new DotFolderManager(this.workspacePath); + const retriever = new KnowledgeRetriever(this.memory, dotFolder); + this.master.setKnowledgeRetriever(retriever); + logger.info('KnowledgeRetriever wired into Master AI'); + } } else { // V0 flow: initialize providers and wire orchestrator for (const providerConfig of this.config.providers) { @@ -92,7 +525,79 @@ export class Bridge { logger.info('Agent orchestrator wired into router'); } - // Initialize connectors + // Set up queue processing BEFORE connectors — so messages are handled + // as soon as any connector is ready (don't wait for slow ones like WhatsApp). + this.queue.onMessage(async (message) => { + // Snapshot daily cost before routing so we can attribute the delta to this user. + let costBefore = 0; + if (this.memory) { + try { + costBefore = await this.memory.getDailyCost(); + } catch { + // best-effort — cost attribution skipped if this fails + } + } + + await this.router.route(message); + + // Increment daily_cost_used by the cost incurred during this task. + if (this.memory) { + try { + const costAfter = await this.memory.getDailyCost(); + const delta = Math.max(0, costAfter - costBefore); + if (delta > 0) { + this.auth.incrementDailyCost(message.sender, message.source, delta); + } + } catch { + // best-effort — cost tracking is non-fatal + } + } + }); + + // Send an error response to the user when their message is permanently failed (DLQ). + this.queue.onDeadLetter(async (message, _error) => { + try { + const connector = this.router.getConnector(message.source); + if (!connector) { + logger.warn( + { source: message.source }, + 'onDeadLetter: connector not found — cannot send error response', + ); + return; + } + const msg: OutboundMessage = { + target: message.source, + recipient: message.sender, + content: + "Sorry, I wasn't able to complete your request. Please try again or simplify your request.", + }; + await connector.sendMessage(msg); + logger.info( + { source: message.source, sender: message.sender }, + 'Sent error response for DLQ message', + ); + } catch (err) { + logger.warn( + { err, source: message.source, sender: message.sender }, + 'onDeadLetter: failed to send error response', + ); + } + }); + + // Notify users when their message must wait behind an in-flight message. + // Includes queue position and estimated wait based on recent processing times. + this.queue.onQueued((message, position, estimatedWaitMs) => { + const waitStr = + estimatedWaitMs < 60_000 + ? `~${Math.ceil(estimatedWaitMs / 1000)}s` + : `~${Math.round(estimatedWaitMs / 60_000)}m`; + const content = `You're #${position} in queue (${waitStr}). I'll get to your message shortly.`; + void this.router.sendDirect(message.source, message.sender, content, message.id); + }); + + // Initialize connectors in parallel — slow connectors (WhatsApp/Puppeteer) + // must not block fast connectors (Console) from starting. + const connectorPromises: Promise[] = []; for (const connectorConfig of this.config.connectors) { if (!connectorConfig.enabled) continue; @@ -113,25 +618,121 @@ export class Bridge { logger.error({ connector: connector.name, error }, 'Connector error'); }); - await connector.initialize(); - this.router.addConnector(connector); - this.connectors.push(connector); - logger.info({ connector: connector.name }, 'Connector initialized'); + // Initialize each connector independently — don't await sequentially + const initPromise = connector + .initialize() + .then(() => { + this.router.addConnector(connector); + this.connectors.push(connector); + logger.info({ connector: connector.name }, 'Connector initialized'); + }) + .catch((error: unknown) => { + logger.error( + { connector: connector.name, error }, + 'Connector initialization failed — other connectors continue', + ); + }); + connectorPromises.push(initPromise); } + // Wait for all connectors to finish (success or failure) + await Promise.allSettled(connectorPromises); - // Set up queue processing - this.queue.onMessage(async (message) => { - await this.router.route(message); - }); + // Wire memory into connectors that support the sessions REST API (e.g. WebChat /api/sessions) + if (this.memory) { + for (const connector of this.connectors) { + const c = connector as { setMemory?: (m: MemoryManager) => void }; + if (typeof c.setMemory === 'function') { + c.setMemory(this.memory); + } + } + } + + // Wire discovered AI tools into connectors that support the /api/discovery endpoint (e.g. WebChat) + if (this.discoveredTools.length > 0) { + for (const connector of this.connectors) { + const c = connector as { setDiscoveryResult?: (t: DiscoveredTool[]) => void }; + if (typeof c.setDiscoveryResult === 'function') { + c.setDiscoveryResult(this.discoveredTools); + } + } + } + + // Wire MCP registry into connectors that support the /api/mcp/servers endpoints (e.g. WebChat) + if (this.mcpRegistry) { + for (const connector of this.connectors) { + const c = connector as { setMcpRegistry?: (r: McpRegistry) => void }; + if (typeof c.setMcpRegistry === 'function') { + c.setMcpRegistry(this.mcpRegistry); + } + } + } + + // Wire MediaManager into connectors that support incoming media download (e.g. WhatsApp) + if (this.workspacePath) { + const mediaManager = createMediaManager(this.workspacePath); + for (const connector of this.connectors) { + const c = connector as { setMediaManager?: (m: MediaManager) => void }; + if (typeof c.setMediaManager === 'function') { + c.setMediaManager(mediaManager); + } + } + } + + // Start agent status broadcasting to WebChat dashboard (2s poll — best-effort) + if (this.memory) { + this.agentStatusInterval = setInterval(() => { + void this.broadcastAgentStatusToWebChat(); + }, 2000); + } // Start health check endpoint this.healthServer.setDataProvider(() => this.getHealthStatus()); + this.healthServer.setMetricsProvider(() => this.metrics.snapshot()); + this.healthServer.setReadinessProvider( + () => this.master !== null && this.master.getState() === 'ready', + ); await this.healthServer.start(); // Start metrics endpoint this.metricsServer.setDataProvider(() => this.metrics.snapshot()); await this.metricsServer.start(); + // Start local file server for generated content (non-fatal) + if (this.fileServer) { + try { + await this.fileServer.start(); + logger.info( + { url: this.fileServer.baseUrl, dir: this.fileServer.directory }, + 'File server started — generated content available at /shared/:filename', + ); + // Wire file server into router so [SHARE:FILE] markers create shareable links + this.router.setFileServer(this.fileServer); + } catch (error) { + logger.warn({ err: error }, 'File server failed to start — continuing without it'); + this.fileServer = null; + } + } + + // Start tunnel if configured — expose file server to the internet (non-fatal) + if (this.tunnelManager && this.fileServer) { + this.registerTunnelShutdownHandlers(); + const fileServerPort = this.getFileServerPort(); + if (fileServerPort !== null) { + try { + this.tunnelPublicUrl = await this.tunnelManager.start(fileServerPort); + logger.info( + { url: this.tunnelPublicUrl }, + 'Tunnel started — file server accessible at public URL', + ); + } catch (error) { + logger.warn( + { err: error }, + 'Tunnel failed to start — continuing with localhost access only', + ); + } + } + } + // Start config file watcher for hot-reload if (this.configPath) { this.configWatcher = new ConfigWatcher(this.configPath); @@ -144,11 +745,29 @@ export class Bridge { /** Stop the bridge gracefully — drains in-flight messages, then shuts down connectors and providers */ async stop(): Promise { + if (this.stopped) { + logger.warn('Bridge.stop() called again — already stopped, skipping'); + return; + } + this.stopped = true; logger.info('Stopping OpenBridge...'); logger.info('Draining message queue...'); - await this.queue.drain(); - logger.info('Message queue drained'); + const drainTimeout = new Promise<'timeout'>((resolve) => + setTimeout(() => resolve('timeout'), this.drainTimeoutMs), + ); + const result = await Promise.race([ + this.queue.drain().then(() => 'done' as const), + drainTimeout, + ]); + if (result === 'timeout') { + logger.warn( + { drainTimeoutMs: this.drainTimeoutMs }, + `Queue drain timed out after ${this.drainTimeoutMs}ms — proceeding with shutdown`, + ); + } else { + logger.info('Message queue drained'); + } // Shut down Master AI if set if (this.master) { @@ -156,6 +775,20 @@ export class Bridge { logger.info('Master AI shut down'); } + // Shut down IntegrationHub — gracefully tears down all registered integrations + try { + await this.integrationHub.shutdown(); + logger.info('IntegrationHub shut down'); + } catch (error) { + logger.warn({ err: error }, 'IntegrationHub shutdown failed — continuing'); + } + + // Stop all running apps before tearing down connectors + if (this.appServer) { + this.appServer.stopAll(); + logger.info('AppServer shut down'); + } + // Shut down orchestrator — cancels active agents before providers are torn down const activeAgents = this.orchestrator.getActiveAgents().length; if (activeAgents > 0) { @@ -182,11 +815,61 @@ export class Bridge { } } + if (this.agentStatusInterval) { + clearInterval(this.agentStatusInterval); + this.agentStatusInterval = null; + } + + if (this.evictionInterval) { + clearInterval(this.evictionInterval); + this.evictionInterval = null; + } + + this.rateLimiter.dispose(); + + this.dockerHealthMonitor?.stop(); + + // Force-remove any Docker containers that are still tracked (OB-F111). + if (this.securityConfig?.sandbox?.mode === 'docker') { + await cleanupSandboxContainers(); + } + this.configWatcher?.stop(); await this.healthServer.stop(); await this.metricsServer.stop(); + if (this.fileServer) { + try { + await this.fileServer.stop(); + } catch (error) { + logger.warn({ err: error }, 'Error stopping file server'); + } + } + + if (this.tunnelManager) { + this.tunnelManager.stop(); + this.tunnelPublicUrl = null; + this.clearTunnelShutdownHandlers(); + logger.info('Tunnel stopped'); + } + + if (this.memory) { + try { + await this.memory.closeActiveSessions(); + logger.info('Active sessions closed'); + } catch (error) { + logger.warn({ err: error }, 'Error closing active sessions — continuing with DB close'); + } + try { + await this.memory.close(); + logger.info('MemoryManager closed'); + } catch (error) { + logger.error({ err: error }, 'Error closing MemoryManager'); + } + this.memory = null; + } + logger.info('OpenBridge stopped'); } @@ -196,9 +879,46 @@ export class Bridge { this.auth.updateConfig(newConfig.auth); this.rateLimiter.updateConfig(newConfig.auth.rateLimit); + // Propagate MCP server changes to McpRegistry and MasterManager. + // config.json may have been updated by McpRegistry.persistToConfig() (via the WebChat + // MCP management API) or by the user editing the file directly. Either way we re-parse + // the raw file to extract the V2Config MCP section and synchronise runtime state. + if (this.configPath && (this.mcpRegistry !== null || this.master !== null)) { + try { + const raw = readFileSync(this.configPath, 'utf-8'); + const parsed: unknown = JSON.parse(raw); + const v2Result = V2ConfigSchema.safeParse(parsed); + if (v2Result.success) { + const newMcpServers = + v2Result.data.mcp?.enabled !== false ? (v2Result.data.mcp?.servers ?? []) : []; + this.mcpRegistry?.reload(newMcpServers); + this.master?.reloadMcpServers(newMcpServers); + logger.info({ count: newMcpServers.length }, 'MCP server list hot-reloaded'); + } + } catch (err) { + logger.warn({ err }, 'Failed to extract MCP servers from hot-reloaded config'); + } + } + logger.info('Configuration hot-reload complete'); } + /** Poll active agent activity and broadcast to any connector that supports it (e.g. WebChat). */ + private async broadcastAgentStatusToWebChat(): Promise { + if (!this.memory) return; + try { + const agents = await this.memory.getActiveAgents(); + for (const connector of this.connectors) { + const c = connector as { broadcastAgentStatus?: (agents: unknown[]) => void }; + if (typeof c.broadcastAgentStatus === 'function') { + c.broadcastAgentStatus(agents); + } + } + } catch { + // Non-fatal — dashboard updates are best-effort + } + } + private getHealthStatus(): HealthStatus { const connectorStatuses: ComponentStatus[] = this.connectors.map((c) => ({ name: c.name, @@ -226,7 +946,12 @@ export class Bridge { return { status: overall, - uptime: Math.floor((Date.now() - this.startedAt) / 1000), + uptime_seconds: Math.floor((Date.now() - this.startedAt) / 1000), + memory_mb: Math.round((process.memoryUsage().heapUsed / 1024 / 1024) * 10) / 10, + active_workers: orchestratorSnapshot.activeAgents, + master_status: this.master ? this.master.getState() : 'not_configured', + db_status: this.memory ? 'connected' : 'disconnected', + last_message_at: this.lastMessageAt, timestamp: new Date().toISOString(), connectors: connectorStatuses, providers: providerStatuses, @@ -239,10 +964,41 @@ export class Bridge { }; } - private handleIncomingMessage(message: InboundMessage, _connector?: Connector): void { + /** + * Connectors where every message is an AI command (no shared conversation). + * Messages from these connectors get the prefix auto-prepended if missing. + */ + private static readonly DIRECT_AI_CONNECTORS = new Set(['webchat', 'console']); + + private handleIncomingMessage(incomingMessage: InboundMessage, connector?: Connector): void { + // Cap rawContent length before any further processing to protect queue, auth, and prefix checks + let message = incomingMessage; + if (incomingMessage.rawContent.length > MAX_INBOUND_LENGTH) { + logger.warn( + { sender: incomingMessage.sender, originalLength: incomingMessage.rawContent.length }, + `Inbound message truncated from ${incomingMessage.rawContent.length} to ${MAX_INBOUND_LENGTH} chars`, + ); + message = { + ...incomingMessage, + rawContent: incomingMessage.rawContent.slice(0, MAX_INBOUND_LENGTH), + }; + } + + // Auto-prepend prefix for direct AI connectors (webchat, console) where every + // message is an AI command. Shared channels (WhatsApp, Telegram, Discord) still + // require the explicit prefix to distinguish AI commands from normal chat. + if ( + Bridge.DIRECT_AI_CONNECTORS.has(message.source) && + !this.auth.hasPrefix(message.rawContent) + ) { + const prefix = this.auth.commandPrefix; + message = { ...message, rawContent: `${prefix} ${message.rawContent}` }; + } + this.metrics.recordReceived(); + this.lastMessageAt = new Date().toISOString(); - if (!this.auth.isAuthorized(message.sender)) { + if (!this.auth.isAuthorized(message.sender, message.source)) { logger.warn({ sender: message.sender }, 'Unauthorized sender'); void this.auditLogger.logAuthDenied(message.sender); return; @@ -264,6 +1020,37 @@ export class Bridge { const strippedContent = this.auth.stripPrefix(message.rawContent); const metadata: Record = { ...message.metadata }; + // Auto-create access_control entry for whitelisted user on first command. + // Ensures checkAccessControl always finds an entry rather than silently defaulting to owner. + this.auth.ensureAccessEntry(message.sender, message.source); + + // Access control check: verify role, action, scope, and daily budget. + // Runs after prefix-stripping so the classifier receives the actual command text. + const accessResult = this.auth.checkAccessControl( + message.sender, + message.source, + strippedContent, + ); + if (!accessResult.allowed) { + logger.warn( + { sender: message.sender, reason: accessResult.reason }, + 'Message blocked by access control', + ); + this.metrics.recordCommandBlocked(); + void this.auditLogger.logAuthDenied(message.sender); + // Send a denial message so the user knows why their request was rejected. + if (connector) { + const denialMsg: OutboundMessage = { + target: message.source, + recipient: message.sender, + content: accessResult.reason ?? 'You do not have permission to perform that action.', + replyTo: message.id, + }; + void connector.sendMessage(denialMsg); + } + return; + } + const filterResult = this.auth.filterCommand(strippedContent); if (!filterResult.allowed) { logger.warn({ sender: message.sender }, 'Message blocked by command filter'); @@ -278,6 +1065,123 @@ export class Bridge { }; void this.auditLogger.logInbound(cleaned); - void this.queue.enqueue(cleaned); + const priority = classifyMessagePriority(cleaned.content); + void this.queue.enqueue(cleaned, priority); + } + + /** + * Scan the workspace root for sensitive files (name-check only, 1 level deep). + * Logs a warning listing every detected file, then records each as a relative-path + * exclude pattern in sessionExcludePatterns so it is hidden from AI visibility. + * Errors are non-fatal — a scan failure must not prevent the bridge from starting. + */ + private async runSecretScan(workspacePath: string): Promise { + try { + const scanner = new SecretScanner(undefined, this.securityConfig?.sensitiveFileExceptions); + const matches = await scanner.scanWorkspace(workspacePath); + + if (matches.length === 0) return; + + logger.warn( + { + count: matches.length, + paths: matches.map((m) => m.path), + }, + 'Sensitive files detected in workspace — auto-excluding from AI visibility for this session', + ); + + for (const match of matches) { + this.detectedSecrets.push(match); + // Convert absolute path to a workspace-relative pattern for isFileVisible() + const relative = path.relative(workspacePath, match.path); + if (relative && !this.sessionExcludePatterns.includes(relative)) { + this.sessionExcludePatterns.push(relative); + } + } + } catch (err) { + logger.warn({ err }, 'Workspace secret scan failed — continuing without session excludes'); + } + } + + /** + * Remove legacy .openbridge/ artifacts that have been migrated to SQLite. + * Called on startup after memory.init() and memory.migrate() succeed. + * Only deletes files/dirs when the corresponding data exists in the DB, + * to avoid data loss on the first run before migration has occurred. + */ + private async cleanLegacyDotFolderArtifacts( + workspacePath: string, + memory: MemoryManager, + ): Promise { + const dotFolderPath = path.join(workspacePath, '.openbridge'); + + try { + await fs.access(dotFolderPath); + } catch { + return; // .openbridge/ doesn't exist yet — nothing to clean + } + + // 1. exploration.log — logging is now in the DB + try { + await fs.unlink(path.join(dotFolderPath, 'exploration.log')); + logger.info('Removed legacy .openbridge/exploration.log'); + } catch { + // File doesn't exist — no-op + } + + // 2. exploration/ directory — only delete when exploration is completed or missing. + // Active exploration writes incremental state here; deleting mid-run corrupts it. + try { + await fs.access(path.join(dotFolderPath, 'exploration')); + const statePath = path.join(dotFolderPath, 'exploration', 'exploration-state.json'); + let safeToDelete = false; + try { + const stateRaw = await fs.readFile(statePath, 'utf-8'); + const state = JSON.parse(stateRaw) as { status?: string }; + if (state.status === 'completed') { + safeToDelete = true; + } else { + logger.debug( + { status: state.status }, + 'Skipping exploration/ cleanup — exploration still in progress', + ); + } + } catch { + // exploration-state.json missing — safe to delete the directory + safeToDelete = true; + } + if (safeToDelete) { + await fs.rm(path.join(dotFolderPath, 'exploration'), { recursive: true, force: true }); + logger.info('Removed legacy .openbridge/exploration/ directory'); + } + } catch { + // Directory doesn't exist — no-op + } + + // 3. prompts/ directory — only remove when memory already holds the manifest, + // to avoid deleting prompt data before it has been migrated to the DB. + try { + await fs.access(path.join(dotFolderPath, 'prompts')); + const manifest = await memory.getPromptManifest(); + if (manifest !== null) { + await fs.rm(path.join(dotFolderPath, 'prompts'), { recursive: true, force: true }); + logger.info('Removed legacy .openbridge/prompts/ directory'); + } + } catch { + // Directory doesn't exist — no-op + } + + // 4. *.migrated backup files — safe to delete (migration has already run) + try { + const entries = await fs.readdir(dotFolderPath); + for (const entry of entries) { + if (entry.endsWith('.migrated')) { + await fs.unlink(path.join(dotFolderPath, entry)); + logger.info({ file: entry }, 'Removed legacy .migrated backup file'); + } + } + } catch { + // Best-effort — ignore readdir or unlink errors + } } } diff --git a/src/core/cli-adapter.ts b/src/core/cli-adapter.ts new file mode 100644 index 00000000..6683ef8c --- /dev/null +++ b/src/core/cli-adapter.ts @@ -0,0 +1,103 @@ +/** + * CLIAdapter — Provider-agnostic interface for spawning AI CLI tools. + * + * Each CLI tool (claude, codex, aider) implements this interface to translate + * the provider-neutral SpawnOptions into tool-specific binary, args, and env. + * + * The adapter sits between AgentRunner and child_process.spawn(): + * SpawnOptions → adapter.buildSpawnConfig() → spawn(config.binary, config.args, { env: config.env }) + * + * Lossy translation is intentional: if a CLI doesn't support a feature + * (e.g. codex has no --max-turns), the adapter silently drops it. + */ + +import type { SpawnOptions } from './agent-runner.js'; + +/** The output of a CLIAdapter: everything needed to call child_process.spawn() */ +export interface CLISpawnConfig { + /** Command name or absolute path to the binary */ + binary: string; + /** CLI arguments array */ + args: string[]; + /** Environment variables (already cleaned of conflicting vars) */ + env: Record; + /** + * stdin behavior: 'ignore' closes stdin (Claude), 'pipe' provides a writable stream + * (needed by CLIs that check for TTY). Defaults to 'ignore'. + */ + stdin?: 'ignore' | 'pipe'; + /** + * Optional post-processor for raw stdout. When set, AgentRunner applies this + * function to the accumulated stdout after the process exits. Used by adapters + * that emit structured output (e.g. Codex `--json` JSONL) to extract the final + * human-readable message content before returning the AgentResult. + * Falls back to raw stdout if the function returns undefined or throws. + */ + parseOutput?: (stdout: string) => string; + /** + * Optional incremental chunk parser for streaming output. When set, + * AgentRunner applies this function to each stdout chunk as it arrives + * during `spawnWithStreamingHandle()`. Returns human-readable text for + * user-visible events, or null for events that have no visible content + * (e.g. thread.started, item.started). Enables real-time readable output + * for adapters that emit structured streaming formats (e.g. Codex `--json`). + */ + parseStreamChunk?: (chunk: string) => string | null; +} + +/** + * Capability level — maps tool profiles to CLI-specific access mechanisms. + * Claude uses --allowedTools, Codex uses --sandbox modes, Aider uses --yes. + */ +export type CapabilityLevel = 'read-only' | 'code-edit' | 'full-access'; + +export interface CLIAdapter { + /** Provider name matching DiscoveredTool.name (e.g. 'claude', 'codex', 'aider') */ + readonly name: string; + + /** + * Build the spawn configuration from provider-neutral options. + * Translates SpawnOptions into the binary, args, and env for this CLI. + */ + buildSpawnConfig(opts: SpawnOptions): CLISpawnConfig; + + /** + * Clean the process environment for this CLI tool. + * Removes env vars that would cause nested-session or other conflicts. + */ + cleanEnv(env: Record): Record; + + /** + * Map a capability level to whatever mechanism this CLI uses for access control. + * For Claude: tool name lists. For Codex: approval modes. For Aider: flags. + * Returns undefined if the CLI doesn't have a capability restriction mechanism. + */ + mapCapabilityLevel(level: CapabilityLevel): string[] | undefined; + + /** + * Validate a model string for this provider. + * Returns true if the model is recognized or can be passed through. + */ + isValidModel(model: string): boolean; + + /** + * Declare which CapabilityLevel profiles this adapter enforces via native + * CLI mechanisms (e.g. Claude's --allowedTools named tool lists). + * Profiles not in this list are handled via sandbox modes or system-prompt + * constraints — less strictly enforced than named tool restrictions. + * + * Optional — adapters that don't declare this are treated as having no + * native profile enforcement (all profiles emulated via other mechanisms). + */ + supportedProfiles?(): readonly CapabilityLevel[]; + + /** + * Declare the prompt size budget for this adapter (and optionally a specific model). + * Returns conservative char-based estimates of token limits so that + * PromptAssembler can fit content within provider constraints. + * + * Optional — adapters that don't implement this fall back to the defaults: + * { maxPromptChars: 32_768, maxSystemPromptChars: 100_000 } + */ + getPromptBudget?(model?: string): { maxPromptChars: number; maxSystemPromptChars: number }; +} diff --git a/src/core/cloudflared-adapter.ts b/src/core/cloudflared-adapter.ts new file mode 100644 index 00000000..3aa51fa6 --- /dev/null +++ b/src/core/cloudflared-adapter.ts @@ -0,0 +1,45 @@ +/** + * CloudflaredAdapter — TunnelAdapter for cloudflared (Cloudflare Tunnel). + * + * Spawns: cloudflared tunnel --url localhost:{port} + * URL format: https://xxx.trycloudflare.com + * + * This is the preferred tunnel option — free, no signup required, fast setup. + * Auto-registers with TunnelManager when this module is imported. + */ + +import type { TunnelAdapter, TunnelConfig } from './tunnel-manager.js'; +import { TunnelManager } from './tunnel-manager.js'; + +/** Regex to extract the trycloudflare.com public URL from cloudflared output */ +const TRYCLOUDFLARE_URL_RE = /https:\/\/[a-zA-Z0-9-]+\.trycloudflare\.com/; + +export class CloudflaredAdapter implements TunnelAdapter { + readonly toolName = 'cloudflared'; + + /** + * Build CLI args for a cloudflared quick tunnel. + * Produces: cloudflared tunnel --url localhost:{port} + * The subdomain config is not supported for quick tunnels (trycloudflare.com + * assigns a random subdomain). authToken is ignored — no account required. + */ + buildArgs(port: number, _config?: TunnelConfig): string[] { + return ['tunnel', '--url', `localhost:${port}`]; + } + + /** + * Extract the public URL from one line of cloudflared stdout/stderr. + * + * Cloudflared quick tunnel announces its URL in a formatted box: + * 2024-01-15T10:00:00Z INF | https://example-word.trycloudflare.com | + * + * Returns the URL string if found, null otherwise. + */ + parseUrl(line: string): string | null { + const match = TRYCLOUDFLARE_URL_RE.exec(line); + return match !== null ? match[0] : null; + } +} + +// Auto-register when this module is imported +TunnelManager.registerAdapter(new CloudflaredAdapter()); diff --git a/src/core/command-handlers.ts b/src/core/command-handlers.ts new file mode 100644 index 00000000..ee111f41 --- /dev/null +++ b/src/core/command-handlers.ts @@ -0,0 +1,4452 @@ +/** CommandHandlers — extracted from Router (OB-1283, OB-F159). */ + +import type { AIProvider } from '../types/provider.js'; +import type { InboundMessage } from '../types/message.js'; +import type { Connector } from '../types/connector.js'; +import type { AppServer } from './app-server.js'; +import type { SecretMatch } from './secret-scanner.js'; +import type { + MemoryManager, + ActivityRecord, + ConversationEntry, + ExplorationProgressRow, + SessionSummary, +} from '../memory/index.js'; +import type { AccessRole } from '../memory/access-store.js'; +import type { MessageQueue } from './queue.js'; +import type { MasterManager } from '../master/master-manager.js'; +import type { AuthService } from './auth.js'; +import type { RiskLevel, ExecutionProfile, DeepPhase } from '../types/agent.js'; +import { BuiltInProfileNameSchema } from '../types/agent.js'; +import type { SkillManager } from '../master/skill-manager.js'; +import type { ParsedSpawnMarker } from '../master/spawn-parser.js'; +import type { IntegrationHub } from '../integrations/hub.js'; +import type { CredentialStore } from '../integrations/credential-store.js'; +import type { WorkflowStore } from '../workflows/workflow-store.js'; +import type { WorkflowEngine } from '../workflows/engine.js'; +import type { SecurityConfig } from '../types/config.js'; +import { CHECKS } from '../cli/doctor.js'; +import type { CheckResult } from '../cli/doctor.js'; +import { loadAllSkillPacks } from '../master/skill-pack-loader.js'; +import type { ProcessedDocument, ExtractedEntity } from '../types/intelligence.js'; +import type { FullDocType } from '../intelligence/doctype-store.js'; +import type { IntegrationCapability } from '../types/integration.js'; +import type { OpenAPI, OpenAPIV3 } from 'openapi-types'; +import type Database from 'better-sqlite3'; +import { createLogger } from './logger.js'; + +const logger = createLogger('command-handlers'); + +// --------------------------------------------------------------------------- +// Module-level utility functions (exported for testing) +// --------------------------------------------------------------------------- + +/** Format a millisecond duration as a human-readable string (e.g. "2h 14m", "45s"). */ +export function formatDuration(ms: number): string { + const totalSeconds = Math.floor(ms / 1000); + if (totalSeconds < 60) return `${totalSeconds}s`; + const minutes = Math.floor(totalSeconds / 60); + if (minutes < 60) return `${minutes}m ${totalSeconds % 60}s`; + const hours = Math.floor(minutes / 60); + return `${hours}h ${minutes % 60}m`; +} + +/** Render a simple Unicode progress bar of the given width (default 5 blocks). */ +export function makeProgressBar(pct: number, width = 5): string { + const filled = Math.max(0, Math.min(width, Math.round((pct / 100) * width))); + return '█'.repeat(filled) + '░'.repeat(width - filled); +} + +/** Escape special HTML characters for safe WebChat output. */ +export function escapeHtml(str: string): string { + return str + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} + +// --------------------------------------------------------------------------- +// Pending confirmation / escalation types (re-exported from router.ts) +// --------------------------------------------------------------------------- + +/** Pending stop-all confirmation entry, keyed by sender ID. */ +export interface PendingConfirmation { + action: 'kill-all'; + expiresAt: number; +} + +/** A pending spawn confirmation entry — queued when a high-risk SPAWN is intercepted. */ +export interface PendingSpawnEntry { + /** The SPAWN markers that need user confirmation before dispatch */ + markers: ParsedSpawnMarker[]; + /** The original inbound message that triggered the SPAWN */ + message: InboundMessage; + /** Connector used to send the confirmation prompt */ + connector: Connector; + /** One-line summaries of each high-risk task */ + taskSummaries: string[]; + /** Profile of the highest-risk SPAWN marker */ + profile: string; + /** Risk level of the highest-risk marker */ + riskLevel: RiskLevel; + /** Auto-cancel timeout handle — cleared when user replies or entry is consumed */ + timeoutHandle: ReturnType; +} + +/** A pending tool escalation request — queued when a worker needs additional tool access. */ +export interface PendingEscalation { + /** ID of the worker requesting additional tools */ + workerId: string; + /** Tool names the worker is requesting access to */ + requestedTools: string[]; + /** Current tool profile the worker is running under */ + currentProfile: string; + /** Reason from the worker failure explaining why additional tools are needed */ + reason: string; + /** The original inbound message that triggered this worker */ + message: InboundMessage; + /** Connector used to send the escalation prompt and receive the reply */ + connector: Connector; + /** Auto-deny timeout handle — cleared when user replies with /allow or /deny */ + timeoutHandle: ReturnType; + /** + * Optional callback to re-spawn the worker with upgraded tools after the grant is + * approved (OB-1594). Provided by MasterManager when calling requestToolEscalation(). + * Called with the granted tool/profile name(s) so the worker can retry. + */ + respawn?: (grantedTools: string[]) => Promise; +} + +// --------------------------------------------------------------------------- +// Dependencies interface — callbacks + shared references from Router +// --------------------------------------------------------------------------- + +export interface CommandHandlerDeps { + // Mutable references via getters + getMaster: () => MasterManager | undefined; + getMemory: () => MemoryManager | undefined; + getQueue: () => MessageQueue | undefined; + getAuth: () => AuthService | undefined; + getAppServer: () => AppServer | undefined; + getSkillManager: () => SkillManager | undefined; + getWorkspacePath: () => string | undefined; + getIntegrationHub: () => IntegrationHub | undefined; + getCredentialStore: () => CredentialStore | undefined; + getWorkflowStore: () => WorkflowStore | undefined; + getWorkflowEngine: () => WorkflowEngine | undefined; + getConnectors: () => Map; + getProviders: () => Map; + getSecurityConfig: () => SecurityConfig | undefined; + + // Pending confirmations/escalations + getPendingStopConfirmations: () => Map; + getSessionGrantedTools: () => Map>; + + // Scope/visibility state + getDetectedSecrets: () => readonly SecretMatch[]; + getSessionExcludePatterns: () => readonly string[]; + getWorkspaceInclude: () => readonly string[]; + getWorkspaceExclude: () => readonly string[]; + + // Delegated actions + takePendingSpawnConfirmation: (sender: string) => PendingSpawnEntry | undefined; + takePendingEscalation: (sender: string) => PendingEscalation | undefined; + takeAllPendingEscalations: (sender: string) => PendingEscalation[]; + pendingEscalationCount: (sender: string) => number; + route: (message: InboundMessage) => Promise; +} + +// --------------------------------------------------------------------------- +// CommandHandlers class +// --------------------------------------------------------------------------- + +export class CommandHandlers { + private deps: CommandHandlerDeps; + + /** + * Per-user cURL accumulation buffer. Users can send multiple cURL commands as separate + * messages and then trigger connection with "done" or "connect". Maps sender → curl lines. + */ + private readonly curlBuffers: Map = new Map(); + + constructor(deps: CommandHandlerDeps) { + this.deps = deps; + } + + /** Update mutable dependency references (e.g. after memory init). */ + updateDeps(partial: Partial): void { + this.deps = { ...this.deps, ...partial }; + } + + // ------------------------------------------------------------------------- + // cURL accumulation buffer — used by router to support multi-message cURL + // ------------------------------------------------------------------------- + + /** Returns true if the sender has one or more pending cURL commands accumulated. */ + hasPendingCurls(sender: string): boolean { + const buf = this.curlBuffers.get(sender); + return buf !== undefined && buf.length > 0; + } + + /** + * Appends a cURL command to the sender's buffer. + * @returns The new buffer length after adding the command. + */ + addCurlToBuffer(sender: string, curl: string): number { + const existing = this.curlBuffers.get(sender) ?? []; + existing.push(curl.trim()); + this.curlBuffers.set(sender, existing); + return existing.length; + } + + /** Flushes and returns all accumulated cURL commands joined by newline, then clears buffer. */ + flushCurlBuffer(sender: string): string { + const buf = this.curlBuffers.get(sender) ?? []; + this.curlBuffers.delete(sender); + return buf.join('\n'); + } + + /** Clears the cURL buffer for a sender without returning content. */ + clearCurlBuffer(sender: string): void { + this.curlBuffers.delete(sender); + } + + /** + * handleCurlAccumulationMessage — called when a standalone "curl …" message is received. + * Adds the cURL to the sender's buffer and sends an acknowledgement. + */ + async handleCurlAccumulationMessage( + message: InboundMessage, + connector: Connector, + ): Promise { + const count = this.addCurlToBuffer(message.sender, message.content.trim()); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `cURL command ${count} saved. Send more cURL commands or say *"done"* / *"connect"* to create the API integration.`, + replyTo: message.id, + }); + } + + // ------------------------------------------------------------------------- + // handleStatusCommand + // ------------------------------------------------------------------------- + + async handleStatusCommand(message: InboundMessage, connector: Connector): Promise { + const memory = this.deps.getMemory(); + const lines: string[] = ['*OpenBridge Status*']; + + if (!memory) { + lines.push('Status tracking not available — memory system not initialized.'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + return; + } + + let agents: ActivityRecord[] = []; + let exploration: ExplorationProgressRow[] = []; + try { + [agents, exploration] = await Promise.all([ + memory.getActiveAgents(), + memory.getExplorationProgress(), + ]); + } catch (err) { + logger.warn({ err }, 'handleStatusCommand: failed to query agent activity'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Status temporarily unavailable — could not query agent activity.', + replyTo: message.id, + }); + return; + } + + const masters = agents.filter((a) => a.type === 'master'); + const workers = agents.filter((a) => a.type !== 'master'); + + // Master AI section + if (masters.length > 0) { + const m = masters[0]!; + const uptime = formatDuration(Date.now() - new Date(m.started_at).getTime()); + lines.push(`\n🤖 Master AI: ACTIVE (${m.model ?? 'unknown'})`); + lines.push(` Uptime: ${uptime}`); + if (m.task_summary) lines.push(` Current: ${m.task_summary}`); + } else { + lines.push('\n🤖 Master AI: idle'); + } + + // Active workers section + if (workers.length > 0) { + lines.push(`\nWorkers (${workers.length} active):`); + for (const w of workers) { + const elapsed = formatDuration(Date.now() - new Date(w.started_at).getTime()); + const bar = makeProgressBar(w.progress_pct ?? 0); + const shortId = w.id.length > 8 ? `…${w.id.slice(-6)}` : w.id; + const task = (w.task_summary ?? 'working...').slice(0, 32); + lines.push( + ` • ${shortId} | ${w.model ?? '?'} | ${w.profile ?? '?'} | ${task} | ${bar} | ${elapsed}`, + ); + } + } else { + lines.push('\nWorkers: none active'); + } + + // Exploration progress section + if (exploration.length > 0) { + lines.push('\nExploration:'); + for (const ep of exploration) { + const bar = makeProgressBar(ep.progress_pct ?? 0); + const label = ep.target ?? ep.phase; + lines.push(` ${label}: [${bar}] ${ep.progress_pct ?? 0}%`); + } + } + + // Queue depth + estimated completion time section (OB-923) + const queue = this.deps.getQueue(); + if (queue) { + const queueSnapshot = queue.getQueueSnapshot(); + if (queueSnapshot.length > 0) { + lines.push('\nQueue:'); + for (const entry of queueSnapshot) { + const waitStr = + entry.estimatedWaitMs < 60_000 + ? `~${Math.ceil(entry.estimatedWaitMs / 1000)}s` + : `~${Math.round(entry.estimatedWaitMs / 60_000)}m`; + const shortSender = + entry.sender.length > 12 + ? `${entry.sender.slice(0, 6)}…${entry.sender.slice(-4)}` + : entry.sender; + lines.push( + ` • ${shortSender}: ${entry.pending} message${entry.pending !== 1 ? 's' : ''} waiting (est. ${waitStr})`, + ); + } + } else { + lines.push('\nQueue: idle'); + } + } + + // Cost summary — use getDailyCost to include all completed workers today (OB-746) + let dailyCost = 0; + try { + dailyCost = await memory.getDailyCost(); + } catch (costErr) { + logger.warn({ costErr }, 'handleStatusCommand: failed to query daily cost'); + // Fall back to summing active agents' costs + dailyCost = agents.reduce((sum, a) => sum + (a.cost_usd ?? 0), 0); + } + const workerCount = agents.filter((a) => a.type === 'worker').length; + lines.push( + `\nCost: $${dailyCost.toFixed(4)} today | ${workerCount} worker${workerCount !== 1 ? 's' : ''} spawned`, + ); + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + logger.info({ sender: message.sender }, 'Status command handled'); + } + + // ------------------------------------------------------------------------- + // handleConfirmCommand + // ------------------------------------------------------------------------- + + /** + * Handle a "confirm" message that follows a pending "stop all" request. + * Executes killAllWorkers() if the confirmation arrived within the 30-second window; + * otherwise reports that it has expired. + */ + async handleConfirmCommand( + message: InboundMessage, + connector: Connector, + pending: PendingConfirmation, + ): Promise { + // Always remove the pending entry — one shot regardless of outcome + this.deps.getPendingStopConfirmations().delete(message.sender); + + if (Date.now() > pending.expiresAt) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: "Confirmation expired. Send 'stop all' again to retry.", + replyTo: message.id, + }); + logger.info({ sender: message.sender }, 'Stop all confirmation expired'); + return; + } + + const master = this.deps.getMaster(); + if (!master) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Stop command not available — Master AI not initialized.', + replyTo: message.id, + }); + return; + } + + const result = await master.killAllWorkers(message.sender); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: result.message, + replyTo: message.id, + }); + logger.info({ sender: message.sender }, 'Stop all confirmed and executed'); + } + + // ------------------------------------------------------------------------- + // handleConfirmSpawnCommand + // ------------------------------------------------------------------------- + + /** + * Handle "go" / "/confirm" — approve a pending high-risk spawn confirmation. + * + * Retrieves and removes the pending spawn entry for the sender. If found, + * re-routes the original message so the Master AI dispatches the workers. + * If no confirmation is pending, responds with "No pending confirmation." + */ + async handleConfirmSpawnCommand(message: InboundMessage, connector: Connector): Promise { + const entry = this.deps.takePendingSpawnConfirmation(message.sender); + if (!entry) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'No pending confirmation.', + replyTo: message.id, + }); + logger.info({ sender: message.sender }, 'Spawn confirm: no pending entry'); + return; + } + + logger.info( + { sender: message.sender, markerCount: entry.markers.length }, + 'Spawn confirmation approved — dispatching workers', + ); + + // Re-route the original message so the Master AI re-processes and dispatches workers + await this.deps.route(entry.message); + } + + // ------------------------------------------------------------------------- + // handleSkipSpawnCommand + // ------------------------------------------------------------------------- + + /** + * Handle "skip" / "/skip" — cancel a pending high-risk spawn confirmation. + * + * Retrieves and removes the pending spawn entry for the sender. If found, + * notifies the user that the spawn has been cancelled. If no confirmation + * is pending, responds with "No pending confirmation." + */ + async handleSkipSpawnCommand(message: InboundMessage, connector: Connector): Promise { + const entry = this.deps.takePendingSpawnConfirmation(message.sender); + if (!entry) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'No pending confirmation.', + replyTo: message.id, + }); + logger.info({ sender: message.sender }, 'Spawn skip: no pending entry'); + return; + } + + logger.info( + { sender: message.sender, markerCount: entry.markers.length }, + 'Spawn confirmation rejected — spawn cancelled', + ); + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Spawn cancelled.', + replyTo: message.id, + }); + } + + // ------------------------------------------------------------------------- + // handleAllowCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "/allow" command — grant a pending tool escalation (OB-1586). + * + * Syntax: + * /allow -> grant single tool, scope: once (default) + * /allow -> upgrade to named profile, scope: once + * /allow --session -> grant for the entire session + * /allow --permanent -> grant permanently (stored in DB) + * + * Clears the pending escalation for the sender, sends a confirmation, and + * stores the grant in the appropriate backing store (session Map or DB) per scope (OB-1588). + */ + async handleAllowCommand(message: InboundMessage, connector: Connector): Promise { + const trustLevel = this.deps.getSecurityConfig()?.trustLevel ?? 'standard'; + if (trustLevel === 'sandbox') { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: '⛔ Sandbox mode — tool escalation is disabled.', + replyTo: message.id, + }); + return; + } + if (trustLevel === 'trusted') { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'ℹ️ Trusted mode — all tools are already available.', + replyTo: message.id, + }); + return; + } + + // Parse: /allow all — grant all pending escalations at once (OB-1632) + const trimmed = message.content.trim(); + const rest = trimmed.slice('/allow'.length).trim(); + + if (/^all$/i.test(rest)) { + await this.handleAllowAllCommand(message, connector); + return; + } + + const entry = this.deps.takePendingEscalation(message.sender); + if (!entry) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'No pending tool escalation.', + replyTo: message.id, + }); + logger.info({ sender: message.sender }, 'Allow: no pending escalation'); + return; + } + + // Extract scope suffix + let scope: 'once' | 'session' | 'permanent' = 'once'; + let grantArg = rest; + if (/--permanent$/i.test(rest)) { + scope = 'permanent'; + grantArg = rest.replace(/\s*--permanent$/i, '').trim(); + } else if (/--session$/i.test(rest)) { + scope = 'session'; + grantArg = rest.replace(/\s*--session$/i, '').trim(); + } + + // Determine whether grantArg is a built-in profile name or a single tool + const isProfile = BuiltInProfileNameSchema.safeParse(grantArg).success; + + const scopeLabel = + scope === 'once' ? 'this request' : scope === 'session' ? 'this session' : 'permanently'; + const grantDescription = isProfile ? `profile upgrade to *${grantArg}*` : `tool *${grantArg}*`; + + const remaining = this.deps.pendingEscalationCount(message.sender); + const remainingLine = + remaining > 0 + ? `\n${remaining} more pending escalation(s) — reply /allow for next or /allow all for all` + : ''; + const confirmText = + `✅ Granted ${grantDescription} to worker ${entry.workerId} for ${scopeLabel}.\n` + + `Worker will be notified to retry with the granted access.${remainingLine}`; + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: confirmText, + replyTo: message.id, + }); + + logger.info( + { sender: message.sender, workerId: entry.workerId, grantArg, scope, isProfile }, + 'Tool escalation granted via /allow', + ); + + // Wire scope-specific grant storage (OB-1588) + const sessionGrantedTools = this.deps.getSessionGrantedTools(); + const memory = this.deps.getMemory(); + if (scope === 'session') { + const existing = sessionGrantedTools.get(message.sender) ?? new Set(); + existing.add(grantArg); + sessionGrantedTools.set(message.sender, existing); + logger.debug({ sender: message.sender, grantArg }, 'Session tool grant stored'); + } else if (scope === 'permanent' && memory) { + try { + const accessEntry = await memory.getAccess(message.sender, message.source); + const existingActions = accessEntry?.allowed_actions ?? []; + if (!existingActions.includes(grantArg)) { + await memory.setAccess({ + ...(accessEntry ?? {}), + user_id: message.sender, + channel: message.source, + role: accessEntry?.role ?? 'custom', + allowed_actions: [...existingActions, grantArg], + }); + logger.debug({ sender: message.sender, grantArg }, 'Permanent tool grant stored in DB'); + } + } catch (err) { + logger.warn( + { err, sender: message.sender, grantArg }, + 'Failed to persist permanent tool grant', + ); + } + } + + // OB-1594: Re-spawn the worker with the granted tools/profile. + // The respawn callback was registered by MasterManager when requestToolEscalation() + // was called. Errors are non-fatal — the grant is already recorded. + if (entry.respawn) { + logger.info( + { workerId: entry.workerId, grantArg }, + 'Triggering worker re-spawn after tool grant', + ); + entry.respawn([grantArg]).catch((err: unknown) => { + logger.warn( + { err, workerId: entry.workerId, grantArg }, + 'Worker re-spawn after tool grant failed', + ); + }); + } + } + + // ------------------------------------------------------------------------- + // handleAllowAllCommand + // ------------------------------------------------------------------------- + + /** + * Handle the "/allow all" command — grant all pending tool escalations for the sender + * at once (OB-1632). + * + * Drains the escalation queue, grants each worker's originally-requested tools, and + * triggers their respawn callbacks sequentially. Scope is always "once" — no + * --session / --permanent modifiers are supported for bulk grants. + */ + async handleAllowAllCommand(message: InboundMessage, connector: Connector): Promise { + const trustLevel = this.deps.getSecurityConfig()?.trustLevel ?? 'standard'; + if (trustLevel === 'sandbox') { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: '⛔ Sandbox mode — tool escalation is disabled.', + replyTo: message.id, + }); + return; + } + if (trustLevel === 'trusted') { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'ℹ️ Trusted mode — all tools are already available.', + replyTo: message.id, + }); + return; + } + + const entries = this.deps.takeAllPendingEscalations(message.sender); + if (entries.length === 0) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'No pending tool escalations.', + replyTo: message.id, + }); + logger.info({ sender: message.sender }, 'Allow all: no pending escalations'); + return; + } + + const workerIds = entries.map((e) => e.workerId).join(', '); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `✅ Granted all pending escalations (${entries.length} worker(s): ${workerIds}).\nEach worker will be notified to retry with the granted access.`, + replyTo: message.id, + }); + + logger.info( + { sender: message.sender, count: entries.length, workerIds }, + 'All pending tool escalations granted via /allow all', + ); + + for (const entry of entries) { + if (entry.respawn) { + logger.info( + { workerId: entry.workerId, requestedTools: entry.requestedTools }, + 'Triggering worker re-spawn after /allow all grant', + ); + entry.respawn(entry.requestedTools).catch((err: unknown) => { + logger.warn({ err, workerId: entry.workerId }, 'Worker re-spawn after /allow all failed'); + }); + } + } + } + + // ------------------------------------------------------------------------- + // handleDenyCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "/deny" command — reject a pending tool escalation (OB-1587). + * + * Removes the pending escalation for the sender and notifies the user. + * The Master AI is notified to continue the worker without the requested tools + * or abort the worker if the task cannot proceed without them. + */ + async handleDenyCommand(message: InboundMessage, connector: Connector): Promise { + const entry = this.deps.takePendingEscalation(message.sender); + if (!entry) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'No pending tool escalation.', + replyTo: message.id, + }); + logger.info({ sender: message.sender }, 'Deny: no pending escalation'); + return; + } + + const denyText = + `❌ Tool escalation denied for worker ${entry.workerId}.\n` + + `The worker will continue with its current profile (*${entry.currentProfile}*) or abort if unable to proceed.`; + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: denyText, + replyTo: message.id, + }); + + logger.info( + { sender: message.sender, workerId: entry.workerId, requestedTools: entry.requestedTools }, + 'Tool escalation denied via /deny', + ); + } + + // ------------------------------------------------------------------------- + // handleDenyAllCommand + // ------------------------------------------------------------------------- + + /** + * Handle the "/deny all" command — deny all pending tool escalations for the sender + * at once (OB-1634). + * + * Drains the escalation queue and marks all queued workers as denied. + */ + async handleDenyAllCommand(message: InboundMessage, connector: Connector): Promise { + const entries = this.deps.takeAllPendingEscalations(message.sender); + if (entries.length === 0) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'No pending tool escalations.', + replyTo: message.id, + }); + logger.info({ sender: message.sender }, 'Deny all: no pending escalations'); + return; + } + + const workerIds = entries.map((e) => e.workerId).join(', '); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `❌ Denied all pending escalations (${entries.length} worker(s): ${workerIds}).\nEach worker will continue with its current profile or abort if unable to proceed.`, + replyTo: message.id, + }); + + logger.info( + { sender: message.sender, count: entries.length, workerIds }, + 'All pending tool escalations denied via /deny all', + ); + } + + // ------------------------------------------------------------------------- + // handlePermissionsCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "/permissions" command — show the current user's tool grants and consent + * mode (OB-1589). + * + * Output: + * - Consent mode (always-ask / auto-approve-read / auto-approve-all) + * - Session grants (from in-memory sessionGrantedTools Map) + * - Permanent grants (from access_control DB via allowed_actions) + */ + async handlePermissionsCommand(message: InboundMessage, connector: Connector): Promise { + const memory = this.deps.getMemory(); + const sessionGrantedTools = this.deps.getSessionGrantedTools(); + const lines: string[] = ['*Your Permissions*', '']; + + // Consent mode + let consentMode: string = 'always-ask'; + if (memory) { + try { + consentMode = await memory.getConsentMode(message.sender, message.source); + } catch { + consentMode = 'always-ask'; + } + } + lines.push(`*Consent mode:* ${consentMode}`); + lines.push(''); + + // Session grants (in-memory, cleared on restart) + const sessionGrants = sessionGrantedTools.get(message.sender); + if (sessionGrants && sessionGrants.size > 0) { + lines.push('*Session grants* (active until restart):'); + for (const grant of sessionGrants) { + lines.push(` • ${grant}`); + } + } else { + lines.push('*Session grants:* none'); + } + lines.push(''); + + // Permanent grants (stored in DB) + if (memory) { + try { + const entry = await memory.getAccess(message.sender, message.source); + const permanentGrants = entry?.allowed_actions ?? []; + if (permanentGrants.length > 0) { + lines.push('*Permanent grants* (stored in DB):'); + for (const grant of permanentGrants) { + lines.push(` • ${grant}`); + } + } else { + lines.push('*Permanent grants:* none'); + } + } catch { + lines.push('*Permanent grants:* (unavailable — DB error)'); + } + } else { + lines.push('*Permanent grants:* (unavailable — memory not initialised)'); + } + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + + logger.info({ sender: message.sender }, 'Permissions displayed via /permissions'); + } + + // ------------------------------------------------------------------------- + // handleTrustCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "/trust" command — change the user's consent mode. + * + * Usage: + * /trust — show current trust level + * /trust auto — auto-approve all escalations (no prompts) + * /trust edit — auto-approve up to code-edit (prompt for full-access) + * /trust ask — always ask (default, safest) + * + * Inspired by OpenClaw's "identity first" security model: establish trust + * level once at the identity layer, then let operations flow without + * per-action confirmation friction. + */ + async handleTrustCommand(message: InboundMessage, connector: Connector): Promise { + const memory = this.deps.getMemory(); + const sender = message.sender; + const channel = message.source; + + const arg = message.content + .trim() + .replace(/^\/trust\s*/i, '') + .trim() + .toLowerCase(); + + // Map shorthand aliases to ConsentMode values + const TRUST_ALIASES: Record< + string, + 'always-ask' | 'auto-approve-up-to-edit' | 'auto-approve-all' + > = { + auto: 'auto-approve-all', + all: 'auto-approve-all', + edit: 'auto-approve-up-to-edit', + 'code-edit': 'auto-approve-up-to-edit', + ask: 'always-ask', + default: 'always-ask', + off: 'always-ask', + // Full names also accepted + 'auto-approve-all': 'auto-approve-all', + 'auto-approve-up-to-edit': 'auto-approve-up-to-edit', + 'always-ask': 'always-ask', + }; + + // No argument — show current trust level + if (!arg) { + let current = 'always-ask'; + if (memory) { + try { + current = await memory.getConsentMode(sender, channel); + } catch { + current = 'always-ask'; + } + } + const levelLabel = + current === 'auto-approve-all' + ? 'auto (CLI adapter — no per-tool prompts)' + : current === 'auto-approve-up-to-edit' + ? 'edit (SDK adapter — prompt for Bash/Write only)' + : 'ask (SDK adapter — every tool call relayed to you)'; + + await connector.sendMessage({ + target: channel, + recipient: sender, + content: + `*Trust level:* ${levelLabel}\n\n` + + 'Change with:\n' + + '• `/trust auto` — approve all escalations automatically\n' + + '• `/trust edit` — auto-approve up to code-edit, prompt for full-access\n' + + '• `/trust ask` — always ask before granting tools (safest)', + replyTo: message.id, + }); + return; + } + + const newMode = TRUST_ALIASES[arg]; + if (!newMode) { + await connector.sendMessage({ + target: channel, + recipient: sender, + content: `Unknown trust level: *${arg}*\n\n` + 'Valid options: `auto`, `edit`, `ask`', + replyTo: message.id, + }); + return; + } + + if (!memory) { + await connector.sendMessage({ + target: channel, + recipient: sender, + content: 'Cannot update trust level — memory system not initialised.', + replyTo: message.id, + }); + return; + } + + await memory.setConsentMode(sender, channel, newMode); + + const confirmLabel = + newMode === 'auto-approve-all' + ? 'auto — CLI adapter, pre-approved tools, no per-tool prompts' + : newMode === 'auto-approve-up-to-edit' + ? 'edit — SDK adapter, auto-approve reads/edits, prompt for Bash/Write' + : 'ask — SDK adapter, every tool call relayed to you for approval'; + + await connector.sendMessage({ + target: channel, + recipient: sender, + content: `✅ Trust level set to *${confirmLabel}*`, + replyTo: message.id, + }); + + logger.info({ sender, channel, newMode }, 'Trust level updated via /trust command'); + } + + // ------------------------------------------------------------------------- + // handleWhoamiCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "/whoami" command. + * + * Shows the user their role, channel, allowed actions, daily cost usage, and consent mode. + * Requires no elevated permissions — any user can see their own identity info (OB-1720). + */ + async handleWhoamiCommand(message: InboundMessage, connector: Connector): Promise { + const memory = this.deps.getMemory(); + const lines: string[] = ['*Who Am I*', '']; + + // Sender (truncated for display) + const sender = message.sender; + const displaySender = sender.length > 20 ? `${sender.slice(0, 8)}…${sender.slice(-6)}` : sender; + lines.push(`*User:* ${displaySender}`); + lines.push(`*Channel:* ${message.source}`); + lines.push(''); + + // Role + allowed actions from access_control + let role = 'owner'; // default fallback + let allowedActions: string[] | null = null; + let dailyCostUsed: number | null = null; + let dailyCostLimit: number | null = null; + + if (memory) { + try { + const entry = await memory.getAccess(sender, message.source); + if (entry) { + role = entry.role; + // Resolve effective allowed actions: explicit list takes precedence over role default + if (entry.allowed_actions && entry.allowed_actions.length > 0) { + allowedActions = entry.allowed_actions; + } else { + const roleDefaults: Record = { + owner: null, + admin: null, + developer: ['read', 'edit', 'test'], + viewer: ['read'], + custom: null, + }; + allowedActions = roleDefaults[entry.role] ?? null; + } + dailyCostUsed = entry.daily_cost_used ?? null; + dailyCostLimit = entry.max_cost_per_day_usd ?? null; + } + } catch { + // leave defaults + } + } + + lines.push(`*Role:* ${role}`); + + if (allowedActions === null) { + lines.push('*Allowed actions:* all (no restrictions)'); + } else { + lines.push(`*Allowed actions:* ${allowedActions.join(', ')}`); + } + lines.push(''); + + // Daily cost usage + if (dailyCostUsed !== null) { + const usedStr = `$${dailyCostUsed.toFixed(4)}`; + const limitStr = dailyCostLimit != null ? ` / $${dailyCostLimit.toFixed(2)} limit` : ''; + lines.push(`*Daily cost:* ${usedStr}${limitStr}`); + } else if (memory) { + try { + const totalCost = await memory.getDailyCost(); + lines.push(`*Daily cost (shared):* $${totalCost.toFixed(4)}`); + } catch { + lines.push('*Daily cost:* unavailable'); + } + } else { + lines.push('*Daily cost:* unavailable'); + } + lines.push(''); + + // Consent mode + let consentMode = 'always-ask'; + if (memory) { + try { + consentMode = await memory.getConsentMode(sender, message.source); + } catch { + consentMode = 'always-ask'; + } + } + lines.push(`*Consent mode:* ${consentMode}`); + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + + logger.info({ sender: message.sender }, 'Identity info displayed via /whoami'); + } + + // ------------------------------------------------------------------------- + // handleRoleCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "/role " command. + * + * Syntax: /role + * Valid roles: owner, admin, developer, viewer, custom + * + * Only owner or admin callers may use this command. + * Sets the role for the target user on the same channel (OB-1721). + */ + async handleRoleCommand(message: InboundMessage, connector: Connector): Promise { + const memory = this.deps.getMemory(); + const VALID_ROLES = new Set(['owner', 'admin', 'developer', 'viewer', 'custom']); + + const send = (content: string): Promise => + connector.sendMessage({ + target: message.source, + recipient: message.sender, + content, + replyTo: message.id, + }); + + // Check caller has owner or admin role + if (memory) { + try { + const callerEntry = await memory.getAccess(message.sender, message.source); + const callerRole = callerEntry?.role ?? 'owner'; + if (callerRole !== 'owner' && callerRole !== 'admin') { + await send( + `Permission denied. The /role command requires owner or admin role. Your current role is *${callerRole}*.`, + ); + return; + } + } catch { + // allow fallthrough — default is owner + } + } + + // Parse: /role + const parts = message.content.trim().split(/\s+/); + // parts[0] = "/role", parts[1] = user_id, parts[2] = role + const [, rawTargetUserId, rawNewRole] = parts; + if (!rawTargetUserId || !rawNewRole) { + await send( + 'Usage: /role \nValid roles: owner, admin, developer, viewer, custom', + ); + return; + } + + const targetUserId = rawTargetUserId; + const newRole = rawNewRole.toLowerCase(); + + if (!VALID_ROLES.has(newRole)) { + await send(`Invalid role *${newRole}*. Valid roles: owner, admin, developer, viewer, custom`); + return; + } + + if (!memory) { + await send('Role management is unavailable: memory system not initialised.'); + return; + } + + try { + const existing = await memory.getAccess(targetUserId, message.source); + if (existing) { + await memory.setAccess({ ...existing, role: newRole as AccessRole }); + } else { + await memory.setAccess({ + user_id: targetUserId, + channel: message.source, + role: newRole as AccessRole, + }); + } + + const displayTarget = + targetUserId.length > 20 + ? `${targetUserId.slice(0, 8)}…${targetUserId.slice(-6)}` + : targetUserId; + await send( + `Role updated: *${displayTarget}* is now *${newRole}* on channel *${message.source}*.`, + ); + logger.info( + { sender: message.sender, targetUserId, newRole, channel: message.source }, + 'Role updated via /role command', + ); + } catch (err) { + logger.error({ err }, 'Failed to update role via /role command'); + await send('Failed to update role. Please try again.'); + } + } + + // ------------------------------------------------------------------------- + // handleApproveCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "/approve " command. + * + * Owner or admin users can approve a pending pairing request by submitting + * the 6-digit code that the unknown sender received. On success the sender + * is added to access_control with the viewer role and the pending pairing + * entry is removed (OB-1698). + */ + async handleApproveCommand(message: InboundMessage, connector: Connector): Promise { + const memory = this.deps.getMemory(); + const auth = this.deps.getAuth(); + + const send = (content: string): Promise => + connector.sendMessage({ + target: message.source, + recipient: message.sender, + content, + replyTo: message.id, + }); + + // Only owner or admin may approve pairing requests + if (memory) { + try { + const callerEntry = await memory.getAccess(message.sender, message.source); + const callerRole = callerEntry?.role ?? 'owner'; + if (callerRole !== 'owner' && callerRole !== 'admin') { + await send( + `Permission denied. The /approve command requires owner or admin role. Your current role is *${callerRole}*.`, + ); + return; + } + } catch { + // allow fallthrough — default is owner + } + } + + // Parse: /approve + const parts = message.content.trim().split(/\s+/); + const code = parts[1]; + if (!code) { + await send('Usage: /approve \nExample: /approve 482916'); + return; + } + + if (!auth) { + await send('Pairing approval unavailable: auth service not initialised.'); + return; + } + + const pairing = auth.getPairing(code); + if (!pairing) { + await send( + `No pending pairing found for code *${code}*. It may have already been used or expired.`, + ); + return; + } + + // Grant access — role is configurable via auth.channelRoles / auth.defaultRole + if (!memory) { + await send('Pairing approval unavailable: memory system not initialised.'); + return; + } + + try { + const role = (auth.getRoleForChannel(pairing.channel) as AccessRole) ?? 'viewer'; + await memory.approvePairing(pairing.senderId, pairing.channel, role); + + auth.removePairing(code); + + const displayId = + pairing.senderId.length > 20 + ? `${pairing.senderId.slice(0, 8)}…${pairing.senderId.slice(-6)}` + : pairing.senderId; + await send( + `Pairing approved. *${displayId}* has been granted *viewer* access on channel *${pairing.channel}*.`, + ); + logger.info( + { approver: message.sender, senderId: pairing.senderId, channel: pairing.channel, code }, + 'Pairing approved via /approve command', + ); + } catch (err) { + logger.error({ err }, 'Failed to approve pairing via /approve command'); + await send('Failed to approve pairing. Please try again.'); + } + } + + // ------------------------------------------------------------------------- + // handleStopCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "stop" command. + * + * Syntax: + * "stop" -> request confirmation then kill all running workers + * "stop all" -> request confirmation then kill all running workers + * "stop " -> kill the worker whose ID ends with (partial match, no confirmation) + * + * Requires the Master AI to be configured. Returns a plain-text response + * that works on all channels. + */ + async handleStopCommand(message: InboundMessage, connector: Connector): Promise { + const master = this.deps.getMaster(); + const auth = this.deps.getAuth(); + + if (!master) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Stop command not available — Master AI not initialized.', + replyTo: message.id, + }); + return; + } + + // Access control — only owner and admin may stop workers + if (auth) { + const accessResult = auth.checkAccessControl(message.sender, message.source, 'stop'); + if (!accessResult.allowed) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: accessResult.reason ?? 'You do not have permission to use the stop command.', + replyTo: message.id, + }); + return; + } + } + + const trimmed = message.content.trim(); + // Extract everything after "stop" (and optional whitespace) + const rest = trimmed.slice(4).trim().toLowerCase(); + + let responseText: string; + + if (rest === '' || rest === 'all') { + // "stop" or "stop all" — require confirmation before killing all workers + const running = master.getWorkerRegistry().getRunningWorkers(); + if (running.length === 0) { + responseText = 'No workers are currently running.'; + } else { + this.deps.getPendingStopConfirmations().set(message.sender, { + action: 'kill-all', + expiresAt: Date.now() + 30_000, + }); + responseText = `This will terminate ${running.length} running worker${running.length !== 1 ? 's' : ''}. Reply 'confirm' within 30 seconds to proceed.`; + } + } else { + // "stop " — find a worker whose ID ends with the partial ID + const partialId = rest; + const registry = master.getWorkerRegistry(); + const allWorkers = registry.getAllWorkers(); + + // Match: exact ID, or ID ends with "-", or ID contains + const matched = allWorkers.find( + (w) => w.id === partialId || w.id.endsWith(`-${partialId}`) || w.id.includes(partialId), + ); + + if (!matched) { + responseText = `Worker '${partialId}' not found. Use 'status' to list active workers.`; + } else { + const result = await master.killWorker(matched.id, message.sender); + responseText = result.message; + } + } + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: responseText, + replyTo: message.id, + }); + logger.info({ sender: message.sender, command: trimmed }, 'Stop command handled'); + } + + // ------------------------------------------------------------------------- + // handleExploreCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "explore" command. + * Triggers workspace re-exploration from any channel. + * + * Syntax: + * "explore" -> quick re-exploration via Master session prompt + * "explore full" -> full 5-phase re-exploration with ExplorationCoordinator + * "explore status" -> show current exploration state and progress + */ + async handleExploreCommand(message: InboundMessage, connector: Connector): Promise { + const master = this.deps.getMaster(); + const auth = this.deps.getAuth(); + + if (!master) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Explore command not available — Master AI not initialized.', + replyTo: message.id, + }); + return; + } + + // Access control — exploration classifies as 'read' action + if (auth) { + const accessResult = auth.checkAccessControl(message.sender, message.source, 'explore'); + if (!accessResult.allowed) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: accessResult.reason ?? 'You do not have permission to trigger exploration.', + replyTo: message.id, + }); + return; + } + } + + const trimmed = message.content.trim(); + const rest = trimmed.slice(7).trim().toLowerCase(); // slice past "explore" + + if (rest === 'status') { + await this.handleExploreStatusSubcommand(message, connector); + return; + } + + const currentState = master.getState(); + + if (currentState === 'exploring') { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Exploration is already in progress. Use "explore status" to check progress.', + replyTo: message.id, + }); + return; + } + + if (currentState !== 'ready') { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Cannot start exploration — Master is currently in '${currentState}' state. Please wait until it is ready.`, + replyTo: message.id, + }); + return; + } + + const isFull = rest === 'full'; + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: isFull + ? 'Starting full workspace re-exploration (5 phases). This may take a few minutes...' + : 'Starting quick workspace re-exploration...', + replyTo: message.id, + }); + + try { + if (isFull) { + await master.fullReExplore(); + } else { + await master.reExplore(); + } + + const summary = master.getExplorationSummary(); + const completionParts = ['Workspace re-exploration completed.']; + if (summary?.projectType) { + completionParts.push(`Project type: ${summary.projectType}`); + } + if (summary?.frameworks && summary.frameworks.length > 0) { + completionParts.push(`Frameworks: ${summary.frameworks.join(', ')}`); + } + if (summary?.directoriesExplored !== undefined) { + completionParts.push(`Directories explored: ${summary.directoriesExplored}`); + } + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: completionParts.join('\n'), + replyTo: message.id, + }); + } catch (error) { + const errorMsg = error instanceof Error ? error.message : String(error); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Exploration failed: ${errorMsg}`, + replyTo: message.id, + }); + } + + logger.info( + { sender: message.sender, mode: isFull ? 'full' : 'quick' }, + 'Explore command handled', + ); + } + + // ------------------------------------------------------------------------- + // handleExploreStatusSubcommand + // ------------------------------------------------------------------------- + + /** + * Handle the "explore status" subcommand. + * Shows last exploration timestamp, project type, and any in-progress phases. + */ + async handleExploreStatusSubcommand( + message: InboundMessage, + connector: Connector, + ): Promise { + const master = this.deps.getMaster()!; + const memory = this.deps.getMemory(); + const lines: string[] = ['*Exploration Status*']; + + const state = master.getState(); + lines.push(`Master state: ${state}`); + + const summary = master.getExplorationSummary(); + if (summary) { + if (summary.completedAt) { + lines.push(`Last exploration: ${summary.completedAt}`); + } + if (summary.projectType) { + lines.push(`Project type: ${summary.projectType}`); + } + if (summary.frameworks && summary.frameworks.length > 0) { + lines.push(`Frameworks: ${summary.frameworks.join(', ')}`); + } + lines.push(`Directories explored: ${summary.directoriesExplored}`); + lines.push(`Files scanned: ${summary.filesScanned}`); + } else { + lines.push('No exploration has been completed yet.'); + } + + if (memory) { + try { + const progress = await memory.getExplorationProgress(); + if (progress.length > 0) { + lines.push('\nPhase progress:'); + for (const ep of progress) { + const bar = makeProgressBar(ep.progress_pct ?? 0); + const label = ep.target ?? ep.phase; + lines.push(` ${label}: [${bar}] ${ep.progress_pct ?? 0}%`); + } + } + } catch { + // Silently skip if DB query fails + } + } + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + } + + // ------------------------------------------------------------------------- + // handleAuditCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "/audit" command. + * Shows the last 10 worker spawns with task ID, profile, duration, estimated cost, and result status. + * Reads from the agent_activity table via MemoryManager. + * + * Syntax: + * "/audit" -> list last 10 worker spawns + */ + async handleAuditCommand(message: InboundMessage, connector: Connector): Promise { + const memory = this.deps.getMemory(); + if (!memory) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Audit log not available — memory system not initialized.', + replyTo: message.id, + }); + return; + } + + let spawns: ActivityRecord[]; + try { + spawns = await memory.getRecentWorkerSpawns(10); + } catch (err) { + logger.warn({ err }, 'handleAuditCommand: failed to query worker spawns'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Audit log temporarily unavailable — could not query activity records.', + replyTo: message.id, + }); + return; + } + + const content = this.formatAuditLog(spawns, connector.name); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content, + replyTo: message.id, + }); + logger.info({ sender: message.sender }, 'Audit command handled'); + } + + // ------------------------------------------------------------------------- + // handleDeepCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "/deep" command — starts Deep Mode, toggles it, or shows status. + * + * Syntax: + * /deep -> toggle (start thorough if inactive, show status if active) + * /deep thorough -> start automatic multi-phase execution + * /deep manual -> start with pause between phases for user review + * /deep off -> deactivate Deep Mode, abort all active sessions + */ + async handleDeepCommand(message: InboundMessage, connector: Connector): Promise { + const trimmed = message.content.trim(); + const rest = trimmed.slice(5).trim().toLowerCase(); // Remove "/deep" + const master = this.deps.getMaster(); + + if (!master) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Deep Mode not available — Master AI not initialized.', + replyTo: message.id, + }); + return; + } + + const deepMode = master.getDeepModeManager(); + const activeSessions = deepMode.getActiveSessions(); + + // /deep off — abort all active sessions + if (rest === 'off') { + if (activeSessions.length === 0) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'No active Deep Mode session to deactivate.', + replyTo: message.id, + }); + return; + } + for (const sessionId of activeSessions) { + deepMode.abort(sessionId); + } + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Deep Mode deactivated — ${activeSessions.length} session(s) aborted.`, + replyTo: message.id, + }); + logger.info( + { sender: message.sender, count: activeSessions.length }, + 'Deep Mode aborted via /deep off', + ); + return; + } + + // Unknown argument + if (rest !== '' && rest !== 'thorough' && rest !== 'manual') { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: [ + 'Unknown Deep Mode option. Usage:', + ' /deep — toggle (or show status if active)', + ' /deep thorough — start automatic multi-phase execution', + ' /deep manual — start with pause between phases', + ' /deep off — deactivate Deep Mode', + ].join('\n'), + replyTo: message.id, + }); + return; + } + + // If already active, show current status + if (activeSessions.length > 0) { + const lines: string[] = ['*Deep Mode Status*', '']; + for (const sessionId of activeSessions) { + const state = deepMode.getSessionState(sessionId); + if (!state) continue; + const paused = deepMode.isPaused(sessionId); + const phase = state.currentPhase ?? 'done'; + const completedPhases = Object.keys(state.phaseResults); + lines.push(`Profile: ${state.profile}`); + lines.push( + `Current phase: ${phase}${paused ? ' (paused — send /proceed to continue)' : ''}`, + ); + if (completedPhases.length > 0) { + lines.push(`Completed: ${completedPhases.join(', ')}`); + } + lines.push(`Task: ${state.taskSummary.slice(0, 100)}`); + } + lines.push(''); + lines.push('Send /deep off to deactivate.'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + return; + } + + // Start a new session — default to thorough when no arg provided (toggle on) + const effectiveProfile: ExecutionProfile = rest === 'manual' ? 'manual' : 'thorough'; + const sessionId = deepMode.startSession( + `User-initiated via /deep ${effectiveProfile}`, + effectiveProfile, + ); + + if (!sessionId) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Deep Mode not started — fast profile skips multi-phase execution.', + replyTo: message.id, + }); + return; + } + + const currentPhase = deepMode.getCurrentPhase(sessionId); + const profileDescription = + effectiveProfile === 'manual' + ? 'I will pause after each phase for your review. Send /proceed to advance.' + : 'I will run all phases automatically and report when complete.'; + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: [ + `*Deep Mode started* (${effectiveProfile} profile)`, + `Current phase: *${currentPhase ?? 'investigate'}*`, + '', + profileDescription, + '', + 'Send your task description and Deep Mode will guide multi-phase execution.', + ].join('\n'), + replyTo: message.id, + }); + logger.info( + { sender: message.sender, sessionId, profile: effectiveProfile }, + 'Deep Mode activated via /deep command', + ); + } + + // ------------------------------------------------------------------------- + // handleProceedCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "/proceed" command — advances to the next Deep Mode phase. + * + * Behaviour per profile: + * manual -> if the session is paused, resume it so the next phase can run. + * if the session is not paused (phase still running), inform the user. + * thorough -> no-op; Deep Mode auto-advances through phases without user input. + * + * Responds with "No active Deep Mode session" when no session exists. + */ + async handleProceedCommand(message: InboundMessage, connector: Connector): Promise { + const master = this.deps.getMaster(); + if (!master) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Deep Mode not available — Master AI not initialized.', + replyTo: message.id, + }); + return; + } + + const deepMode = master.getDeepModeManager(); + const activeSessions = deepMode.getActiveSessions(); + + if (activeSessions.length === 0) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'No active Deep Mode session.', + replyTo: message.id, + }); + return; + } + + const lines: string[] = []; + + for (const sessionId of activeSessions) { + const state = deepMode.getSessionState(sessionId); + if (!state) continue; + + if (state.profile === 'thorough') { + // Thorough profile auto-advances — /proceed is a no-op + const phase = state.currentPhase ?? 'done'; + lines.push( + `Deep Mode (thorough) is running automatically — no action needed.\nCurrent phase: *${phase}*`, + ); + } else if (state.profile === 'manual') { + if (deepMode.isPaused(sessionId)) { + deepMode.resume(sessionId); + const phase = state.currentPhase ?? 'done'; + lines.push(`Proceeding with Deep Mode — resuming *${phase}* phase.`); + logger.info( + { sender: message.sender, sessionId, phase }, + 'Deep Mode resumed via /proceed', + ); + } else { + const phase = state.currentPhase ?? 'done'; + lines.push( + `Deep Mode *${phase}* phase is still running — please wait for it to complete before sending /proceed.`, + ); + } + } + } + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n\n') || 'No actionable Deep Mode session found.', + replyTo: message.id, + }); + } + + // ------------------------------------------------------------------------- + // handleFocusCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "/focus N" command — digs deeper into finding number N. + * + * Behaviour: + * 1. Records the focused item in the active Deep Mode session via focusOnItem(). + * 2. Builds a focused investigation message for the Master AI. + * 3. Confirms to the user that investigation is starting. + * 4. Routes the investigation to Master AI via processMessage() and sends the result. + * + * Responds with "No active Deep Mode session" when no session exists. + * Responds with usage guidance when N is missing or invalid. + */ + async handleFocusCommand(message: InboundMessage, connector: Connector): Promise { + const master = this.deps.getMaster(); + if (!master) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Deep Mode not available — Master AI not initialized.', + replyTo: message.id, + }); + return; + } + + // Parse item number N from "/focus N" + const trimmed = message.content.trim(); + const match = /^\/focus\s+(\d+)/i.exec(trimmed); + if (!match) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Usage: /focus N — provide a finding number (e.g., /focus 3)', + replyTo: message.id, + }); + return; + } + + const itemIndex = parseInt(match[1]!, 10); + + const deepMode = master.getDeepModeManager(); + const activeSessions = deepMode.getActiveSessions(); + + if (activeSessions.length === 0) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'No active Deep Mode session.', + replyTo: message.id, + }); + return; + } + + // Use the first active session + const sessionId = activeSessions[0]!; + const state = deepMode.getSessionState(sessionId); + + if (!state) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'No active Deep Mode session.', + replyTo: message.id, + }); + return; + } + + // Record the focused item in the session state + deepMode.focusOnItem(sessionId, itemIndex); + + // Get the most recent phase result to provide context for the investigation + const reportResult = deepMode.getPhaseResult(sessionId, 'report'); + const investigateResult = deepMode.getPhaseResult(sessionId, 'investigate'); + const latestResult = reportResult ?? investigateResult; + + const contextSnippet = latestResult + ? `\n\nContext from ${latestResult.phase} phase:\n${latestResult.output.slice(0, 800)}` + : ''; + + // Build focused investigation prompt for the Master AI + const focusContent = + `[Deep Mode — Focused Investigation on Finding #${itemIndex}]\n` + + `The user requested a deep-dive on finding #${itemIndex} from the current analysis.\n` + + `Task: "${state.taskSummary}"${contextSnippet}\n\n` + + `Please investigate finding #${itemIndex} thoroughly:\n` + + `- Trace all code paths, dependencies, and side effects related to this finding\n` + + `- Identify the root cause, not just the symptom\n` + + `- List all files affected (with file:line references)\n` + + `- Assess the severity and impact\n` + + `- Suggest specific remediation steps`; + + // Confirm immediately to the user before the investigation begins + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Investigating finding #${itemIndex} in depth — spawning focused worker.\nTask: "${state.taskSummary}"`, + replyTo: message.id, + }); + + logger.info( + { sender: message.sender, sessionId, itemIndex, phase: state.currentPhase }, + 'Deep Mode focused investigation requested via /focus', + ); + + // Spawn the focused investigation via Master AI and send the result back + try { + const focusMessage: InboundMessage = { ...message, content: focusContent }; + const response = await master.processMessage(focusMessage); + if (response) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: response, + }); + } + } catch (err: unknown) { + const errMsg = err instanceof Error ? err.message : String(err); + logger.error( + { sender: message.sender, sessionId, itemIndex, err: errMsg }, + 'Deep Mode focused investigation failed', + ); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Focused investigation of finding #${itemIndex} failed: ${errMsg}`, + }); + } + } + + // ------------------------------------------------------------------------- + // handleSkipItemCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "/skip N" command — marks plan item N as skipped in Deep Mode. + * + * Behaviour: + * 1. Parses N from "/skip N". + * 2. Marks item N as skipped in the active Deep Mode session via skipItem(). + * 3. Confirms to the user that the item has been skipped. + * 4. The execute phase will not process skipped items. + * + * Responds with "No active Deep Mode session" when no session exists. + * Responds with usage guidance when N is missing or invalid. + */ + async handleSkipItemCommand(message: InboundMessage, connector: Connector): Promise { + const master = this.deps.getMaster(); + if (!master) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Deep Mode not available — Master AI not initialized.', + replyTo: message.id, + }); + return; + } + + // Parse item number N from "/skip N" + const trimmed = message.content.trim(); + const match = /^\/skip\s+(\d+)/i.exec(trimmed); + if (!match) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Usage: /skip N — provide a task number (e.g., /skip 3)', + replyTo: message.id, + }); + return; + } + + const itemIndex = parseInt(match[1]!, 10); + + const deepMode = master.getDeepModeManager(); + const activeSessions = deepMode.getActiveSessions(); + + if (activeSessions.length === 0) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'No active Deep Mode session.', + replyTo: message.id, + }); + return; + } + + // Use the first active session + const sessionId = activeSessions[0]!; + const state = deepMode.getSessionState(sessionId); + + if (!state) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'No active Deep Mode session.', + replyTo: message.id, + }); + return; + } + + // Mark the item as skipped in the session state + deepMode.skipItem(sessionId, itemIndex); + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Task #${itemIndex} marked as skipped — it will not be processed in the execute phase.\nTask: "${state.taskSummary}"`, + replyTo: message.id, + }); + + logger.info( + { sender: message.sender, sessionId, itemIndex, phase: state.currentPhase }, + 'Deep Mode item skipped via /skip command', + ); + } + + // ------------------------------------------------------------------------- + // handlePhaseCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "/phase" command — shows current phase and progress for Deep Mode. + * + * Displays for each active Deep Mode session: + * - Profile name and task summary + * - Current phase (or "complete" if all phases done) + * - Completed phases with a brief output summary (first 200 chars) + * - Pending phases not yet started + * - Skipped items (if any) + * + * Responds with "No active Deep Mode session" when none exists. + * Responds with "Deep Mode not available" when Master AI is not initialized. + */ + async handlePhaseCommand(message: InboundMessage, connector: Connector): Promise { + const master = this.deps.getMaster(); + if (!master) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Deep Mode not available — Master AI not initialized.', + replyTo: message.id, + }); + return; + } + + const deepMode = master.getDeepModeManager(); + const activeSessions = deepMode.getActiveSessions(); + + if (activeSessions.length === 0) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'No active Deep Mode session.', + replyTo: message.id, + }); + return; + } + + const phaseOrder: DeepPhase[] = ['investigate', 'report', 'plan', 'execute', 'verify']; + const sessionBlocks: string[] = []; + + for (const sessionId of activeSessions) { + const state = deepMode.getSessionState(sessionId); + if (!state) continue; + + const currentPhase = state.currentPhase; + const completedPhaseNames = phaseOrder.filter((p) => state.phaseResults[p] !== undefined); + const currentIndex = currentPhase ? phaseOrder.indexOf(currentPhase) : phaseOrder.length; + const pendingPhases = phaseOrder.slice(currentPhase ? currentIndex + 1 : phaseOrder.length); + + const headerLines = [ + `*Deep Mode — Phase Status*`, + `Profile: ${state.profile}`, + `Task: "${state.taskSummary}"`, + ``, + `Current phase: ${currentPhase ? `*${currentPhase}*` : 'complete'}`, + ]; + + const completedLines: string[] = []; + if (completedPhaseNames.length > 0) { + completedLines.push('', 'Completed phases:'); + for (const phaseName of completedPhaseNames) { + const result = deepMode.getPhaseResult(sessionId, phaseName); + const snippet = result ? result.output.slice(0, 200).replace(/\n+/g, ' ').trim() : ''; + const ellipsis = result && result.output.length > 200 ? '…' : ''; + completedLines.push(`• ${phaseName} ✓${snippet ? ` — "${snippet}${ellipsis}"` : ''}`); + } + } + + const pendingLines: string[] = []; + if (pendingPhases.length > 0) { + pendingLines.push('', 'Pending phases:'); + for (const p of pendingPhases) { + pendingLines.push(`• ${p}`); + } + } + + const skippedNote: string[] = []; + if (state.skippedItems.length > 0) { + skippedNote.push('', `Skipped items: ${state.skippedItems.join(', ')}`); + } + + sessionBlocks.push( + [...headerLines, ...completedLines, ...pendingLines, ...skippedNote].join('\n'), + ); + } + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: sessionBlocks.join('\n\n---\n\n') || 'No Deep Mode session details available.', + replyTo: message.id, + }); + + logger.info( + { sender: message.sender, sessionCount: activeSessions.length }, + 'Deep Mode phase status shown via /phase', + ); + } + + // ------------------------------------------------------------------------- + // handleModelOverrideCommand + // ------------------------------------------------------------------------- + + /** + * Handle natural language model override requests in Deep Mode (OB-1412). + * + * Parses phrases like: + * - "use opus for task 1" -> override task 1 with 'powerful' tier + * - "use haiku for this" -> override current task (index 0) with 'fast' tier + * - "use balanced for task 3" -> override task 3 with 'balanced' tier + * + * Model name -> tier mapping: + * opus -> powerful + * sonnet / claude -> balanced + * haiku -> fast + * powerful / balanced / fast -> pass-through tier names + * + * Responds with a confirmation message that echoes the override back to the user. + * Responds with an error if no active Deep Mode session exists. + */ + async handleModelOverrideCommand(message: InboundMessage, connector: Connector): Promise { + const master = this.deps.getMaster(); + if (!master) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Deep Mode not available — Master AI not initialized.', + replyTo: message.id, + }); + return; + } + + const deepMode = master.getDeepModeManager(); + const activeSessions = deepMode.getActiveSessions(); + + if (activeSessions.length === 0) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'No active Deep Mode session — model override has no effect.', + replyTo: message.id, + }); + return; + } + + const text = message.content.trim(); + + // Extract model name from the message + const modelMatch = + /\b(?:use|switch\s+to|change\s+to)\s+(?:\w+[-\s]?)?(opus|sonnet|haiku|fast|balanced|powerful)\b/i.exec( + text, + ); + if (!modelMatch) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Could not parse model name. Try: "use opus for task 1" or "use haiku for this".', + replyTo: message.id, + }); + return; + } + + const modelName = modelMatch[1]!.toLowerCase(); + + // Map model name to ModelTier + const MODEL_NAME_TO_TIER: Record = { + opus: 'powerful', + sonnet: 'balanced', + claude: 'balanced', + haiku: 'fast', + fast: 'fast', + balanced: 'balanced', + powerful: 'powerful', + }; + const modelTier = MODEL_NAME_TO_TIER[modelName]; + if (!modelTier) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Unknown model "${modelName}". Supported: opus (powerful), sonnet (balanced), haiku (fast).`, + replyTo: message.id, + }); + return; + } + + // Extract task index from "for task N" or "for this" + const taskMatch = /\bfor\s+task\s*#?\s*(\d+)\b/i.exec(text); + const isThis = /\bfor\s+this\b|\bthis\s+task\b/i.test(text) && !taskMatch; + const taskIndex = taskMatch ? parseInt(taskMatch[1]!, 10) : 0; // 0 = current task sentinel + + // Apply the override to all active sessions (typically only one) + for (const sessionId of activeSessions) { + deepMode.setTaskModelOverride(sessionId, taskIndex, modelTier); + } + + // Build a human-readable confirmation + const tierLabel = + modelTier === 'powerful' + ? 'opus (powerful)' + : modelTier === 'balanced' + ? 'sonnet (balanced)' + : 'haiku (fast)'; + const scopeLabel = taskMatch + ? `task ${taskIndex}` + : isThis + ? 'the current task' + : 'the current task'; + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Model override set — ${scopeLabel} will use *${tierLabel}*.\n\nThis applies when the execute phase processes that task. Use /phase to see current Deep Mode status.`, + replyTo: message.id, + }); + + logger.info( + { sender: message.sender, modelTier, taskIndex, sessionCount: activeSessions.length }, + 'Deep Mode model override applied via chat (OB-1412)', + ); + } + + // ------------------------------------------------------------------------- + // handleHistoryCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "history" command. + * Lists the last 10 conversation sessions with title, message count, and date. + * Formats output per channel (WhatsApp/Telegram/Discord = numbered list, Console = table, WebChat = HTML). + * + * Syntax: + * "history" -> list last 10 sessions + * "history search " -> search sessions by keyword (OB-1034) + * "history " -> show full transcript (OB-1035) + */ + async handleHistoryCommand(message: InboundMessage, connector: Connector): Promise { + const memory = this.deps.getMemory(); + const trimmed = message.content.trim(); + const rest = trimmed.slice(7).trim(); // slice past "history" + + // history search — OB-1034 + if (rest.toLowerCase().startsWith('search')) { + const query = rest.slice(6).trim(); // slice past "search" + if (!query) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Usage: history search ', + replyTo: message.id, + }); + return; + } + + if (!memory) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'History search not available — memory system not initialized.', + replyTo: message.id, + }); + return; + } + + let sessions: SessionSummary[]; + try { + sessions = await memory.searchSessions(query, 10); + } catch (err) { + logger.warn({ err }, 'handleHistoryCommand: failed to search sessions'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'History search temporarily unavailable — could not query sessions.', + replyTo: message.id, + }); + return; + } + + const content = + sessions.length === 0 + ? `*Conversation History*\n\nNo sessions found matching "${query}".` + : this.formatSessionList(sessions, connector.name); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content, + replyTo: message.id, + }); + logger.info({ sender: message.sender, query }, 'History search command handled'); + return; + } + + // history — OB-1035 + if (rest.length > 0) { + if (!memory) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'History not available — memory system not initialized.', + replyTo: message.id, + }); + return; + } + + const sessionId = rest; + let entries: ConversationEntry[]; + try { + entries = await memory.getSessionHistory(sessionId, 50); + } catch (err) { + logger.warn({ err }, 'handleHistoryCommand: failed to get session history'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'History temporarily unavailable — could not query session.', + replyTo: message.id, + }); + return; + } + + const content = + entries.length === 0 + ? `No conversation found for session: ${sessionId}` + : this.formatSessionTranscript(entries, connector.name); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content, + replyTo: message.id, + }); + logger.info({ sender: message.sender, sessionId }, 'History transcript command handled'); + return; + } + + // bare "history" — list last 10 sessions + if (!memory) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'History not available — memory system not initialized.', + replyTo: message.id, + }); + return; + } + + let sessions: SessionSummary[]; + try { + sessions = await memory.listSessions(10, 0); + } catch (err) { + logger.warn({ err }, 'handleHistoryCommand: failed to list sessions'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'History temporarily unavailable — could not query sessions.', + replyTo: message.id, + }); + return; + } + + const content = this.formatSessionList(sessions, connector.name); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content, + replyTo: message.id, + }); + logger.info({ sender: message.sender }, 'History command handled'); + } + + // ------------------------------------------------------------------------- + // handleAppsCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "/apps" command — list running app instances with URLs. + * + * Shows each running app's URL and public URL (if tunnel is active). + * Responds with "No apps running" when there are no active instances. + */ + async handleAppsCommand(message: InboundMessage, connector: Connector): Promise { + const appServer = this.deps.getAppServer(); + if (!appServer) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'App server is not enabled.', + replyTo: message.id, + }); + return; + } + + const apps = appServer.listApps(); + + if (apps.length === 0) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'No apps running.', + replyTo: message.id, + }); + return; + } + + const lines: string[] = ['*Running Apps*', '']; + for (const app of apps) { + const displayUrl = app.publicUrl ?? app.url; + lines.push(`• ${app.id.slice(0, 8)} — ${displayUrl}`); + if (app.publicUrl) { + lines.push(` Local: ${app.url}`); + } + } + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + + logger.info({ sender: message.sender, appCount: apps.length }, 'Apps listed via /apps'); + } + + // ------------------------------------------------------------------------- + // handleScopeCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "/scope" command — show workspace visibility rules and detected secrets. + * + * Output sections: + * Visibility Rules — include (if set) and exclude patterns + * Sensitive Files — detected files with severity, or "No sensitive files detected" + */ + async handleScopeCommand(message: InboundMessage, connector: Connector): Promise { + const detectedSecrets = this.deps.getDetectedSecrets(); + const sessionExcludePatterns = this.deps.getSessionExcludePatterns(); + const workspaceInclude = this.deps.getWorkspaceInclude(); + const workspaceExclude = this.deps.getWorkspaceExclude(); + + const lines: string[] = ['*Workspace Scope*', '']; + + // --- Visibility Rules --- + lines.push('*Visibility Rules*'); + + if (workspaceInclude.length > 0) { + lines.push('Include (only these visible):'); + for (const pattern of workspaceInclude) { + lines.push(` • ${pattern}`); + } + } else { + lines.push('Include: all files (no include filter set)'); + } + + lines.push(''); + + // Combine session-detected excludes with user-configured excludes for display + const allUserExcludes = [...sessionExcludePatterns, ...workspaceExclude]; + if (allUserExcludes.length > 0) { + lines.push('Exclude (hidden from AI):'); + for (const pattern of allUserExcludes) { + lines.push(` • ${pattern}`); + } + } else { + lines.push('Exclude: default patterns only'); + } + + lines.push(''); + + // --- Sensitive Files --- + lines.push('*Sensitive Files*'); + + if (detectedSecrets.length === 0) { + lines.push('No sensitive files detected.'); + } else { + for (const secret of detectedSecrets) { + const basename = secret.path.split('/').pop() ?? secret.path; + const severityLabel = + secret.severity === 'critical' + ? '🔴 critical' + : secret.severity === 'high' + ? '🟠 high' + : '🟡 medium'; + lines.push(` • ${basename} — ${severityLabel} (pattern: ${secret.pattern})`); + } + } + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + + logger.info( + { sender: message.sender, secretCount: detectedSecrets.length }, + 'Scope info shown via /scope', + ); + } + + // ------------------------------------------------------------------------- + // handleSkillsCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "/skills" command — list available skills with descriptions and usage counts. + * + * Output sections: + * Built-in skills — shipped with OpenBridge (code-review, test-runner, etc.) + * User-defined — workspace-specific skills from .openbridge/skills/ + * + * Each skill line shows: name, description, tool profile, and usage count (if > 0). + */ + async handleSkillsCommand(message: InboundMessage, connector: Connector): Promise { + const skillManager = this.deps.getSkillManager(); + if (!skillManager) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Skills not available — skill manager not initialized.', + replyTo: message.id, + }); + return; + } + + // Reload user-defined skills to pick up any new files dropped since startup + await skillManager.load(); + + const allSkills = skillManager.getAll(); + + if (allSkills.length === 0) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'No skills available.', + replyTo: message.id, + }); + return; + } + + // Fetch usage stats (persisted to .skill-stats.json) + const statsArray = await skillManager.getSkillStats(); + const statsMap = new Map(statsArray.map((s) => [s.name, s])); + + const builtIn = allSkills.filter((s) => !s.isUserDefined); + const userDefined = allSkills.filter((s) => s.isUserDefined); + + const lines: string[] = ['*Available Skills*', '']; + + const formatSkill = (skill: { + name: string; + description: string; + toolProfile: string; + }): string => { + const stats = statsMap.get(skill.name); + const usageSuffix = stats && stats.usageCount > 0 ? ` (used ${stats.usageCount}×)` : ''; + return `• *${skill.name}* [${skill.toolProfile}]${usageSuffix} — ${skill.description}`; + }; + + if (builtIn.length > 0) { + lines.push('*Built-in*'); + for (const skill of builtIn) { + lines.push(formatSkill(skill)); + } + } + + if (userDefined.length > 0) { + if (builtIn.length > 0) lines.push(''); + lines.push('*Workspace Skills*'); + for (const skill of userDefined) { + lines.push(formatSkill(skill)); + } + } + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + + logger.info( + { sender: message.sender, skillCount: allSkills.length }, + 'Skills listed via /skills', + ); + } + + // ------------------------------------------------------------------------- + // handleSkillPacksCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "/skill-packs" command — list available skill packs with descriptions. + * + * Shows built-in packs first, then user-defined overrides from + * `.openbridge/skill-packs/`. Each entry shows the pack name, tool profile, + * and description. User-defined packs are labelled with "(custom)". + */ + async handleSkillPacksCommand(message: InboundMessage, connector: Connector): Promise { + const workspacePath = this.deps.getWorkspacePath(); + if (!workspacePath) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Skill packs not available — workspace path not configured.', + replyTo: message.id, + }); + return; + } + + let packs: Awaited>['packs']; + let userDefinedCount: number; + try { + ({ packs, userDefinedCount } = await loadAllSkillPacks(workspacePath)); + } catch (err) { + logger.warn({ err }, 'handleSkillPacksCommand: failed to load skill packs'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Failed to load skill packs.', + replyTo: message.id, + }); + return; + } + + if (packs.length === 0) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'No skill packs available.', + replyTo: message.id, + }); + return; + } + + const builtIn = packs.filter((p) => !p.isUserDefined); + const userDefined = packs.filter((p) => p.isUserDefined); + + const lines: string[] = ['*Available Skill Packs*', '']; + + if (builtIn.length > 0) { + lines.push('*Built-in*'); + for (const pack of builtIn) { + lines.push(`• *${pack.name}* [${pack.toolProfile}] — ${pack.description}`); + } + } + + if (userDefined.length > 0) { + if (builtIn.length > 0) lines.push(''); + lines.push('*Workspace (custom)*'); + for (const pack of userDefined) { + lines.push(`• *${pack.name}* [${pack.toolProfile}] — ${pack.description}`); + } + } + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + + logger.info( + { sender: message.sender, total: packs.length, userDefinedCount }, + 'Skill packs listed via /skill-packs', + ); + } + + // ------------------------------------------------------------------------- + // handleWorkersCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "/workers" command. + * Lists all active workers (pending + running) with ID, status, profile, duration, and PID. + * Also shows the count of orphaned workers. + * Users can follow up with /kill to force-stop a stuck worker. + */ + async handleWorkersCommand(message: InboundMessage, connector: Connector): Promise { + const master = this.deps.getMaster(); + if (!master) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Workers command not available — Master AI not initialized.', + replyTo: message.id, + }); + return; + } + + const registry = master.getWorkerRegistry(); + const allWorkers = registry.getAllWorkers(); + const activeWorkers = allWorkers.filter( + (w) => w.status === 'pending' || w.status === 'running', + ); + const orphaned = registry.getOrphanedWorkers(); + + const lines: string[] = ['*Active Workers*']; + + if (activeWorkers.length === 0) { + lines.push('No workers are currently active.'); + } else { + lines.push(`${activeWorkers.length} active worker${activeWorkers.length !== 1 ? 's' : ''}:`); + lines.push(''); + for (const w of activeWorkers) { + const elapsed = formatDuration(Date.now() - new Date(w.startedAt).getTime()); + const profile = w.taskManifest.profile ?? 'unknown'; + // Show last 8 chars of ID for readability + const shortId = w.id.length > 16 ? `…${w.id.slice(-12)}` : w.id; + const pidStr = w.pid !== undefined ? ` PID:${w.pid}` : ''; + lines.push(` • ${shortId} | ${w.status} | ${profile} | ${elapsed}${pidStr}`); + } + } + + if (orphaned.length > 0) { + lines.push(''); + lines.push( + `⚠️ ${orphaned.length} orphaned worker${orphaned.length !== 1 ? 's' : ''} (pending/running with no recent progress)`, + ); + } + + if (activeWorkers.length > 0) { + lines.push(''); + lines.push('Use /kill to force-stop a stuck worker.'); + } + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + + logger.info( + { sender: message.sender, activeCount: activeWorkers.length }, + '/workers command handled', + ); + } + + // ------------------------------------------------------------------------- + // handleKillWorkerCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "/kill " command. + * Force-stops a worker by partial or full ID match. + * Delegates to master.killWorker(). + */ + async handleKillWorkerCommand(message: InboundMessage, connector: Connector): Promise { + const master = this.deps.getMaster(); + if (!master) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Kill command not available — Master AI not initialized.', + replyTo: message.id, + }); + return; + } + + const trimmed = message.content.trim(); + // Extract the worker ID argument after "/kill " + const partialId = trimmed.slice(5).trim(); + + if (!partialId) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Usage: /kill . Use /workers to list active workers.', + replyTo: message.id, + }); + return; + } + + const registry = master.getWorkerRegistry(); + const allWorkers = registry.getAllWorkers(); + + // Match: exact ID, or ID ends with "-", or ID contains + const matched = allWorkers.find( + (w) => w.id === partialId || w.id.endsWith(`-${partialId}`) || w.id.includes(partialId), + ); + + let responseText: string; + if (!matched) { + responseText = `Worker '${partialId}' not found. Use /workers to list active workers.`; + } else { + const result = await master.killWorker(matched.id, message.sender); + responseText = result.message; + } + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: responseText, + replyTo: message.id, + }); + + logger.info({ sender: message.sender, partialId }, '/kill command handled'); + } + + // ------------------------------------------------------------------------- + // handleStatsCommand + // ------------------------------------------------------------------------- + + /** + * Handle the built-in "/stats" command — show exploration ROI summary. + * + * Queries the token_economics table for aggregate stats and formats a + * human-readable message: + * "Explored with ~50K tokens, saved ~200K tokens across 15 retrievals (4x ROI)" + * + * Falls back gracefully when token_economics data is not yet available. + */ + async handleStatsCommand(message: InboundMessage, connector: Connector): Promise { + const memory = this.deps.getMemory(); + if (!memory) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Stats not available — memory not initialized.', + replyTo: message.id, + }); + return; + } + + let content: string; + + try { + const stats = await memory.getTokenEconomicsStats(); + + if (!stats || stats.chunksTracked === 0) { + content = + '*Exploration Stats*\n\nNo data yet — stats are collected as the workspace is explored and queried.'; + } else { + const { totalDiscoveryTokens, totalReadTokens, totalRetrievals, chunksTracked } = stats; + + // Format large numbers as "~50K" or "~1.2M" + const fmt = (n: number): string => { + if (n >= 1_000_000) return `~${(n / 1_000_000).toFixed(1)}M`; + if (n >= 1_000) return `~${Math.round(n / 1_000)}K`; + return `${n}`; + }; + + const roi = + totalDiscoveryTokens > 0 ? (totalReadTokens / totalDiscoveryTokens).toFixed(1) : null; + + const roiStr = roi !== null ? ` (${roi}x ROI)` : ''; + + const lines = [ + '*Exploration Stats*', + '', + `Explored with ${fmt(totalDiscoveryTokens)} tokens, saved ${fmt(totalReadTokens)} tokens across ${totalRetrievals} retrieval${totalRetrievals !== 1 ? 's' : ''}${roiStr}`, + `Chunks tracked: ${chunksTracked}`, + ]; + + content = lines.join('\n'); + } + } catch (err) { + logger.warn({ err }, 'handleStatsCommand: failed to fetch token economics'); + content = 'Stats unavailable — could not read token economics data.'; + } + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content, + replyTo: message.id, + }); + + logger.info({ sender: message.sender }, '/stats command handled'); + } + + // ------------------------------------------------------------------------- + // handleDoctorCommand + // ------------------------------------------------------------------------- + + async handleDoctorCommand(message: InboundMessage, connector: Connector): Promise { + const lines: string[] = ['*OpenBridge Health*', '']; + + let failCount = 0; + let warnCount = 0; + + for (const check of CHECKS) { + let result: CheckResult; + try { + result = check.run(); + } catch (err) { + result = { pass: false, message: `check threw: ${(err as Error).message}` }; + } + + let icon: string; + if (result.pass === true) { + icon = '✓'; + } else if (result.pass === 'warn') { + icon = '⚠'; + warnCount++; + } else { + icon = '✗'; + failCount++; + } + + lines.push(`${icon} ${check.label.padEnd(14)} ${result.message}`); + if (result.pass !== true && result.fixHint) { + lines.push(` → ${result.fixHint}`); + } + } + + lines.push(''); + if (failCount === 0 && warnCount === 0) { + lines.push('All checks passed.'); + } else if (failCount === 0) { + lines.push(`${warnCount} warning${warnCount !== 1 ? 's' : ''} (non-critical).`); + } else { + lines.push(`${failCount} failed, ${warnCount} warning${warnCount !== 1 ? 's' : ''}.`); + } + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + + logger.info({ sender: message.sender }, '/doctor command handled'); + } + + // ------------------------------------------------------------------------- + // handleProcessCommand + // ------------------------------------------------------------------------- + + async handleProcessCommand(message: InboundMessage, connector: Connector): Promise { + // Parse file path from "/process " + const match = /^\/process\s+(.+)$/i.exec(message.content.trim()); + if (!match) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Usage: /process \nExample: /process /path/to/invoice.pdf', + replyTo: message.id, + }); + return; + } + + const filePath = (match[1] ?? '').trim(); + + // Send processing acknowledgement + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Processing document: ${filePath}...`, + replyTo: message.id, + }); + + let doc: ProcessedDocument; + try { + const { processDocument } = await import('../intelligence/document-processor.js'); + doc = await processDocument(filePath); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + logger.warn({ filePath, err }, '/process command: document processing failed'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Failed to process document: ${msg}`, + replyTo: message.id, + }); + return; + } + + // Run entity extraction to get docType + entities + let docType = doc.docType; + let entities: ExtractedEntity[] = []; + try { + const { extractEntities } = await import('../intelligence/entity-extractor.js'); + const extraction = await extractEntities( + { + rawText: doc.rawText, + tables: doc.tables, + images: doc.images, + metadata: doc.metadata, + }, + `File: ${doc.filename}`, + ); + docType = extraction.docType; + entities = extraction.entities; + } catch (err) { + logger.warn( + { filePath, err }, + '/process command: entity extraction failed, using raw result', + ); + } + + // Format user-friendly summary + const lines: string[] = [`*Document: ${doc.filename}*`, '']; + lines.push(`Type: ${docType}`); + lines.push(`Format: ${doc.mimeType}`); + + if (doc.tables.length > 0) { + lines.push(`Tables: ${doc.tables.length}`); + } + + if (entities.length > 0) { + // Group entities by type + const byType = new Map(); + for (const e of entities) { + const group = byType.get(e.type) ?? []; + group.push(e.name); + byType.set(e.type, group); + } + + lines.push(''); + lines.push('*Extracted Entities*'); + for (const [type, names] of byType) { + const label = type.charAt(0).toUpperCase() + type.slice(1); + lines.push(`• ${label}: ${names.join(', ')}`); + } + } else if (doc.rawText.trim().length > 0) { + // No entities — show a short text excerpt + const excerpt = doc.rawText.trim().slice(0, 200); + lines.push(''); + lines.push('*Preview*'); + lines.push(excerpt + (doc.rawText.length > 200 ? '…' : '')); + } else { + lines.push(''); + lines.push('No text content extracted.'); + } + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + + logger.info({ sender: message.sender, filePath, docType }, '/process command handled'); + } + + // ------------------------------------------------------------------------- + // handleDoctypesCommand — /doctypes (list all registered DocTypes) + // ------------------------------------------------------------------------- + + async handleDoctypesCommand(message: InboundMessage, connector: Connector): Promise { + const memory = this.deps.getMemory(); + const db = memory?.getDb(); + if (!db) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'DocType registry unavailable — memory system not initialized.', + replyTo: message.id, + }); + return; + } + + let doctypes: Array<{ name: string; label_plural: string; icon?: string | null }> = []; + try { + const { listDocTypes } = await import('../intelligence/doctype-store.js'); + doctypes = listDocTypes(db); + } catch (err) { + logger.warn({ err }, '/doctypes command: failed to list doctypes'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Failed to load DocTypes — database may not be initialized.', + replyTo: message.id, + }); + return; + } + + if (doctypes.length === 0) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: + '*DocTypes*\n\nNo DocTypes registered yet.\nAsk the AI to create one: "I need to track invoices"', + replyTo: message.id, + }); + return; + } + + const lines: string[] = ['*Registered DocTypes*', '']; + for (const dt of doctypes) { + const icon = dt.icon ? `${dt.icon} ` : ''; + lines.push(`• ${icon}${dt.label_plural} (/doctype ${dt.name})`); + } + lines.push(''); + lines.push('Use /doctype to see fields and states.'); + lines.push('Use /dt list to browse records.'); + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + + logger.info({ sender: message.sender, count: doctypes.length }, '/doctypes command handled'); + } + + // ------------------------------------------------------------------------- + // handleDoctypeCommand — /doctype {name} (show DocType details) + // ------------------------------------------------------------------------- + + async handleDoctypeCommand(message: InboundMessage, connector: Connector): Promise { + const match = /^\/doctype\s+(\S+)/i.exec(message.content.trim()); + if (!match) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Usage: /doctype \nExample: /doctype invoice\n\nUse /doctypes to see all.', + replyTo: message.id, + }); + return; + } + + const name = (match[1] ?? '').trim(); + const memory = this.deps.getMemory(); + const db = memory?.getDb(); + if (!db) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'DocType registry unavailable — memory system not initialized.', + replyTo: message.id, + }); + return; + } + + let full: FullDocType | null = null; + try { + const { getDocTypeByName } = await import('../intelligence/doctype-store.js'); + full = getDocTypeByName(db, name); + } catch (err) { + logger.warn({ err, name }, '/doctype command: failed to load doctype'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Failed to load DocType "${name}".`, + replyTo: message.id, + }); + return; + } + + if (!full) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `DocType "${name}" not found.\nUse /doctypes to see all registered DocTypes.`, + replyTo: message.id, + }); + return; + } + + const dt = full.doctype; + const icon = dt.icon ? `${dt.icon} ` : ''; + const lines: string[] = [`*${icon}${dt.label_singular}* (${dt.name})`, '']; + + // Fields + if (full.fields.length > 0) { + lines.push('*Fields*'); + for (const f of full.fields) { + const req = f.required ? ' ✱' : ''; + const computed = f.formula ? ' (computed)' : ''; + lines.push(`• ${f.label} [${f.field_type}]${req}${computed}`); + } + lines.push(''); + } + + // States + if (full.states.length > 0) { + lines.push('*States*'); + const stateList = full.states.map((s) => s.label).join(' → '); + lines.push(stateList); + lines.push(''); + } + + // Transitions + if (full.transitions.length > 0) { + lines.push('*Actions*'); + for (const t of full.transitions) { + lines.push(`• ${t.action_label} (${t.from_state} → ${t.to_state})`); + } + lines.push(''); + } + + lines.push(`Use /dt ${dt.name} list to browse records.`); + lines.push(`Use /dt ${dt.name} create to add a new record.`); + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + + logger.info({ sender: message.sender, name }, '/doctype command handled'); + } + + // ------------------------------------------------------------------------- + // handleDtCommand — /dt {doctype} list|create|{id} + // ------------------------------------------------------------------------- + + async handleDtCommand(message: InboundMessage, connector: Connector): Promise { + // Parse: /dt + const match = /^\/dt\s+(\S+)(?:\s+(.+))?$/i.exec(message.content.trim()); + if (!match) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: + 'Usage:\n' + + ' /dt list — list records\n' + + ' /dt create — start creation flow\n' + + ' /dt — show record details\n' + + '\nExample: /dt invoice list', + replyTo: message.id, + }); + return; + } + + const doctypeName = (match[1] ?? '').trim(); + const sub = (match[2] ?? '').trim().toLowerCase(); + + const memory = this.deps.getMemory(); + const db = memory?.getDb(); + if (!db) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'DocType registry unavailable — memory system not initialized.', + replyTo: message.id, + }); + return; + } + + let full: FullDocType | null = null; + try { + const { getDocTypeByName } = await import('../intelligence/doctype-store.js'); + full = getDocTypeByName(db, doctypeName); + } catch (err) { + logger.warn({ err, doctypeName }, '/dt command: failed to load doctype'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Failed to load DocType "${doctypeName}".`, + replyTo: message.id, + }); + return; + } + + if (!full) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `DocType "${doctypeName}" not found.\nUse /doctypes to see all registered DocTypes.`, + replyTo: message.id, + }); + return; + } + + const dt = full.doctype; + const tableName = `"${dt.table_name.replace(/"/g, '""')}"`; + + /** Safely convert an unknown SQLite column value to a display string. */ + const toStr = (v: unknown, fallback = ''): string => { + if (v === null || v === undefined) return fallback; + if (typeof v === 'string') return v; + if (typeof v === 'number' || typeof v === 'boolean') return String(v); + return JSON.stringify(v); + }; + + // ── /dt list ────────────────────────────────────────────────── + if (!sub || sub === 'list') { + try { + const rows = db + .prepare(`SELECT * FROM ${tableName} ORDER BY created_at DESC LIMIT 20`) + .all() as Record[]; + + if (rows.length === 0) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `*${dt.label_plural}*\n\nNo records found.\nUse /dt ${dt.name} create to add one.`, + replyTo: message.id, + }); + return; + } + + // Pick a display column: prefer 'name', 'title', 'subject', then first text field + const textFields = full.fields.filter( + (f) => f.field_type === 'text' || f.field_type === 'email' || f.field_type === 'link', + ); + const displayField = + textFields.find((f) => ['name', 'title', 'subject', 'label'].includes(f.name)) ?? + textFields[0]; + + const lines: string[] = [`*${dt.label_plural}* (${rows.length} shown)`, '']; + for (const row of rows) { + const id = toStr(row['id']).slice(-8); + const label = displayField + ? toStr(row[displayField.name], '—').slice(0, 50) + : `Record ${id}`; + const status = 'status' in row ? ` [${toStr(row['status']).slice(0, 20)}]` : ''; + lines.push(`• ${label}${status} — /dt ${dt.name} ${toStr(row['id'])}`); + } + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + } catch (err) { + logger.warn({ err, doctypeName }, '/dt list: failed to query records'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Failed to list ${dt.label_plural} — table may not be initialized yet.`, + replyTo: message.id, + }); + } + + logger.info({ sender: message.sender, doctypeName, sub: 'list' }, '/dt list handled'); + return; + } + + // ── /dt create ──────────────────────────────────────────────── + if (sub === 'create') { + const requiredFields = full.fields.filter((f) => f.required && !f.formula); + const optionalFields = full.fields.filter((f) => !f.required && !f.formula); + + const lines: string[] = [`*Create ${dt.label_singular}*`, '']; + + if (requiredFields.length > 0) { + lines.push('*Required fields:*'); + for (const f of requiredFields) { + const hint = f.options?.length + ? ` (${f.options.slice(0, 4).join(' | ')})` + : f.field_type !== 'text' + ? ` [${f.field_type}]` + : ''; + lines.push(`• ${f.label}${hint}`); + } + } + + if (optionalFields.length > 0) { + lines.push(''); + lines.push('*Optional fields:*'); + for (const f of optionalFields.slice(0, 6)) { + lines.push(`• ${f.label} [${f.field_type}]`); + } + if (optionalFields.length > 6) { + lines.push(` … and ${optionalFields.length - 6} more`); + } + } + + lines.push(''); + lines.push(`Tell the AI: "Create a ${dt.label_singular} with "`); + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + + logger.info({ sender: message.sender, doctypeName, sub: 'create' }, '/dt create handled'); + return; + } + + // ── /dt {id} ────────────────────────────────────────────────── + const recordId = (match[2] ?? '').trim(); + try { + const row = db.prepare(`SELECT * FROM ${tableName} WHERE id = ?`).get(recordId) as + | Record + | undefined; + + if (!row) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Record "${recordId}" not found in ${dt.label_plural}.\nUse /dt ${dt.name} list to browse records.`, + replyTo: message.id, + }); + return; + } + + const lines: string[] = [`*${dt.label_singular} — ${recordId.slice(-8)}*`, '']; + for (const f of full.fields) { + const val = row[f.name]; + if (val !== null && val !== undefined && val !== '') { + lines.push(`${f.label}: ${toStr(val).slice(0, 100)}`); + } + } + + // Include any system columns not in field list + for (const col of ['status', 'created_at', 'updated_at', 'created_by']) { + if (col in row && !full.fields.some((f) => f.name === col)) { + lines.push(`${col}: ${toStr(row[col]).slice(0, 60)}`); + } + } + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + } catch (err) { + logger.warn({ err, doctypeName, recordId }, '/dt {id}: failed to fetch record'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Failed to fetch record — table may not be initialized yet.`, + replyTo: message.id, + }); + } + + logger.info({ sender: message.sender, doctypeName, sub: 'record' }, '/dt record handled'); + } + + // ------------------------------------------------------------------------- + // handleHelpCommand + // ------------------------------------------------------------------------- + + async handleHelpCommand(message: InboundMessage, connector: Connector): Promise { + const lines: string[] = [ + '*OpenBridge Commands*', + '', + '*General*', + '• status — show active workers, exploration progress, and daily cost', + '• stop — stop all active workers', + '• explore — re-explore the workspace', + '• history — show recent conversation history', + '• /audit — list recent worker spawns', + '• /apps — list running app instances with URLs', + '• /scope — show workspace visibility rules and detected sensitive files', + '• /workers — list active workers with ID, status, profile, duration, and PID', + '• /kill — force-stop a stuck worker by ID (partial match supported)', + '• /stats — show exploration ROI: tokens spent vs tokens saved across all retrievals', + '• /doctor — run health checks (Node.js, AI tools, config, SQLite, channels) and show summary', + '• /process — process a document file (PDF, DOCX, XLSX, image, etc.) and extract key entities, amounts, and dates', + '• /doctypes — list all registered DocTypes (business data schemas)', + '• /doctype — show DocType details: fields, states, and available actions', + '• /dt list — list records for a DocType (most recent 20)', + '• /dt create — show creation form with required and optional fields', + '• /dt — show a specific record by ID', + '• /skills — list available skills with descriptions and usage counts', + '• /skill-packs — list available skill packs (built-in + workspace custom)', + '', + '*Tool Escalation*', + '• /allow — grant a pending tool escalation (scope: once by default)', + '• /allow --session — grant for the entire session', + '• /allow --permanent — grant permanently', + '• /allow all — grant all pending escalations at once', + '• /deny — reject a pending tool escalation', + '• /deny all — reject all pending escalations at once', + '• /whoami — show your role, channel, allowed actions, daily cost, and consent mode', + '• /role — (owner/admin) set role for another user on this channel', + '• /approve — (owner/admin) approve a pairing request using the 6-digit code', + '• /permissions — show your consent mode, session grants, and permanent grants', + '• /trust [auto|edit|ask] — set trust level (auto = no prompts, edit = prompt for full-access only, ask = always prompt)', + '', + '*Batch Control*', + '• /batch — show batch status: current item, progress, cost, elapsed time, failed items', + '• /pause — pause active batch (in-progress workers finish, no new items started)', + '• /continue — resume a paused batch from where it left off', + '• /batch abort — cancel remaining batch items and show completion summary', + '• /batch skip — skip the current failed item and continue with the next', + '', + '*Deep Mode*', + '• /deep — start a deep analysis session (investigate → report → plan → execute → verify)', + '• /proceed — advance to the next Deep Mode phase', + '• /focus N — dig deeper into finding number N from the current plan', + '• /skip N — skip item N from the current Deep Mode plan', + '• /phase — show current Deep Mode phase and progress', + ]; + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + + logger.info({ sender: message.sender }, 'Help command shown via /help'); + } + + // ------------------------------------------------------------------------- + // Formatting helpers + // ------------------------------------------------------------------------- + + /** + * Format the last N worker spawns for the given channel. + * webchat -> HTML table + * console -> ASCII table + * all other -> numbered list (WhatsApp, Telegram, Discord) + */ + formatAuditLog(spawns: ActivityRecord[], channel: string): string { + if (spawns.length === 0) { + return '*Worker Audit Log*\n\nNo worker spawns recorded yet.'; + } + + const formatDate = (iso: string): string => iso.slice(0, 16).replace('T', ' '); + const formatDurationMs = (startIso: string, endIso: string | undefined): string => { + if (!endIso) return 'running'; + const ms = new Date(endIso).getTime() - new Date(startIso).getTime(); + return formatDuration(ms); + }; + const formatCost = (costUsd: number | undefined): string => + costUsd !== undefined ? `$${costUsd.toFixed(3)}` : 'n/a'; + const shortId = (id: string): string => id.slice(-8); + + if (channel === 'webchat') { + const rows = spawns + .map( + (s) => + `${escapeHtml(shortId(s.id))}` + + `${escapeHtml(s.profile ?? '—')}` + + `${escapeHtml(s.model ?? '—')}` + + `${formatDurationMs(s.started_at, s.completed_at)}` + + `${formatCost(s.cost_usd)}` + + `${escapeHtml(s.status)}` + + `${escapeHtml((s.task_summary ?? '').slice(0, 60))}`, + ) + .join(''); + return ( + 'Worker Audit Log' + + '' + + '' + + rows + + '
IDProfileModelDurationCostStatusTask
' + ); + } + + if (channel === 'console') { + const header = ' # | ID | Profile | Duration | Cost | Status | Task'; + const sep = '-'.repeat(header.length); + const rows = spawns.map((s, i) => { + const idx = String(i + 1).padStart(2); + const id = shortId(s.id).padEnd(8); + const profile = (s.profile ?? '—').padEnd(10).slice(0, 10); + const dur = formatDurationMs(s.started_at, s.completed_at).padEnd(8); + const cost = formatCost(s.cost_usd).padEnd(7); + const status = (s.status ?? '—').padEnd(9).slice(0, 9); + const task = (s.task_summary ?? '').slice(0, 40); + return ` ${idx} | ${id} | ${profile} | ${dur} | ${cost} | ${status} | ${task}`; + }); + return ['*Worker Audit Log*', header, sep, ...rows, sep].join('\n'); + } + + // Default: WhatsApp, Telegram, Discord — numbered list + const rowLines = spawns.map((s, i) => { + const idx = i + 1; + const id = shortId(s.id); + const profile = s.profile ?? '—'; + const model = s.model ?? '—'; + const dur = formatDurationMs(s.started_at, s.completed_at); + const cost = formatCost(s.cost_usd); + const status = s.status; + const task = (s.task_summary ?? '').slice(0, 80); + const date = formatDate(s.started_at); + return `${idx}. [${id}] ${profile} (${model})\n ${date} · ${dur} · ${cost} · ${status}\n ${task}`; + }); + return ['*Worker Audit Log*', '', ...rowLines].join('\n'); + } + + /** + * Format a session list for the given channel. + * webchat -> HTML table + * console -> ASCII table + * all other -> numbered list (WhatsApp, Telegram, Discord) + */ + formatSessionList(sessions: SessionSummary[], channel: string): string { + if (sessions.length === 0) { + return '*Conversation History*\n\nNo past sessions found.'; + } + + const formatDate = (iso: string): string => iso.slice(0, 10); // YYYY-MM-DD + + if (channel === 'webchat') { + const rows = sessions + .map( + (s, i) => + `${i + 1}${escapeHtml(s.title ?? 'Untitled')}` + + `${s.message_count}${formatDate(s.last_message_at)}`, + ) + .join(''); + return ( + 'Conversation History' + + '' + + rows + + '
#TitleMsgsDate
' + ); + } + + if (channel === 'console') { + const header = ' # | Title | Msgs | Date '; + const sep = '---|----------------------|------|----------'; + const rowLines = sessions.map((s, i) => { + const num = String(i + 1).padStart(2, ' '); + const title = (s.title ?? 'Untitled').slice(0, 20).padEnd(20, ' '); + const msgs = String(s.message_count).padStart(4, ' '); + const date = formatDate(s.last_message_at); + return `${num} | ${title} | ${msgs} | ${date}`; + }); + return ['*Conversation History*', '', header, sep, ...rowLines].join('\n'); + } + + // Default: numbered list (WhatsApp, Telegram, Discord) + const rowLines = sessions.map((s, i) => { + const title = s.title ?? 'Untitled'; + const msgWord = s.message_count === 1 ? 'msg' : 'msgs'; + return `${i + 1}. ${title} — ${s.message_count} ${msgWord} — ${formatDate(s.last_message_at)}`; + }); + return ['*Conversation History*', '', ...rowLines].join('\n'); + } + + // ------------------------------------------------------------------------- + // handleConnectCommand — /connect [] + // ------------------------------------------------------------------------- + + async handleConnectCommand(message: InboundMessage, connector: Connector): Promise { + const trimmed = message.content.trim(); + // Parse: /connect [ []] + // Use [\s\S]+ to capture multiline credentials (e.g. multi-line cURL commands) + const match = /^\/connect(?:\s+(\S+)(?:\s+([\s\S]+))?)?$/i.exec(trimmed); + + const sendHelp = async (): Promise => { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: + '*Connect an Integration*\n\n' + + 'Usage:\n' + + ' /connect stripe — Stripe payments\n' + + ' /connect google-drive — Google Drive storage\n' + + ' /connect api — Any OpenAPI/REST service\n\n' + + 'To get started, send the command without a credential to see what is required:\n' + + ' /connect stripe', + replyTo: message.id, + }); + }; + + if (!match || !match[1]) { + await sendHelp(); + return; + } + + const integrationName = match[1].toLowerCase(); + const credential = match[2]?.trim() ?? ''; + + // If no credential provided for `api`, check if a file was attached — use it as the spec + if (!credential && integrationName === 'api' && message.processedDocument) { + const workspacePath = this.deps.getWorkspacePath(); + const memory = this.deps.getMemory(); + const db = memory?.getDb(); + if (workspacePath && db) { + await this.handleConnectApiCommand(message, connector, '', workspacePath, db); + return; + } + } + + // If no credential provided, show integration-specific instructions + if (!credential) { + const instructions: Record = { + stripe: + '*Connect Stripe*\n\nProvide your Stripe secret API key:\n /connect stripe \n\nFind it at: https://dashboard.stripe.com/apikeys', + 'google-drive': + '*Connect Google Drive*\n\nProvide your Google service account JSON key or OAuth2 token:\n /connect google-drive \n\nSee Google Cloud Console for credentials.', + api: + '*Connect any REST/OpenAPI Service*\n\nSend one of the following:\n' + + ' /connect api \n — URL to an OpenAPI/Swagger spec or Postman collection\n' + + ' /connect api { "info": { ... }, "item": [...] }\n — Paste a Postman collection JSON\n' + + " /connect api curl 'https://api.example.com/v1/users' -H 'Authorization: Bearer '\n — One or more cURL commands (paste multi-line cURL as a single message)\n\n" + + 'You can also *send a file* (Postman JSON, Swagger YAML, PDF docs) along with /connect api.\n' + + 'After connecting, a skill pack will be auto-generated and capabilities listed.', + }; + + const msg = + instructions[integrationName] ?? + `*Connect ${integrationName}*\n\nProvide your API key or credential:\n /connect ${integrationName} `; + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: msg, + replyTo: message.id, + }); + return; + } + + // Credential provided — encrypt and store + const workspacePath = this.deps.getWorkspacePath(); + const memory = this.deps.getMemory(); + const db = memory?.getDb(); + + if (!workspacePath || !db) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: + 'Integration credentials cannot be stored — workspace not initialized. Start the bridge first.', + replyTo: message.id, + }); + return; + } + + // Use credential store from deps if available, otherwise create a temporary one + let credStore = this.deps.getCredentialStore(); + if (!credStore) { + const { CredentialStore: CS } = await import('../integrations/credential-store.js'); + credStore = new CS(workspacePath); + } + + // For `api` integrations, use multi-format detection and parsing + if (integrationName === 'api') { + await this.handleConnectApiCommand(message, connector, credential, workspacePath, db); + return; + } + + const credData: Record = { apiKey: credential }; + + try { + credStore.storeCredential(db, integrationName, credData); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + logger.warn({ integrationName, err }, '/connect: failed to store credential'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Failed to store credential for "${integrationName}": ${msg}`, + replyTo: message.id, + }); + return; + } + + logger.info({ integrationName, sender: message.sender }, '/connect: credential stored'); + + // Attempt to initialize via IntegrationHub if the integration is registered + const hub = this.deps.getIntegrationHub(); + let connectionStatus = 'Credential stored and encrypted.'; + + if (hub) { + try { + const config = { + name: integrationName, + credentialKey: integrationName, + options: credData, + }; + await hub.initialize(integrationName, config); + connectionStatus = 'Connected and verified successfully.'; + logger.info({ integrationName }, '/connect: integration initialized via hub'); + } catch (err) { + // Integration may not be registered yet (adapters ship in Phase 120) + const errMsg = err instanceof Error ? err.message : String(err); + if (errMsg.includes('not found')) { + connectionStatus = + 'Credential stored and encrypted. (Adapter not yet installed — will activate when available.)'; + } else { + connectionStatus = `Credential stored, but connection test failed: ${errMsg}`; + logger.warn({ integrationName, err }, '/connect: hub initialization failed'); + } + } + } + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `*${integrationName}* integration: ${connectionStatus}`, + replyTo: message.id, + }); + + logger.info({ sender: message.sender, integrationName }, '/connect command handled'); + } + + // ------------------------------------------------------------------------- + // handleConnectApiCommand — multi-format /connect api + // ------------------------------------------------------------------------- + + private async handleConnectApiCommand( + message: InboundMessage, + connector: Connector, + input: string, + workspacePath: string, + db: Database.Database, + ): Promise { + const { detectInputFormat, parseInputToOpenAPI } = + await import('../integrations/adapters/openapi-adapter.js'); + + // ── File upload path ──────────────────────────────────────────────────── + // When the user sends a file (Postman JSON, Swagger YAML, PDF API docs) + // with /connect api, use the processed document's raw text instead of + // the text input. This covers three cases: + // 1. /connect api + attached file (input is empty or irrelevant text) + // 2. /connect api + attached file (file takes precedence) + // 3. Accumulated doc detected before command routing + if (message.processedDocument) { + const doc = message.processedDocument; + const rawText = doc.rawText?.trim() ?? ''; + + if (!rawText) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: + 'The attached file appears to be empty or could not be read. Please try again with a Postman JSON, Swagger YAML, or PDF/text API documentation.', + replyTo: message.id, + }); + return; + } + + // Try direct format detection on the raw text first (handles JSON/YAML specs) + const docFormat = detectInputFormat(rawText); + + if (docFormat !== 'unknown') { + // Known structured format — parse directly + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Detected *${docFormat}* format in attached file. Parsing API spec…`, + replyTo: message.id, + }); + + let spec: OpenAPI.Document; + try { + spec = await parseInputToOpenAPI(rawText); + } catch (err) { + const errMsg = err instanceof Error ? err.message : String(err); + logger.warn({ docFormat, err }, '/connect api: failed to parse document input'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Could not parse the API spec from the attached file (${docFormat}): ${errMsg}`, + replyTo: message.id, + }); + return; + } + + await this.finalizeApiConnection(message, connector, spec, workspacePath, db); + return; + } + + // Unknown structured format — fall back to AI-powered doc extraction + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Extracting API endpoints from the attached document using AI…', + replyTo: message.id, + }); + + try { + const { docsToOpenAPI } = await import('../integrations/parsers/doc-parser.js'); + const spec = await docsToOpenAPI(rawText); + await this.finalizeApiConnection(message, connector, spec, workspacePath, db); + } catch (err) { + const errMsg = err instanceof Error ? err.message : String(err); + logger.warn({ err }, '/connect api: AI doc extraction failed'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Could not extract API endpoints from the document: ${errMsg}\n\nPlease try a Postman collection JSON or Swagger/OpenAPI YAML/JSON file.`, + replyTo: message.id, + }); + } + return; + } + + // ── Text input path ───────────────────────────────────────────────────── + const format = detectInputFormat(input); + + if (format === 'unknown') { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: + "I couldn't recognise the API input format. Please send one of:\n" + + '• A URL to a Swagger/OpenAPI spec or Postman collection\n' + + '• Postman collection JSON (paste the full JSON)\n' + + '• One or more cURL commands starting with `curl `\n' + + '• Attach a file (Postman JSON, Swagger YAML, PDF API docs)\n\n' + + 'Example:\n /connect api https://petstore.swagger.io/v2/swagger.json', + replyTo: message.id, + }); + return; + } + + // Acknowledge receipt and show progress for slow operations + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Detected format: *${format}*. Parsing API spec…`, + replyTo: message.id, + }); + + let spec: OpenAPI.Document; + try { + spec = await parseInputToOpenAPI(input); + } catch (err) { + const errMsg = err instanceof Error ? err.message : String(err); + logger.warn({ format, err }, '/connect api: failed to parse input'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: + `Could not parse the API spec (${format}): ${errMsg}\n\n` + + 'Please check the input and try again. For cURL, ensure the command starts with `curl `.', + replyTo: message.id, + }); + return; + } + + await this.finalizeApiConnection(message, connector, spec, workspacePath, db); + } + + /** + * finalizeApiConnection — shared finalization for all /connect api paths. + * Persists the spec, registers the adapter, reports capabilities, and + * kicks off async skill pack generation. + */ + private async finalizeApiConnection( + message: InboundMessage, + connector: Connector, + spec: OpenAPI.Document, + workspacePath: string, + db: Database.Database, + ): Promise { + const { OpenAPIAdapter } = await import('../integrations/adapters/openapi-adapter.js'); + const specDoc = spec as OpenAPIV3.Document; + const rawTitle = specDoc.info?.title ?? ''; + const apiSlug = rawTitle + ? rawTitle + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, '') + .slice(0, 40) || 'custom-api' + : 'custom-api'; + + const specJson = JSON.stringify(spec); + + // Persist credential so the adapter can be re-created on restart + let credStore = this.deps.getCredentialStore(); + if (!credStore) { + const { CredentialStore: CS } = await import('../integrations/credential-store.js'); + credStore = new CS(workspacePath); + } + + try { + credStore.storeCredential(db, apiSlug, { specJson }); + } catch (err) { + logger.warn({ apiSlug, err }, '/connect api: failed to store credential'); + // Non-fatal — proceed without persistence + } + + // Register + initialise the OpenAPIAdapter in the hub + const hub = this.deps.getIntegrationHub(); + const adapter = new OpenAPIAdapter(apiSlug); + + let capabilities: IntegrationCapability[] = []; + let initError: string | null = null; + + if (hub) { + hub.register(adapter); + try { + await hub.initialize(apiSlug, { + name: apiSlug, + credentialKey: apiSlug, + options: { specJson }, + }); + capabilities = adapter.describeCapabilities(); + logger.info({ apiSlug, capabilities: capabilities.length }, '/connect api: adapter ready'); + } catch (err) { + initError = err instanceof Error ? err.message : String(err); + logger.warn({ apiSlug, err }, '/connect api: adapter initialization failed'); + } + } else { + try { + await adapter.initialize({ name: apiSlug, credentialKey: apiSlug, options: { specJson } }); + capabilities = adapter.describeCapabilities(); + } catch { + // Ignore — initError stays null, capabilities stays empty + } + } + + // Build the response message + const lines: string[] = [`*${rawTitle || apiSlug}* API connected`]; + + if (initError) { + lines.push(`⚠️ Connection test failed: ${initError}`); + } else { + lines.push('✅ Spec parsed and adapter registered.'); + } + + if (capabilities.length > 0) { + lines.push(''); + lines.push(`*Capabilities (${capabilities.length}):*`); + const shown = capabilities.slice(0, 10); + for (const cap of shown) { + lines.push(`• ${cap.name} — ${cap.description}`); + } + if (capabilities.length > 10) { + lines.push(`… and ${capabilities.length - 10} more`); + } + } else { + lines.push('No capabilities could be extracted from the spec.'); + } + + lines.push(''); + lines.push('Generating skill pack in the background…'); + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + + // Async skill pack + workflow template generation — do not block the response + import('../integrations/skill-pack-generator.js') + .then(async ({ generateSkillPack }) => { + const { AgentRunner } = await import('./agent-runner.js'); + const runner = new AgentRunner(); + const packPath = await generateSkillPack(spec, workspacePath, runner); + if (packPath) { + logger.info({ apiSlug, packPath }, '/connect api: skill pack generated'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `✅ Skill pack for *${rawTitle || apiSlug}* saved. The Master AI can now use this API conversationally.`, + }); + } else { + logger.warn({ apiSlug }, '/connect api: skill pack generation returned null'); + } + + // Suggest pre-built workflows based on the API spec + const { generateDefaultWorkflows } = await import('../integrations/workflow-templates.js'); + const suggestions = generateDefaultWorkflows(spec); + if (suggestions.length > 0) { + const lines: string[] = [`🔧 *Suggested workflows for ${rawTitle || apiSlug}:*`, '']; + const icons = ['📬', '📊', '🔔']; + suggestions.forEach((wf, i) => { + lines.push(`${icons[i] ?? '•'} *${wf.name}*`); + lines.push(wf.description ?? ''); + lines.push(''); + }); + lines.push( + 'Reply *approve 1*, *approve 2*, *approve 3*, or *approve all* to enable workflows.', + ); + lines.push('Reply *skip* to skip workflow setup.'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + }); + logger.info( + { apiSlug, count: suggestions.length }, + '/connect api: workflow suggestions sent', + ); + } + }) + .catch((err: unknown) => { + logger.warn({ apiSlug, err }, '/connect api: skill pack generation failed'); + }); + + logger.info({ sender: message.sender, apiSlug }, '/connect api command handled'); + } + + // ------------------------------------------------------------------------- + // handleIntegrationsCommand — /integrations (list all registered integrations) + // ------------------------------------------------------------------------- + + async handleIntegrationsCommand(message: InboundMessage, connector: Connector): Promise { + const hub = this.deps.getIntegrationHub(); + + if (!hub) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Integration system not available.', + replyTo: message.id, + }); + return; + } + + try { + const integrations = hub.list(); + + if (integrations.length === 0) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: + 'No integrations registered.\n\nUse `/connect ` to connect a service.', + replyTo: message.id, + }); + return; + } + + // Load last health check times from DB if available + const db = this.deps.getMemory()?.getDb(); + const lastCheckedMap = new Map(); + if (db) { + try { + const rows = db + .prepare( + `SELECT integration_name, MAX(checked_at) AS last_checked + FROM integration_health_log + GROUP BY integration_name`, + ) + .all() as { integration_name: string; last_checked: string }[]; + for (const row of rows) { + lastCheckedMap.set(row.integration_name, row.last_checked); + } + } catch { + // health log table may not exist yet — ignore + } + } + + // Build formatted list of integrations + const lines: string[] = ['*Connected Integrations*', '']; + + for (const integration of integrations) { + const status = integration.connected ? '✅ Connected' : '❌ Disconnected'; + const healthEmoji = + integration.healthStatus === 'healthy' + ? '💚' + : integration.healthStatus === 'degraded' + ? '🟡' + : integration.healthStatus === 'unhealthy' + ? '❌' + : '❓'; + const healthLabel = `${healthEmoji} ${integration.healthStatus}`; + const capCount = integration.capabilityCount; + const capLabel = capCount === 1 ? 'capability' : 'capabilities'; + + lines.push(`• *${integration.name}* (${integration.type})`); + lines.push(` Status: ${status}`); + lines.push(` Health: ${healthLabel}`); + lines.push(` Capabilities: ${capCount} ${capLabel}`); + + const lastChecked = lastCheckedMap.get(integration.name); + if (lastChecked) { + const date = new Date(lastChecked); + lines.push(` Last checked: ${date.toLocaleString()}`); + } + + lines.push(''); + } + + lines.push(`Use \`/connect \` to connect a new integration.`); + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + + logger.info( + { sender: message.sender, count: integrations.length }, + '/integrations command handled', + ); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + logger.warn({ err }, '/integrations: failed to list integrations'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Failed to list integrations: ${msg}`, + replyTo: message.id, + }); + } + } + + // ------------------------------------------------------------------------- + // handleWorkflowsCommand — /workflows (list/enable/disable/runs/delete) + // ------------------------------------------------------------------------- + + async handleWorkflowsCommand(message: InboundMessage, connector: Connector): Promise { + const store = this.deps.getWorkflowStore(); + const engine = this.deps.getWorkflowEngine(); + + if (!store) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Workflow engine unavailable — not initialized.', + replyTo: message.id, + }); + return; + } + + const trimmed = message.content.trim(); + // Parse subcommand: /workflows [list|enable|disable|runs|delete] [id] + const match = /^\/workflows(?:\s+(list|enable|disable|runs|delete)(?:\s+(\S+))?)?$/i.exec( + trimmed, + ); + + if (!match) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: [ + '*Workflow Commands*', + '', + '/workflows list — show all workflows', + '/workflows enable {id} — enable a workflow', + '/workflows disable {id} — disable a workflow', + '/workflows runs {id} — show run history', + '/workflows delete {id} — delete a workflow', + ].join('\n'), + replyTo: message.id, + }); + return; + } + + const sub = (match[1] ?? 'list').toLowerCase(); + const id = match[2] ?? ''; + + // /workflows list (default) + if (sub === 'list') { + let workflows; + try { + workflows = store.listWorkflows(); + } catch (err) { + logger.warn({ err }, '/workflows list: failed'); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'Failed to list workflows.', + replyTo: message.id, + }); + return; + } + + if (workflows.length === 0) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: + '*Workflows*\n\nNo workflows defined yet.\nAsk the AI to create one: "remind me every morning about overdue invoices"', + replyTo: message.id, + }); + return; + } + + const lines: string[] = ['*Workflows*', '']; + for (const wf of workflows) { + const status = wf.status === 'active' ? '✅' : '⏸'; + const trigger = wf.trigger.type; + const runs = wf.run_count ?? 0; + lines.push(`${status} *${wf.name}* (\`${wf.id}\`)`); + lines.push(` Trigger: ${trigger} | Runs: ${runs}`); + } + lines.push(''); + lines.push('Use /workflows enable {id} or /workflows disable {id} to toggle.'); + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + return; + } + + // Subcommands that require an id + if (!id) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Usage: /workflows ${sub} {workflow-id}`, + replyTo: message.id, + }); + return; + } + + if (sub === 'enable') { + try { + if (engine) { + await engine.enableWorkflow(id); + } else { + store.updateWorkflow(id, { status: 'active' }); + } + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Workflow \`${id}\` enabled.`, + replyTo: message.id, + }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Failed to enable workflow: ${msg}`, + replyTo: message.id, + }); + } + return; + } + + if (sub === 'disable') { + try { + if (engine) { + await engine.disableWorkflow(id); + } else { + store.updateWorkflow(id, { status: 'inactive' }); + } + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Workflow \`${id}\` disabled.`, + replyTo: message.id, + }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Failed to disable workflow: ${msg}`, + replyTo: message.id, + }); + } + return; + } + + if (sub === 'runs') { + let runs; + try { + runs = store.listRuns(id, 10); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Failed to fetch runs: ${msg}`, + replyTo: message.id, + }); + return; + } + + if (runs.length === 0) { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `No runs found for workflow \`${id}\`.`, + replyTo: message.id, + }); + return; + } + + const lines: string[] = [`*Run History — \`${id}\`*`, '']; + for (const run of runs) { + const statusIcon = + run.status === 'completed' + ? '✅' + : run.status === 'failed' + ? '❌' + : run.status === 'running' + ? '⏳' + : '⏸'; + const started = run.started_at ? run.started_at.slice(0, 16).replace('T', ' ') : '—'; + const duration = + run.started_at && run.completed_at + ? formatDuration( + new Date(run.completed_at).getTime() - new Date(run.started_at).getTime(), + ) + : '—'; + lines.push(`${statusIcon} \`${run.id}\` — ${started} (${duration})`); + if (run.error) lines.push(` Error: ${run.error.slice(0, 80)}`); + } + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: lines.join('\n'), + replyTo: message.id, + }); + return; + } + + if (sub === 'delete') { + try { + store.deleteWorkflow(id); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Workflow \`${id}\` deleted.`, + replyTo: message.id, + }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: `Failed to delete workflow: ${msg}`, + replyTo: message.id, + }); + } + return; + } + } + + /** + * Format a session transcript for the given channel. + * webchat -> HTML message bubbles + * console -> plain text with separator line + * all other -> plain text list (WhatsApp, Telegram, Discord) + */ + formatSessionTranscript(entries: ConversationEntry[], channel: string): string { + const formatTime = (iso: string): string => iso.slice(0, 16).replace('T', ' '); // YYYY-MM-DD HH:MM + const formatRole = (role: string): string => { + if (role === 'user') return 'You'; + if (role === 'master' || role === 'worker') return 'AI'; + return 'System'; + }; + + if (channel === 'webchat') { + const items = entries + .map((e) => { + const time = formatTime(e.created_at ?? ''); + const role = formatRole(e.role); + const content = escapeHtml(e.content.slice(0, 500)); + const cls = e.role === 'user' ? 'user' : 'ai'; + return ( + `
${escapeHtml(role)} ` + + `${time}

${content}

` + ); + }) + .join(''); + return `Conversation Transcript
${items}
`; + } + + const rows = entries.map((e) => { + const time = formatTime(e.created_at ?? ''); + const role = formatRole(e.role); + const snippet = e.content.slice(0, 300); + return `[${time}] ${role}: ${snippet}`; + }); + + if (channel === 'console') { + return ['*Conversation Transcript*', '─'.repeat(40), ...rows, '─'.repeat(40)].join('\n'); + } + + // Default: WhatsApp, Telegram, Discord + return ['*Conversation Transcript*', '', ...rows].join('\n'); + } +} diff --git a/src/core/config-watcher.ts b/src/core/config-watcher.ts index 1b0ba967..a1bbbaac 100644 --- a/src/core/config-watcher.ts +++ b/src/core/config-watcher.ts @@ -1,8 +1,7 @@ import { watch, type FSWatcher } from 'node:fs'; -import { readFile } from 'node:fs/promises'; import { resolve } from 'node:path'; -import { AppConfigSchema } from '../types/config.js'; import type { AppConfig } from '../types/config.js'; +import { loadConfig } from './config.js'; import { createLogger } from './logger.js'; const logger = createLogger('config-watcher'); @@ -62,15 +61,15 @@ export class ConfigWatcher { } this.debounceTimer = setTimeout(() => { this.debounceTimer = null; - void this.reload(); + this.reload().catch((err) => { + logger.error({ err }, 'Unhandled config reload error'); + }); }, this.debounceMs); } private async reload(): Promise { try { - const raw = await readFile(this.configPath, 'utf-8'); - const parsed: unknown = JSON.parse(raw); - const config = AppConfigSchema.parse(parsed); + const config = await loadConfig(this.configPath); logger.info('Config file reloaded successfully'); diff --git a/src/core/config.ts b/src/core/config.ts index b68ff4a7..d2487d66 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -1,14 +1,145 @@ -import { readFile } from 'node:fs/promises'; -import { resolve } from 'node:path'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { getConfigDir } from '../cli/utils.js'; import { AppConfigSchema, V2ConfigSchema } from '../types/config.js'; import type { AppConfig, V2Config } from '../types/config.js'; import { createLogger } from './logger.js'; +const SUPPORTED_LOG_LEVELS = ['trace', 'debug', 'info', 'warn', 'error', 'fatal'] as const; + const logger = createLogger('config'); +/** Path to the written MCP config file, set by writeMcpConfig(). */ +let _mcpConfigPath: string | null = null; + +/** + * Returns the path to the global MCP config file written on startup, or null + * if no MCP config was written (no mcp section in config, disabled, or no servers). + */ +export function getMcpConfigPath(): string | null { + return _mcpConfigPath; +} + +/** Claude CLI MCP config format: { mcpServers: { [name]: { command, args?, env? } } } */ +type ClaudeMcpServerEntry = { + command: string; + args?: string[]; + env?: Record; +}; + +/** + * Write global MCP config to {workspacePath}/.openbridge/mcp-config.json on Bridge startup. + * + * When V2Config.mcp is set and enabled: + * - If configPath is set, reads and validates the external MCP config JSON (Claude Desktop format) + * - If inline servers are defined, converts to Claude CLI format + * - Merges both; inline servers override same-name imports + * - Writes merged config to {workspacePath}/.openbridge/mcp-config.json + * + * @returns Path to the written config file, or null if no MCP config is needed + */ +export async function writeMcpConfig( + v2Config: V2Config, + workspacePath: string, +): Promise { + const mcpConfig = v2Config.mcp; + if (!mcpConfig || mcpConfig.enabled === false) return null; + + const hasServers = mcpConfig.servers.length > 0; + const hasConfigPath = Boolean(mcpConfig.configPath); + + if (!hasServers && !hasConfigPath) return null; + + const merged: Record = {}; + + // Step 1: Load from external configPath (base layer — may be overridden by inline servers) + if (hasConfigPath) { + const expandedPath = expandTilde(mcpConfig.configPath!); + let raw: string; + try { + raw = await readFile(expandedPath, 'utf-8'); + } catch (err) { + throw new Error( + `MCP configPath "${mcpConfig.configPath}" not found or unreadable: ${(err as Error).message}`, + ); + } + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (err) { + throw new Error( + `MCP configPath "${mcpConfig.configPath}" is not valid JSON: ${(err as Error).message}`, + ); + } + + // Support Claude Desktop format { mcpServers: {...} } or direct { [name]: {...} } object + const externalConfig = parsed as Record; + const externalServers = + typeof externalConfig['mcpServers'] === 'object' && externalConfig['mcpServers'] !== null + ? (externalConfig['mcpServers'] as Record) + : externalConfig; + + for (const [name, serverConfig] of Object.entries(externalServers)) { + if ( + typeof serverConfig === 'object' && + serverConfig !== null && + 'command' in serverConfig && + typeof (serverConfig as Record)['command'] === 'string' + ) { + merged[name] = serverConfig as ClaudeMcpServerEntry; + } + } + + logger.info( + { configPath: mcpConfig.configPath, count: Object.keys(merged).length }, + 'MCP: loaded servers from configPath', + ); + } + + // Step 2: Apply inline servers (override same-name imports) + if (hasServers) { + for (const server of mcpConfig.servers) { + const entry: ClaudeMcpServerEntry = { command: server.command }; + if (server.args && server.args.length > 0) entry.args = server.args; + if (server.env && Object.keys(server.env).length > 0) entry.env = server.env; + merged[server.name] = entry; + } + logger.info({ count: mcpConfig.servers.length }, 'MCP: applied inline servers'); + } + + if (Object.keys(merged).length === 0) return null; + + // Step 3: Write to .openbridge/mcp-config.json + const dotFolderPath = join(workspacePath, '.openbridge'); + await mkdir(dotFolderPath, { recursive: true }); + const outputPath = join(dotFolderPath, 'mcp-config.json'); + await writeFile(outputPath, JSON.stringify({ mcpServers: merged }, null, 2), 'utf-8'); + + _mcpConfigPath = outputPath; + logger.info({ path: outputPath, servers: Object.keys(merged) }, 'MCP config written'); + + return outputPath; +} + +export function expandTilde(filePath: string): string { + if (filePath === '~' || filePath.startsWith('~/') || filePath.startsWith('~\\')) { + return homedir() + filePath.slice(1); + } + return filePath; +} + export function resolveConfigPath(configPath?: string): string { - const path = configPath ?? process.env['CONFIG_PATH'] ?? './config.json'; - return resolve(path); + if (configPath) { + return resolve(configPath); + } + if (process.env['CONFIG_PATH']) { + return resolve(process.env['CONFIG_PATH']); + } + // In packaged mode (pkg binary), getConfigDir() returns ~/.openbridge/ + // In dev mode, getConfigDir() returns process.cwd() + return join(getConfigDir(), 'config.json'); } export function isV2Config(parsed: unknown): parsed is V2Config { @@ -34,7 +165,7 @@ export function convertV2ToInternal(v2Config: V2Config): AppConfig { workspaces: [ { name: 'default', - path: v2Config.workspacePath, + path: expandTilde(v2Config.workspacePath), }, ], defaultWorkspace: 'default', @@ -51,6 +182,9 @@ export function convertV2ToInternal(v2Config: V2Config): AppConfig { denyPatterns: [], denyMessage: 'That command is not allowed.', }, + defaultRole: v2Config.auth.defaultRole, + channelRoles: v2Config.auth.channelRoles, + pairingEnabled: v2Config.auth.pairingEnabled ?? true, }, queue: v2Config.queue ?? { maxRetries: 3, @@ -58,6 +192,7 @@ export function convertV2ToInternal(v2Config: V2Config): AppConfig { }, router: v2Config.router ?? { progressIntervalMs: 15_000, + escalationTimeoutMs: 180_000, }, audit: v2Config.audit ?? { enabled: false, @@ -75,17 +210,182 @@ export function convertV2ToInternal(v2Config: V2Config): AppConfig { }; } +/** + * In non-production environments, automatically add the WebChat connector if + * not already configured. Also adds 'webchat-user' to the auth whitelist so + * local connector senders (webchat-user, console-user) can authenticate. + * + * Call this after loadConfig() in startup flows. + */ +export function injectDevConnectors(config: AppConfig): void { + if (process.env['NODE_ENV'] === 'production') return; + + const hasWebChat = config.connectors.some((c) => c.type === 'webchat'); + if (hasWebChat) return; + + config.connectors.push({ type: 'webchat', enabled: true, options: {} }); + + // Allow non-numeric senders (webchat-user, console-user) through auth. + // normalizeNumber() strips non-digits → 'webchat-user' → ''. Adding any + // non-numeric entry to the whitelist puts '' in the normalized set, which + // authorises all local-connector senders in dev mode. + if (!config.auth.whitelist.includes('webchat-user')) { + config.auth.whitelist.push('webchat-user'); + } + + logger.info('Dev mode: WebChat connector auto-injected (localhost:3000)'); +} + +/** + * Apply environment variable overrides to a parsed V2Config. + * ENV vars take precedence over values in config.json. + * + * Supported variables: + * OPENBRIDGE_WORKSPACE_PATH — overrides workspacePath + * OPENBRIDGE_CHANNELS — JSON array string, overrides channels + * OPENBRIDGE_AUTH_WHITELIST — comma-separated phone numbers, overrides auth.whitelist + * OPENBRIDGE_AUTH_PREFIX — overrides auth.prefix + * OPENBRIDGE_LOG_LEVEL — overrides logLevel + */ +export function applyEnvOverrides(v2Config: V2Config): V2Config { + const overridden: V2Config = { ...v2Config, auth: { ...v2Config.auth } }; + + if (process.env['OPENBRIDGE_WORKSPACE_PATH']) { + overridden.workspacePath = process.env['OPENBRIDGE_WORKSPACE_PATH']; + } + + if (process.env['OPENBRIDGE_CHANNELS']) { + try { + overridden.channels = JSON.parse(process.env['OPENBRIDGE_CHANNELS']) as V2Config['channels']; + } catch { + throw new Error( + 'OPENBRIDGE_CHANNELS must be a valid JSON array, e.g.: \'[{"type":"console","enabled":true}]\'', + ); + } + } + + if (process.env['OPENBRIDGE_AUTH_WHITELIST']) { + overridden.auth.whitelist = process.env['OPENBRIDGE_AUTH_WHITELIST'] + .split(',') + .map((s) => s.trim()) + .filter(Boolean); + } + + if (process.env['OPENBRIDGE_AUTH_PREFIX']) { + overridden.auth.prefix = process.env['OPENBRIDGE_AUTH_PREFIX']; + } + + if (process.env['OPENBRIDGE_LOG_LEVEL']) { + const level = process.env['OPENBRIDGE_LOG_LEVEL']; + if (!SUPPORTED_LOG_LEVELS.includes(level as (typeof SUPPORTED_LOG_LEVELS)[number])) { + throw new Error( + `OPENBRIDGE_LOG_LEVEL must be one of: ${SUPPORTED_LOG_LEVELS.join(', ')}. Got: "${level}"`, + ); + } + overridden.logLevel = level as V2Config['logLevel']; + } + + return overridden; +} + +/** + * Build a complete V2Config from environment variables alone (no config.json required). + * Throws a descriptive error if required variables are missing. + */ +export function buildV2ConfigFromEnv(): V2Config { + const workspacePath = process.env['OPENBRIDGE_WORKSPACE_PATH']; + const whitelistRaw = process.env['OPENBRIDGE_AUTH_WHITELIST']; + + const missing: string[] = []; + if (!workspacePath) missing.push('OPENBRIDGE_WORKSPACE_PATH'); + if (!whitelistRaw) missing.push('OPENBRIDGE_AUTH_WHITELIST'); + + if (missing.length > 0) { + throw new Error( + `No config.json found and required environment variables are not set: ${missing.join(', ')}.\n` + + 'Either create a config.json (run "npx openbridge init") or set:\n' + + ' OPENBRIDGE_WORKSPACE_PATH=/absolute/path/to/your/project\n' + + ' OPENBRIDGE_AUTH_WHITELIST=+1234567890,+0987654321\n' + + ' OPENBRIDGE_CHANNELS=\'[{"type":"console","enabled":true}]\' # optional, defaults to console\n' + + ' OPENBRIDGE_AUTH_PREFIX=/ai # optional\n' + + ' OPENBRIDGE_LOG_LEVEL=info # optional', + ); + } + + let channels: V2Config['channels']; + const channelsRaw = process.env['OPENBRIDGE_CHANNELS']; + if (channelsRaw) { + try { + channels = JSON.parse(channelsRaw) as V2Config['channels']; + } catch { + throw new Error( + 'OPENBRIDGE_CHANNELS must be a valid JSON array, e.g.: \'[{"type":"console","enabled":true}]\'', + ); + } + } else { + channels = [{ type: 'console', enabled: true }]; + } + + const whitelist = (whitelistRaw as string) + .split(',') + .map((s) => s.trim()) + .filter(Boolean); + + const result: Record = { + workspacePath, + channels, + auth: { + whitelist, + prefix: process.env['OPENBRIDGE_AUTH_PREFIX'] ?? '/ai', + }, + }; + + if (process.env['OPENBRIDGE_LOG_LEVEL']) { + const level = process.env['OPENBRIDGE_LOG_LEVEL']; + if (!SUPPORTED_LOG_LEVELS.includes(level as (typeof SUPPORTED_LOG_LEVELS)[number])) { + throw new Error( + `OPENBRIDGE_LOG_LEVEL must be one of: ${SUPPORTED_LOG_LEVELS.join(', ')}. Got: "${level}"`, + ); + } + result['logLevel'] = level; + } + + return V2ConfigSchema.parse(result); +} + export async function loadConfig(configPath?: string): Promise { const absolutePath = resolveConfigPath(configPath); logger.info({ path: absolutePath }, 'Loading configuration'); - const raw = await readFile(absolutePath, 'utf-8'); + let raw: string; + try { + raw = await readFile(absolutePath, 'utf-8'); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') { + logger.info('No config.json found — loading configuration from environment variables'); + const v2Config = buildV2ConfigFromEnv(); + const internalConfig = convertV2ToInternal(v2Config); + logger.info( + { + workspacePath: v2Config.workspacePath, + channels: v2Config.channels.length, + whitelist: v2Config.auth.whitelist.length, + source: 'env', + }, + 'Configuration loaded from environment variables', + ); + return internalConfig; + } + throw err; + } + const parsed: unknown = JSON.parse(raw); if (isV2Config(parsed)) { logger.info('Detected V2 config format'); - const v2Config = V2ConfigSchema.parse(parsed); + let v2Config = V2ConfigSchema.parse(parsed); + v2Config = applyEnvOverrides(v2Config); const internalConfig = convertV2ToInternal(v2Config); logger.info( diff --git a/src/core/content-redactor.ts b/src/core/content-redactor.ts new file mode 100644 index 00000000..8bef9ecb --- /dev/null +++ b/src/core/content-redactor.ts @@ -0,0 +1,124 @@ +import { createLogger } from './logger.js'; + +const logger = createLogger('content-redactor'); + +export interface RedactionResult { + /** The content with sensitive values replaced by REDACTED: tokens */ + redacted: string; + /** Number of replacements made across all patterns */ + redactionCount: number; +} + +interface RedactionPattern { + /** Human-readable name used in the replacement token, e.g. "openai_key" */ + name: string; + /** Regex that matches the sensitive value. Must use the global flag (g) */ + pattern: RegExp; +} + +export interface ContentRedactorOptions { + /** Whether redaction is active. Defaults to false — opt-in feature */ + enabled?: boolean; + /** Override the built-in pattern list */ + patterns?: RedactionPattern[]; +} + +/** + * Scans text content for sensitive values (API keys, connection strings, private key blocks) + * and replaces them with REDACTED: tokens. + * + * This is an opt-in feature — disabled by default. + * Enable by constructing with { enabled: true }. + * + * Redaction patterns are defined in DEFAULT_PATTERNS below and extended by OB-1471. + */ +export class ContentRedactor { + private readonly enabled: boolean; + private readonly patterns: readonly RedactionPattern[]; + + constructor(options: ContentRedactorOptions = {}) { + this.enabled = options.enabled ?? false; + this.patterns = options.patterns ?? DEFAULT_PATTERNS; + } + + /** + * Scan content for sensitive values and replace them with redaction tokens. + * + * If the redactor is disabled, returns the original content unchanged with count 0. + * + * @param content Raw text to scan (e.g. file content, worker output, prompt text) + * @returns { redacted, redactionCount } + */ + public redact(content: string): RedactionResult { + if (!this.enabled) { + return { redacted: content, redactionCount: 0 }; + } + + let redacted = content; + let redactionCount = 0; + + for (const { name, pattern } of this.patterns) { + const replacement = `REDACTED:${name}`; + const matches = redacted.match(pattern); + if (matches) { + redactionCount += matches.length; + redacted = redacted.replace(pattern, replacement); + } + } + + if (redactionCount > 0) { + logger.debug({ redactionCount }, 'ContentRedactor: redacted sensitive values from content'); + } + + return { redacted, redactionCount }; + } +} + +/** + * Default redaction patterns. + * Ordered from most specific to least specific to avoid partial matches shadowing + * more-precise ones. + */ +export const DEFAULT_PATTERNS: RedactionPattern[] = [ + // PEM private key blocks — matched first (multi-line, most specific) + { + name: 'pem_private_key', + pattern: + /-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----/g, + }, + // OpenAI API keys: sk- or sk-proj- followed by 20+ base64url chars + { + name: 'openai_key', + pattern: /sk-(?:proj-)?[A-Za-z0-9\-_]{20,}/g, + }, + // AWS Access Key IDs: AKIA + 16 uppercase alphanumeric chars + { + name: 'aws_key', + pattern: /(?:A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}/g, + }, + // GitHub Personal Access Tokens (classic) + { + name: 'github_pat', + pattern: /ghp_[A-Za-z0-9]{36}/g, + }, + // GitHub Server-to-Server tokens + { + name: 'github_server_token', + pattern: /ghs_[A-Za-z0-9]{36}/g, + }, + // MongoDB connection strings + { + name: 'mongodb_connection_string', + pattern: /mongodb:\/\/[^\s"'`\])}]+/g, + }, + // PostgreSQL connection strings + { + name: 'postgres_connection_string', + pattern: /postgres(?:ql)?:\/\/[^\s"'`\])}]+/g, + }, + // MySQL connection strings + { + name: 'mysql_connection_string', + pattern: /mysql:\/\/[^\s"'`\])}]+/g, + }, +]; diff --git a/src/core/cost-manager.ts b/src/core/cost-manager.ts new file mode 100644 index 00000000..26950d6e --- /dev/null +++ b/src/core/cost-manager.ts @@ -0,0 +1,229 @@ +/** + * Cost management helpers for worker cost tracking, estimation, and cap enforcement. + * + * Extracted from agent-runner.ts (OB-1286) so cost management logic can be + * imported independently without pulling in the full process-execution machinery. + */ + +import { createLogger } from './logger.js'; +import type { WorkspaceTrustLevel } from '../types/config.js'; + +const logger = createLogger('cost-manager'); + +/** + * Cost multipliers per trust level. Trusted mode allows higher spending; + * sandbox mode constrains it; standard mode is the baseline (1×). + */ +const TRUST_COST_MULTIPLIER: Record = { + sandbox: 0.5, + standard: 1, + trusted: 3, +}; + +/** + * Default per-profile cost caps in USD. + * If a worker's estimated cost exceeds the cap for its profile, + * the agent is aborted and a WARNING is logged (OB-F101). + */ +export const PROFILE_COST_CAPS: Record = { + 'read-only': 0.5, + 'code-edit': 1.0, + 'code-audit': 1.0, + 'full-access': 2.0, +}; + +/** + * Get the cost cap in USD for a given tool profile. + * Scales the base cap by `trustLevel` multiplier (sandbox: 0.5×, standard: 1×, trusted: 3×). + * User-configured `overrides` always win over the scaled value. + * Returns `undefined` if the profile is unknown or no cap is configured. + */ +export function getProfileCostCap( + profile: string | undefined, + overrides?: Record, + trustLevel?: WorkspaceTrustLevel, +): number | undefined { + if (!profile) return undefined; + const baseCap = PROFILE_COST_CAPS[profile]; + if (baseCap === undefined) return undefined; + const scaledCap = baseCap * TRUST_COST_MULTIPLIER[trustLevel ?? 'standard']; + if (overrides && Object.prototype.hasOwnProperty.call(overrides, profile)) { + return overrides[profile]; + } + return scaledCap; +} + +/** + * Module-level in-memory running average of worker costs per profile (OB-1673). + * Tracks { sum, count } so we can compute mean without storing all values. + * Resets on process restart — this is intentional (short-lived diagnostic signal). + */ +const _profileCostAccumulator: Map = new Map(); + +/** + * Record a completed worker cost for its profile and warn if the cost is + * more than 10x the running average for that profile. + * + * Only called after the agent has successfully produced a result (not on abort). + * The average is updated AFTER the spike check so extreme outliers do not + * immediately skew the baseline. + */ +export function checkProfileCostSpike(profile: string | undefined, costUsd: number): void { + if (!profile || costUsd <= 0) return; + + const acc = _profileCostAccumulator.get(profile); + if (acc && acc.count > 0) { + const avg = acc.sum / acc.count; + if (avg > 0 && costUsd > avg * 10) { + const multiplier = (costUsd / avg).toFixed(0); + logger.warn( + { cost: costUsd, average: avg, multiplier: Number(multiplier), profile }, + `Worker cost $${costUsd.toFixed(4)} is ${multiplier}x average $${avg.toFixed(4)} for ${profile} profile`, + ); + } + } + + // Update accumulator after the check to keep baseline stable + if (acc) { + acc.sum += costUsd; + acc.count += 1; + } else { + _profileCostAccumulator.set(profile, { sum: costUsd, count: 1 }); + } +} + +/** + * Return a snapshot of the current profile cost averages (for testing). + */ +export function getProfileCostAverages(): Record { + const result: Record = {}; + for (const [profile, acc] of _profileCostAccumulator) { + result[profile] = { avg: acc.count > 0 ? acc.sum / acc.count : 0, count: acc.count }; + } + return result; +} + +/** + * Reset the cost accumulator (exposed for tests only). + */ +export function resetProfileCostAverages(): void { + _profileCostAccumulator.clear(); +} + +/** + * Check whether a worker's cumulative cost has exceeded its cap. + * + * Returns `true` when `currentCost >= maxCost`, indicating the worker should + * be killed. Returns `false` when under the cap. + */ +export function checkCostCap(currentCost: number, maxCost: number): boolean { + return currentCost >= maxCost; +} + +/** + * Build a consistent cost-cap warning message for logging and result summaries. + * + * @param workerId Identifier for the worker (e.g. "worker-abc123") + * @param currentCost Cost accumulated so far in USD + * @param maxCost Cap threshold in USD + * @param model Model name (e.g. "claude-opus-4-6") or undefined + */ +export function formatCostWarning( + workerId: string, + currentCost: number, + maxCost: number, + model: string | undefined, +): string { + const modelStr = model ?? 'unknown-model'; + return ( + `Worker ${workerId} cost-capped: $${currentCost.toFixed(4)} >= $${maxCost.toFixed(4)}` + + ` (model: ${modelStr}) — output may be incomplete` + ); +} + +/** + * Estimate the cost in USD for a single agent call. + * Uses a simple per-call heuristic scaled by output size. + * + * Pricing tiers (Anthropic pricing as of v0.1.0): + * Haiku 4.5 (claude-haiku-4-5-*): $1/MTok input, $5/MTok output + * → base $0.001 + $0.00128 per KB output + * Sonnet 4.6 (claude-sonnet-4-6): $3/MTok input, $15/MTok output + * → base $0.003 + $0.00384 per KB output + * Opus 4.6 (claude-opus-4-6): $5/MTok input, $25/MTok output + * → base $0.005 + $0.0064 per KB output + * + * Per-KB multiplier derivation: price/MTok × (1024 bytes / 4 bytes-per-token) / 1_000_000 + * Falls back to Sonnet 4.6 pricing for unknown / undefined models. + */ +export function estimateCostUsd(model: string | undefined, outputBytes: number): number { + const outputKb = outputBytes / 1024; + const modelKey = (model ?? '').toLowerCase(); + + // Haiku 4.5: $1/MTok input, $5/MTok output + if (modelKey.includes('haiku') || /haiku.*4[.-]5/.test(modelKey)) { + return 0.001 + outputKb * 0.00128; + } + // Opus 4.6: $5/MTok input, $25/MTok output + if (modelKey.includes('opus') || /opus.*4[.-]6/.test(modelKey)) { + return 0.005 + outputKb * 0.0064; + } + // Codex / OpenAI models (gpt-5.x): ~2.5x Claude Sonnet pricing + if (modelKey.includes('codex') || modelKey.includes('gpt-5') || modelKey.includes('gpt-4o')) { + return 0.008 + outputKb * 0.0096; + } + // Default / Sonnet 4.6: $3/MTok input, $15/MTok output + return 0.003 + outputKb * 0.00384; +} + +/** + * Result returned by estimateCost(). + * Provides rough pre-execution cost and time estimates for display in the + * user confirmation prompt before a high-risk worker is spawned. + */ +export interface CostEstimate { + /** Expected number of agent turns (equals maxTurns — pessimistic worst-case estimate) */ + estimatedTurns: number; + /** Human-readable cost estimate string, e.g. "~$0.30" */ + costString: string; + /** Human-readable time estimate string, e.g. "~5 min" */ + timeString: string; +} + +/** + * Estimate the pre-execution cost and time for a worker spawn. + * + * Uses rough per-turn costs (pessimistic — assumes all turns are used): + * haiku / fast = $0.01 / turn + * sonnet / balanced = $0.03 / turn + * opus / powerful = $0.10 / turn + * + * Time estimate: ~10 seconds per turn. + * + * @param _profile Profile name — reserved for future profile-based adjustments + * @param maxTurns Maximum turns the worker is allowed (used as the turn estimate) + * @param modelTier Model tier ('fast', 'balanced', 'powerful') or alias ('haiku', 'sonnet', 'opus') + */ +export function estimateCost(_profile: string, maxTurns: number, modelTier: string): CostEstimate { + const tier = modelTier.toLowerCase(); + + let costPerTurn: number; + if (tier === 'fast' || tier.includes('haiku')) { + costPerTurn = 0.01; + } else if (tier === 'powerful' || tier.includes('opus')) { + costPerTurn = 0.1; + } else { + // balanced / sonnet / unknown — default to sonnet pricing + costPerTurn = 0.03; + } + + const estimatedTurns = maxTurns; + const totalCost = costPerTurn * estimatedTurns; + const totalSeconds = estimatedTurns * 10; + const totalMinutes = Math.ceil(totalSeconds / 60); + + const costString = `~$${totalCost.toFixed(2)}`; + const timeString = totalMinutes <= 1 ? '~1 min' : `~${totalMinutes} min`; + + return { estimatedTurns, costString, timeString }; +} diff --git a/src/core/docker-sandbox.ts b/src/core/docker-sandbox.ts new file mode 100644 index 00000000..c7150761 --- /dev/null +++ b/src/core/docker-sandbox.ts @@ -0,0 +1,642 @@ +/** + * DockerSandbox — run worker agents inside Docker containers. + * + * Uses the docker CLI via child_process (no SDK dependency). + * Provides container lifecycle management (create/start/stop/remove/exec) + * and an availability check that verifies both the CLI and daemon. + * + * @see OB-1545 + */ + +import { execFile, execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; +import { createLogger } from './logger.js'; + +const execFileAsync = promisify(execFile); + +const logger = createLogger('docker-sandbox'); + +// ─── Container lifecycle tracking ───────────────────────────────────────────── + +/** + * Module-level set of container IDs that are currently running. + * Populated by DockerSandbox.trackContainer() and cleared by untrackContainer(). + * Process-exit and SIGINT handlers use this set to force-remove orphaned containers. + */ +const _activeContainerIds = new Set(); + +/** + * Synchronously remove all tracked containers via execFileSync. + * Called from process.on('exit') and process.on('SIGINT') where async is unavailable + * or unreliable. Best-effort — errors are silently ignored. + */ +function _syncCleanupContainers(): void { + if (_activeContainerIds.size === 0) return; + const ids = [..._activeContainerIds]; + _activeContainerIds.clear(); + for (const id of ids) { + try { + execFileSync('docker', ['rm', '--force', id], { stdio: 'ignore', timeout: 10_000 }); + } catch { + // best-effort: ignore errors during emergency cleanup + } + } +} + +// Synchronous safety net — fires when the Node process is about to exit for any reason +// (normal exit, uncaughtException, or after SIGINT/SIGTERM handlers complete). +process.on('exit', _syncCleanupContainers); + +// SIGINT safety net — cleans up containers if the bridge does not reach graceful shutdown. +// Does NOT call process.exit() so that other SIGINT handlers (e.g. graceful shutdown in +// index.ts) can still fire and perform their own cleanup before the process terminates. +process.on('SIGINT', _syncCleanupContainers); + +/** + * Async cleanup of all currently-tracked containers — intended for graceful shutdown paths + * (e.g. Bridge.shutdown()) where async operations are available. + * + * Clears the tracking set immediately so that the synchronous exit handler is a no-op if + * this function already ran. + */ +export async function cleanupSandboxContainers(): Promise { + if (_activeContainerIds.size === 0) return; + const ids = [..._activeContainerIds]; + _activeContainerIds.clear(); + for (const id of ids) { + try { + await execFileAsync('docker', ['rm', '--force', id], { timeout: 30_000 }); + } catch { + // best-effort + } + } +} + +// ─── Types ──────────────────────────────────────────────────────────────────── + +/** Volume mount specification for a container */ +export interface VolumeMount { + /** Host path (absolute) */ + host: string; + /** Container path (absolute) */ + container: string; + /** If true, mount is read-only inside the container */ + readOnly?: boolean; +} + +/** Options for creating a new container */ +export interface ContainerOptions { + /** Docker image to use */ + image: string; + /** Optional container name (auto-generated if not set) */ + name?: string; + /** Mounts to attach to the container */ + mounts?: VolumeMount[]; + /** Environment variables to pass in */ + env?: Record; + /** Network mode: 'none' (default, most secure), 'host', or 'bridge' */ + network?: 'none' | 'host' | 'bridge'; + /** Memory limit in megabytes (default: 512) */ + memoryMB?: number; + /** CPU limit (default: 1) */ + cpus?: number; + /** Maximum number of PIDs (default: 100) */ + pidsLimit?: number; + /** Working directory inside the container */ + workdir?: string; + /** Command to run (for run-and-remove flows; exec() ignores this) */ + command?: string[]; +} + +/** Options for exec() calls */ +export interface ExecOptions { + /** Working directory inside the container */ + cwd?: string; + /** Execution timeout in milliseconds */ + timeout?: number; +} + +/** Result from an exec() call */ +export interface ExecResult { + stdout: string; + stderr: string; + exitCode: number; +} + +/** Options for buildImage() */ +export interface BuildImageOptions { + /** + * Rebuild the image even if it already exists locally. + * Default: false — skips build when `openbridge-worker:latest` is present. + */ + force?: boolean; + /** + * Absolute path to the project root used as the Docker build context. + * Default: auto-resolved from this module's location (two directories up + * from dist/core/docker-sandbox.js → project root). + */ + context?: string; +} + +/** Options for buildWorkspaceMounts() */ +export interface WorkspaceMountOptions { + /** + * Absolute path to the workspace (target project) on the host. + * Mounted at /workspace inside the container as read-only. + */ + workspacePath: string; + /** + * Absolute path to the .openbridge/ folder on the host. + * Mounted at /workspace/.openbridge as read-write so agents can write outputs. + * Defaults to `{workspacePath}/.openbridge`. + */ + openbridgePath?: string; +} + +// ─── DockerHealthMonitor ─────────────────────────────────────────────────────── + +/** + * Monitors Docker daemon availability at startup and on a periodic interval. + * + * On `start()`: + * 1. Performs an immediate availability check and logs the result. + * 2. Schedules a recheck every `recheckIntervalMs` milliseconds (default: 5 min). + * + * Callers can read the cached result via `isDockerAvailable()` at any time without + * incurring a blocking `docker info` call. + * + * Call `stop()` to cancel the interval (e.g. on bridge shutdown). + * + * @see OB-1557 + */ +export class DockerHealthMonitor { + private readonly sandbox: DockerSandbox; + private readonly recheckIntervalMs: number; + private available = false; + private interval: ReturnType | null = null; + private readonly monitorLogger = createLogger('docker-health-monitor'); + + constructor(sandbox: DockerSandbox, recheckIntervalMs = 5 * 60 * 1_000) { + this.sandbox = sandbox; + this.recheckIntervalMs = recheckIntervalMs; + } + + /** + * Run the initial health check and start the periodic recheck interval. + * + * Resolves after the first check completes so callers know the initial + * availability state before proceeding with startup. + */ + async start(): Promise { + await this._check(); + + this.interval = setInterval(() => { + this._check().catch((err: unknown) => { + this.monitorLogger.warn({ err }, 'Docker health recheck failed unexpectedly'); + }); + }, this.recheckIntervalMs); + + // Prevent the interval from blocking the Node process from exiting. + if (this.interval.unref) { + this.interval.unref(); + } + } + + /** Stop the periodic recheck interval. */ + stop(): void { + if (this.interval) { + clearInterval(this.interval); + this.interval = null; + } + } + + /** Returns the most recently cached Docker availability result. */ + isDockerAvailable(): boolean { + return this.available; + } + + private async _check(): Promise { + const wasAvailable = this.available; + this.available = await this.sandbox.isAvailable(); + + if (!this.available && wasAvailable) { + // Docker became unavailable after a previous success. + this.monitorLogger.warn( + 'Docker daemon became unavailable — sandbox mode disabled; falling back to direct (unsandboxed) spawn', + ); + } else if (!this.available && !wasAvailable) { + // Docker is still unavailable (state transition already logged). + this.monitorLogger.debug('Docker daemon still unavailable — sandbox mode remains disabled'); + } else if (!wasAvailable && this.available) { + // Docker became available after a previous failure. + this.monitorLogger.info('Docker daemon is now available — sandbox mode enabled'); + } else { + this.monitorLogger.debug('Docker daemon health check passed'); + } + } +} + +// ─── DockerSandbox ──────────────────────────────────────────────────────────── + +/** + * DockerSandbox wraps the docker CLI to manage containers for sandboxed + * worker execution. All public methods reject on unexpected errors. + * + * Typical flow: + * const sandbox = new DockerSandbox(); + * if (!await sandbox.isAvailable()) { ... fallback ... } + * const id = await sandbox.createContainer({ image: 'openbridge-worker:latest', ... }); + * await sandbox.startContainer(id); + * const result = await sandbox.exec(id, ['claude', '--print', ...]); + * await sandbox.stopContainer(id); + * await sandbox.removeContainer(id); + */ +export class DockerSandbox { + /** + * Check whether Docker is available on this machine. + * + * Returns true only when: + * 1. The `docker` binary is in PATH, AND + * 2. The Docker daemon is running (`docker info` succeeds). + * + * Note: This method is silent — it does not log. The DockerHealthMonitor handles + * logging on state transitions. Direct callers should log at appropriate levels + * based on their context (e.g., DEBUG when falling back, ERROR on unexpected failures). + * @see OB-1666 + */ + async isAvailable(): Promise { + try { + await execFileAsync('docker', ['info'], { timeout: 10_000 }); + return true; + } catch { + // Silent — let the caller and DockerHealthMonitor handle logging + return false; + } + } + + /** + * Register a container ID as active so it is cleaned up on process crash. + * Call this immediately after createContainer() succeeds. + */ + trackContainer(containerId: string): void { + _activeContainerIds.add(containerId); + logger.debug({ containerId: containerId.slice(0, 12) }, 'Container tracked for crash cleanup'); + } + + /** + * Remove a container ID from the crash-cleanup tracking set. + * Call this before attempting normal cleanup so the exit handler skips it. + */ + untrackContainer(containerId: string): void { + _activeContainerIds.delete(containerId); + } + + /** + * Create a container (does NOT start it). + * + * Returns the full container ID string. + */ + async createContainer(options: ContainerOptions): Promise { + const args = this._buildCreateArgs(options); + + logger.debug({ args }, 'Creating Docker container'); + + const { stdout, stderr } = await execFileAsync('docker', args, { + timeout: 60_000, + }); + + const id = stdout.trim(); + if (!id) { + throw new Error(`docker create produced no container ID. stderr: ${stderr}`); + } + + logger.info({ containerId: id.slice(0, 12), image: options.image }, 'Container created'); + return id; + } + + /** + * Start a previously created (stopped or new) container. + */ + async startContainer(containerId: string): Promise { + logger.debug({ containerId: containerId.slice(0, 12) }, 'Starting container'); + + await execFileAsync('docker', ['start', containerId], { timeout: 30_000 }); + + logger.info({ containerId: containerId.slice(0, 12) }, 'Container started'); + } + + /** + * Stop a running container. + * + * @param containerId - Full or short container ID + * @param timeoutSeconds - Seconds to wait for graceful stop (default: 10) + */ + async stopContainer(containerId: string, timeoutSeconds = 10): Promise { + logger.debug({ containerId: containerId.slice(0, 12), timeoutSeconds }, 'Stopping container'); + + await execFileAsync('docker', ['stop', '--time', String(timeoutSeconds), containerId], { + timeout: (timeoutSeconds + 5) * 1_000, + }); + + logger.info({ containerId: containerId.slice(0, 12) }, 'Container stopped'); + } + + /** + * Remove a container. + * + * @param containerId - Full or short container ID + * @param force - Force removal even if running (default: false) + */ + async removeContainer(containerId: string, force = false): Promise { + logger.debug({ containerId: containerId.slice(0, 12), force }, 'Removing container'); + + const args = ['rm']; + if (force) args.push('--force'); + args.push(containerId); + + await execFileAsync('docker', args, { timeout: 30_000 }); + + logger.info({ containerId: containerId.slice(0, 12) }, 'Container removed'); + } + + /** + * Execute a command inside a running container. + * + * Returns stdout, stderr, and the exit code. Does NOT throw on non-zero + * exit codes — callers are responsible for interpreting the exit code. + */ + async exec( + containerId: string, + command: string[], + options: ExecOptions = {}, + ): Promise { + const args: string[] = ['exec']; + + if (options.cwd) { + args.push('--workdir', options.cwd); + } + + args.push(containerId, ...command); + + logger.debug({ containerId: containerId.slice(0, 12), command }, 'Executing in container'); + + try { + const { stdout, stderr } = await execFileAsync('docker', args, { + timeout: options.timeout ?? 300_000, + }); + + return { stdout, stderr, exitCode: 0 }; + } catch (err) { + // execFile rejects on non-zero exit codes — extract the details. + const execErr = err as NodeJS.ErrnoException & { + stdout?: string; + stderr?: string; + code?: string; + status?: number; + }; + const exitCode: number = execErr.status ?? 1; + + // Exit code 137 = SIGKILL, typically the Docker OOM killer. + if (exitCode === 137) { + logger.warn( + { containerId: containerId.slice(0, 12) }, + 'Container was OOM-killed (exit code 137) — memory limit exceeded; container force-stopped', + ); + } else { + logger.debug( + { containerId: containerId.slice(0, 12), exitCode }, + 'Container exec returned non-zero exit code', + ); + } + + return { + stdout: execErr.stdout ?? '', + stderr: execErr.stderr ?? execErr.message, + exitCode, + }; + } + } + + /** + * Build the worker image from `docker/Dockerfile.worker`. + * + * Tags the image as `openbridge-worker:latest`. Docker layer caching keeps + * incremental rebuilds fast when the Dockerfile has not changed. + * + * Skips the build when the image already exists locally unless `force` is + * set to `true`. + * + * @param options.force - Rebuild even when the image exists (default: false) + * @param options.context - Project root build context (default: auto-resolved) + */ + async buildImage(options: BuildImageOptions = {}): Promise { + const tag = 'openbridge-worker:latest'; + const force = options.force ?? false; + + if (!force && (await this._imageExists(tag))) { + logger.info({ tag }, 'Worker image already exists — skipping build'); + return; + } + + const projectRoot = options.context ?? this._resolveProjectRoot(); + const dockerfile = path.join(projectRoot, 'docker', 'Dockerfile.worker'); + + logger.info({ tag, dockerfile }, 'Building worker image'); + + await execFileAsync('docker', ['build', '--file', dockerfile, '--tag', tag, projectRoot], { + timeout: 600_000, // 10 minutes for a full image build + }); + + logger.info({ tag }, 'Worker image built successfully'); + } + + // ─── Workspace mounting ───────────────────────────────────────────────────── + + /** + * Build the standard workspace volume mounts for a sandboxed worker. + * + * Produces two mounts: + * - `workspacePath` → `/workspace` (read-only) + * - `.openbridge/` → `/workspace/.openbridge` (read-write, for outputs) + * + * If `.openbridge/` does not exist it is created automatically so Docker + * does not create it as root-owned (which would break non-root writes). + * + * @throws if either resolved path is not absolute. + */ + static buildWorkspaceMounts(options: WorkspaceMountOptions): VolumeMount[] { + const { workspacePath } = options; + const openbridgePath = options.openbridgePath ?? path.join(workspacePath, '.openbridge'); + + DockerSandbox._validateMountPath(workspacePath, 'workspacePath'); + DockerSandbox._validateMountPath(openbridgePath, 'openbridgePath'); + + // Ensure .openbridge/ exists before Docker mounts it. If the directory is + // absent Docker creates it as root, preventing the non-root container user + // from writing outputs. + if (!fs.existsSync(openbridgePath)) { + fs.mkdirSync(openbridgePath, { recursive: true }); + logger.debug({ openbridgePath }, 'Created .openbridge directory for container mount'); + } + + return [ + { + host: workspacePath, + container: '/workspace', + readOnly: true, + }, + { + host: openbridgePath, + container: '/workspace/.openbridge', + readOnly: false, + }, + ]; + } + + // ─── Cleanup ────────────────────────────────────────────────────────────── + + /** + * Remove stopped, created, or dead containers whose names start with `ob-worker-`. + * + * Called on bridge startup to purge any containers that were left behind by a + * previous run (e.g. after a hard crash). The operation is best-effort — + * failures are logged as warnings but never propagated to the caller. + * + * @returns The number of containers successfully removed. + */ + async cleanupDanglingContainers(): Promise { + const listArgs = [ + 'ps', + '-a', + '--filter', + 'name=ob-worker-', + '--filter', + 'status=exited', + '--filter', + 'status=created', + '--filter', + 'status=dead', + '--format', + '{{.ID}}', + ]; + + let ids: string[]; + try { + const { stdout } = await execFileAsync('docker', listArgs, { timeout: 30_000 }); + ids = stdout + .split('\n') + .map((s) => s.trim()) + .filter(Boolean); + } catch (err) { + logger.warn({ err }, 'Failed to list dangling ob-worker containers — cleanup skipped'); + return 0; + } + + if (ids.length === 0) { + logger.debug('No dangling ob-worker containers found'); + return 0; + } + + logger.info({ count: ids.length }, 'Cleaning up dangling ob-worker containers'); + + let removed = 0; + for (const id of ids) { + try { + await execFileAsync('docker', ['rm', '--force', id], { timeout: 15_000 }); + removed++; + } catch (err) { + logger.warn({ containerId: id.slice(0, 12), err }, 'Failed to remove dangling container'); + } + } + + if (removed > 0) { + logger.info({ removed }, 'Dangling ob-worker containers removed'); + } + + return removed; + } + + // ─── Private helpers ─────────────────────────────────────────────────────── + + /** Validate that a host path is a non-empty absolute path. */ + private static _validateMountPath(hostPath: string, label: string): void { + if (!hostPath || typeof hostPath !== 'string') { + throw new Error(`DockerSandbox: ${label} must be a non-empty string`); + } + if (!path.isAbsolute(hostPath)) { + throw new Error(`DockerSandbox: ${label} must be an absolute path, got: ${hostPath}`); + } + } + + /** Returns true if a Docker image with the given tag exists locally. */ + private async _imageExists(tag: string): Promise { + try { + await execFileAsync('docker', ['image', 'inspect', tag], { timeout: 10_000 }); + return true; + } catch { + return false; + } + } + + /** + * Resolve the project root from this module's compiled location. + * + * At runtime the compiled file lives at `dist/core/docker-sandbox.js`. + * Two directory levels up yields the project root. + */ + private _resolveProjectRoot(): string { + return path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); + } + + private _buildCreateArgs(options: ContainerOptions): string[] { + const args: string[] = ['create']; + + // Container name + if (options.name) { + args.push('--name', options.name); + } + + // Volume mounts + for (const mount of options.mounts ?? []) { + const spec = `${mount.host}:${mount.container}${mount.readOnly ? ':ro' : ''}`; + args.push('--volume', spec); + } + + // Environment variables + for (const [key, value] of Object.entries(options.env ?? {})) { + args.push('--env', `${key}=${value}`); + } + + // Network mode (default: none for maximum isolation) + const network = options.network ?? 'none'; + args.push('--network', network); + + // Resource limits + const memoryMB = options.memoryMB ?? 512; + args.push('--memory', `${memoryMB}m`); + + const cpus = options.cpus ?? 1; + args.push('--cpus', String(cpus)); + + const pidsLimit = options.pidsLimit ?? 100; + args.push('--pids-limit', String(pidsLimit)); + + // Working directory + if (options.workdir) { + args.push('--workdir', options.workdir); + } + + // Image + args.push(options.image); + + // Optional command + if (options.command && options.command.length > 0) { + args.push(...options.command); + } + + return args; + } +} diff --git a/src/core/email-sender.ts b/src/core/email-sender.ts new file mode 100644 index 00000000..da100fe5 --- /dev/null +++ b/src/core/email-sender.ts @@ -0,0 +1,60 @@ +import nodemailer from 'nodemailer'; +import { createLogger } from './logger.js'; + +const logger = createLogger('email-sender'); + +export interface EmailConfig { + host: string; + port: number; + user: string; + pass: string; + from: string; + /** Allowlist of email addresses that can receive emails */ + allowlist: string[]; +} + +export interface EmailAttachment { + filename: string; + content: Buffer; + contentType: string; +} + +/** + * Send an email via SMTP using the provided config. + * Only sends to addresses in the config allowlist. + */ +export async function sendEmail( + config: EmailConfig, + to: string, + subject: string, + body: string, + attachments?: EmailAttachment[], +): Promise { + if (!config.allowlist.includes(to)) { + logger.warn({ to }, 'Email blocked — recipient not in allowlist'); + throw new Error(`Recipient ${to} is not in the email allowlist`); + } + + const transporter = nodemailer.createTransport({ + host: config.host, + port: config.port, + auth: { + user: config.user, + pass: config.pass, + }, + }); + + await transporter.sendMail({ + from: config.from, + to, + subject, + text: body, + attachments: attachments?.map((att) => ({ + filename: att.filename, + content: att.content, + contentType: att.contentType, + })), + }); + + logger.info({ to, subject }, 'Email sent'); +} diff --git a/src/core/env-sanitizer.ts b/src/core/env-sanitizer.ts new file mode 100644 index 00000000..61bc2007 --- /dev/null +++ b/src/core/env-sanitizer.ts @@ -0,0 +1,82 @@ +/** + * Environment Variable Sanitizer + * + * Prevents sensitive secrets (API keys, DB credentials, tokens) from leaking + * to spawned worker processes. All CLI adapters use this shared module. + * + * @see OB-F70 — Environment variables leak sensitive secrets to workers + */ + +import { createLogger } from './logger.js'; +import type { SecurityConfig } from '../types/config.js'; + +const logger = createLogger('env-sanitizer'); + +/** Convert a simple glob pattern (with * wildcards) to a regex */ +function globToRegex(pattern: string): RegExp { + const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, '\\$&'); + const regexStr = '^' + escaped.replace(/\*/g, '.*') + '$'; + return new RegExp(regexStr); +} + +function matchesAnyPattern(name: string, patterns: readonly string[]): boolean { + return patterns.some((pattern) => globToRegex(pattern).test(name)); +} + +/** + * Sanitize environment variables for a worker process. + * + * For each variable name: + * 1. If it matches any pattern in `config.envDenyPatterns`, it is a candidate for removal. + * 2. If it also matches any pattern in `config.envAllowPatterns`, it is kept (allow overrides deny). + * 3. Otherwise it is stripped from the returned object. + * + * `process.env` is never mutated; a new object is always returned. + */ +export function sanitizeEnv( + env: Record, + config: SecurityConfig, +): Record { + const { envDenyPatterns, envAllowPatterns } = config; + const result: Record = {}; + + for (const [key, value] of Object.entries(env)) { + const denied = matchesAnyPattern(key, envDenyPatterns); + if (denied) { + const allowed = envAllowPatterns.length > 0 && matchesAnyPattern(key, envAllowPatterns); + if (!allowed) { + continue; // strip + } + } + result[key] = value; + } + + return result; +} + +/** + * Scan the current environment for known secret patterns and log warnings. + * Called once at bridge startup for transparency. + */ +export function warnAboutExposedSecrets( + env: Record, + denyPatterns: readonly string[], +): string[] { + const found: string[] = []; + + for (const key of Object.keys(env)) { + if (matchesAnyPattern(key, denyPatterns) && env[key] !== undefined) { + found.push(key); + } + } + + if (found.length > 0) { + logger.warn( + { count: found.length, vars: found }, + 'Detected %d environment variable(s) matching secret patterns — these will be stripped from worker environments', + found.length, + ); + } + + return found; +} diff --git a/src/core/error-classifier.ts b/src/core/error-classifier.ts new file mode 100644 index 00000000..594a6660 --- /dev/null +++ b/src/core/error-classifier.ts @@ -0,0 +1,130 @@ +/** + * Error classification helpers for worker exit analysis. + * + * Extracted from agent-runner.ts (OB-1285) so the classification + * logic can be imported independently without pulling in the full + * process-execution machinery. + */ + +/** + * Heuristic patterns that indicate a rate-limit or model-unavailability error. + * Matched case-insensitively against stderr output. + */ +const RATE_LIMIT_PATTERNS = [ + 'rate limit', + 'rate_limit', + 'too many requests', + '429', + 'overloaded', + 'capacity', + 'unavailable', + 'model_not_available', +]; + +/** + * Patterns in stdout that indicate the Claude CLI hit its --max-turns limit. + * Claude exits with code 0 when max-turns is reached, so we must scan stdout + * to distinguish a complete run from a turn-budget exhaustion. + * Matched case-insensitively against stdout. + */ +export const MAX_TURNS_PATTERNS = [ + 'max turns reached', + 'maximum turns reached', + 'turn limit', + 'turn budget', + 'turns exhausted', + 'max_turns', +]; + +/** + * Patterns indicating authentication / authorization failures. + * These are non-retryable — retrying with the same credentials will fail again. + */ +const AUTH_PATTERNS = [ + 'api key', + 'api_key', + 'invalid api', + 'unauthorized', + 'unauthenticated', + 'authentication failed', + 'permission denied', + 'access denied', + 'invalid token', + 'forbidden', + '401', + '403', + 'auth-required', + 'interactive authentication', +]; + +/** + * Patterns indicating the prompt or context exceeded the model's context window. + * These are non-retryable with the same prompt — the task must be split. + */ +const CONTEXT_OVERFLOW_PATTERNS = [ + 'context too long', + 'context window', + 'context length', + 'context_length_exceeded', + 'prompt too long', + 'maximum context', + 'token limit', + 'too many tokens', + 'context overflow', + 'context_overflow', +]; + +/** + * Categories of worker exit errors. + * Used by callers to decide retry strategy: + * retryable: 'rate-limit', 'timeout', 'crash' + * non-retryable: 'auth', 'context-overflow', 'unknown' + */ +export type ErrorCategory = + | 'rate-limit' + | 'auth' + | 'timeout' + | 'crash' + | 'context-overflow' + | 'unknown'; + +/** + * Check whether the stderr output from a failed attempt indicates a rate-limit + * or model-unavailability error that warrants falling back to a different model. + */ +export function isRateLimitError(stderr: string): boolean { + const lower = stderr.toLowerCase(); + return RATE_LIMIT_PATTERNS.some((pattern) => lower.includes(pattern)); +} + +/** + * Check whether the stdout from a completed agent run indicates that the + * Claude CLI hit its --max-turns limit. Claude exits with code 0 when + * max-turns is reached, so we must inspect stdout to detect incomplete work. + */ +export function isMaxTurnsExhausted(stdout: string): boolean { + const lower = stdout.toLowerCase(); + return MAX_TURNS_PATTERNS.some((pattern) => lower.includes(pattern)); +} + +/** + * Classify the category of a worker exit error from stderr output and exit code. + * + * Priority order (highest to lowest): + * 1. rate-limit — recoverable, retry with model fallback + * 2. auth — non-retryable, report to user + * 3. context-overflow — non-retryable, split the task + * 4. timeout — exit code 143/137 or "timeout" in stderr + * 5. crash — any other non-zero exit + * 6. unknown — exit code 0 with unrecognised stderr + */ +export function classifyError(stderr: string, exitCode: number): ErrorCategory { + const lower = stderr.toLowerCase(); + + if (isRateLimitError(stderr)) return 'rate-limit'; + if (AUTH_PATTERNS.some((p) => lower.includes(p))) return 'auth'; + if (CONTEXT_OVERFLOW_PATTERNS.some((p) => lower.includes(p))) return 'context-overflow'; + if (exitCode === 143 || exitCode === 137 || lower.includes('timeout')) return 'timeout'; + if (exitCode !== 0) return 'crash'; + return 'unknown'; +} diff --git a/src/core/fast-path-responder.ts b/src/core/fast-path-responder.ts new file mode 100644 index 00000000..d1725634 --- /dev/null +++ b/src/core/fast-path-responder.ts @@ -0,0 +1,172 @@ +import type { AgentRunner } from './agent-runner.js'; +import { TOOLS_READ_ONLY } from './agent-runner.js'; +import type { MemoryManager } from '../memory/index.js'; +import { createLogger } from './logger.js'; + +const logger = createLogger('fast-path-responder'); + +/** Configuration for the FastPathResponder pool. */ +export interface FastPathOptions { + /** Maximum number of fast-path agents that may run concurrently. Default: 2. */ + maxConcurrent?: number; + /** Maximum turns per fast-path agent call. Default: 3. */ + maxTurns?: number; + /** Per-call timeout in milliseconds. Default: 30 000. */ + timeoutMs?: number; +} + +/** Parameters for a single fast-path answer request. */ +export interface FastPathRequest { + /** The user's question to answer. */ + question: string; + /** Absolute path to the workspace the agent will read. */ + workspacePath: string; + /** + * Pre-built workspace map summary string (project name, frameworks, commands, + * key files). Populated by the caller from MasterManager.getWorkspaceMap(). + * When provided the FastPathResponder injects it as context. + */ + workspaceContext?: string; +} + +/** + * Manages a pool of short-lived read-only agent sessions for quick-answer + * responses while the Master AI is busy processing a complex task. + * + * Key features + * ───────────── + * • Concurrency pool — at most `maxConcurrent` (default 2) fast-path agents + * run at the same time. Extra requests are rejected immediately with a + * "busy" message rather than queuing indefinitely. + * • DB context augmentation — when a MemoryManager is provided the responder + * performs a hybrid search on `context_chunks` and prepends the top results + * to the agent prompt (read-only DB access; no writes). + * • Workspace-map context — the caller may supply a compact summary string + * derived from the cached workspace map so the agent already knows the + * project structure without extra file reads. + */ +export class FastPathResponder { + private readonly maxConcurrent: number; + private readonly maxTurns: number; + private readonly timeoutMs: number; + private readonly runner: AgentRunner; + private memory?: MemoryManager; + private activeCount = 0; + + constructor(runner: AgentRunner, memory?: MemoryManager, options?: FastPathOptions) { + this.runner = runner; + this.memory = memory; + this.maxConcurrent = options?.maxConcurrent ?? 2; + this.maxTurns = options?.maxTurns ?? 3; + this.timeoutMs = options?.timeoutMs ?? 30_000; + } + + /** Attach or replace the MemoryManager used for context chunk retrieval. */ + setMemory(memory: MemoryManager): void { + this.memory = memory; + } + + /** Returns true when the pool is at full capacity (activeCount >= maxConcurrent). */ + get isBusy(): boolean { + return this.activeCount >= this.maxConcurrent; + } + + /** Number of fast-path agent calls currently in flight. */ + get activeSessions(): number { + return this.activeCount; + } + + /** Configured maximum concurrent fast-path agents. */ + get maxSessions(): number { + return this.maxConcurrent; + } + + /** + * Answer a quick question using a read-only agent session. + * + * Returns the agent's response as a plain string. Falls back to a graceful + * "busy" or "failed" message instead of throwing, so callers can forward the + * text to the user without error handling. + */ + async answer(request: FastPathRequest): Promise { + if (this.isBusy) { + logger.warn( + { activeCount: this.activeCount, maxConcurrent: this.maxConcurrent }, + 'FastPathResponder: pool at capacity, rejecting request', + ); + return 'The AI is busy processing multiple requests. Please wait a moment and try again.'; + } + + this.activeCount++; + logger.info( + { activeCount: this.activeCount, maxConcurrent: this.maxConcurrent }, + 'FastPathResponder: starting fast-path agent', + ); + + try { + const context = await this.buildContext(request); + const promptParts = [ + 'You are a helpful assistant with read-only access to a codebase. Answer the question concisely (1–3 paragraphs). Do not modify any files.', + ]; + if (context) { + promptParts.push(`\nWorkspace context:\n${context}`); + } + promptParts.push(`\nUser question: ${request.question}`); + const prompt = promptParts.join(''); + + const result = await this.runner.spawn({ + prompt, + workspacePath: request.workspacePath, + allowedTools: [...TOOLS_READ_ONLY], + maxTurns: this.maxTurns, + retries: 0, + timeout: this.timeoutMs, + }); + + const reply = result.stdout.trim() || 'No response — please try again.'; + logger.info( + { activeCount: this.activeCount }, + 'FastPathResponder: fast-path agent completed', + ); + return reply; + } catch (err) { + logger.warn({ err }, 'FastPathResponder: fast-path agent failed'); + return 'The AI is busy and could not answer right now. Your question will be processed shortly.'; + } finally { + this.activeCount--; + } + } + + /** + * Build the context string to inject into the fast-path agent prompt. + * + * Combines (in order): + * 1. The pre-built workspace map summary from the caller (if provided). + * 2. The top-3 context chunks from the MemoryManager (if available), + * retrieved via a hybrid search on the user's question. + * + * Returns an empty string when no context is available. + */ + private async buildContext(request: FastPathRequest): Promise { + const parts: string[] = []; + + if (request.workspaceContext) { + parts.push(request.workspaceContext); + } + + if (this.memory) { + try { + const chunks = await this.memory.searchContext(request.question, 3); + if (chunks.length > 0) { + const chunkText = chunks.map((c) => c.content).join('\n---\n'); + parts.push(`Relevant context:\n${chunkText}`); + } + } catch (err) { + // DB search is best-effort — a missing or locked DB must not block the fast-path + logger.debug({ err }, 'FastPathResponder: context chunk search failed (non-fatal)'); + } + } + + return parts.join('\n\n'); + } +} diff --git a/src/core/file-server.ts b/src/core/file-server.ts new file mode 100644 index 00000000..85313199 --- /dev/null +++ b/src/core/file-server.ts @@ -0,0 +1,517 @@ +import { createServer, type Server, type IncomingMessage, type ServerResponse } from 'node:http'; +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { createLogger } from './logger.js'; +import type MemoryManager from '../memory/index.js'; +import type { RenderOptions, RenderResult } from './html-renderer.js'; + +const logger = createLogger('file-server'); + +/** MIME type map for supported file extensions */ +const MIME_TYPES: Record = { + '.html': 'text/html; charset=utf-8', + '.htm': 'text/html; charset=utf-8', + '.pdf': 'application/pdf', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + '.csv': 'text/csv; charset=utf-8', + '.json': 'application/json; charset=utf-8', + '.md': 'text/markdown; charset=utf-8', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.png': 'image/png', + '.gif': 'image/gif', + '.webp': 'image/webp', + '.svg': 'image/svg+xml; charset=utf-8', + '.txt': 'text/plain; charset=utf-8', +}; + +/** Default port for the file server (separate from WebChat's 3000) */ +const DEFAULT_PORT = 3001; + +/** Default expiry for shareable links: 24 hours in ms */ +const DEFAULT_EXPIRY_HOURS = 24; + +/** system_config key used to persist shareable link mappings */ +const SHARED_LINKS_CONFIG_KEY = 'shared_links'; + +/** CORS headers for local development */ +const CORS_HEADERS: Record = { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'GET, OPTIONS', + 'Access-Control-Allow-Headers': 'Content-Type', +}; + +/** One entry in the shared-links map */ +interface SharedLinkEntry { + filename: string; + filePath: string; + expiresAt: string; // ISO-8601 +} + +/** Full map stored in system_config under SHARED_LINKS_CONFIG_KEY */ +type SharedLinksMap = Record; + +/** + * FileServer — serves AI-generated content from `.openbridge/generated/` via HTTP. + * + * Routes: + * GET /shared/:filename — serve a file directly by name + * GET /shared/:uuid/:filename — serve a file via a shareable UUID link (expiry checked) + * + * Usage: + * const server = new FileServer('/path/to/workspace'); + * await server.start(); + * // Serve by name: + * // http://localhost:3001/shared/report.html + * // Create a shareable link: + * const url = await server.createShareableLink('report.html'); + * // http://localhost:3001/shared//report.html (expires in 24 h) + * await server.stop(); + */ +export class FileServer { + private readonly workspacePath: string; + private readonly generatedDir: string; + private readonly port: number; + private server: Server | null = null; + + /** Optional MemoryManager for persistent UUID → file mappings */ + private readonly memory: MemoryManager | null; + + /** + * In-memory fallback when MemoryManager is unavailable. + * Keys are UUIDs; values are SharedLinkEntry objects. + */ + private readonly inMemoryLinks: SharedLinksMap = {}; + + /** Public tunnel URL when a tunnel is active, null otherwise */ + private publicUrl: string | null = null; + + constructor( + workspacePath: string, + port: number = DEFAULT_PORT, + memory: MemoryManager | null = null, + ) { + this.workspacePath = workspacePath; + this.generatedDir = path.join(workspacePath, '.openbridge', 'generated'); + this.port = port; + this.memory = memory; + } + + /** Returns the base URL for the file server */ + get baseUrl(): string { + return `http://localhost:${this.port}`; + } + + /** + * Returns a URL to the interactive preview page for the given filename. + * The preview page wraps the file in an iframe with a toolbar showing the + * filename and a link to open the file directly. + * + * Uses the public tunnel URL when active, otherwise localhost. + * + * @param filename Name of the file in `.openbridge/generated/` (no path separators) + */ + getPreviewUrl(filename: string): string { + return `${this.getFileUrl()}/preview/${encodeURIComponent(filename)}`; + } + + /** Set the public tunnel URL. Pass null to clear and fall back to localhost. */ + setPublicUrl(url: string | null): void { + this.publicUrl = url; + logger.info({ publicUrl: url }, 'File server public URL updated'); + } + + /** Returns the public URL when a tunnel is active, localhost URL otherwise. */ + getFileUrl(): string { + return this.publicUrl ?? this.baseUrl; + } + + /** Returns the path to the generated files directory */ + get directory(): string { + return this.generatedDir; + } + + async start(): Promise { + // Ensure the generated directory exists + await fs.mkdir(this.generatedDir, { recursive: true }); + + this.server = createServer((req: IncomingMessage, res: ServerResponse) => { + void this.handleRequest(req, res); + }); + + await new Promise((resolve, reject) => { + this.server!.listen(this.port, 'localhost', () => { + logger.info( + { port: this.port, dir: this.generatedDir }, + 'File server started — serving generated content', + ); + resolve(); + }); + this.server!.on('error', reject); + }); + } + + async stop(): Promise { + if (!this.server) return; + await new Promise((resolve, reject) => { + this.server!.close((err?: Error) => { + if (err) reject(err); + else resolve(); + }); + }); + this.server = null; + logger.info('File server stopped'); + } + + /** + * Create a shareable UUID link for a file in the generated directory. + * + * @param filename Name of the file in `.openbridge/generated/` (no path separators) + * @param expiryHours Hours until the link expires (default 24) + * @returns Full URL like `http://localhost:3001/shared//report.html` + */ + async createShareableLink( + filename: string, + expiryHours: number = DEFAULT_EXPIRY_HOURS, + ): Promise { + // Security: reject path traversal + if (filename.includes('..') || filename.includes('/') || filename.includes('\\')) { + throw new Error('Invalid filename'); + } + + const filePath = path.join(this.generatedDir, filename); + + // Verify the file exists + await fs.access(filePath); + + const uuid = randomUUID(); + const expiresAt = new Date(Date.now() + expiryHours * 60 * 60 * 1000).toISOString(); + + const entry: SharedLinkEntry = { filename, filePath, expiresAt }; + + await this.saveLink(uuid, entry); + + const url = `${this.getFileUrl()}/shared/${uuid}/${encodeURIComponent(filename)}`; + logger.info({ uuid, filename, expiresAt }, 'Shareable link created'); + return url; + } + + /** + * Save raw SVG markup to the generated directory. + * + * @param svgContent Full SVG markup string + * @param filename Desired filename (must end in `.svg`). Auto-generated if omitted. + * @returns Absolute path to the saved file + */ + async saveSvgContent(svgContent: string, filename?: string): Promise { + const resolvedFilename = filename ?? `svg-${randomUUID()}.svg`; + if ( + resolvedFilename.includes('..') || + resolvedFilename.includes('/') || + resolvedFilename.includes('\\') + ) { + throw new Error('Invalid filename'); + } + + await fs.mkdir(this.generatedDir, { recursive: true }); + const filePath = path.join(this.generatedDir, resolvedFilename); + await fs.writeFile(filePath, svgContent, 'utf-8'); + + logger.info( + { filePath, filename: resolvedFilename }, + 'SVG content saved to generated directory', + ); + return filePath; + } + + /** + * Convert a stored SVG file to a raster image (PNG/JPEG) using HTMLRenderer + Puppeteer. + * + * Returns null when Puppeteer is unavailable. Callers should check `HTMLRenderer.isAvailable()` + * first if they need a hard guarantee. + * + * @param svgFilename Name of the SVG file already in `.openbridge/generated/` + * @param options Rendering options passed to HTMLRenderer + * @returns RenderResult (outputPath, format, sizeBytes) or null if Puppeteer unavailable + */ + async renderSvgToImage( + svgFilename: string, + options: RenderOptions = {}, + ): Promise { + if (svgFilename.includes('..') || svgFilename.includes('/') || svgFilename.includes('\\')) { + throw new Error('Invalid filename'); + } + + const svgPath = path.join(this.generatedDir, svgFilename); + // Ensure the file exists + await fs.access(svgPath); + + // Lazy-import to keep Puppeteer optional + const { HTMLRenderer } = await import('./html-renderer.js'); + + if (!(await HTMLRenderer.isAvailable())) { + logger.warn({ svgFilename }, 'Puppeteer not available — SVG-to-image conversion skipped'); + return null; + } + + const renderer = new HTMLRenderer(this.workspacePath); + const result = await renderer.renderSvgFile(svgPath, options); + + logger.info( + { svgFilename, outputPath: result.outputPath, format: result.format }, + 'SVG rendered to raster image', + ); + return result; + } + + // --------------------------------------------------------------------------- + // Private helpers + // --------------------------------------------------------------------------- + + /** Load the full shared-links map from storage */ + private async loadLinks(): Promise { + if (this.memory) { + try { + const raw = await this.memory.getSystemConfig(SHARED_LINKS_CONFIG_KEY); + if (raw) return JSON.parse(raw) as SharedLinksMap; + } catch { + // Fall through to empty map + } + return {}; + } + return { ...this.inMemoryLinks }; + } + + /** Persist the full shared-links map to storage */ + private async persistLinks(map: SharedLinksMap): Promise { + if (this.memory) { + await this.memory.setSystemConfig(SHARED_LINKS_CONFIG_KEY, JSON.stringify(map)); + } else { + // Update in-memory store + for (const [k, v] of Object.entries(map)) { + this.inMemoryLinks[k] = v; + } + // Remove keys that are no longer present + for (const k of Object.keys(this.inMemoryLinks)) { + if (!(k in map)) delete this.inMemoryLinks[k]; + } + } + } + + /** Save a single link entry */ + private async saveLink(uuid: string, entry: SharedLinkEntry): Promise { + const map = await this.loadLinks(); + map[uuid] = entry; + await this.persistLinks(map); + } + + /** Retrieve and validate a link entry; returns null if not found or expired */ + private async resolveLink(uuid: string): Promise { + const map = await this.loadLinks(); + const entry = map[uuid]; + if (!entry) return null; + + if (new Date(entry.expiresAt) < new Date()) { + // Clean up expired entry + delete map[uuid]; + await this.persistLinks(map); + return null; + } + + return entry; + } + + private async handleRequest(req: IncomingMessage, res: ServerResponse): Promise { + const url = req.url ?? '/'; + + // Handle CORS preflight + if (req.method === 'OPTIONS') { + res.writeHead(204, CORS_HEADERS); + res.end(); + return; + } + + if (req.method !== 'GET') { + res.writeHead(404, { 'Content-Type': 'text/plain', ...CORS_HEADERS }); + res.end('Not found'); + return; + } + + // Route: GET /shared/:uuid/:filename (UUID-based shareable link) + const uuidMatch = url.match(/^\/shared\/([^/]+)\/([^/]+)$/); + if (uuidMatch) { + await this.handleShareableLink(res, uuidMatch[1]!, uuidMatch[2]!); + return; + } + + // Route: GET /shared/:filename (direct filename) + const directMatch = url.match(/^\/shared\/([^/]+)$/); + if (directMatch) { + await this.handleDirectFile(res, directMatch[1]!); + return; + } + + // Route: GET /preview/:filename (interactive HTML preview wrapper) + const previewMatch = url.match(/^\/preview\/([^/]+)$/); + if (previewMatch) { + await this.handlePreview(res, previewMatch[1]!); + return; + } + + res.writeHead(404, { 'Content-Type': 'text/plain', ...CORS_HEADERS }); + res.end('Not found'); + } + + /** Serve a file via its UUID shareable link */ + private async handleShareableLink( + res: ServerResponse, + uuid: string, + rawFilename: string, + ): Promise { + // Security: reject path traversal in either segment + if (uuid.includes('/') || uuid.includes('\\') || uuid.includes('..')) { + res.writeHead(400, { 'Content-Type': 'text/plain', ...CORS_HEADERS }); + res.end('Invalid link'); + return; + } + + const entry = await this.resolveLink(uuid); + if (!entry) { + res.writeHead(404, { 'Content-Type': 'text/plain', ...CORS_HEADERS }); + res.end('Link not found or expired'); + return; + } + + // Decode the filename from the URL and verify it matches the stored entry + const decodedFilename = decodeURIComponent(rawFilename); + if (decodedFilename !== entry.filename) { + res.writeHead(404, { 'Content-Type': 'text/plain', ...CORS_HEADERS }); + res.end('File not found'); + return; + } + + await this.serveFile(res, entry.filePath, entry.filename); + } + + /** Serve a file directly by name from the generated directory */ + private async handleDirectFile(res: ServerResponse, rawFilename: string): Promise { + // Security: reject path traversal attempts + if (rawFilename.includes('..') || rawFilename.includes('/') || rawFilename.includes('\\')) { + res.writeHead(400, { 'Content-Type': 'text/plain', ...CORS_HEADERS }); + res.end('Invalid filename'); + return; + } + + const filePath = path.join(this.generatedDir, rawFilename); + await this.serveFile(res, filePath, rawFilename); + } + + /** + * Serve an interactive preview page for an HTML file. + * The preview page embeds the raw file in a full-window iframe with a + * minimal toolbar showing the filename and a direct download/open link. + * Non-HTML files are redirected to the direct /shared/:filename route. + */ + private async handlePreview(res: ServerResponse, rawFilename: string): Promise { + // Security: reject path traversal attempts + if (rawFilename.includes('..') || rawFilename.includes('/') || rawFilename.includes('\\')) { + res.writeHead(400, { 'Content-Type': 'text/plain', ...CORS_HEADERS }); + res.end('Invalid filename'); + return; + } + + const filename = decodeURIComponent(rawFilename); + const filePath = path.join(this.generatedDir, filename); + + // Verify the file exists + try { + await fs.access(filePath); + } catch { + res.writeHead(404, { 'Content-Type': 'text/plain', ...CORS_HEADERS }); + res.end('File not found'); + return; + } + + const ext = path.extname(filename).toLowerCase(); + const directUrl = `${this.getFileUrl()}/shared/${encodeURIComponent(filename)}`; + + // For non-HTML files, redirect to the direct serve route + if (ext !== '.html' && ext !== '.htm') { + res.writeHead(302, { Location: directUrl, ...CORS_HEADERS }); + res.end(); + return; + } + + const escapedFilename = filename + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); + const html = ` + + + + + Preview: ${escapedFilename} + + + +
+ OpenBridge Preview + ${escapedFilename} + Open full screen ↗ +
+ + +`; + + const buf = Buffer.from(html, 'utf-8'); + res.writeHead(200, { + 'Content-Type': 'text/html; charset=utf-8', + 'Content-Length': buf.length, + ...CORS_HEADERS, + }); + res.end(buf); + } + + /** Read and write a file to the response */ + private async serveFile(res: ServerResponse, filePath: string, filename: string): Promise { + const ext = path.extname(filename).toLowerCase(); + const mimeType = MIME_TYPES[ext] ?? 'application/octet-stream'; + + let data: Buffer; + try { + data = await fs.readFile(filePath); + } catch { + res.writeHead(404, { 'Content-Type': 'text/plain', ...CORS_HEADERS }); + res.end('File not found'); + return; + } + + res.writeHead(200, { + 'Content-Type': mimeType, + 'Content-Length': data.length, + ...CORS_HEADERS, + }); + res.end(data); + } +} diff --git a/src/core/github-publisher.ts b/src/core/github-publisher.ts new file mode 100644 index 00000000..89a38cbc --- /dev/null +++ b/src/core/github-publisher.ts @@ -0,0 +1,129 @@ +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { mkdtemp, copyFile, rm } from 'node:fs/promises'; +import path from 'node:path'; +import os from 'node:os'; +import { createLogger } from './logger.js'; + +const execFileAsync = promisify(execFile); +const logger = createLogger('github-publisher'); + +async function runGit(args: string[], cwd: string): Promise { + const { stdout } = await execFileAsync('git', args, { cwd }); + return stdout.trim(); +} + +function extractGitHubPagesUrl(remoteUrl: string, filename: string): string { + // HTTPS: https://github.com/owner/repo.git + const httpsMatch = remoteUrl.match(/https?:\/\/github\.com\/([^/]+)\/([^/]+?)(?:\.git)?$/); + if (httpsMatch) { + const [, owner, repo] = httpsMatch; + return `https://${owner}.github.io/${repo}/${filename}`; + } + + // SSH: git@github.com:owner/repo.git + const sshMatch = remoteUrl.match(/git@github\.com:([^/]+)\/([^/]+?)(?:\.git)?$/); + if (sshMatch) { + const [, owner, repo] = sshMatch; + return `https://${owner}.github.io/${repo}/${filename}`; + } + + return ''; +} + +/** + * Publish a file to the gh-pages branch of the workspace git repository. + * Creates the gh-pages branch as an orphan if it does not yet exist. + * When the branch already exists its content is preserved and the file is added or replaced. + * Requires git to be configured with push access to the remote. + * + * @param filePath - Absolute path to the file to publish. + * @param repoUrl - Remote URL override. If omitted, uses the `origin` remote from the workspace. + * @returns The GitHub Pages URL for the published file, or empty string if URL cannot be determined. + */ +export async function publishToGitHubPages(filePath: string, repoUrl?: string): Promise { + const resolvedFilePath = path.resolve(filePath); + const fileDir = path.dirname(resolvedFilePath); + const filename = path.basename(resolvedFilePath); + + // Locate the git root from the file's directory + let gitRoot: string; + try { + gitRoot = await runGit(['rev-parse', '--show-toplevel'], fileDir); + } catch (err) { + throw new Error( + `Cannot locate git repository for path "${resolvedFilePath}": ${err instanceof Error ? err.message : String(err)}`, + ); + } + + // Resolve remote URL + let remote: string; + if (repoUrl) { + remote = repoUrl; + } else { + try { + remote = await runGit(['remote', 'get-url', 'origin'], gitRoot); + } catch (err) { + throw new Error( + `No remote "origin" configured in git repo at "${gitRoot}": ${err instanceof Error ? err.message : String(err)}`, + ); + } + } + + // Create a temp directory for the gh-pages working tree + const tmpDir = await mkdtemp(path.join(os.tmpdir(), 'openbridge-ghpages-')); + + try { + // Check if gh-pages already exists on the remote + let branchExists = false; + try { + const lsOutput = await runGit(['ls-remote', '--heads', remote, 'gh-pages'], gitRoot); + branchExists = lsOutput.length > 0; + } catch { + // ls-remote failed — treat as branch not existing + branchExists = false; + } + + // Initialize a fresh git repo in the temp dir + await runGit(['init'], tmpDir); + await runGit(['remote', 'add', 'origin', remote], tmpDir); + + // Set a local git identity so commits don't fail in headless environments + await runGit(['config', 'user.email', 'openbridge@localhost'], tmpDir); + await runGit(['config', 'user.name', 'OpenBridge'], tmpDir); + + if (branchExists) { + // Fetch the existing gh-pages branch (shallow for speed) and check it out + await runGit(['fetch', '--depth=1', 'origin', 'gh-pages'], tmpDir); + await runGit(['checkout', '-b', 'gh-pages', 'FETCH_HEAD'], tmpDir); + } else { + // Create a fresh orphan branch — no history, no parent commits + await runGit(['checkout', '--orphan', 'gh-pages'], tmpDir); + } + + // Copy the file into the working tree + await copyFile(resolvedFilePath, path.join(tmpDir, filename)); + + // Stage and commit + await runGit(['add', filename], tmpDir); + await runGit(['commit', '-m', `chore: publish ${filename} to GitHub Pages`], tmpDir); + + // Push to the gh-pages branch on the remote + await runGit(['push', 'origin', 'HEAD:gh-pages'], tmpDir); + + const pagesUrl = extractGitHubPagesUrl(remote, filename); + logger.info( + { filename, remote, pagesUrl: pagesUrl || '(unknown)' }, + 'Published to GitHub Pages', + ); + + return pagesUrl; + } finally { + // Clean up temp directory (best effort) + try { + await rm(tmpDir, { recursive: true, force: true }); + } catch { + // ignore + } + } +} diff --git a/src/core/health.ts b/src/core/health.ts index 2d293ecd..c288c398 100644 --- a/src/core/health.ts +++ b/src/core/health.ts @@ -1,5 +1,10 @@ +import { execFileSync } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; import { createServer, type Server, type IncomingMessage, type ServerResponse } from 'node:http'; -import type { HealthConfig } from '../types/config.js'; +import { resolve } from 'node:path'; +import { V2ConfigSchema } from '../types/config.js'; +import type { HealthConfig, MCPServer, V2Config } from '../types/config.js'; +import type { MetricsSnapshot } from './metrics.js'; import { createLogger } from './logger.js'; const logger = createLogger('health'); @@ -9,9 +14,20 @@ export interface ComponentStatus { status: 'healthy' | 'degraded' | 'unhealthy'; } +export interface McpServerStatus { + name: string; + status: 'configured' | 'error'; + command: string; +} + export interface HealthStatus { status: 'healthy' | 'degraded' | 'unhealthy'; - uptime: number; + uptime_seconds: number; + memory_mb: number; + active_workers: number; + master_status: string; + db_status: 'connected' | 'disconnected'; + last_message_at: string | null; timestamp: string; connectors: ComponentStatus[]; providers: ComponentStatus[]; @@ -26,6 +42,182 @@ export interface HealthStatus { taskAgents: number; byStatus: Record; }; + mcp?: { + enabled: boolean; + servers: McpServerStatus[]; + }; +} + +/** + * Check whether a command exists on PATH using `which` (Unix) or `where` (Windows). + * Returns true if found, false if not. + */ +export function checkCommandOnPath(command: string): boolean { + try { + const whichCmd = process.platform === 'win32' ? 'where' : 'which'; + execFileSync(whichCmd, [command], { stdio: 'ignore' }); + return true; + } catch { + return false; + } +} + +/** + * Check health of a list of MCP servers by verifying their commands exist on PATH. + * Returns the mcp health section ready to include in HealthStatus. + */ +export function checkMcpServersHealth(servers: MCPServer[]): HealthStatus['mcp'] { + if (servers.length === 0) { + return { enabled: true, servers: [] }; + } + return { + enabled: true, + servers: servers.map((server) => ({ + name: server.name, + command: server.command, + status: checkCommandOnPath(server.command) ? 'configured' : 'error', + })), + }; +} + +export interface HealthCheckItem { + name: string; + passed: boolean; + message: string; +} + +export interface HealthCheckResult { + passed: boolean; + checks: HealthCheckItem[]; +} + +/** + * Run a pre-flight health check suitable for use from both the CLI wizard and the HTTP endpoint. + * Checks: config file validity, AI tool availability, workspace path accessibility, + * and connector-specific prerequisites (Telegram token, Discord credentials). + * + * @param configPath - Absolute path to config.json. Defaults to `/config.json`. + */ +export function runHealthCheck(configPath?: string): HealthCheckResult { + const checks: HealthCheckItem[] = []; + const resolvedPath = configPath ?? resolve(process.cwd(), 'config.json'); + + // Check 1: Config file exists and is valid + let config: V2Config | null = null; + if (!existsSync(resolvedPath)) { + checks.push({ + name: 'Config file', + passed: false, + message: `config.json not found at ${resolvedPath}`, + }); + } else { + try { + const raw = readFileSync(resolvedPath, 'utf-8'); + const json = JSON.parse(raw) as unknown; + const result = V2ConfigSchema.safeParse(json); + if (result.success) { + config = result.data; + checks.push({ + name: 'Config file', + passed: true, + message: `config.json is valid`, + }); + } else { + const firstIssue = result.error.issues[0]?.message ?? 'unknown error'; + checks.push({ + name: 'Config file', + passed: false, + message: `config.json has validation errors: ${firstIssue}`, + }); + } + } catch (err) { + checks.push({ + name: 'Config file', + passed: false, + message: `config.json is not valid JSON: ${(err as Error).message}`, + }); + } + } + + // Check 2: At least one AI tool available + const aiTools = ['claude', 'codex', 'aider']; + const foundTools = aiTools.filter((tool) => checkCommandOnPath(tool)); + if (foundTools.length > 0) { + checks.push({ + name: 'AI tools', + passed: true, + message: `Found: ${foundTools.join(', ')}`, + }); + } else { + checks.push({ + name: 'AI tools', + passed: false, + message: 'No AI tools found. Install claude, codex, or aider.', + }); + } + + if (config) { + // Check 3: Workspace path accessible + if (existsSync(config.workspacePath)) { + checks.push({ + name: 'Workspace path', + passed: true, + message: `Workspace accessible: ${config.workspacePath}`, + }); + } else { + checks.push({ + name: 'Workspace path', + passed: false, + message: `Workspace path not found: ${config.workspacePath}`, + }); + } + + // Check 4: Connector-specific prerequisites + for (const channel of config.channels) { + if (!channel.enabled) continue; + const opts = channel.options ?? {}; + + if (channel.type === 'telegram') { + if (opts['token']) { + checks.push({ + name: 'Telegram token', + passed: true, + message: 'Telegram bot token is configured', + }); + } else { + checks.push({ + name: 'Telegram token', + passed: false, + message: 'Telegram connector requires a bot token (options.token)', + }); + } + } else if (channel.type === 'discord') { + const hasToken = Boolean(opts['token']); + const hasAppId = Boolean(opts['applicationId'] ?? opts['appId']); + if (hasToken && hasAppId) { + checks.push({ + name: 'Discord credentials', + passed: true, + message: 'Discord bot token and application ID are configured', + }); + } else { + const missing: string[] = []; + if (!hasToken) missing.push('token'); + if (!hasAppId) missing.push('applicationId'); + checks.push({ + name: 'Discord credentials', + passed: false, + message: `Discord connector requires: ${missing.join(', ')} in options`, + }); + } + } + } + } + + return { + passed: checks.every((c) => c.passed), + checks, + }; } export type HealthDataProvider = () => HealthStatus; @@ -39,6 +231,9 @@ export class HealthServer { private readonly config: HealthConfig; private server: Server | null = null; private dataProvider: HealthDataProvider | null = null; + private metricsProvider: (() => MetricsSnapshot) | null = null; + private readinessProvider: (() => boolean) | null = null; + private mcpServers: MCPServer[] = []; constructor(config: Partial = {}) { this.config = { ...DEFAULT_CONFIG, ...config }; @@ -48,6 +243,24 @@ export class HealthServer { this.dataProvider = provider; } + setMetricsProvider(provider: () => MetricsSnapshot): void { + this.metricsProvider = provider; + } + + setReadinessProvider(provider: () => boolean): void { + this.readinessProvider = provider; + } + + /** + * Configure the MCP servers to health-check on each /health request. + * Call this after Bridge.start() with servers from V2Config.mcp.servers. + * When servers are set, the /health response includes an `mcp` section + * reporting whether each server's command exists on PATH. + */ + setMcpServers(servers: MCPServer[]): void { + this.mcpServers = servers; + } + async start(): Promise { if (!this.config.enabled) { logger.debug('Health check endpoint disabled'); @@ -83,7 +296,23 @@ export class HealthServer { }); } - private handleRequest(_req: IncomingMessage, res: ServerResponse): void { + private handleRequest(req: IncomingMessage, res: ServerResponse): void { + const url = req.url ?? '/'; + const path = url.split('?')[0]; + + if (path === '/health' || path === '/') { + this.handleHealthRequest(res); + } else if (path === '/metrics') { + this.handleMetricsRequest(res); + } else if (path === '/ready') { + this.handleReadyRequest(res); + } else { + res.writeHead(404, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Not found' })); + } + } + + private handleHealthRequest(res: ServerResponse): void { if (!this.dataProvider) { res.writeHead(503, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ status: 'unhealthy', error: 'Not initialized' })); @@ -91,9 +320,91 @@ export class HealthServer { } const health = this.dataProvider(); - const statusCode = health.status === 'healthy' ? 200 : health.status === 'degraded' ? 200 : 503; + + if (this.mcpServers.length > 0) { + health.mcp = checkMcpServersHealth(this.mcpServers); + } + + const statusCode = health.status === 'unhealthy' ? 503 : 200; res.writeHead(statusCode, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(health)); } + + private handleMetricsRequest(res: ServerResponse): void { + const lines: string[] = []; + + if (this.metricsProvider) { + const snap = this.metricsProvider(); + lines.push( + '# HELP openbridge_messages_received_total Total inbound messages received', + '# TYPE openbridge_messages_received_total counter', + `openbridge_messages_received_total ${snap.messages.received}`, + '', + '# HELP openbridge_messages_processed_total Total messages successfully processed', + '# TYPE openbridge_messages_processed_total counter', + `openbridge_messages_processed_total ${snap.messages.processed}`, + '', + '# HELP openbridge_messages_failed_total Total messages that failed processing', + '# TYPE openbridge_messages_failed_total counter', + `openbridge_messages_failed_total ${snap.messages.failed}`, + '', + '# HELP openbridge_errors_total Total error count', + '# TYPE openbridge_errors_total counter', + `openbridge_errors_total ${snap.errors.total}`, + '', + '# HELP openbridge_response_latency_ms_avg Average response latency in milliseconds', + '# TYPE openbridge_response_latency_ms_avg gauge', + `openbridge_response_latency_ms_avg ${snap.latency.avgMs}`, + '', + '# HELP openbridge_response_latency_ms_min Minimum response latency in milliseconds', + '# TYPE openbridge_response_latency_ms_min gauge', + `openbridge_response_latency_ms_min ${snap.latency.minMs}`, + '', + '# HELP openbridge_response_latency_ms_max Maximum response latency in milliseconds', + '# TYPE openbridge_response_latency_ms_max gauge', + `openbridge_response_latency_ms_max ${snap.latency.maxMs}`, + '', + '# HELP openbridge_response_latency_ms_total Cumulative response latency in milliseconds', + '# TYPE openbridge_response_latency_ms_total counter', + `openbridge_response_latency_ms_total ${snap.latency.totalMs}`, + '', + '# HELP openbridge_response_latency_count_total Total number of measured responses', + '# TYPE openbridge_response_latency_count_total counter', + `openbridge_response_latency_count_total ${snap.latency.count}`, + '', + '# HELP openbridge_queue_enqueued_total Total messages enqueued', + '# TYPE openbridge_queue_enqueued_total counter', + `openbridge_queue_enqueued_total ${snap.queue.enqueued}`, + '', + '# HELP openbridge_uptime_seconds Bridge uptime in seconds', + '# TYPE openbridge_uptime_seconds gauge', + `openbridge_uptime_seconds ${snap.uptime}`, + ); + } + + if (this.dataProvider) { + const health = this.dataProvider(); + lines.push( + '', + '# HELP openbridge_workers_active Current number of active worker agents', + '# TYPE openbridge_workers_active gauge', + `openbridge_workers_active ${health.active_workers}`, + ); + } + + res.writeHead(200, { 'Content-Type': 'text/plain; version=0.0.4; charset=utf-8' }); + res.end(lines.join('\n') + '\n'); + } + + private handleReadyRequest(res: ServerResponse): void { + if (!this.readinessProvider || !this.readinessProvider()) { + res.writeHead(503, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ready: false, reason: 'Master AI not initialized' })); + return; + } + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ready: true })); + } } diff --git a/src/core/html-renderer.ts b/src/core/html-renderer.ts new file mode 100644 index 00000000..ae2522e9 --- /dev/null +++ b/src/core/html-renderer.ts @@ -0,0 +1,514 @@ +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { createLogger } from './logger.js'; + +const logger = createLogger('html-renderer'); + +export type ImageFormat = 'png' | 'jpeg'; + +/** Parsed natural dimensions from an SVG element */ +export interface SvgDimensions { + width?: number; + height?: number; +} + +/** + * Parse the natural width/height from an SVG string. + * Reads `viewBox="minX minY width height"` first; falls back to `width`/`height` attributes. + */ +export function parseSvgDimensions(svg: string): SvgDimensions { + // viewBox="minX minY width height" — capture the 3rd and 4th values + const vbMatch = /viewBox\s*=\s*["']\s*[\d.]+\s+[\d.]+\s+([\d.]+)\s+([\d.]+)\s*["']/.exec(svg); + if (vbMatch) { + return { + width: Math.round(parseFloat(vbMatch[1]!)), + height: Math.round(parseFloat(vbMatch[2]!)), + }; + } + const wMatch = /\bwidth\s*=\s*["']([\d.]+)(?:px)?["']/.exec(svg); + const hMatch = /\bheight\s*=\s*["']([\d.]+)(?:px)?["']/.exec(svg); + return { + width: wMatch ? Math.round(parseFloat(wMatch[1]!)) : undefined, + height: hMatch ? Math.round(parseFloat(hMatch[1]!)) : undefined, + }; +} + +export interface RenderOptions { + /** Output image format. Default: 'png' */ + format?: ImageFormat; + /** Viewport width in pixels. Default: 1280 */ + width?: number; + /** Viewport height in pixels. Default: 720 */ + height?: number; + /** JPEG quality (1-100, ignored for PNG). Default: 90 */ + quality?: number; + /** Wait for network idle before capturing (slower but more reliable). Default: false */ + waitForNetworkIdle?: boolean; + /** Full-page screenshot (captures content beyond viewport). Default: false */ + fullPage?: boolean; +} + +export interface RenderResult { + /** Absolute path to the rendered image file */ + outputPath: string; + /** File format */ + format: ImageFormat; + /** File size in bytes */ + sizeBytes: number; +} + +// --------------------------------------------------------------------------- +// Minimal Puppeteer typings — avoids requiring @types/puppeteer at build time +// --------------------------------------------------------------------------- + +interface PuppeteerPage { + setViewport(opts: { width: number; height: number }): Promise; + setContent(html: string, opts: { waitUntil: string }): Promise; + goto(url: string, opts: { waitUntil: string }): Promise; + screenshot(opts: { + path: string; + type: 'png' | 'jpeg'; + quality?: number; + fullPage?: boolean; + }): Promise; +} + +interface PuppeteerBrowser { + newPage(): Promise; + close(): Promise; +} + +interface PuppeteerModule { + launch(opts: { headless: boolean; args: string[] }): Promise; +} + +/** + * HTMLRenderer — converts HTML content to PNG/JPEG images using Puppeteer. + * + * Puppeteer is an optional dependency. If not installed, `isAvailable()` returns false + * and `render*` methods throw a descriptive error. + * + * Usage: + * const renderer = new HTMLRenderer('/path/to/workspace'); + * if (await HTMLRenderer.isAvailable()) { + * const result = await renderer.renderHtmlString('

Hello

'); + * console.log(result.outputPath); + * } + */ +export class HTMLRenderer { + private readonly outputDir: string; + + constructor(workspacePath: string) { + this.outputDir = path.join(workspacePath, '.openbridge', 'generated'); + } + + /** + * Check whether Puppeteer is installed and usable. + * This is a fast check — no browser launch. + */ + static async isAvailable(): Promise { + try { + await import('puppeteer'); + return true; + } catch { + return false; + } + } + + /** + * Render an HTML string to an image file. + * + * @param html Full HTML document string (or fragment — wrapped in a minimal document if no tag) + * @param options Rendering options + * @returns RenderResult with the output file path and metadata + */ + async renderHtmlString(html: string, options: RenderOptions = {}): Promise { + const trimmed = html.trim(); + const normalized = + trimmed.toLowerCase().startsWith('${html}`; + + return this.captureWithPuppeteer({ html: normalized }, options); + } + + /** + * Render an HTML file to an image file. + * + * @param htmlFilePath Absolute path to the HTML file to render + * @param options Rendering options + * @returns RenderResult with the output file path and metadata + */ + async renderHtmlFile(htmlFilePath: string, options: RenderOptions = {}): Promise { + const resolved = path.resolve(htmlFilePath); + // Ensure the file exists before launching browser + await fs.access(resolved); + return this.captureWithPuppeteer({ filePath: resolved }, options); + } + + /** + * Render an SVG string to an image file. + * + * The SVG is wrapped in a minimal HTML document sized to the SVG's natural dimensions + * (derived from `viewBox` or `width`/`height` attributes). Dimensions can be overridden + * via `options.width` / `options.height`. + * + * @param svg Full SVG markup string + * @param options Rendering options (width/height default to SVG's natural size, or 800×600) + * @returns RenderResult with the output file path and metadata + */ + async renderSvgString(svg: string, options: RenderOptions = {}): Promise { + const dims = parseSvgDimensions(svg); + const width = options.width ?? dims.width ?? 800; + const height = options.height ?? dims.height ?? 600; + + // Wrap SVG in a minimal HTML page sized to match + const html = `${svg}`; + return this.captureWithPuppeteer({ html }, { ...options, width, height }); + } + + /** + * Render an SVG file to an image file. + * + * @param svgFilePath Absolute path to the SVG file to render + * @param options Rendering options + * @returns RenderResult with the output file path and metadata + */ + async renderSvgFile(svgFilePath: string, options: RenderOptions = {}): Promise { + const resolved = path.resolve(svgFilePath); + await fs.access(resolved); + const svgContent = await fs.readFile(resolved, 'utf-8'); + return this.renderSvgString(svgContent, options); + } + + // --------------------------------------------------------------------------- + // Private helpers + // --------------------------------------------------------------------------- + + private async loadPuppeteer(): Promise { + try { + const mod = (await import('puppeteer')) as { default?: PuppeteerModule } & PuppeteerModule; + // Handle both CommonJS-wrapped and ESM puppeteer exports + return mod.default ?? mod; + } catch { + throw new Error( + 'Puppeteer is not installed. Run `npm install puppeteer` to enable HTML-to-image rendering.', + ); + } + } + + private async captureWithPuppeteer( + source: { html: string } | { filePath: string }, + options: RenderOptions, + ): Promise { + const { + format = 'png', + width = 1280, + height = 720, + quality = 90, + waitForNetworkIdle = false, + fullPage = false, + } = options; + + const puppeteer = await this.loadPuppeteer(); + + await fs.mkdir(this.outputDir, { recursive: true }); + + const ext = format === 'jpeg' ? 'jpg' : format; + const outputFilename = `render-${randomUUID()}.${ext}`; + const outputPath = path.join(this.outputDir, outputFilename); + + const browser = await puppeteer.launch({ + headless: true, + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage', + '--disable-gpu', + ], + }); + + try { + const page = await browser.newPage(); + await page.setViewport({ width, height }); + + const waitUntil = waitForNetworkIdle ? 'networkidle0' : 'load'; + + if ('html' in source) { + await page.setContent(source.html, { waitUntil }); + } else { + await page.goto(`file://${source.filePath}`, { waitUntil }); + } + + if (format === 'jpeg') { + await page.screenshot({ path: outputPath, type: 'jpeg', quality, fullPage }); + } else { + await page.screenshot({ path: outputPath, type: 'png', fullPage }); + } + + const stat = await fs.stat(outputPath); + + logger.info( + { outputPath, format, sizeBytes: stat.size, width, height }, + 'HTML rendered to image', + ); + + return { outputPath, format, sizeBytes: stat.size }; + } finally { + await browser.close(); + } + } +} + +/** + * Convenience function — render an HTML string to an image in the workspace's generated folder. + * + * @param workspacePath Absolute path to the workspace (`.openbridge/generated/` is created inside) + * @param html HTML content to render + * @param options Rendering options + * @returns RenderResult + */ +export async function renderHtmlToImage( + workspacePath: string, + html: string, + options: RenderOptions = {}, +): Promise { + const renderer = new HTMLRenderer(workspacePath); + return renderer.renderHtmlString(html, options); +} + +/** + * Convenience function — render an SVG string to an image in the workspace's generated folder. + * + * @param workspacePath Absolute path to the workspace (`.openbridge/generated/` is created inside) + * @param svg SVG markup string + * @param options Rendering options (width/height default to SVG's natural dimensions) + * @returns RenderResult + */ +export async function renderSvgToImage( + workspacePath: string, + svg: string, + options: RenderOptions = {}, +): Promise { + const renderer = new HTMLRenderer(workspacePath); + return renderer.renderSvgString(svg, options); +} + +// --------------------------------------------------------------------------- +// Mermaid rendering support +// --------------------------------------------------------------------------- + +export interface MermaidRenderOptions extends RenderOptions { + /** Mermaid theme. Default: 'default' */ + theme?: 'default' | 'dark' | 'forest' | 'neutral'; + /** Background color (CSS color string). Default: 'white' */ + backgroundColor?: string; +} + +export interface MermaidRenderResult extends RenderResult { + /** The Mermaid definition that was rendered */ + definition: string; + /** Which backend produced the image */ + backend: 'mermaid-ink' | 'puppeteer'; +} + +/** + * MermaidRenderer — converts Mermaid diagram definitions to PNG images. + * + * Rendering backends (tried in order): + * 1. mermaid.ink HTTP API — no installation required, needs network access + * 2. Puppeteer + Mermaid CDN — requires Puppeteer, renders locally via browser + * + * Usage: + * const renderer = new MermaidRenderer('/path/to/workspace'); + * const result = await renderer.renderDefinition('graph TD; A-->B;'); + * console.log(result.outputPath); + */ +export class MermaidRenderer { + private readonly workspacePath: string; + private readonly outputDir: string; + + private static readonly MERMAID_INK_BASE = 'https://mermaid.ink'; + + constructor(workspacePath: string) { + this.workspacePath = workspacePath; + this.outputDir = path.join(workspacePath, '.openbridge', 'generated'); + } + + /** + * Check whether any Mermaid rendering backend is potentially available. + * Always returns true because mermaid.ink API requires no local installation. + * Puppeteer availability is checked separately via HTMLRenderer.isAvailable(). + */ + static isAvailable(): Promise { + return Promise.resolve(true); + } + + /** + * Render a Mermaid diagram definition to an image file. + * + * Tries mermaid.ink API first; falls back to Puppeteer + Mermaid.js CDN. + * + * @param definition Mermaid diagram definition string (e.g. "graph TD; A-->B;") + * @param options Rendering options + * @returns MermaidRenderResult with output path, metadata, and backend used + */ + async renderDefinition( + definition: string, + options: MermaidRenderOptions = {}, + ): Promise { + await fs.mkdir(this.outputDir, { recursive: true }); + + // Backend 1: mermaid.ink HTTP API + try { + return await this.renderViaMermaidInk(definition, options); + } catch (inkErr) { + logger.warn({ err: inkErr }, 'mermaid.ink API unavailable, falling back to Puppeteer'); + } + + // Backend 2: Puppeteer + Mermaid.js CDN + try { + return await this.renderViaPuppeteer(definition, options); + } catch (puppeteerErr) { + throw new Error( + `Mermaid rendering failed. mermaid.ink is unreachable and Puppeteer is not available.\n` + + `Install Puppeteer with \`npm install puppeteer\` for local rendering.\n` + + `Puppeteer error: ${(puppeteerErr as Error).message}`, + ); + } + } + + /** + * Render a Mermaid definition to an SVG string using the mermaid.ink API. + * + * @param definition Mermaid diagram definition string + * @param theme Mermaid theme. Default: 'default' + * @returns SVG markup string + */ + async renderToSvgString( + definition: string, + theme: MermaidRenderOptions['theme'] = 'default', + ): Promise { + const payload = JSON.stringify({ code: definition, mermaid: { theme } }); + const encoded = Buffer.from(payload).toString('base64url'); + const url = `${MermaidRenderer.MERMAID_INK_BASE}/svg/${encoded}`; + + const response = await fetch(url, { + headers: { Accept: 'image/svg+xml,*/*' }, + signal: AbortSignal.timeout(15_000), + }); + + if (!response.ok) { + throw new Error(`mermaid.ink SVG request failed: ${response.status} ${response.statusText}`); + } + + return response.text(); + } + + // --------------------------------------------------------------------------- + // Private helpers + // --------------------------------------------------------------------------- + + private async renderViaMermaidInk( + definition: string, + options: MermaidRenderOptions, + ): Promise { + const { theme = 'default', backgroundColor = 'white' } = options; + + // mermaid.ink accepts a JSON payload base64url-encoded in the URL path. + // The `bgColor` param uses '!' instead of '#' for hex colors. + const payload = JSON.stringify({ code: definition, mermaid: { theme } }); + const encoded = Buffer.from(payload).toString('base64url'); + const bgParam = backgroundColor.startsWith('#') + ? `!${backgroundColor.slice(1)}` + : backgroundColor; + const url = `${MermaidRenderer.MERMAID_INK_BASE}/img/${encoded}?bgColor=${bgParam}`; + + const response = await fetch(url, { + headers: { Accept: 'image/png,*/*' }, + signal: AbortSignal.timeout(20_000), + }); + + if (!response.ok) { + throw new Error(`mermaid.ink request failed: ${response.status} ${response.statusText}`); + } + + const outputFilename = `mermaid-${randomUUID()}.png`; + const outputPath = path.join(this.outputDir, outputFilename); + const buffer = Buffer.from(await response.arrayBuffer()); + await fs.writeFile(outputPath, buffer); + + const stat = await fs.stat(outputPath); + + logger.info({ outputPath, sizeBytes: stat.size }, 'Mermaid diagram rendered via mermaid.ink'); + + return { + outputPath, + format: 'png', + sizeBytes: stat.size, + definition, + backend: 'mermaid-ink', + }; + } + + private async renderViaPuppeteer( + definition: string, + options: MermaidRenderOptions, + ): Promise { + const { theme = 'default', format = 'png', width = 1280, height = 720, quality = 90 } = options; + + // Escape the definition for safe injection into an HTML attribute/script context + const escaped = escapeHtmlAttribute(definition); + + const html = ` + + + + + +
${escaped}
+ +`; + + const htmlRenderer = new HTMLRenderer(this.workspacePath); + const result = await htmlRenderer.renderHtmlString(html, { + format, + width, + height, + quality, + waitForNetworkIdle: true, + }); + + logger.info({ outputPath: result.outputPath }, 'Mermaid diagram rendered via Puppeteer'); + + return { ...result, definition, backend: 'puppeteer' }; + } +} + +/** Escape a string for safe embedding as HTML text content (not attribute). */ +function escapeHtmlAttribute(str: string): string { + return str + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +/** + * Convenience function — render a Mermaid diagram definition to a PNG image. + * + * @param workspacePath Absolute path to the workspace (`.openbridge/generated/` is created inside) + * @param definition Mermaid diagram definition string + * @param options Rendering options (theme, backgroundColor, width, height) + * @returns MermaidRenderResult with output file path and metadata + */ +export async function renderMermaidToImage( + workspacePath: string, + definition: string, + options: MermaidRenderOptions = {}, +): Promise { + const renderer = new MermaidRenderer(workspacePath); + return renderer.renderDefinition(definition, options); +} diff --git a/src/core/index.ts b/src/core/index.ts index b2ccaee5..25e65866 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -11,7 +11,7 @@ export type { ConnectorPluginModule, ProviderPluginModule, } from './registry.js'; -export { createLogger } from './logger.js'; +export { createLogger, setLogLevel } from './logger.js'; export { loadConfig, resolveConfigPath, isV2Config, convertV2ToInternal } from './config.js'; export { AuditLogger } from './audit-logger.js'; export { HealthServer } from './health.js'; diff --git a/src/core/interaction-relay.ts b/src/core/interaction-relay.ts new file mode 100644 index 00000000..e4a7fe93 --- /dev/null +++ b/src/core/interaction-relay.ts @@ -0,0 +1,314 @@ +import { createServer } from 'node:net'; +import { randomUUID } from 'node:crypto'; +import { createLogger } from './logger.js'; + +const logger = createLogger('interaction-relay'); + +const DEFAULT_PORT = 3099; +const WS_OPEN = 1; + +/** Minimal WS client interface — avoids importing ws types at module level */ +interface WsClient { + readyState: number; + send(data: string): void; + on(event: 'message', listener: (data: Buffer | string) => void): void; + on(event: 'close', listener: () => void): void; + on(event: 'error', listener: (err: Error) => void): void; + close(code?: number, reason?: string): void; +} + +/** Minimal WebSocketServer interface */ +interface WssServer { + on( + event: 'connection', + listener: ( + socket: WsClient, + request: { headers: Record; url?: string }, + ) => void, + ): void; + close(callback?: () => void): void; +} + +export interface RelayMessage { + appId: string; + type: string; + data: unknown; + id?: string; + timestamp?: string; +} + +export type AppMessageHandler = (message: RelayMessage) => void | Promise; + +interface AppConnection { + appId: string; + socket: WsClient; + connectedAt: string; +} + +/** + * InteractionRelay — WebSocket server on port 3099. + * Accepts connections from served apps and routes messages between apps and Master AI. + * + * Security: when any app tokens are registered (via registerApp), the relay requires + * a valid ?token= query parameter on every WebSocket connection. + * Connections without a valid token are immediately closed. + * When no tokens are registered the relay operates in open mode (development). + */ +export class InteractionRelay { + private wss: WssServer | null = null; + private readonly connections = new Map(); + private messageHandlers: AppMessageHandler[] = []; + private readonly port: number; + private running = false; + /** token → appId — populated by registerApp() */ + private readonly allowedTokens = new Map(); + + constructor(port = DEFAULT_PORT) { + this.port = port; + } + + /** + * Register an app token generated by AppServer.startApp(). + * Once any token is registered the relay switches to authenticated mode. + */ + registerApp(appId: string, token: string): void { + this.allowedTokens.set(token, appId); + logger.debug({ appId }, 'App token registered with relay'); + } + + /** + * Unregister all tokens for a given appId. + * Called by AppServer.stopApp(). + */ + unregisterApp(appId: string): void { + for (const [token, id] of this.allowedTokens) { + if (id === appId) { + this.allowedTokens.delete(token); + } + } + logger.debug({ appId }, 'App token unregistered from relay'); + } + + /** + * Start the WebSocket relay server. + * Resolves when the server is listening. + */ + async start(): Promise { + if (this.running) return; + + const inUse = await isPortInUse(this.port); + if (inUse) { + throw new Error(`InteractionRelay port ${this.port} is already in use`); + } + + const { WebSocketServer } = (await import('ws')) as { + WebSocketServer: new (options: { port: number }) => WssServer; + }; + + const wss = new WebSocketServer({ port: this.port }); + this.wss = wss; + this.running = true; + + wss.on('connection', (socket, request) => { + // Authenticate the connection when the relay is in secured mode + const authResult = this.authenticateConnection(request.url); + if (authResult === null) { + // Tokens are registered but none matched — reject connection + logger.warn({ url: request.url }, 'Relay rejected unauthenticated connection'); + socket.close(1008, 'Unauthorized'); + return; + } + + const connId = randomUUID(); + // If authentication returned a known appId use it; otherwise fall back to header/uuid + const appId = authResult ?? this.resolveAppId(request.headers, connId); + + const conn: AppConnection = { + appId, + socket, + connectedAt: new Date().toISOString(), + }; + this.connections.set(connId, conn); + logger.info({ connId, appId }, 'App connected to relay'); + + socket.on('message', (raw) => { + let message: RelayMessage; + try { + message = JSON.parse(raw.toString()) as RelayMessage; + // Ensure appId is always set from the connection + message.appId = appId; + if (!message.timestamp) { + message.timestamp = new Date().toISOString(); + } + } catch (err) { + logger.warn({ connId, appId, err }, 'Failed to parse relay message — ignoring'); + return; + } + + logger.debug({ connId, appId, type: message.type }, 'Relay received message from app'); + + for (const handler of this.messageHandlers) { + Promise.resolve(handler(message)).catch((handlerErr) => { + logger.error({ err: handlerErr, appId }, 'Error in relay message handler'); + }); + } + }); + + socket.on('close', () => { + this.connections.delete(connId); + logger.info({ connId, appId }, 'App disconnected from relay'); + }); + + socket.on('error', (err) => { + logger.error({ connId, appId, err }, 'Relay socket error'); + }); + }); + + logger.info({ port: this.port }, 'InteractionRelay started'); + } + + /** + * Stop the relay server and close all connections. + */ + stop(): Promise { + return new Promise((resolve) => { + if (!this.wss || !this.running) { + resolve(); + return; + } + + this.running = false; + this.connections.clear(); + this.wss.close(() => { + this.wss = null; + logger.info({ port: this.port }, 'InteractionRelay stopped'); + resolve(); + }); + }); + } + + /** + * Send a message to a specific app by appId. + * Returns true if the message was sent, false if the app is not connected. + */ + sendToApp(appId: string, type: string, data: unknown): boolean { + const conn = this.findConnectionByAppId(appId); + if (!conn || conn.socket.readyState !== WS_OPEN) { + logger.warn({ appId }, 'Cannot send to app — not connected'); + return false; + } + + const message: RelayMessage = { + appId, + type, + data, + id: randomUUID(), + timestamp: new Date().toISOString(), + }; + + try { + conn.socket.send(JSON.stringify(message)); + logger.debug({ appId, type }, 'Relay sent message to app'); + return true; + } catch (err) { + logger.error({ appId, type, err }, 'Failed to send message to app'); + return false; + } + } + + /** + * Register a handler for messages received from apps. + * Multiple handlers can be registered — all are called in registration order. + */ + onAppMessage(handler: AppMessageHandler): void { + this.messageHandlers.push(handler); + } + + /** + * Returns the number of currently connected apps. + */ + get connectionCount(): number { + return this.connections.size; + } + + /** + * Returns whether the relay server is currently running. + */ + get isRunning(): boolean { + return this.running; + } + + /** + * Authenticate an incoming WebSocket connection. + * + * Returns: + * - non-null string — the registered appId for the token (authenticated), or + * empty string when in open mode (no tokens registered) + * - `null` — tokens are registered but none matched; reject the connection + */ + private authenticateConnection(url: string | undefined): string | null { + // Open mode — no tokens registered yet + if (this.allowedTokens.size === 0) { + return ''; + } + + // Parse token from ?token= query string + const token = extractQueryParam(url, 'token'); + if (token) { + const appId = this.allowedTokens.get(token); + if (appId !== undefined) { + return appId; + } + } + + // Secured mode: no valid token found — reject + return null; + } + + /** Resolve appId from connection headers or generate one from connId */ + private resolveAppId( + headers: Record, + connId: string, + ): string { + const header = headers['x-app-id']; + if (typeof header === 'string' && header.trim().length > 0) { + return header.trim(); + } + return connId; + } + + /** Find the first connection with the given appId */ + private findConnectionByAppId(appId: string): AppConnection | null { + for (const conn of this.connections.values()) { + if (conn.appId === appId) { + return conn; + } + } + return null; + } +} + +/** Extract a single query parameter value from a URL path string (e.g. "/?token=abc"). */ +function extractQueryParam(url: string | undefined, param: string): string | null { + if (!url) return null; + try { + // Use a dummy base so URL() can parse path-only strings + const parsed = new URL(url, 'http://localhost'); + return parsed.searchParams.get(param); + } catch { + return null; + } +} + +function isPortInUse(port: number): Promise { + return new Promise((resolve) => { + const server = createServer(); + server.once('error', (err: NodeJS.ErrnoException) => { + resolve(err.code === 'EADDRINUSE'); + }); + server.once('listening', () => { + server.close(() => resolve(false)); + }); + server.listen(port, '127.0.0.1'); + }); +} diff --git a/src/core/knowledge-retriever.ts b/src/core/knowledge-retriever.ts new file mode 100644 index 00000000..f6000f74 --- /dev/null +++ b/src/core/knowledge-retriever.ts @@ -0,0 +1,758 @@ +import * as nodePath from 'node:path'; +import type { MemoryManager, Chunk } from '../memory/index.js'; +import { QACacheStore } from '../memory/qa-cache-store.js'; +import { + searchIndex as _searchIndex, + getDetails as _getDetails, + type SearchOptions, + type IndexResult, +} from '../memory/retrieval.js'; +import type { DotFolderManager } from '../master/dotfolder-manager.js'; +import type { WorkspaceMap } from '../types/master.js'; +import { createLogger } from './logger.js'; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface KnowledgeResult { + chunks: Chunk[]; + confidence: number; + sources: string[]; + needsWorker?: boolean; +} + +export interface ExtractedEntities { + filePaths: string[]; + functionNames: string[]; + moduleNames: string[]; +} + +// --------------------------------------------------------------------------- +// Stop words for FTS5 query parsing +// --------------------------------------------------------------------------- +// Only true function words — articles, conjunctions, prepositions, auxiliary +// verbs, and pronouns. Domain-relevant short terms like "api", "ui", "db", +// "cli", "ai" are intentionally NOT listed here so they survive the filter. +const STOP_WORDS = new Set([ + // Articles + 'a', + 'an', + 'the', + // Conjunctions + 'and', + 'or', + 'but', + // Prepositions + 'in', + 'on', + 'at', + 'to', + 'for', + 'of', + 'with', + 'by', + 'from', + 'into', + // Verb forms + 'is', + 'are', + 'was', + 'were', + 'be', + 'been', + 'being', + 'have', + 'has', + 'had', + 'do', + 'does', + 'did', + // Modal verbs + 'will', + 'would', + 'could', + 'should', + 'may', + 'might', + 'can', + // Pronouns / determiners + 'it', + 'its', + 'this', + 'that', + 'these', + 'those', + 'my', + 'your', + 'their', + // Other common function words + 'so', + 'if', + 'as', +]); + +// --------------------------------------------------------------------------- +// Entity extraction +// --------------------------------------------------------------------------- + +/** + * Extract structured entities from a block of text using regex patterns. + * Finds file paths (src/…/*.ts), function names (function X / const X =), + * and module names (import … from '…'). Used to enrich stored chunk metadata + * so that FTS5 searches can surface relevant chunks without re-spawning workers. + * + * OB-1363 + */ +export function extractEntities(text: string): ExtractedEntities { + // ── File paths ──────────────────────────────────────────────────────────── + // Match src/… paths with a file extension (e.g. src/core/router.ts) + const filePathPattern = /\bsrc\/[\w/.-]+\.\w{1,5}\b/g; + const filePaths = [...new Set(text.match(filePathPattern) ?? [])]; + + // ── Function names ──────────────────────────────────────────────────────── + const functionNames = new Set(); + + // function foo / async function foo (with optional type-params) + const funcDeclPattern = /\b(?:async\s+)?function\s+(\w+)\s*(?:<[^>]*>)?\s*\(/g; + let m: RegExpExecArray | null; + while ((m = funcDeclPattern.exec(text)) !== null) { + if (m[1]) functionNames.add(m[1]); + } + + // const foo = (async) (function|arrow) + const constFuncPattern = + /\bconst\s+(\w+)\s*(?::\s*[\w<>[\]|&, ]+\s*)?=\s*(?:async\s+)?(?:function|\()/g; + while ((m = constFuncPattern.exec(text)) !== null) { + if (m[1]) functionNames.add(m[1]); + } + + // ── Module names ────────────────────────────────────────────────────────── + const moduleNames = new Set(); + // import … from 'module' / import type … from "module" + const importPattern = /\bimport\s+(?:type\s+)?(?:[\s\S]*?)\bfrom\s+['"]([^'"]+)['"]/g; + while ((m = importPattern.exec(text)) !== null) { + if (m[1]) moduleNames.add(m[1]); + } + + return { + filePaths, + functionNames: [...functionNames], + moduleNames: [...moduleNames], + }; +} + +// --------------------------------------------------------------------------- +// KnowledgeRetriever +// --------------------------------------------------------------------------- + +/** + * RAG query orchestrator — searches existing knowledge (FTS5 chunks, workspace + * map, dir-dive JSONs) before spawning workers. Reduces redundant file reads + * for questions about already-analysed code. + */ +export class KnowledgeRetriever { + private readonly memoryManager: MemoryManager; + private readonly dotFolderManager: DotFolderManager; + private qaCache: QACacheStore | null = null; + private readonly logger = createLogger('knowledge-retriever'); + + constructor(memoryManager: MemoryManager, dotFolderManager: DotFolderManager) { + this.memoryManager = memoryManager; + this.dotFolderManager = dotFolderManager; + } + + private getQACache(): QACacheStore | null { + if (this.qaCache) return this.qaCache; + const db = this.memoryManager.getDb(); + if (!db) return null; + this.qaCache = new QACacheStore(db); + return this.qaCache; + } + + /** + * Parse a natural-language question into a cleaned FTS5 search query string. + * Splits on whitespace, removes stop words and very short tokens (< 2 chars), + * and returns the remaining terms joined by spaces for FTS5 MATCH. + * Domain-relevant 2-char terms (api, ui, db, cli, ai) are preserved. + * Fallback: if all tokens are filtered out, returns the original query so + * FTS5 always has something to search with. + */ + private buildSearchQuery(question: string): string { + const rawTokens = question.toLowerCase().split(/\s+/).filter(Boolean); + const tokens = rawTokens.filter((term) => term.length > 1 && !STOP_WORDS.has(term)); + + if (tokens.length === 0 && rawTokens.length > 0) { + const tooShort = rawTokens.filter((t) => t.length <= 1); + const stopWordsFiltered = rawTokens.filter((t) => t.length > 1 && STOP_WORDS.has(t)); + let reason: string; + if (tooShort.length > 0 && stopWordsFiltered.length > 0) { + reason = 'mixed: some tokens too short, some are stop words'; + } else if (tooShort.length > 0) { + reason = 'all tokens too short (length <= 1)'; + } else { + reason = 'all tokens are stop words'; + } + this.logger.warn( + { + originalQuestion: question, + filteredTokens: rawTokens, + tooShort, + stopWordsFiltered, + reason, + }, + 'buildSearchQuery: all tokens filtered — falling back to original query', + ); + } + + const finalQuery = tokens.length > 0 ? tokens.join(' ') : question.trim(); + + this.logger.debug( + { + originalQuestion: question, + rawTokens, + filteredTokens: tokens, + finalQuery, + tokenCount: tokens.length, + isSingleChar: question.trim().length === 1, + isMultiWord: rawTokens.length > 1, + }, + 'buildSearchQuery: FTS5 query constructed', + ); + + return finalQuery; + } + + /** + * Search directory dive JSONs for content relevant to the question. + * Returns synthetic Chunk objects for matching dives. + * + * OB-1337: dir-dive JSON loading — scans exploration/dirs/*.json for + * directory dive results whose path, key files, or insights match the + * question search terms. + */ + private async searchDirDives( + question: string, + dives: Array<{ dirPath: string; resultPath: string }>, + ): Promise { + const searchTerms = this.buildSearchQuery(question).split(/\s+/).filter(Boolean); + if (searchTerms.length === 0) return []; + + const questionLower = question.toLowerCase(); + const chunks: Chunk[] = []; + + for (const { dirPath } of dives) { + const dive = await this.dotFolderManager.readDirectoryDive(dirPath); + if (!dive) continue; + + const diveLower = dive.path.toLowerCase(); + const purposeLower = dive.purpose.toLowerCase(); + + // Directory path or purpose matches a search term, or question explicitly references the dir + const dirMatches = + questionLower.includes(diveLower) || + searchTerms.some((t) => diveLower.includes(t) || purposeLower.includes(t)); + + // Key files in this dive relevant to the question + const relevantFiles = dive.keyFiles.filter((kf) => { + const kfLower = kf.path.toLowerCase(); + const kfPurposeLower = kf.purpose.toLowerCase(); + return searchTerms.some((t) => kfLower.includes(t) || kfPurposeLower.includes(t)); + }); + + // Insights in this dive relevant to the question + const relevantInsights = dive.insights.filter((insight) => + searchTerms.some((t) => insight.toLowerCase().includes(t)), + ); + + if (!dirMatches && relevantFiles.length === 0 && relevantInsights.length === 0) continue; + + // Build a compact content string for this dive + const lines: string[] = [`${dive.path}: ${dive.purpose}`]; + for (const kf of relevantFiles.slice(0, 5)) { + lines.push(` ${kf.path} (${kf.type}): ${kf.purpose}`); + } + for (const insight of relevantInsights.slice(0, 3)) { + lines.push(` ${insight}`); + } + + chunks.push({ + scope: dive.path, + category: 'structure' as const, + content: lines.join('\n'), + }); + } + + return chunks; + } + + /** + * Match workspace map key files against terms extracted from the question. + * Returns synthetic Chunk objects for each matched file. + * + * OB-1336: workspace map key-file matching — scans keyFiles for filenames or + * module names mentioned in the question. + */ + private matchKeyFiles( + question: string, + keyFiles: Array<{ path: string; type: string; purpose: string }>, + ): Chunk[] { + // Extract explicit file references like "router.ts", "auth.js" + const fileRefPattern = /\b[\w-]+\.\w{1,5}\b/g; + const explicitRefs = (question.match(fileRefPattern) ?? []).map((r) => r.toLowerCase()); + + // Extract module-name terms (non-stop-words, >= 3 chars) + const moduleTerms = question + .toLowerCase() + .split(/[\s.,!?;:'"()[\]{}]+/) + .filter((term) => term.length >= 3 && !STOP_WORDS.has(term)); + + return keyFiles + .filter((kf) => { + const kfLower = kf.path.toLowerCase(); + const baseName = nodePath.basename(kfLower); + const baseNoExt = baseName.replace(/\.[^.]+$/, ''); + + // Explicit file reference match (e.g., "router.ts" in question matches path) + if (explicitRefs.some((ref) => kfLower.includes(ref))) return true; + + // Module name match (e.g., "router" matches basename "router.ts") + if (moduleTerms.some((term) => baseNoExt === term || baseNoExt.includes(term))) return true; + + return false; + }) + .map((kf) => ({ + scope: kf.path, + category: 'structure' as const, + content: `${kf.path} (${kf.type}): ${kf.purpose}`, + })); + } + + /** + * Compute confidence score (0.0–1.0) based on chunk count and source diversity. + * + * OB-1338: + * - Chunk count component: 0 chunks → 0.0, 5+ chunks → 0.8 (linear). + * - Source diversity bonus: +0.1 per additional source type beyond first, + * capped at +0.2 (so 3 sources gives the full bonus). + * - Total is capped at 1.0. + */ + private computeConfidence(chunkCount: number, sourceCount: number): number { + // Chunk count component: scales linearly from 0 (0 chunks) to 0.8 (5+ chunks) + const countComponent = Math.min(chunkCount / 5, 1) * 0.8; + + // Diversity bonus: +0.1 per additional source beyond the first, max +0.2 + const diversityBonus = Math.min(Math.max(sourceCount - 1, 0) * 0.1, 0.2); + + return Math.min(countComponent + diversityBonus, 1.0); + } + + /** + * Format a {@link KnowledgeResult} into a concise context string suitable + * for injection into a system prompt. + * + * OB-1339: Produces a "Relevant Knowledge" header followed by source-grouped + * sections (FTS5 chunks, workspace-map entries, dir-dive summaries) and a + * confidence percentage footer. Output is hard-truncated to 4000 characters. + */ + formatKnowledgeContext(result: KnowledgeResult): string { + if (result.chunks.length === 0) { + return ''; + } + + const lines: string[] = ['## Relevant Knowledge']; + + // Assign each chunk to its source group based on the chunk's category / + // scope heuristic. Chunks carry no explicit source tag, so we infer: + // • FTS5 chunks → any category (they come from the DB, scope is a path) + // • workspace-map chunks → category === 'structure' AND scope ends in a + // filename (single path segment after the last slash) + // • dir-dive chunks → category === 'structure' AND scope is a dir path + // + // For simplicity we use the order in which sources were populated: the + // first `result.sources.length` groups are valid; within each group we + // include up to 5 chunks. + + const fts5Chunks: Chunk[] = []; + const mapChunks: Chunk[] = []; + const diveChunks: Chunk[] = []; + + for (const chunk of result.chunks) { + // Workspace-map synthetic chunks have a single-line content like + // "path/to/file.ts (type): purpose" + if ( + chunk.category === 'structure' && + /^[^\n]+ \(\w+\): .+$/.test(chunk.content) && + chunk.content.split('\n').length === 1 + ) { + mapChunks.push(chunk); + } else if ( + chunk.category === 'structure' && + chunk.content.includes('\n') && + !chunk.scope.match(/\.\w{1,5}$/) + ) { + // Dir-dive chunks are multi-line and scope is a directory path + diveChunks.push(chunk); + } else { + fts5Chunks.push(chunk); + } + } + + // FTS5 section + if (result.sources.includes('fts5') && fts5Chunks.length > 0) { + lines.push('\n### Code Chunks'); + for (const chunk of fts5Chunks.slice(0, 5)) { + lines.push(`\n**${chunk.scope}** (${chunk.category}):\n${chunk.content.trim()}`); + } + } + + // Workspace-map section + if (result.sources.includes('workspace-map') && mapChunks.length > 0) { + lines.push('\n### Key Files'); + for (const chunk of mapChunks.slice(0, 5)) { + lines.push(`- ${chunk.content.trim()}`); + } + } + + // Dir-dive section + if (result.sources.includes('dir-dive') && diveChunks.length > 0) { + lines.push('\n### Directory Summaries'); + for (const chunk of diveChunks.slice(0, 3)) { + lines.push(`\n${chunk.content.trim()}`); + } + } + + // Confidence footer + const pct = Math.round(result.confidence * 100); + lines.push(`\n*Confidence: ${pct}%*`); + + const full = lines.join('\n'); + // Hard-truncate to 4000 chars + if (full.length <= 4000) return full; + return full.slice(0, 3997) + '...'; + } + + /** + * Suggest target files for a focused read-only worker when RAG confidence + * is low. Applies three heuristics in priority order: + * 1. Explicit file references found in the question (highest weight) + * 2. Key files whose basename or purpose matches question keywords + * 3. Files under directories whose name or purpose matches question keywords + * + * Returns up to 10 file paths, de-duplicated and ordered by relevance score. + * + * OB-1352 + */ + suggestTargetFiles(question: string, workspaceMap: WorkspaceMap): string[] { + // ── Build term sets ─────────────────────────────────────────────────────── + // Explicit file references like "router.ts", "src/core/router.ts" + const fileRefPattern = /\b[\w/.-]+\.\w{1,5}\b/g; + const explicitRefs = (question.match(fileRefPattern) ?? []).map((r) => r.toLowerCase()); + + // Module terms: non-stop-words >= 3 chars + const moduleTerms = question + .toLowerCase() + .split(/[\s.,!?;:'"()[\]{}]+/) + .filter((t) => t.length >= 3 && !STOP_WORDS.has(t)); + + // Score map: filePath → cumulative score + const scores = new Map(); + const addScore = (path: string, points: number): void => { + scores.set(path, (scores.get(path) ?? 0) + points); + }; + + // ── Heuristic 1: explicit file references ───────────────────────────────── + for (const kf of workspaceMap.keyFiles) { + const kfLower = kf.path.toLowerCase(); + for (const ref of explicitRefs) { + if (kfLower.endsWith(ref) || kfLower.includes(ref)) { + addScore(kf.path, 10); + break; + } + } + } + for (const ep of workspaceMap.entryPoints) { + const epLower = ep.toLowerCase(); + for (const ref of explicitRefs) { + if (epLower.endsWith(ref) || epLower.includes(ref)) { + addScore(ep, 10); + break; + } + } + } + + // ── Heuristic 2: key files matching question keywords ───────────────────── + for (const kf of workspaceMap.keyFiles) { + const kfLower = kf.path.toLowerCase(); + const baseName = nodePath.basename(kfLower); + const baseNoExt = baseName.replace(/\.[^.]+$/, ''); + const purposeLower = kf.purpose.toLowerCase(); + for (const term of moduleTerms) { + if (baseNoExt === term) { + addScore(kf.path, 6); // exact basename match + } else if (baseNoExt.includes(term) || kfLower.includes(term)) { + addScore(kf.path, 3); // partial path match + } + if (purposeLower.includes(term)) { + addScore(kf.path, 2); // purpose text match + } + } + } + + // ── Heuristic 3: directories matching keywords → their key files ────────── + for (const dir of Object.values(workspaceMap.structure)) { + const dirPathLower = dir.path.toLowerCase(); + const dirNameLower = nodePath.basename(dirPathLower); + const purposeLower = dir.purpose.toLowerCase(); + + const dirMatch = moduleTerms.some( + (t) => dirNameLower.includes(t) || purposeLower.includes(t), + ); + if (!dirMatch) continue; + + // Promote key files that live under the matching directory + for (const kf of workspaceMap.keyFiles) { + const kfLower = kf.path.toLowerCase(); + if (kfLower.startsWith(dirPathLower + '/') || kfLower.includes('/' + dirPathLower + '/')) { + addScore(kf.path, 4); + } + } + // Promote entry points that reference the matching directory + for (const ep of workspaceMap.entryPoints) { + if (ep.toLowerCase().includes(dirPathLower)) { + addScore(ep, 4); + } + } + } + + // ── Sort, de-duplicate, limit to 10 ────────────────────────────────────── + return [...scores.entries()] + .filter(([, score]) => score > 0) + .sort(([, a], [, b]) => b - a) + .slice(0, 10) + .map(([path]) => path); + } + + /** + * Store a worker's read result as a new chunk in the workspace knowledge + * base. The chunk is tagged with source `'worker-read'` via `source_hash` + * and embeds the originating question and file paths as a metadata header so + * future FTS5 searches can surface the answer without re-spawning a worker. + * + * OB-1359 + */ + async storeWorkerResult( + workerOutput: string, + question: string, + filePaths: string[], + ): Promise { + if (!workerOutput.trim()) return; + + const entities = extractEntities(workerOutput); + const allFilePaths = [...new Set([...filePaths, ...entities.filePaths])]; + + const metaLines: string[] = []; + if (question) metaLines.push(`Q: ${question}`); + if (allFilePaths.length > 0) metaLines.push(`Files: ${allFilePaths.join(', ')}`); + if (entities.functionNames.length > 0) { + metaLines.push(`Functions: ${entities.functionNames.slice(0, 10).join(', ')}`); + } + if (entities.moduleNames.length > 0) { + metaLines.push(`Modules: ${entities.moduleNames.slice(0, 10).join(', ')}`); + } + + const content = + metaLines.length > 0 ? `${metaLines.join('\n')}\n---\n${workerOutput}` : workerOutput; + + const scope = filePaths[0] ?? entities.filePaths[0] ?? 'worker-read'; + + await this.memoryManager.storeChunks([ + { + scope, + category: 'patterns', + content, + source_hash: 'worker-read', + }, + ]); + } + + /** + * Query the local knowledge store for information relevant to the given + * question. Returns a {@link KnowledgeResult} with matched chunks, a + * confidence score, and the source types that contributed results. + * + * OB-1335: FTS5 chunk search — searches workspace_chunks via FTS5 MATCH and + * returns up to 10 results ordered by BM25 rank score. + * OB-1336: workspace map key-file matching. + * OB-1337: dir-dive JSON loading. + * OB-1338: confidence scoring + needsWorker flag. + */ + async query(question: string): Promise { + // --- OB-1656: skip RAG for very short inputs (< 3 chars) --- + // Single-character inputs like "1" or "3" are menu selections, not + // searchable questions. Running FTS5 on them wastes resources and returns + // noise. Return empty results immediately so the router falls through to the + // Master AI directly. + if (question.trim().length < 3) { + this.logger.debug( + { question: question.trim(), length: question.trim().length }, + 'RAG skipped: query shorter than 3 characters', + ); + return { chunks: [], confidence: 0, sources: [] }; + } + + // --- Q&A cache check (OB-1362) --- + const qaCache = this.getQACache(); + if (qaCache) { + const [entry] = qaCache.findSimilar(question, 1); + if (entry !== undefined) { + const ageMs = entry.created_at + ? Date.now() - new Date(entry.created_at).getTime() + : Infinity; + const withinTtl = ageMs <= 24 * 60 * 60 * 1000; + if (entry.confidence >= 0.7 && withinTtl && entry.id !== undefined) { + qaCache.incrementAccess(entry.id); + return { + chunks: [{ scope: 'qa-cache', category: 'patterns', content: entry.answer }], + confidence: entry.confidence, + sources: ['qa-cache'], + needsWorker: false, + }; + } + } + } + + const result: KnowledgeResult = { chunks: [], confidence: 0, sources: [] }; + + // --- FTS5 chunk search (OB-1335) --- + const searchQuery = this.buildSearchQuery(question); + if (searchQuery) { + const fts5Chunks = await this.memoryManager.searchContext(searchQuery, 10); + if (fts5Chunks.length > 0) { + result.chunks.push(...fts5Chunks); + result.sources.push('fts5'); + } else { + // OB-1655: fallback — retry with top-3 content words individually + const keywords = question + .toLowerCase() + .split(/\s+/) + .filter((t) => t.length > 1 && !STOP_WORDS.has(t)) + .sort((a, b) => b.length - a.length) + .slice(0, 3); + if (keywords.length > 0) { + this.logger.info( + { originalQuery: searchQuery, keywords }, + 'RAG fallback: retrying with individual keywords', + ); + const seen = new Set(); + const fallbackChunks: Chunk[] = []; + for (const kw of keywords) { + const kwChunks = await this.memoryManager.searchContext(kw, 10); + for (const c of kwChunks) { + if (!seen.has(c.content)) { + seen.add(c.content); + fallbackChunks.push(c); + } + } + } + if (fallbackChunks.length > 0) { + result.chunks.push(...fallbackChunks); + result.sources.push('fts5'); + } + } + } + } + + // --- Workspace map key-file matching (OB-1336) --- + const workspaceMap = await this.dotFolderManager.readWorkspaceMap(); + if (workspaceMap && workspaceMap.keyFiles.length > 0) { + const mapChunks = this.matchKeyFiles(question, workspaceMap.keyFiles); + if (mapChunks.length > 0) { + result.chunks.push(...mapChunks); + result.sources.push('workspace-map'); + } + } + + // --- Dir-dive JSON loading (OB-1337) --- + const dirDives = await this.dotFolderManager.listDirDiveResults(); + if (dirDives.length > 0) { + const diveChunks = await this.searchDirDives(question, dirDives); + if (diveChunks.length > 0) { + result.chunks.push(...diveChunks); + result.sources.push('dir-dive'); + } + } + + // --- Confidence scoring (OB-1338) --- + result.confidence = this.computeConfidence(result.chunks.length, result.sources.length); + if (result.confidence < 0.3) { + result.needsWorker = true; + } + + return result; + } + + /** + * 2-step retrieval for the RAG flow (OB-1660). + * + * Step 1: `searchIndex(query)` — compact results (~10x fewer tokens than full chunks). + * Step 2: Filter results with score > scoreThreshold (default 0.3). + * Step 3: `getDetails(topIds)` — fetch full content only for the relevant chunks. + * + * This is more token-efficient than `query()` because it avoids retrieving full + * chunk content for low-relevance results. Master should prefer this method when + * assembling context for generation. + * + * @param question Natural-language question to search for + * @param options hybridSearch options forwarded to searchIndex (scope, category, limit, etc.) + * @param scoreThreshold Minimum score to include in getDetails step (default 0.3) + */ + async queryWithIndex( + question: string, + options: SearchOptions = {}, + scoreThreshold = 0.3, + ): Promise { + if (question.trim().length < 3) { + return { chunks: [], confidence: 0, sources: [] }; + } + + const db = this.memoryManager.getDb(); + if (!db) { + return { chunks: [], confidence: 0, sources: [] }; + } + + // Step 1: get compact index results + let indexResults: IndexResult[]; + try { + indexResults = await _searchIndex(db, question, options); + } catch (err) { + this.logger.warn({ err }, 'queryWithIndex: searchIndex failed, returning empty result'); + return { chunks: [], confidence: 0, sources: [] }; + } + + // Step 2: filter by score threshold + const relevant = indexResults.filter((r) => r.score > scoreThreshold); + + if (relevant.length === 0) { + return { chunks: [], confidence: 0, sources: ['index'] }; + } + + // Step 3: fetch full content for relevant IDs + const topIds = relevant.map((r) => r.id); + let chunks: Chunk[]; + try { + chunks = _getDetails(db, topIds); + } catch (err) { + this.logger.warn({ err }, 'queryWithIndex: getDetails failed, returning empty result'); + return { chunks: [], confidence: 0, sources: ['index'] }; + } + + const confidence = this.computeConfidence(chunks.length, 1); + return { + chunks, + confidence, + sources: ['index'], + needsWorker: confidence < 0.3, + }; + } +} diff --git a/src/core/logger.ts b/src/core/logger.ts index 3e7f6598..a3a8099c 100644 --- a/src/core/logger.ts +++ b/src/core/logger.ts @@ -1,12 +1,57 @@ +import { mkdirSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { join } from 'node:path'; import pino from 'pino'; -export function createLogger(name: string, level = 'info'): pino.Logger { - return pino({ - name, - level, - transport: - process.env['NODE_ENV'] !== 'production' - ? { target: 'pino-pretty', options: { colorize: true } } - : undefined, - }); +// Single root logger with one transport shared across the entire app. +// Using child() for each module avoids the MaxListenersExceededWarning: +// each pino instance with transport registers a process.on('exit') handler, +// so N modules × N loggers = N handlers (exceeds the default limit of 10). +// With a singleton + child(), only one transport exists → one handler total. +const initialLevel = process.env['LOG_LEVEL'] ?? 'info'; + +/** Returns true when running inside a pkg-compiled binary. */ +function isPackagedMode(): boolean { + return (process as { pkg?: unknown }).pkg !== undefined; +} + +/** + * Returns the logs directory for file transport. + * In packaged mode: ~/.openbridge/logs/ + * In dev/production mode: not used (logs go to stdout). + */ +function getLogDir(): string { + const logsDir = join(homedir(), '.openbridge', 'logs'); + mkdirSync(logsDir, { recursive: true }); + return logsDir; +} + +function createRootLogger(): pino.Logger { + if (isPackagedMode()) { + const logFile = join(getLogDir(), 'openbridge.log'); + return pino({ level: initialLevel }, pino.destination({ dest: logFile, sync: false })); + } + if (process.env['NODE_ENV'] === 'production') { + return pino({ level: initialLevel }); + } + try { + return pino({ + level: initialLevel, + transport: { target: 'pino-pretty', options: { colorize: true } }, + }); + } catch { + // pino-pretty not installed — fall back to plain JSON logs + return pino({ level: initialLevel }); + } +} + +const rootLogger = createRootLogger(); + +export function createLogger(name: string): pino.Logger { + return rootLogger.child({ name }); +} + +/** Apply a log level from config or env at runtime (call after loadConfig). */ +export function setLogLevel(level: string): void { + rootLogger.level = level; } diff --git a/src/core/mcp-catalog.ts b/src/core/mcp-catalog.ts new file mode 100644 index 00000000..10e49a4d --- /dev/null +++ b/src/core/mcp-catalog.ts @@ -0,0 +1,243 @@ +import type { MCPCatalogEntry } from '../types/config.js'; + +/** + * Built-in catalog of known MCP servers users can browse and install. + * Each entry describes a well-known MCP server package with its configuration requirements. + */ +export const MCP_CATALOG: MCPCatalogEntry[] = [ + { + name: 'Filesystem', + description: + 'Read and write files on the local filesystem. Gives AI workers access to read, write, and list files within allowed directories.', + category: 'code', + command: 'npx', + args: ['-y', '@modelcontextprotocol/server-filesystem'], + envVars: [], + docsUrl: 'https://github.com/modelcontextprotocol/servers/tree/main/src/filesystem', + }, + { + name: 'GitHub', + description: + 'Interact with GitHub repositories — read files, create issues, open pull requests, search code, and manage workflows.', + category: 'code', + command: 'npx', + args: ['-y', '@modelcontextprotocol/server-github'], + envVars: [ + { + key: 'GITHUB_PERSONAL_ACCESS_TOKEN', + description: 'GitHub personal access token with repo and workflow scopes', + required: true, + }, + ], + docsUrl: 'https://github.com/modelcontextprotocol/servers/tree/main/src/github', + }, + { + name: 'Slack', + description: + 'Read and send Slack messages, list channels, search conversations, and post updates to your Slack workspace.', + category: 'communication', + command: 'npx', + args: ['-y', '@modelcontextprotocol/server-slack'], + envVars: [ + { + key: 'SLACK_BOT_TOKEN', + description: 'Slack bot token (xoxb-...) with channels:read and chat:write scopes', + required: true, + }, + { + key: 'SLACK_TEAM_ID', + description: 'Slack workspace team ID (found in workspace settings)', + required: true, + }, + ], + docsUrl: 'https://github.com/modelcontextprotocol/servers/tree/main/src/slack', + }, + { + name: 'Gmail', + description: + 'Read, search, and send Gmail messages. Lets AI workers process email threads and draft replies on your behalf.', + category: 'communication', + command: 'npx', + args: ['-y', '@modelcontextprotocol/server-gmail'], + envVars: [ + { + key: 'GMAIL_OAUTH_CREDENTIALS', + description: 'Path to OAuth 2.0 credentials JSON file from Google Cloud Console', + required: true, + }, + ], + docsUrl: 'https://github.com/modelcontextprotocol/servers/tree/main/src/gmail', + }, + { + name: 'Canva', + description: + 'Create and edit Canva designs programmatically. Generate presentations, social media graphics, and documents via the Canva API.', + category: 'design', + command: 'npx', + args: ['-y', '@modelcontextprotocol/server-canva'], + envVars: [ + { + key: 'CANVA_API_TOKEN', + description: 'Canva API token from your Canva developer account', + required: true, + }, + ], + docsUrl: 'https://www.canva.com/developers/', + }, + { + name: 'Brave Search', + description: + 'Search the web using Brave Search. Provides real-time web search results without tracking.', + category: 'productivity', + command: 'npx', + args: ['-y', '@modelcontextprotocol/server-brave-search'], + envVars: [ + { + key: 'BRAVE_API_KEY', + description: 'Brave Search API key from the Brave developer portal', + required: true, + }, + ], + docsUrl: 'https://github.com/modelcontextprotocol/servers/tree/main/src/brave-search', + }, + { + name: 'Puppeteer', + description: + 'Control a headless Chromium browser — navigate pages, take screenshots, extract content, and interact with web UIs.', + category: 'code', + command: 'npx', + args: ['-y', '@modelcontextprotocol/server-puppeteer'], + envVars: [], + docsUrl: 'https://github.com/modelcontextprotocol/servers/tree/main/src/puppeteer', + }, + { + name: 'PostgreSQL', + description: + 'Query and manage PostgreSQL databases. Run SQL queries, inspect schemas, and retrieve data for analysis tasks.', + category: 'data', + command: 'npx', + args: ['-y', '@modelcontextprotocol/server-postgres'], + envVars: [ + { + key: 'POSTGRES_CONNECTION_STRING', + description: 'PostgreSQL connection string (e.g. postgresql://user:pass@host:5432/dbname)', + required: true, + }, + ], + docsUrl: 'https://github.com/modelcontextprotocol/servers/tree/main/src/postgres', + }, + { + name: 'SQLite', + description: + 'Read and write SQLite database files. Useful for local data storage, prototypes, and embedded databases.', + category: 'data', + command: 'npx', + args: ['-y', '@modelcontextprotocol/server-sqlite'], + envVars: [ + { + key: 'SQLITE_DB_PATH', + description: 'Absolute path to the SQLite database file', + required: false, + }, + ], + docsUrl: 'https://github.com/modelcontextprotocol/servers/tree/main/src/sqlite', + }, + { + name: 'Sentry', + description: + 'Access Sentry error tracking — retrieve issues, stack traces, releases, and performance data for debugging tasks.', + category: 'code', + command: 'npx', + args: ['-y', '@modelcontextprotocol/server-sentry'], + envVars: [ + { + key: 'SENTRY_AUTH_TOKEN', + description: 'Sentry authentication token with project:read scope', + required: true, + }, + { + key: 'SENTRY_ORG', + description: 'Sentry organization slug', + required: true, + }, + ], + docsUrl: 'https://github.com/modelcontextprotocol/servers/tree/main/src/sentry', + }, + { + name: 'GitLab', + description: + 'Interact with GitLab repositories — read files, manage merge requests, issues, and CI/CD pipelines.', + category: 'code', + command: 'npx', + args: ['-y', '@modelcontextprotocol/server-gitlab'], + envVars: [ + { + key: 'GITLAB_PERSONAL_ACCESS_TOKEN', + description: 'GitLab personal access token with api scope', + required: true, + }, + { + key: 'GITLAB_API_URL', + description: 'GitLab API base URL (defaults to https://gitlab.com/api/v4)', + required: false, + }, + ], + docsUrl: 'https://github.com/modelcontextprotocol/servers/tree/main/src/gitlab', + }, + { + name: 'Notion', + description: + 'Read and write Notion pages and databases. Useful for knowledge management, task tracking, and documentation workflows.', + category: 'productivity', + command: 'npx', + args: ['-y', '@modelcontextprotocol/server-notion'], + envVars: [ + { + key: 'NOTION_API_TOKEN', + description: 'Notion integration token from your Notion workspace settings', + required: true, + }, + ], + docsUrl: 'https://developers.notion.com/', + }, + { + name: 'Google Drive', + description: + 'Upload, download, and manage files in Google Drive. Create folders, share files, and generate public links for document collaboration.', + category: 'productivity', + command: 'npx', + args: ['-y', '@modelcontextprotocol/server-google-drive'], + envVars: [ + { + key: 'GOOGLE_DRIVE_CREDENTIALS', + description: + 'Path to Google service account JSON key file or OAuth2 credentials from Google Cloud Console', + required: true, + }, + { + key: 'GOOGLE_DRIVE_FOLDER_ID', + description: + 'Optional: Google Drive folder ID to restrict file operations to a specific folder (defaults to root)', + required: false, + }, + ], + docsUrl: 'https://github.com/modelcontextprotocol/servers/tree/main/src/google-drive', + }, + { + name: 'Dropbox', + description: + 'Upload, download, and organize files in Dropbox. Create shared links, manage folders, and synchronize files for team collaboration.', + category: 'productivity', + command: 'npx', + args: ['-y', '@modelcontextprotocol/server-dropbox'], + envVars: [ + { + key: 'DROPBOX_ACCESS_TOKEN', + description: + 'Dropbox OAuth2 access token from your Dropbox App Console (requires files.content.write and files.content.read scopes)', + required: true, + }, + ], + docsUrl: 'https://github.com/modelcontextprotocol/servers/tree/main/src/dropbox', + }, +]; diff --git a/src/core/mcp-registry.ts b/src/core/mcp-registry.ts new file mode 100644 index 00000000..ec216272 --- /dev/null +++ b/src/core/mcp-registry.ts @@ -0,0 +1,184 @@ +import { readFileSync, writeFileSync } from 'node:fs'; +import type { MCPServer } from '../types/config.js'; +import { checkCommandOnPath } from './health.js'; + +export type McpServerStatus = 'healthy' | 'error' | 'unknown'; + +export interface McpServerEntry extends MCPServer { + enabled: boolean; +} + +export interface McpServerWithStatus extends McpServerEntry { + status: McpServerStatus; +} + +/** + * Runtime registry of MCP servers with enable/disable support and health status. + * Config persistence is handled separately via persistToConfig() (OB-1173). + */ +export class McpRegistry { + private readonly servers: Map; + // configPath is stored for use by persistToConfig() in OB-1173 + readonly configPath: string; + private onChange: ((servers: McpServerWithStatus[]) => void) | null = null; + + constructor(configPath: string, initialServers: MCPServer[]) { + this.configPath = configPath; + this.servers = new Map(); + for (const server of initialServers) { + this.servers.set(server.name, { ...server, enabled: true }); + } + } + + /** + * Register a callback to be called after every mutation (add/remove/toggle). + * Used to broadcast mcp-status WebSocket events to connected clients. + */ + setOnChange(callback: (servers: McpServerWithStatus[]) => void): void { + this.onChange = callback; + } + + private notifyChange(): void { + if (this.onChange) { + this.onChange(this.listServers()); + } + } + + /** + * Add a new MCP server to the registry. + * Throws if a server with the same name already exists. + */ + addServer(server: MCPServer): void { + if (this.servers.has(server.name)) { + throw new Error(`MCP server "${server.name}" already exists`); + } + this.servers.set(server.name, { ...server, enabled: true }); + this.persistToConfig(); + this.notifyChange(); + } + + /** + * Remove an MCP server from the registry by name. + * Throws if no server with that name is found. + */ + removeServer(name: string): void { + if (!this.servers.has(name)) { + throw new Error(`MCP server "${name}" not found`); + } + this.servers.delete(name); + this.persistToConfig(); + this.notifyChange(); + } + + /** + * Enable or disable an MCP server by name. + * Throws if no server with that name is found. + */ + toggleServer(name: string, enabled: boolean): void { + const entry = this.servers.get(name); + if (!entry) { + throw new Error(`MCP server "${name}" not found`); + } + this.servers.set(name, { ...entry, enabled }); + this.persistToConfig(); + this.notifyChange(); + } + + /** + * Mask all values in an env record for safe API/WebSocket exposure. + * Shows first 4 chars + '****', or just '****' if value is shorter than 4 chars. + */ + private maskEnv(env: Record | undefined): Record | undefined { + if (env === undefined) return undefined; + const masked: Record = {}; + for (const [key, value] of Object.entries(env)) { + masked[key] = value.length >= 4 ? `${value.slice(0, 4)}****` : '****'; + } + return masked; + } + + /** + * List all servers with their enabled state and health status. + * Status is determined by checking whether the server's command exists on PATH: + * - 'healthy' — server enabled and command found on PATH + * - 'error' — server enabled but command not found on PATH + * - 'unknown' — server is disabled (command not checked) + * + * Env var values are masked (first 4 chars + ****) to prevent credential leakage + * in API responses and WebSocket broadcasts. + */ + listServers(): McpServerWithStatus[] { + return Array.from(this.servers.values()).map((entry) => { + let status: McpServerStatus; + if (!entry.enabled) { + status = 'unknown'; + } else { + status = checkCommandOnPath(entry.command) ? 'healthy' : 'error'; + } + return { ...entry, env: this.maskEnv(entry.env), status }; + }); + } + + /** + * Get a single server entry by name. + * Returns undefined if not found. + */ + getServer(name: string): McpServerEntry | undefined { + return this.servers.get(name); + } + + /** + * Replace the internal server list with a new set of servers. + * Used by hot-reload: called from Bridge.onConfigChange() when config.json is updated + * externally (e.g., by a user or CI pipeline). Does NOT persist to config — the new + * servers are already on disk; this just synchronises the runtime state. + */ + reload(servers: MCPServer[]): void { + this.servers.clear(); + for (const server of servers) { + this.servers.set(server.name, { ...server, enabled: true }); + } + } + + /** + * Persist current server list to config.json. + * Reads the file, merges mcp.servers from internal state, and writes back. + * Called after every mutation (addServer, removeServer, toggleServer). + */ + private persistToConfig(): void { + let raw: string; + try { + raw = readFileSync(this.configPath, 'utf-8'); + } catch (err) { + throw new Error( + `McpRegistry: cannot read config file "${this.configPath}": ${(err as Error).message}`, + ); + } + + let parsed: Record; + try { + parsed = JSON.parse(raw) as Record; + } catch (err) { + throw new Error( + `McpRegistry: config file "${this.configPath}" is not valid JSON: ${(err as Error).message}`, + ); + } + + // Build MCPServer array from internal state, stripping the runtime-only `enabled` field + const servers: MCPServer[] = Array.from(this.servers.values()).map((entry) => { + const server: MCPServer = { name: entry.name, command: entry.command }; + if (entry.args !== undefined) server.args = entry.args; + if (entry.env !== undefined) server.env = entry.env; + return server; + }); + + // Preserve existing mcp fields (enabled, configPath) and replace servers + const existingMcp = + typeof parsed['mcp'] === 'object' && parsed['mcp'] !== null + ? (parsed['mcp'] as Record) + : {}; + parsed['mcp'] = { ...existingMcp, servers }; + + writeFileSync(this.configPath, JSON.stringify(parsed, null, 2), 'utf-8'); + } +} diff --git a/src/core/media-manager.ts b/src/core/media-manager.ts new file mode 100644 index 00000000..f64cbb25 --- /dev/null +++ b/src/core/media-manager.ts @@ -0,0 +1,196 @@ +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { createLogger } from './logger.js'; + +const logger = createLogger('media-manager'); + +/** Default TTL for media files: 1 hour in milliseconds */ +const DEFAULT_TTL_MS = 60 * 60 * 1000; + +/** Default size cap for the media directory: 100 MB in bytes */ +const DEFAULT_SIZE_CAP_BYTES = 100 * 1024 * 1024; + +/** Maps MIME types to file extensions */ +const MIME_TO_EXT: Record = { + 'image/jpeg': '.jpg', + 'image/jpg': '.jpg', + 'image/png': '.png', + 'image/gif': '.gif', + 'image/webp': '.webp', + 'image/heic': '.heic', + 'image/heif': '.heif', + 'image/bmp': '.bmp', + 'image/tiff': '.tiff', + 'image/svg+xml': '.svg', + 'video/mp4': '.mp4', + 'video/mpeg': '.mpeg', + 'video/quicktime': '.mov', + 'video/webm': '.webm', + 'video/ogg': '.ogv', + 'audio/mpeg': '.mp3', + 'audio/mp4': '.m4a', + 'audio/ogg': '.oga', + 'audio/wav': '.wav', + 'audio/webm': '.weba', + 'audio/opus': '.opus', + 'application/pdf': '.pdf', + 'application/msword': '.doc', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx', + 'application/vnd.ms-excel': '.xls', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx', + 'application/zip': '.zip', + 'text/plain': '.txt', + 'text/csv': '.csv', +}; + +/** Result of saving a media file */ +export interface SaveMediaResult { + filePath: string; + sizeBytes: number; +} + +/** + * MediaManager — manages a TTL-based, size-capped temp directory for incoming media. + * + * Files are saved to `/.openbridge/media/` with UUID-based names. + * Expired files (older than TTL) and excess files (when total size > cap) are + * removed by `cleanExpired()`. + */ +export class MediaManager { + private readonly mediaDir: string; + private readonly ttlMs: number; + private readonly sizeCapBytes: number; + + constructor( + workspacePath: string, + ttlMs: number = DEFAULT_TTL_MS, + sizeCapBytes: number = DEFAULT_SIZE_CAP_BYTES, + ) { + this.mediaDir = path.join(workspacePath, '.openbridge', 'media'); + this.ttlMs = ttlMs; + this.sizeCapBytes = sizeCapBytes; + } + + /** Returns the path to the managed media directory */ + get directory(): string { + return this.mediaDir; + } + + /** + * Save a media buffer to disk with a generated unique filename. + * + * @param data Raw media bytes + * @param mimeType MIME type (used to derive the file extension) + * @param filename Optional original filename; if provided, its extension is preferred + * @returns The absolute file path and size in bytes + */ + async saveMedia(data: Buffer, mimeType: string, filename?: string): Promise { + await fs.mkdir(this.mediaDir, { recursive: true }); + + const ext = this.resolveExtension(mimeType, filename); + const uniqueName = `${Date.now()}-${randomUUID()}${ext}`; + const filePath = path.join(this.mediaDir, uniqueName); + + await fs.writeFile(filePath, data); + + const sizeBytes = data.length; + logger.debug({ filePath, sizeBytes, mimeType }, 'Media file saved'); + + return { filePath, sizeBytes }; + } + + /** + * Remove expired files (older than TTL) and trim the directory if total size + * exceeds the size cap (oldest files deleted first). + */ + async cleanExpired(): Promise { + let entries: Array<{ filePath: string; mtime: Date; size: number }>; + try { + entries = await this.listEntries(); + } catch { + // Directory doesn't exist yet — nothing to clean + return; + } + + const cutoff = Date.now() - this.ttlMs; + let totalSize = 0; + const keepers: Array<{ filePath: string; mtime: Date; size: number }> = []; + + for (const entry of entries) { + if (entry.mtime.getTime() < cutoff) { + await this.deleteFile(entry.filePath); + } else { + totalSize += entry.size; + keepers.push(entry); + } + } + + // Enforce size cap — delete oldest keepers first + if (totalSize > this.sizeCapBytes) { + const sorted = [...keepers].sort((a, b) => a.mtime.getTime() - b.mtime.getTime()); + for (const entry of sorted) { + if (totalSize <= this.sizeCapBytes) break; + await this.deleteFile(entry.filePath); + totalSize -= entry.size; + } + } + } + + // --------------------------------------------------------------------------- + // Private helpers + // --------------------------------------------------------------------------- + + /** Derive a file extension from the MIME type or the original filename */ + private resolveExtension(mimeType: string, filename?: string): string { + if (filename) { + const ext = path.extname(filename); + if (ext) return ext.toLowerCase(); + } + + // Normalise MIME type (strip parameters like "; codecs=opus") + const baseMime = mimeType.split(';')[0]?.trim() ?? mimeType; + return MIME_TO_EXT[baseMime] ?? MIME_TO_EXT[mimeType] ?? ''; + } + + /** List all files in the media directory with their mtime and size */ + private async listEntries(): Promise> { + const names = await fs.readdir(this.mediaDir); + const results: Array<{ filePath: string; mtime: Date; size: number }> = []; + + for (const name of names) { + const filePath = path.join(this.mediaDir, name); + try { + const stat = await fs.stat(filePath); + if (stat.isFile()) { + results.push({ filePath, mtime: stat.mtime, size: stat.size }); + } + } catch { + // File may have been removed concurrently — skip + } + } + + return results; + } + + /** Delete a file silently (ignore missing-file errors) */ + private async deleteFile(filePath: string): Promise { + try { + await fs.unlink(filePath); + logger.debug({ filePath }, 'Media file deleted'); + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'ENOENT') { + logger.warn({ filePath, err }, 'Failed to delete media file'); + } + } + } +} + +/** + * Factory function for creating a MediaManager with default settings. + * + * @param workspacePath Absolute path to the target workspace + */ +export function createMediaManager(workspacePath: string): MediaManager { + return new MediaManager(workspacePath); +} diff --git a/src/core/metrics.ts b/src/core/metrics.ts index 387dd68e..c15818d1 100644 --- a/src/core/metrics.ts +++ b/src/core/metrics.ts @@ -32,6 +32,26 @@ export interface MetricsSnapshot { transient: number; permanent: number; }; + promptSize: { + /** Total number of agent runs that measured a prompt size. */ + runs: number; + /** Number of runs where the prompt was truncated (> limit). */ + truncatedRuns: number; + /** Chars of the most recent prompt measured. */ + lastChars: number; + /** Limit (chars) used for the most recent prompt. */ + lastLimit: number; + /** Truncation percentage of the most recent prompt (0 if not truncated). */ + lastTruncatedPct: number; + /** Maximum prompt size (chars) seen across all runs. */ + maxChars: number; + /** Average prompt size (chars) across all runs. */ + avgChars: number; + }; + costCapped: { + /** Total number of workers killed because they exceeded their cost cap. */ + total: number; + }; } export type MetricsDataProvider = () => MetricsSnapshot; @@ -74,6 +94,18 @@ export class MetricsCollector { private _errorsTransient = 0; private _errorsPermanent = 0; + // Cost-capped worker counter + private _costCapped = 0; + + // Prompt-size tracking + private _promptRuns = 0; + private _promptTruncatedRuns = 0; + private _promptLastChars = 0; + private _promptLastLimit = 0; + private _promptLastTruncatedPct = 0; + private _promptMaxChars = 0; + private _promptTotalChars = 0; + recordReceived(): void { this._received++; } @@ -117,6 +149,28 @@ export class MetricsCollector { this._deadLettered++; } + /** Increment the cost-capped worker counter. */ + recordCostCapped(): void { + this._costCapped++; + } + + /** + * Record prompt-size statistics for a single agent run. + * + * @param chars - Actual prompt size in characters (after sanitization, before truncation) + * @param limit - The max-length budget applied to this prompt + * @param truncatedPct - Percentage of the prompt lost to truncation (0 when not truncated) + */ + recordPromptSize(chars: number, limit: number, truncatedPct: number): void { + this._promptRuns++; + this._promptLastChars = chars; + this._promptLastLimit = limit; + this._promptLastTruncatedPct = truncatedPct; + this._promptTotalChars += chars; + if (chars > this._promptMaxChars) this._promptMaxChars = chars; + if (truncatedPct > 0) this._promptTruncatedRuns++; + } + snapshot(): MetricsSnapshot { return { uptime: Math.floor((Date.now() - this.startedAt) / 1000), @@ -146,6 +200,18 @@ export class MetricsCollector { transient: this._errorsTransient, permanent: this._errorsPermanent, }, + promptSize: { + runs: this._promptRuns, + truncatedRuns: this._promptTruncatedRuns, + lastChars: this._promptLastChars, + lastLimit: this._promptLastLimit, + lastTruncatedPct: this._promptLastTruncatedPct, + maxChars: this._promptMaxChars, + avgChars: this._promptRuns > 0 ? Math.round(this._promptTotalChars / this._promptRuns) : 0, + }, + costCapped: { + total: this._costCapped, + }, }; } } diff --git a/src/core/model-registry.ts b/src/core/model-registry.ts new file mode 100644 index 00000000..eb8922e7 --- /dev/null +++ b/src/core/model-registry.ts @@ -0,0 +1,228 @@ +/** + * Model Registry — Provider-Agnostic Model Selection + * + * Maps abstract capability tiers (fast / balanced / powerful) to concrete + * model IDs for whatever AI provider is active. This decouples the model + * selection logic from any specific provider (Claude, Codex, Aider, etc.). + * + * Usage: + * const registry = new ModelRegistry(); + * registry.registerProvider('claude'); // loads Claude defaults + * registry.resolve('fast'); // → { id: 'haiku', tier: 'fast', provider: 'claude' } + * registry.resolveModelOrTier('balanced'); // → 'sonnet' + * registry.resolveModelOrTier('gpt-4o'); // → 'gpt-4o' (passthrough) + */ + +import { createLogger } from './logger.js'; + +const logger = createLogger('model-registry'); + +// ── Types ─────────────────────────────────────────────────────── + +/** Capability tiers — provider-agnostic */ +export type ModelTier = 'fast' | 'balanced' | 'powerful'; + +export const MODEL_TIERS: readonly ModelTier[] = ['fast', 'balanced', 'powerful'] as const; + +/** A single model entry in a provider's model map */ +export interface ModelEntry { + /** Actual model ID passed to the CLI (e.g. 'haiku', 'codex-mini', 'gpt-4o-mini') */ + id: string; + /** Capability tier this model maps to */ + tier: ModelTier; + /** Provider name (e.g. 'claude', 'codex', 'aider') */ + provider: string; + /** Model context window size in tokens (optional — Claude-specific) */ + contextTokens?: number; + /** Maximum output tokens the model can produce (optional — Claude-specific) */ + maxOutputTokens?: number; +} + +/** Tier-based fallback chain: powerful → balanced → fast → (none) */ +export const TIER_FALLBACK: Record = { + powerful: 'balanced', + balanced: 'fast', + fast: undefined, +}; + +// ── Default Model Maps ────────────────────────────────────────── + +/** Built-in model maps for known providers */ +const DEFAULT_MODEL_MAPS: Record = { + claude: [ + { + id: 'haiku', + tier: 'fast', + provider: 'claude', + contextTokens: 200_000, + maxOutputTokens: 64_000, + }, + { + id: 'sonnet', + tier: 'balanced', + provider: 'claude', + contextTokens: 1_000_000, + maxOutputTokens: 64_000, + }, + { + id: 'opus', + tier: 'powerful', + provider: 'claude', + contextTokens: 1_000_000, + maxOutputTokens: 128_000, + }, + ], + codex: [ + // ChatGPT-account auth only supports gpt-5.2-codex (the default). + // o3, o4-mini, codex-mini are rejected with "not supported when using Codex with a ChatGPT account". + // All tiers map to the same model until Codex expands ChatGPT-auth model support. + { id: 'gpt-5.2-codex', tier: 'fast', provider: 'codex' }, + { id: 'gpt-5.2-codex', tier: 'balanced', provider: 'codex' }, + // gpt-5.3-codex is 25% faster at the same pricing as gpt-5.2-codex. + { id: 'gpt-5.3-codex', tier: 'powerful', provider: 'codex' }, + ], + aider: [ + { id: 'gpt-4o-mini', tier: 'fast', provider: 'aider' }, + { id: 'gpt-4.1', tier: 'balanced', provider: 'aider' }, + { id: 'o3', tier: 'powerful', provider: 'aider' }, + ], +}; + +// ── Registry ──────────────────────────────────────────────────── + +export class ModelRegistry { + private models: ModelEntry[] = []; + + /** + * Register a provider's models. If no custom entries are provided, + * loads from the built-in defaults for the provider name. + * Unknown providers with no custom entries result in an empty registry. + */ + registerProvider(provider: string, models?: ModelEntry[]): void { + const entries = models ?? DEFAULT_MODEL_MAPS[provider]; + + if (!entries) { + logger.warn({ provider }, 'No default model map for provider — registry will be empty'); + return; + } + + // Remove any existing entries for this provider + this.models = this.models.filter((m) => m.provider !== provider); + this.models.push(...entries); + + logger.debug( + { provider, modelCount: entries.length, models: entries.map((m) => m.id) }, + 'Registered provider models', + ); + } + + /** + * Resolve a capability tier to a concrete model entry. + * Returns the first matching entry, or undefined if no model is registered for that tier. + */ + resolve(tier: ModelTier): ModelEntry | undefined { + return this.models.find((m) => m.tier === tier); + } + + /** + * Resolve a string that could be either a tier name, a raw model ID, or a + * foreign provider's model alias (e.g. "haiku" passed to a codex registry). + * + * Resolution order: + * 1. Tier name ('fast', 'balanced', 'powerful') → resolve to this provider's model + * 2. Known model from another provider → translate to equivalent tier's model + * 3. Unknown string → pass through as-is (backward compat) + */ + resolveModelOrTier(value: string): string { + // 1. Tier names resolve directly + if (MODEL_TIERS.includes(value as ModelTier)) { + const entry = this.resolve(value as ModelTier); + return entry?.id ?? value; + } + + // 2. If this model belongs to the current provider, pass through + if (this.models.some((m) => m.id === value)) { + return value; + } + + // 3. Check if it's a known model from another provider — translate via tier + const foreignEntry = findForeignModel(value); + if (foreignEntry) { + const localEntry = this.resolve(foreignEntry.tier); + if (localEntry) { + logger.debug( + { from: value, to: localEntry.id, tier: foreignEntry.tier }, + 'Cross-provider model translation', + ); + return localEntry.id; + } + } + + // 4. Unknown — pass through as-is + return value; + } + + /** + * Get the fallback model for a given model ID. + * Finds the model's tier, then looks up the next tier in the fallback chain. + * Returns undefined if no fallback exists (fast tier) or model is unknown. + */ + getFallback(modelId: string): string | undefined { + const entry = this.models.find((m) => m.id === modelId); + if (!entry) return undefined; + + const nextTier = TIER_FALLBACK[entry.tier]; + if (!nextTier) return undefined; + + const fallback = this.resolve(nextTier); + return fallback?.id; + } + + /** + * Check if a model string is valid (known in the registry). + */ + isValid(model: string): boolean { + // Tier names are always valid + if (MODEL_TIERS.includes(model as ModelTier)) return true; + // Check against registered models + return this.models.some((m) => m.id === model); + } + + /** Get all registered model entries */ + getAll(): ModelEntry[] { + return [...this.models]; + } + + /** Get the model entry for a specific tier, with tier fallback if not found */ + resolveWithFallback(tier: ModelTier): ModelEntry | undefined { + const entry = this.resolve(tier); + if (entry) return entry; + + const nextTier = TIER_FALLBACK[tier]; + if (!nextTier) return undefined; + + return this.resolveWithFallback(nextTier); + } +} + +/** + * Look up a model ID across all known providers' default model maps. + * Returns the matching entry (with tier info) if found, undefined otherwise. + */ +function findForeignModel(modelId: string): ModelEntry | undefined { + for (const entries of Object.values(DEFAULT_MODEL_MAPS)) { + const entry = entries.find((m) => m.id === modelId); + if (entry) return entry; + } + return undefined; +} + +/** + * Create a ModelRegistry pre-loaded with a provider's defaults. + * Convenience factory for the common case. + */ +export function createModelRegistry(provider: string, customModels?: ModelEntry[]): ModelRegistry { + const registry = new ModelRegistry(); + registry.registerProvider(provider, customModels); + return registry; +} diff --git a/src/core/model-selector.ts b/src/core/model-selector.ts new file mode 100644 index 00000000..bcabe707 --- /dev/null +++ b/src/core/model-selector.ts @@ -0,0 +1,355 @@ +/** + * Model Selection Strategy + * + * Given a task description and tool profile, recommends a model. + * Uses capability tiers (fast / balanced / powerful) resolved through + * the ModelRegistry to stay provider-agnostic. + * + * Rules: + * - read-only tasks → fast tier (exploration, information gathering) + * - code-edit tasks → balanced tier (implementation, modification) + * - complex reasoning → powerful tier (architecture, debugging, multi-step logic) + * + * The Master AI can call this or ignore it. An explicit model in the + * TaskManifest always takes priority over the recommendation. + */ + +import type { TaskManifest } from '../types/agent.js'; +import type { MemoryManager } from '../memory/index.js'; +import type { LearningEntry } from '../types/master.js'; +import { createModelRegistry } from './model-registry.js'; +import type { ModelTier, ModelRegistry } from './model-registry.js'; +import { createLogger } from './logger.js'; + +const logger = createLogger('model-selector'); + +/** Lazily-created default registry (Claude) for backward compatibility */ +let defaultRegistry: ModelRegistry | null = null; +function getDefaultRegistry(): ModelRegistry { + if (!defaultRegistry) { + defaultRegistry = createModelRegistry('claude'); + } + return defaultRegistry; +} + +/** Resolve a tier to a model ID, falling back through tiers if needed */ +function resolveTier(tier: ModelTier, registry: ModelRegistry): string { + const entry = registry.resolveWithFallback(tier); + return entry?.id ?? tier; +} + +/** + * Keywords that signal complex reasoning (→ powerful tier). + * Matched case-insensitively against the task description. + */ +const COMPLEX_KEYWORDS = [ + 'architect', + 'debug', + 'refactor', + 'redesign', + 'optimize', + 'security', + 'vulnerability', + 'performance', + 'migration', + 'complex', + 'design', + 'strategy', + 'analyze', + 'investigate', + 'diagnose', +] as const; + +/** + * Keywords that signal code editing (→ balanced tier). + * Matched case-insensitively against the task description. + */ +const CODE_EDIT_KEYWORDS = [ + 'implement', + 'create', + 'add', + 'update', + 'modify', + 'fix', + 'write', + 'change', + 'build', + 'edit', + 'remove', + 'delete', + 'replace', + 'rename', + 'test', +] as const; + +export interface ModelRecommendation { + /** The recommended model ID (provider-specific, resolved from tier) */ + model: string; + /** Why this model was chosen */ + reason: string; +} + +/** + * Recommend a model based on the tool profile name. + * + * - 'read-only' → fast tier (cheap, fast) + * - 'code-edit' → balanced tier (implementation) + * - 'full-access' → balanced tier (capability, not complexity) + * - unknown → balanced tier (safe default) + */ +export function recommendByProfile(profile: string, registry?: ModelRegistry): ModelRecommendation { + const reg = registry ?? getDefaultRegistry(); + + switch (profile) { + case 'read-only': + return { model: resolveTier('fast', reg), reason: 'read-only profile — fast and cheap' }; + case 'code-edit': + return { + model: resolveTier('balanced', reg), + reason: 'code-edit profile — balanced for implementation', + }; + case 'full-access': + return { + model: resolveTier('balanced', reg), + reason: 'full-access profile — balanced default', + }; + default: + return { + model: resolveTier('balanced', reg), + reason: `unknown profile "${profile}" — defaulting to balanced`, + }; + } +} + +/** + * Recommend a model based on the task description. + * Scans for keywords that indicate complexity level. + * + * - Complex reasoning keywords → powerful tier + * - Code editing keywords → balanced tier + * - Everything else (exploration, listing) → fast tier + */ +export function recommendByDescription( + description: string, + registry?: ModelRegistry, +): ModelRecommendation { + const reg = registry ?? getDefaultRegistry(); + const lower = description.toLowerCase(); + + for (const keyword of COMPLEX_KEYWORDS) { + if (lower.includes(keyword)) { + return { + model: resolveTier('powerful', reg), + reason: `description contains "${keyword}" — complex reasoning`, + }; + } + } + + for (const keyword of CODE_EDIT_KEYWORDS) { + if (lower.includes(keyword)) { + return { + model: resolveTier('balanced', reg), + reason: `description contains "${keyword}" — code editing`, + }; + } + } + + return { model: resolveTier('fast', reg), reason: 'no complexity signals — fast default' }; +} + +/** Minimum completed tasks required before trusting learning data. */ +const MIN_TASKS_FOR_LEARNING = 5; + +/** Minimum tasks for a specific model before acting on its failure rate. */ +const MIN_TASKS_FOR_AVOIDANCE = 3; + +/** + * Query the learnings store for the best model for a given task type. + * + * Returns a ModelRecommendation when the learnings table has at least + * MIN_TASKS_FOR_LEARNING completed tasks for the given task type. + * Returns null when there is insufficient data or on any error, so the + * caller can fall back to heuristic selection. + */ +export async function getRecommendedModel( + memory: MemoryManager, + taskType: string, +): Promise { + try { + const learned = await memory.getLearnedParams(taskType); + if (!learned || learned.total_tasks < MIN_TASKS_FOR_LEARNING) { + logger.debug( + { taskType, total_tasks: learned?.total_tasks ?? 0, min: MIN_TASKS_FOR_LEARNING }, + 'Insufficient learning data — skipping adaptive model selection', + ); + return null; + } + const successPct = (learned.success_rate * 100).toFixed(0); + logger.debug( + { taskType, model: learned.model, success_rate: learned.success_rate }, + 'Adaptive model selected from learnings', + ); + return { + model: learned.model, + reason: `learned: ${successPct}% success rate over ${learned.total_tasks} tasks`, + }; + } catch { + // No data or memory error — caller will use heuristic fallback + return null; + } +} + +/** + * If the given model has a >50% failure rate for the task type (with at least + * MIN_TASKS_FOR_AVOIDANCE samples), return a ModelRecommendation for a better model. + * + * Returns null when: + * - The model's failure rate is acceptable (≤50%) + * - There is insufficient data for the (taskType, model) pair + * - No better alternative can be found in learnings + */ +export async function avoidHighFailureModel( + memory: MemoryManager, + taskType: string, + currentModel: string, +): Promise { + try { + const stats = await memory.getModelStatsForTask(taskType, currentModel); + if (!stats || stats.total_tasks < MIN_TASKS_FOR_AVOIDANCE) { + return null; // Insufficient data — do not penalise + } + + const failureRate = 1 - stats.success_rate; + if (failureRate <= 0.5) { + return null; // Model is performing acceptably + } + + logger.debug( + { + taskType, + currentModel, + failureRate: failureRate.toFixed(2), + totalTasks: stats.total_tasks, + }, + 'Model has >50% failure rate — seeking alternative', + ); + + // Find the best-performing alternative from learnings + const best = await getRecommendedModel(memory, taskType); + if (best && best.model !== currentModel) { + return { + model: best.model, + reason: `avoided "${currentModel}" (${(failureRate * 100).toFixed(0)}% failure rate for "${taskType}") — using "${best.model}" instead`, + }; + } + + return null; // No better alternative available + } catch { + return null; // Non-blocking — caller continues with original model + } +} + +/** + * Recommend a model based on historical learnings for a specific task type. + * Requires 5+ entries for the task type and 3+ uses per model to be statistically meaningful. + * Returns null if insufficient data — caller should fall back to heuristics. + * + * This function is already provider-agnostic: it picks the best model from + * actual performance data regardless of provider. + */ +export function recommendFromLearnings( + taskType: string, + learnings: LearningEntry[], +): ModelRecommendation | null { + const filtered = learnings.filter((e) => e.taskType === taskType && e.modelUsed); + if (filtered.length < 5) return null; + + // Group by model and compute success rate + const modelStats = new Map(); + for (const entry of filtered) { + const model = entry.modelUsed!; + const stats = modelStats.get(model) ?? { total: 0, success: 0 }; + stats.total++; + if (entry.success) stats.success++; + modelStats.set(model, stats); + } + + // Find the model with the highest success rate (min 3 uses) + let bestModel: string | null = null; + let bestRate = -1; + for (const [model, stats] of modelStats) { + if (stats.total < 3) continue; + const rate = stats.success / stats.total; + if (rate > bestRate) { + bestRate = rate; + bestModel = model; + } + } + + if (!bestModel) return null; + + const stats = modelStats.get(bestModel)!; + const pct = ((stats.success / stats.total) * 100).toFixed(0); + + logger.debug( + { taskType, model: bestModel, successRate: pct, sampleSize: stats.total }, + 'Model recommended from learnings', + ); + + return { + model: bestModel, + reason: `historical performance for "${taskType}" tasks: ${pct}% success (${stats.total} samples)`, + }; +} + +/** + * Recommend a model for a TaskManifest. + * + * Priority: + * 1. If `manifest.model` is set, return it as-is (explicit override wins) + * 2. If learnings data is available, use data-driven recommendation + * 3. If `manifest.profile` is set, use profile-based recommendation + * 4. Fall back to description-based recommendation using the prompt + * + * Returns a ModelRecommendation. The caller decides whether to use it. + */ +export function recommendModel( + manifest: TaskManifest, + options?: { + learnings?: LearningEntry[]; + taskType?: string; + registry?: ModelRegistry; + }, +): ModelRecommendation { + const registry = options?.registry; + + // Explicit model override — respect the caller's choice + if (manifest.model) { + logger.debug({ model: manifest.model }, 'Model explicitly set in manifest — using as-is'); + return { + model: manifest.model, + reason: 'explicitly set in manifest', + }; + } + + // Data-driven recommendation from learnings + if (options?.learnings && options.taskType) { + const rec = recommendFromLearnings(options.taskType, options.learnings); + if (rec) return rec; + } + + // Profile-based recommendation + if (manifest.profile) { + const rec = recommendByProfile(manifest.profile, registry); + logger.debug( + { profile: manifest.profile, recommended: rec.model }, + 'Model recommended by profile', + ); + return rec; + } + + // Description-based recommendation from the prompt + const rec = recommendByDescription(manifest.prompt, registry); + logger.debug({ recommended: rec.model, reason: rec.reason }, 'Model recommended by description'); + return rec; +} diff --git a/src/core/ngrok-adapter.ts b/src/core/ngrok-adapter.ts new file mode 100644 index 00000000..12489de5 --- /dev/null +++ b/src/core/ngrok-adapter.ts @@ -0,0 +1,95 @@ +/** + * NgrokAdapter — TunnelAdapter for ngrok. + * + * Spawns: ngrok http {port} [--authtoken ] [--subdomain ] + * URL detection: Parses "Forwarding https://..." from stdout, or queries the + * ngrok local API at http://127.0.0.1:4040/api/tunnels (whichever resolves first). + * + * Auth token: + * - Pass via TunnelConfig.authToken (takes precedence) + * - OR set the NGROK_AUTHTOKEN environment variable (ngrok picks it up automatically) + * - Free accounts work without a token for basic tunnels + * + * Auto-registers with TunnelManager when this module is imported. + */ + +import type { TunnelAdapter, TunnelConfig } from './tunnel-manager.js'; +import { TunnelManager } from './tunnel-manager.js'; + +/** Regex to match ngrok v3 domain (ngrok-free.app) */ +const NGROK_V3_URL_RE = /https:\/\/[a-zA-Z0-9-]+\.ngrok-free\.app/; + +/** Regex to match ngrok v2 / paid custom domains (ngrok.io or ngrok.app) */ +const NGROK_LEGACY_URL_RE = /https:\/\/[a-zA-Z0-9-]+\.ngrok(?:\.io|\.app)/; + +interface NgrokTunnel { + public_url: string; + proto: string; +} + +interface NgrokTunnelsResponse { + tunnels: NgrokTunnel[]; +} + +export class NgrokAdapter implements TunnelAdapter { + readonly toolName = 'ngrok'; + + /** + * Build CLI args for an ngrok HTTP tunnel. + * Produces: ngrok http {port} [--authtoken ] [--subdomain ] + * + * Auth token is optional for free tunnels. If provided via config, it is + * passed explicitly so the adapter works without a global ngrok auth setup. + * Subdomain requires a paid ngrok plan. + */ + buildArgs(port: number, config?: TunnelConfig): string[] { + const args: string[] = ['http', String(port)]; + if (config?.authToken) { + args.push('--authtoken', config.authToken); + } + if (config?.subdomain) { + args.push('--subdomain', config.subdomain); + } + return args; + } + + /** + * Extract the public HTTPS URL from one line of ngrok stdout/stderr. + * + * ngrok v2 / v3 announce the tunnel URL in a Forwarding line: + * Forwarding https://abc123.ngrok-free.app -> http://localhost:3001 + * Forwarding https://abc123.ngrok.io -> http://localhost:3001 + * + * Returns the URL string if found, null otherwise. + */ + parseUrl(line: string): string | null { + const matchV3 = NGROK_V3_URL_RE.exec(line); + if (matchV3 !== null) return matchV3[0]; + + const matchLegacy = NGROK_LEGACY_URL_RE.exec(line); + return matchLegacy !== null ? matchLegacy[0] : null; + } + + /** + * Query the ngrok local API at http://127.0.0.1:4040/api/tunnels for the + * public HTTPS URL. ngrok exposes this API server while running. + * + * This is polled in parallel with stdout parsing by TunnelManager. + * Returns the HTTPS tunnel URL, or null if the API is not yet available. + */ + async fetchUrl(_port: number): Promise { + try { + const response = await fetch('http://127.0.0.1:4040/api/tunnels'); + if (!response.ok) return null; + const data = (await response.json()) as NgrokTunnelsResponse; + const httpsTunnel = data.tunnels.find((t) => t.proto === 'https'); + return httpsTunnel?.public_url ?? null; + } catch { + // API not available yet (ngrok still starting) — return null to continue polling + return null; + } + } +} + +// Auto-register when this module is imported +TunnelManager.registerAdapter(new NgrokAdapter()); diff --git a/src/core/output-marker-processor.ts b/src/core/output-marker-processor.ts new file mode 100644 index 00000000..72544d43 --- /dev/null +++ b/src/core/output-marker-processor.ts @@ -0,0 +1,825 @@ +/** OutputMarkerProcessor — extracted from Router (OB-1284, OB-F159). */ + +import path from 'node:path'; +import { readFile } from 'node:fs/promises'; +import type { Connector } from '../types/connector.js'; +import type { OutboundMessage } from '../types/message.js'; +import type { EmailConfig } from '../types/config.js'; +import type { AuthService } from './auth.js'; +import type { AppServer } from './app-server.js'; +import type { InteractionRelay } from './interaction-relay.js'; +import type { FileServer } from './file-server.js'; +import { sendEmail } from './email-sender.js'; +import { publishToGitHubPages } from './github-publisher.js'; +import { createLogger } from './logger.js'; +import type { WorkflowStore } from '../workflows/workflow-store.js'; +import type { WorkflowEngine } from '../workflows/engine.js'; +import type { WorkflowScheduler } from '../workflows/scheduler.js'; +import { WorkflowSchema } from '../types/workflow.js'; +import type { IntegrationHub } from '../integrations/hub.js'; + +const logger = createLogger('output-marker-processor'); + +// --------------------------------------------------------------------------- +// Regex patterns for output markers +// --------------------------------------------------------------------------- + +/** Pattern matching [SEND:channel]recipient|content[/SEND] markers in AI output */ +export const SEND_MARKER_RE = /\[SEND:([^\]]+)\]([^|]+)\|([^[]*)\[\/SEND\]/g; + +/** Pattern matching [VOICE]text[/VOICE] markers in AI output */ +export const VOICE_MARKER_RE = /\[VOICE\]([\s\S]*?)\[\/VOICE\]/g; + +/** Pattern matching [SHARE:channel]/path/to/file[/SHARE] markers in AI output */ +export const SHARE_MARKER_RE = /\[SHARE:([^\]]+)\]([^[]*)\[\/SHARE\]/g; + +/** Pattern matching [APP:start]appPath[/APP] markers in AI output */ +export const APP_START_MARKER_RE = /\[APP:start\]([^[]*)\[\/APP\]/g; + +/** Pattern matching [APP:stop]appId[/APP] markers in AI output */ +export const APP_STOP_MARKER_RE = /\[APP:stop\]([^[]*)\[\/APP\]/g; + +/** Pattern matching [APP:update:appId]jsonData[/APP] markers in AI output */ +export const APP_UPDATE_MARKER_RE = /\[APP:update:([^\]]+)\]([^[]*)\[\/APP\]/g; + +/** Pattern matching [WORKFLOW:create]{...json...}[/WORKFLOW] markers in AI output */ +export const WORKFLOW_CREATE_MARKER_RE = /\[WORKFLOW:create\]([\s\S]*?)\[\/WORKFLOW\]/g; + +/** Channels that are remote (phone/desktop app) and cannot access localhost URLs */ +export const REMOTE_CHANNELS = new Set(['telegram', 'whatsapp', 'discord']); + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** Map file extension to MIME type and media category */ +export function getMimeType(filename: string): { + mimeType: string; + mediaType: 'document' | 'image' | 'audio' | 'video'; +} { + const ext = filename.split('.').pop()?.toLowerCase() ?? ''; + const mimeMap: Record< + string, + { mimeType: string; mediaType: 'document' | 'image' | 'audio' | 'video' } + > = { + pdf: { mimeType: 'application/pdf', mediaType: 'document' }, + html: { mimeType: 'text/html', mediaType: 'document' }, + htm: { mimeType: 'text/html', mediaType: 'document' }, + txt: { mimeType: 'text/plain', mediaType: 'document' }, + csv: { mimeType: 'text/csv', mediaType: 'document' }, + json: { mimeType: 'application/json', mediaType: 'document' }, + md: { mimeType: 'text/markdown', mediaType: 'document' }, + docx: { + mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + mediaType: 'document', + }, + xlsx: { + mimeType: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + mediaType: 'document', + }, + pptx: { + mimeType: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + mediaType: 'document', + }, + png: { mimeType: 'image/png', mediaType: 'image' }, + jpg: { mimeType: 'image/jpeg', mediaType: 'image' }, + jpeg: { mimeType: 'image/jpeg', mediaType: 'image' }, + gif: { mimeType: 'image/gif', mediaType: 'image' }, + webp: { mimeType: 'image/webp', mediaType: 'image' }, + svg: { mimeType: 'image/svg+xml', mediaType: 'image' }, + mp4: { mimeType: 'video/mp4', mediaType: 'video' }, + mp3: { mimeType: 'audio/mpeg', mediaType: 'audio' }, + wav: { mimeType: 'audio/wav', mediaType: 'audio' }, + }; + return mimeMap[ext] ?? { mimeType: 'application/octet-stream', mediaType: 'document' }; +} + +// --------------------------------------------------------------------------- +// Dependency interface — mirrors Router's mutable state via getters +// --------------------------------------------------------------------------- + +export interface OutputMarkerDeps { + getWorkspacePath: () => string | undefined; + getEmailConfig: () => EmailConfig | undefined; + getFileServer: () => FileServer | undefined; + getAppServer: () => AppServer | undefined; + getRelay: () => InteractionRelay | undefined; + getConnectors: () => Map; + getAuth: () => AuthService | undefined; + getWorkflowStore: () => WorkflowStore | undefined; + getWorkflowEngine: () => WorkflowEngine | undefined; + getWorkflowScheduler: () => WorkflowScheduler | undefined; + getIntegrationHub: () => IntegrationHub | undefined; + /** On-demand tunnel starter — used for remote channel APP:start delivery (OB-1633) */ + ensureTunnel?: () => Promise; +} + +// --------------------------------------------------------------------------- +// OutputMarkerProcessor class +// --------------------------------------------------------------------------- + +export class OutputMarkerProcessor { + constructor(private readonly deps: OutputMarkerDeps) {} + + /** + * Process all output markers in sequence: WORKFLOW → SHARE → APP → SEND → VOICE. + * Returns the cleaned content with all markers stripped or replaced. + */ + async processAll( + content: string, + connector: Connector, + recipient: string, + replyTo?: string, + source?: string, + ): Promise { + const afterWorkflow = await this.processWorkflowMarkers(content); + const afterShare = await this.processShareMarkers( + afterWorkflow, + connector, + recipient, + replyTo, + source, + ); + const afterApp = await this.processAppMarkers(afterShare, source); + const afterSend = await this.processSendMarkers(afterApp); + return this.processVoiceMarkers(afterSend, connector, recipient); + } + + /** + * Parse [WORKFLOW:create]{...json...}[/WORKFLOW] markers from AI output. + * Creates workflow definitions via WorkflowStore and schedules them if they + * have a schedule trigger. Replaces markers with confirmation text. + */ + async processWorkflowMarkers(content: string): Promise { + const store = this.deps.getWorkflowStore(); + if (!store) return content.replace(new RegExp(WORKFLOW_CREATE_MARKER_RE.source, 'g'), ''); + + let cleaned = content; + const regex = new RegExp(WORKFLOW_CREATE_MARKER_RE.source, 'g'); + let match: RegExpExecArray | null; + + while ((match = regex.exec(content)) !== null) { + const fullMatch = match[0]; + const jsonBody = (match[1] ?? '').trim(); + + if (!jsonBody) { + cleaned = cleaned.replace(fullMatch, ''); + continue; + } + + try { + const raw = JSON.parse(jsonBody) as Record; + + // Generate an ID if not provided + if (!raw['id']) { + const { randomUUID } = await import('node:crypto'); + raw['id'] = randomUUID(); + } + + // Default status to active + if (!raw['status']) { + raw['status'] = 'active'; + } + + const workflow = WorkflowSchema.parse(raw); + store.createWorkflow(workflow); + + // Reload engine so the new workflow is available for execution + const engine = this.deps.getWorkflowEngine(); + if (engine) { + await engine.loadWorkflows(); + } + + // Schedule if it's a schedule trigger + const scheduler = this.deps.getWorkflowScheduler(); + if (scheduler && workflow.trigger.type === 'schedule' && workflow.trigger.cron) { + await scheduler.scheduleWorkflow(workflow); + } + + const replacement = `\u2705 Workflow **${workflow.name}** created (${workflow.trigger.type} trigger, ${workflow.steps.length} steps).`; + cleaned = cleaned.replace(fullMatch, replacement); + + logger.info( + { workflowId: workflow.id, name: workflow.name, trigger: workflow.trigger.type }, + 'Workflow created via WORKFLOW:create marker', + ); + } catch (err: unknown) { + const errorMsg = err instanceof Error ? err.message : String(err); + logger.warn({ error: errorMsg, jsonBody }, 'Failed to parse WORKFLOW:create marker'); + cleaned = cleaned.replace(fullMatch, `\u26a0\ufe0f Failed to create workflow: ${errorMsg}`); + } + } + + return cleaned; + } + + /** + * Parse [SEND:channel]recipient|content[/SEND] markers from AI output, + * dispatch proactive messages to whitelisted recipients, and return + * the response with markers stripped. + */ + async processSendMarkers(content: string): Promise { + let cleaned = content; + const regex = new RegExp(SEND_MARKER_RE.source, 'g'); + let match: RegExpExecArray | null; + + while ((match = regex.exec(content)) !== null) { + const fullMatch = match[0]; + const channel = match[1] ?? ''; + const recipient = match[2] ?? ''; + const body = match[3] ?? ''; + const trimmedRecipient = recipient.trim(); + const trimmedBody = body.trim(); + + if (!channel || !trimmedRecipient) { + cleaned = cleaned.replace(fullMatch, ''); + continue; + } + + const auth = this.deps.getAuth(); + if (auth && !auth.isAuthorized(trimmedRecipient, channel)) { + logger.warn( + { channel, recipient: trimmedRecipient }, + 'SEND marker blocked — recipient not in whitelist', + ); + cleaned = cleaned.replace(fullMatch, ''); + continue; + } + + const connectors = this.deps.getConnectors(); + const connector = connectors.get(channel); + if (!connector) { + logger.warn({ channel }, 'SEND marker: connector not found'); + cleaned = cleaned.replace(fullMatch, ''); + continue; + } + + if (!connector.sendProactive) { + logger.warn({ channel }, 'SEND marker: connector does not support sendProactive'); + cleaned = cleaned.replace(fullMatch, ''); + continue; + } + + try { + await connector.sendProactive(trimmedRecipient, trimmedBody); + logger.info({ channel, recipient: trimmedRecipient }, 'Proactive SEND dispatched'); + } catch (err) { + logger.warn({ channel, recipient: trimmedRecipient, err }, 'SEND marker dispatch failed'); + } + + cleaned = cleaned.replace(fullMatch, ''); + } + + return cleaned.trim(); + } + + /** + * Parse [SHARE:channel]/path/to/file[/SHARE] markers from AI output, read the file, + * validate it is under .openbridge/generated/ (security), send it as a media attachment + * to the inbound message sender, and return the response with markers stripped. + */ + async processShareMarkers( + content: string, + connector: Connector, + recipient: string, + replyTo?: string, + _source?: string, + ): Promise { + const workspacePath = this.deps.getWorkspacePath(); + if (!workspacePath) return content; + + const generatedDir = path.resolve(path.join(workspacePath, '.openbridge', 'generated')); + + let cleaned = content; + const regex = new RegExp(SHARE_MARKER_RE.source, 'g'); + let match: RegExpExecArray | null; + + while ((match = regex.exec(content)) !== null) { + const fullMatch = match[0]; + const channel = match[1] ?? ''; + let filePath = (match[2] ?? '').trim(); + + // Master AI sometimes emits a JSON object instead of a plain path + // e.g. {"path":"/Users/.../file.pdf"} — extract the actual path string + if (filePath.startsWith('{') && filePath.endsWith('}')) { + try { + const parsed = JSON.parse(filePath) as Record; + if (typeof parsed['path'] === 'string') { + filePath = parsed['path'].trim(); + } + } catch { + // Not valid JSON — fall through and use the raw value + } + } + + if (!channel || !filePath) { + cleaned = cleaned.replace(fullMatch, ''); + continue; + } + + // Resolve path: if relative, resolve against the generated dir + const resolvedPath = path.resolve( + path.isAbsolute(filePath) ? filePath : path.join(generatedDir, filePath), + ); + + // Security: file must be strictly under .openbridge/generated/ + if (!resolvedPath.startsWith(generatedDir + path.sep)) { + logger.warn( + { filePath: resolvedPath, generatedDir }, + 'SHARE marker blocked — file not under .openbridge/generated/', + ); + cleaned = cleaned.replace(fullMatch, ''); + continue; + } + + // Handle email channel separately — it doesn't route through a connector + if (channel === 'email') { + await this.handleEmailShare(filePath, recipient, replyTo); + cleaned = cleaned.replace(fullMatch, ''); + continue; + } + + // Handle github-pages channel — push file to the gh-pages branch + if (channel === 'github-pages') { + await this.handleGitHubPagesShare(resolvedPath); + cleaned = cleaned.replace(fullMatch, ''); + continue; + } + + // Handle gdrive channel — upload file to Google Drive and return shareable link + if (channel === 'gdrive') { + const shareLink = await this.handleGDriveShare(resolvedPath); + cleaned = cleaned.replace(fullMatch, shareLink ?? ''); + continue; + } + + // Handle dropbox channel — upload file to Dropbox and return shareable link + if (channel === 'dropbox') { + const shareLink = await this.handleDropboxShare(resolvedPath); + cleaned = cleaned.replace(fullMatch, shareLink ?? ''); + continue; + } + + // Handle FILE channel — create a shareable link via the local file server, + // and additionally send as a native attachment for connectors that support it (WhatsApp, Telegram). + if (channel === 'FILE') { + const fileServer = this.deps.getFileServer(); + if (fileServer) { + const filename = path.basename(resolvedPath); + try { + const url = await fileServer.createShareableLink(filename); + cleaned = cleaned.replace(fullMatch, url); + logger.info({ filename, url }, 'SHARE:FILE link created'); + } catch (err) { + logger.warn( + { filePath: resolvedPath, err }, + 'SHARE:FILE: failed to create shareable link', + ); + cleaned = cleaned.replace(fullMatch, ''); + } + } else { + logger.warn( + { filePath: resolvedPath }, + 'SHARE:FILE marker received but file server is not running — skipping', + ); + cleaned = cleaned.replace(fullMatch, ''); + } + + // Also deliver as a native file attachment when the connector supports it (e.g. WhatsApp, Telegram) + if (connector.supportsFileAttachments) { + try { + const filename = path.basename(resolvedPath); + const ext = path.extname(filename).slice(1).toLowerCase(); + let { mimeType, mediaType } = getMimeType(filename); + let deliveryPath = resolvedPath; + let deliveryFilename = filename; + + // SVG files aren't natively rendered by WhatsApp/Telegram — convert to PNG when possible + if (ext === 'svg' && fileServer) { + try { + const renderResult = await fileServer.renderSvgToImage(filename); + if (renderResult) { + deliveryPath = renderResult.outputPath; + deliveryFilename = path.basename(renderResult.outputPath); + mimeType = 'image/png'; + mediaType = 'image'; + logger.debug( + { svgFile: filename, pngFile: deliveryFilename }, + 'SVG converted to PNG for image delivery', + ); + } + } catch (svgErr) { + logger.debug( + { filename, err: svgErr }, + 'SVG-to-PNG conversion failed — sending SVG as document', + ); + mediaType = 'document'; + } + } + + const data = await readFile(deliveryPath); + await connector.sendMessage({ + target: connector.name, + recipient, + content: '', + replyTo, + media: { type: mediaType, data, mimeType, filename: deliveryFilename }, + }); + logger.info( + { filename: deliveryFilename, recipient, connector: connector.name }, + 'SHARE:FILE attachment dispatched', + ); + } catch (err) { + logger.warn( + { filePath: resolvedPath, err }, + 'SHARE:FILE: failed to send native attachment', + ); + } + } + + continue; + } + + // Route to the named connector if registered, otherwise the inbound connector + const connectors = this.deps.getConnectors(); + const targetConnector = connectors.get(channel) ?? connector; + + // Read the file + let data: Buffer; + try { + data = await readFile(resolvedPath); + } catch (err) { + logger.warn({ filePath: resolvedPath, err }, 'SHARE marker: failed to read file'); + cleaned = cleaned.replace(fullMatch, ''); + continue; + } + + const filename = path.basename(resolvedPath); + const { mimeType, mediaType } = getMimeType(filename); + + const shareMsg: OutboundMessage = { + target: targetConnector.name, + recipient, + content: '', + replyTo, + media: { type: mediaType, data, mimeType, filename }, + }; + + try { + await targetConnector.sendMessage(shareMsg); + logger.info({ channel, filePath: resolvedPath, recipient }, 'SHARE dispatched'); + } catch (err) { + logger.warn({ channel, filePath: resolvedPath, err }, 'SHARE marker dispatch failed'); + } + + cleaned = cleaned.replace(fullMatch, ''); + } + + return cleaned.trim(); + } + + /** + * Parse [VOICE]text[/VOICE] markers from AI output, dispatch TTS voice replies + * via the connector's sendVoiceReply method, and return the response with markers stripped. + * If the connector does not support voice, the text inside the marker is kept as plain text. + */ + async processVoiceMarkers( + content: string, + connector: Connector, + recipient: string, + ): Promise { + let cleaned = content; + const regex = new RegExp(VOICE_MARKER_RE.source, 'g'); + let match: RegExpExecArray | null; + + while ((match = regex.exec(content)) !== null) { + const fullMatch = match[0]; + const voiceText = (match[1] ?? '').trim(); + + if (!voiceText) { + cleaned = cleaned.replace(fullMatch, ''); + continue; + } + + if (!connector.sendVoiceReply) { + // Connector doesn't support voice — keep text but strip marker tags + cleaned = cleaned.replace(fullMatch, voiceText); + continue; + } + + try { + await connector.sendVoiceReply(recipient, voiceText); + logger.info({ connector: connector.name, recipient }, 'VOICE reply dispatched'); + } catch (err) { + logger.warn({ err, connector: connector.name, recipient }, 'VOICE marker dispatch failed'); + } + + cleaned = cleaned.replace(fullMatch, ''); + } + + return cleaned.trim(); + } + + /** + * Handle [SHARE:email]user@example.com|/path/to/file[/SHARE] markers. + * The raw value from the SHARE marker capture group is `email|filePath`. + * Validates the recipient against the email allowlist, reads the file from + * .openbridge/generated/, and sends it as an email attachment. + */ + private async handleEmailShare( + rawValue: string, + _recipient: string, + _replyTo?: string, + ): Promise { + const emailConfig = this.deps.getEmailConfig(); + if (!emailConfig) { + logger.warn('SHARE:email marker received but no email config is set — skipping'); + return; + } + + const workspacePath = this.deps.getWorkspacePath(); + if (!workspacePath) { + logger.warn('SHARE:email marker received but workspacePath is not set — skipping'); + return; + } + + // Parse email address and file path from raw value (format: "email|/path") + const pipeIdx = rawValue.indexOf('|'); + if (pipeIdx === -1) { + logger.warn({ rawValue }, 'SHARE:email marker has no pipe separator — expected email|path'); + return; + } + const emailAddress = rawValue.slice(0, pipeIdx).trim(); + const filePath = rawValue.slice(pipeIdx + 1).trim(); + + if (!emailAddress || !filePath) { + logger.warn({ rawValue }, 'SHARE:email marker: missing email address or file path'); + return; + } + + const generatedDir = path.resolve(path.join(workspacePath, '.openbridge', 'generated')); + const resolvedPath = path.resolve( + path.isAbsolute(filePath) ? filePath : path.join(generatedDir, filePath), + ); + + // Security: file must be strictly under .openbridge/generated/ + if (!resolvedPath.startsWith(generatedDir + path.sep)) { + logger.warn( + { filePath: resolvedPath, generatedDir }, + 'SHARE:email blocked — file not under .openbridge/generated/', + ); + return; + } + + let data: Buffer; + try { + data = await readFile(resolvedPath); + } catch (err) { + logger.warn({ filePath: resolvedPath, err }, 'SHARE:email: failed to read file'); + return; + } + + const filename = path.basename(resolvedPath); + const { mimeType } = getMimeType(filename); + + try { + await sendEmail( + emailConfig, + emailAddress, + `Shared file: ${filename}`, + `Please find the attached file: ${filename}`, + [{ filename, content: data, contentType: mimeType }], + ); + logger.info({ emailAddress, filePath: resolvedPath }, 'SHARE:email dispatched'); + } catch (err) { + logger.warn({ emailAddress, filePath: resolvedPath, err }, 'SHARE:email dispatch failed'); + } + } + + /** + * Handle [SHARE:gdrive]/path/to/file[/SHARE] markers. + * Uploads the file to Google Drive via the GoogleDriveAdapter, makes it publicly + * accessible, and returns the shareable web link. Returns null on failure. + */ + private async handleGDriveShare(filePath: string): Promise { + const hub = this.deps.getIntegrationHub(); + if (!hub) { + logger.warn('SHARE:gdrive marker received but IntegrationHub is not configured — skipping'); + return null; + } + + let driveAdapter; + try { + driveAdapter = hub.get('google-drive'); + } catch { + logger.warn( + 'SHARE:gdrive marker received but google-drive integration is not registered — skipping', + ); + return null; + } + + try { + const uploadResult = (await driveAdapter.execute('upload_file', { filePath })) as { + fileId: string; + webViewLink: string | null; + }; + + // Make the file publicly accessible so anyone with the link can view it + const shareResult = (await driveAdapter.execute('share_file', { + fileId: uploadResult.fileId, + type: 'anyone', + role: 'reader', + })) as { webViewLink: string | null }; + + const link = shareResult.webViewLink ?? uploadResult.webViewLink; + if (link) { + logger.info({ filePath, link }, 'SHARE:gdrive dispatched'); + return link; + } + + logger.warn({ filePath }, 'SHARE:gdrive: no shareable link returned'); + return null; + } catch (err) { + logger.warn({ filePath, err }, 'SHARE:gdrive: upload or share failed'); + return null; + } + } + + /** + * Handle [SHARE:dropbox]/path/to/file[/SHARE] markers. + * Uploads the file to Dropbox via the DropboxAdapter, creates a public shared link, + * and returns that link. Returns null on failure. + */ + private async handleDropboxShare(filePath: string): Promise { + const hub = this.deps.getIntegrationHub(); + if (!hub) { + logger.warn('SHARE:dropbox marker received but IntegrationHub is not configured — skipping'); + return null; + } + + let dropboxAdapter; + try { + dropboxAdapter = hub.get('dropbox'); + } catch { + logger.warn( + 'SHARE:dropbox marker received but dropbox integration is not registered — skipping', + ); + return null; + } + + try { + const uploadResult = (await dropboxAdapter.execute('upload_file', { filePath })) as { + path: string; + }; + + const linkResult = (await dropboxAdapter.execute('create_shared_link', { + dropboxPath: uploadResult.path, + })) as { url: string }; + + logger.info({ filePath, url: linkResult.url }, 'SHARE:dropbox dispatched'); + return linkResult.url; + } catch (err) { + logger.warn({ filePath, err }, 'SHARE:dropbox: upload or share link creation failed'); + return null; + } + } + + /** + * Handle [SHARE:github-pages]/path/to/file[/SHARE] markers. + * Publishes the validated file (already confirmed to be under .openbridge/generated/) + * to the gh-pages branch of the workspace git repository. + */ + private async handleGitHubPagesShare(filePath: string): Promise { + try { + const pagesUrl = await publishToGitHubPages(filePath); + logger.info({ filePath, pagesUrl: pagesUrl || '(unknown)' }, 'SHARE:github-pages dispatched'); + } catch (err) { + logger.warn({ filePath, err }, 'SHARE:github-pages: publish failed'); + } + } + + /** + * Parse [APP:start]appPath[/APP], [APP:stop]appId[/APP], and + * [APP:update:appId]jsonData[/APP] markers from AI output. + * + * - [APP:start]appPath[/APP]: starts an app via AppServer.startApp(). The marker is + * replaced with the app URL (public URL if tunnel is active, otherwise local URL). + * - [APP:stop]appId[/APP]: stops an app via AppServer.stopApp(). The marker is stripped. + * - [APP:update:appId]jsonData[/APP]: sends data to a connected app via InteractionRelay. + * The jsonData body is parsed as JSON (falls back to raw string). The marker is stripped. + * + * APP:start and APP:stop require an AppServer. APP:update requires an InteractionRelay. + * Markers for unconfigured components are stripped silently. + */ + async processAppMarkers(content: string, _source?: string): Promise { + const appServer = this.deps.getAppServer(); + const relay = this.deps.getRelay(); + if (!appServer && !relay) return content; + + let cleaned = content; + + // Handle APP:start markers — replace each marker with the app URL + if (appServer) { + const startRegex = new RegExp(APP_START_MARKER_RE.source, 'g'); + let match: RegExpExecArray | null; + while ((match = startRegex.exec(content)) !== null) { + const fullMatch = match[0]; + const appPath = (match[1] ?? '').trim(); + + if (!appPath) { + cleaned = cleaned.replace(fullMatch, ''); + continue; + } + + try { + const instance = await appServer.startApp(appPath); + let replacement: string; + if (_source && REMOTE_CHANNELS.has(_source) && !instance.publicUrl) { + // Attempt auto-tunnel before falling back to SHARE attachment (OB-1633) + const tunnelUrl = this.deps.ensureTunnel ? await this.deps.ensureTunnel() : null; + if (tunnelUrl) { + const appUrl = `${tunnelUrl.replace(/\/$/, '')}/${appPath.replace(/^\//, '')}`; + replacement = `App started at ${appUrl}`; + logger.info( + { appPath, appUrl, source: _source }, + 'APP:start on remote channel — using auto-tunnel URL', + ); + } else { + logger.info( + { appPath, source: _source }, + 'APP:start on remote channel without tunnel — falling back to SHARE attachment', + ); + replacement = `[SHARE:${_source}]{"path":"${appPath}/index.html"}[/SHARE]`; + } + } else { + const url = instance.publicUrl ?? instance.url; + replacement = `App started at ${url}`; + logger.info({ appPath, url, appId: instance.id }, 'APP:start marker processed'); + } + cleaned = cleaned.replace(fullMatch, replacement); + } catch (err) { + logger.warn({ appPath, err }, 'APP:start marker: failed to start app'); + cleaned = cleaned.replace(fullMatch, `Failed to start app at ${appPath}`); + } + } + } + + // Handle APP:stop markers — strip each marker after stopping the app + if (appServer) { + const stopRegex = new RegExp(APP_STOP_MARKER_RE.source, 'g'); + let stopMatch: RegExpExecArray | null; + while ((stopMatch = stopRegex.exec(cleaned)) !== null) { + const fullMatch = stopMatch[0]; + const appId = (stopMatch[1] ?? '').trim(); + + if (appId) { + try { + appServer.stopApp(appId); + logger.info({ appId }, 'APP:stop marker processed'); + } catch (err) { + logger.warn({ appId, err }, 'APP:stop marker: failed to stop app'); + } + } + + cleaned = cleaned.replace(fullMatch, ''); + // Reset regex index after replacement to avoid skipping matches + stopRegex.lastIndex = 0; + } + } + + // Handle APP:update markers — send JSON data to a connected app via InteractionRelay + const updateRegex = new RegExp(APP_UPDATE_MARKER_RE.source, 'g'); + let updateMatch: RegExpExecArray | null; + while ((updateMatch = updateRegex.exec(cleaned)) !== null) { + const fullMatch = updateMatch[0]; + const appId = (updateMatch[1] ?? '').trim(); + const rawData = (updateMatch[2] ?? '').trim(); + + if (appId) { + let parsedData: unknown; + try { + parsedData = JSON.parse(rawData); + } catch { + parsedData = rawData; + } + + if (relay) { + const sent = relay.sendToApp(appId, 'update', parsedData); + if (sent) { + logger.info({ appId }, 'APP:update marker processed — data sent to app'); + } else { + logger.warn({ appId }, 'APP:update marker: app not connected, data not delivered'); + } + } else { + logger.warn({ appId }, 'APP:update marker: no InteractionRelay configured'); + } + } + + cleaned = cleaned.replace(fullMatch, ''); + // Reset regex index after replacement to avoid skipping matches + updateRegex.lastIndex = 0; + } + + return cleaned.trim(); + } +} diff --git a/src/core/permission-relay.ts b/src/core/permission-relay.ts new file mode 100644 index 00000000..4ffd65c5 --- /dev/null +++ b/src/core/permission-relay.ts @@ -0,0 +1,313 @@ +/** + * Permission Relay Protocol + * + * Routes tool approval requests from the Claude Agent SDK's `canUseTool` + * callback through messaging channels (WebChat, WhatsApp, Telegram, Discord). + * + * When a worker tries to use a tool not in its pre-approved list, this module: + * 1. Formats a user-friendly permission prompt + * 2. Sends it through the appropriate connector + * 3. Awaits the user's YES/NO response (with configurable timeout) + * 4. Returns the approval decision to the SDK + * + * @see OB-F183, OB-1498 + */ + +import type { Connector } from '../types/connector.js'; +import type { WorkspaceTrustLevel } from '../types/config.js'; +import { createLogger } from './logger.js'; + +/** Read-only tools that sandbox mode allows. */ +const SANDBOX_ALLOWED_TOOLS = new Set(['Read', 'Glob', 'Grep']); + +const logger = createLogger('permission-relay'); + +/** Default timeout for permission requests (60 seconds). */ +const DEFAULT_TIMEOUT_MS = 60_000; + +/** Pattern to match affirmative responses. */ +const YES_PATTERN = /^(yes|y|allow|approve|ok|go)$/i; + +/** Pattern to match negative responses. */ +const NO_PATTERN = /^(no|n|deny|reject|cancel|stop)$/i; + +/** + * Parameters for a permission relay request. + */ +export interface PermissionRelayParams { + /** The tool name (e.g. "Bash", "Write", "Edit") */ + toolName: string; + /** The tool input (e.g. { command: "rm -rf ./old-data/" }) */ + input: Record; + /** The user ID (sender) to relay the prompt to */ + userId: string; + /** The channel/connector name (e.g. "webchat", "whatsapp") */ + channel: string; +} + +/** + * A pending permission request awaiting user response. + */ +export interface PendingPermission { + /** Resolve the promise with the user's decision */ + resolve: (approved: boolean) => void; + /** Timeout handle for auto-deny */ + timeout: ReturnType; + /** Tool name for logging */ + toolName: string; + /** Timestamp when the request was created */ + createdAt: number; +} + +/** + * Configuration for the PermissionRelay. + */ +export interface PermissionRelayConfig { + /** Timeout in milliseconds before auto-denying (default: 60000) */ + timeoutMs?: number; + /** Workspace trust level — controls auto-approve/deny behavior (OB-1602) */ + trustLevel?: WorkspaceTrustLevel; +} + +/** + * Formats a user-friendly permission prompt message. + */ +export function formatPermissionPrompt(toolName: string, input: Record): string { + const lines: string[] = []; + lines.push(`🔐 *Permission Request*`); + lines.push(''); + + // Extract the most relevant detail from the input + const command = typeof input['command'] === 'string' ? input['command'] : undefined; + const filePath = typeof input['file_path'] === 'string' ? input['file_path'] : undefined; + const content = typeof input['content'] === 'string' ? input['content'] : undefined; + + if (command) { + lines.push(`The AI wants to run: \`${truncate(command, 200)}\``); + } else if (filePath) { + const verb = toolName === 'Write' ? 'write to' : toolName === 'Edit' ? 'edit' : 'access'; + lines.push(`The AI wants to ${verb}: \`${filePath}\``); + if (content && toolName === 'Write') { + lines.push(`(${content.length} characters)`); + } + } else { + lines.push(`The AI wants to use tool: *${toolName}*`); + // Show a compact summary of the input + const summary = Object.entries(input) + .slice(0, 3) + .map(([k, v]) => `${k}: ${truncate(String(v), 80)}`) + .join(', '); + if (summary) { + lines.push(`Input: ${summary}`); + } + } + + lines.push(''); + lines.push('Reply *YES* to allow or *NO* to deny.'); + + return lines.join('\n'); +} + +/** + * Checks whether a user reply is a permission response (YES/NO). + */ +export function isPermissionResponse(text: string): boolean { + const trimmed = text.trim(); + return YES_PATTERN.test(trimmed) || NO_PATTERN.test(trimmed); +} + +/** + * Parses a user reply into an approval decision. + * Returns `true` for approval, `false` for denial, `undefined` if not a valid response. + */ +export function parsePermissionResponse(text: string): boolean | undefined { + const trimmed = text.trim(); + if (YES_PATTERN.test(trimmed)) return true; + if (NO_PATTERN.test(trimmed)) return false; + return undefined; +} + +/** + * Permission Relay — manages pending permission requests and routes them + * through messaging connectors. + */ +export class PermissionRelay { + /** Pending permission requests keyed by userId. */ + private readonly pending = new Map(); + private readonly timeoutMs: number; + private readonly connectors: () => Map; + private trustLevel: WorkspaceTrustLevel; + + constructor(getConnectors: () => Map, config?: PermissionRelayConfig) { + this.connectors = getConnectors; + this.timeoutMs = config?.timeoutMs ?? DEFAULT_TIMEOUT_MS; + this.trustLevel = config?.trustLevel ?? 'standard'; + } + + /** Update the trust level at runtime (e.g. after config reload). */ + setTrustLevel(level: WorkspaceTrustLevel): void { + this.trustLevel = level; + } + + /** + * Relay a permission request to the user and await their response. + * + * Returns `true` if the user approves, `false` if denied or timed out. + */ + async relayPermission(params: PermissionRelayParams): Promise { + const { toolName, input, userId, channel } = params; + + // Trust level gates (OB-1602) + if (this.trustLevel === 'trusted') { + logger.debug({ toolName, userId }, 'Trusted mode — auto-granting tool permission'); + return true; + } + + if (this.trustLevel === 'sandbox') { + if (SANDBOX_ALLOWED_TOOLS.has(toolName)) { + logger.debug({ toolName, userId }, 'Sandbox mode — auto-approving read tool'); + return true; + } + logger.info({ toolName, userId }, 'Sandbox mode — denied tool: %s', toolName); + return false; + } + + // If there's already a pending permission for this user, auto-deny the new one + // to prevent confusion from overlapping prompts + if (this.pending.has(userId)) { + logger.warn( + { userId, toolName }, + 'Permission request already pending for user — auto-denying new request', + ); + return false; + } + + const connector = this.connectors().get(channel); + if (!connector) { + logger.error({ channel, userId }, 'No connector found for channel — auto-denying'); + return false; + } + + // Format and send the permission prompt + const promptText = formatPermissionPrompt(toolName, input); + + // Extract the most relevant detail for structured UI display + const command = typeof input['command'] === 'string' ? input['command'] : undefined; + const filePath = typeof input['file_path'] === 'string' ? input['file_path'] : undefined; + const detail = command ?? filePath ?? ''; + + try { + await connector.sendMessage({ + target: channel, + recipient: userId, + content: promptText, + metadata: { + permissionRequest: true, + toolName, + detail: truncate(detail, 200), + timeoutMs: this.timeoutMs, + }, + }); + } catch (err) { + logger.error({ err, channel, userId }, 'Failed to send permission prompt — auto-denying'); + return false; + } + + logger.info({ toolName, userId, channel, timeoutMs: this.timeoutMs }, 'Permission prompt sent'); + + // Create a promise that resolves when the user responds or times out + return new Promise((resolve) => { + const timeout = setTimeout(() => { + // Auto-deny on timeout + this.pending.delete(userId); + logger.info({ toolName, userId }, 'Permission request timed out — auto-denying'); + + // Notify the user + connector + .sendMessage({ + target: channel, + recipient: userId, + content: `⏱️ Permission request for *${toolName}* timed out — automatically denied.`, + }) + .catch((err: unknown) => { + logger.error({ err }, 'Failed to send timeout notification'); + }); + + resolve(false); + }, this.timeoutMs); + + this.pending.set(userId, { + resolve, + timeout, + toolName, + createdAt: Date.now(), + }); + }); + } + + /** + * Handle a user's response to a pending permission request. + * + * Called by the router when it detects a YES/NO reply from a user + * with a pending permission request. + * + * Returns `true` if the response was consumed (was a pending permission reply), + * `false` if there was no pending request for this user. + */ + handleResponse(userId: string, text: string): boolean { + const entry = this.pending.get(userId); + if (!entry) return false; + + const decision = parsePermissionResponse(text); + if (decision === undefined) { + // Not a valid YES/NO response — don't consume it + return false; + } + + // Clear the timeout and pending entry + clearTimeout(entry.timeout); + this.pending.delete(userId); + + logger.info( + { userId, toolName: entry.toolName, approved: decision }, + 'Permission response received', + ); + + entry.resolve(decision); + return true; + } + + /** + * Check whether a user has a pending permission request. + */ + hasPending(userId: string): boolean { + return this.pending.has(userId); + } + + /** + * Cancel all pending permission requests (e.g. on shutdown). + */ + cancelAll(): void { + this.pending.forEach((entry, userId) => { + clearTimeout(entry.timeout); + entry.resolve(false); + logger.info({ userId, toolName: entry.toolName }, 'Permission request cancelled'); + }); + this.pending.clear(); + } + + /** + * Get the number of pending permission requests. + */ + get pendingCount(): number { + return this.pending.size; + } +} + +/** + * Truncate a string to a maximum length, adding ellipsis if truncated. + */ +function truncate(str: string, maxLen: number): string { + if (str.length <= maxLen) return str; + return str.slice(0, maxLen - 3) + '...'; +} diff --git a/src/core/prompt-assembler.ts b/src/core/prompt-assembler.ts new file mode 100644 index 00000000..b45fd7ea --- /dev/null +++ b/src/core/prompt-assembler.ts @@ -0,0 +1,116 @@ +import { createLogger } from './logger.js'; + +const logger = createLogger('prompt-assembler'); + +/** Priority constants for PromptAssembler sections (higher = kept first). */ +export const PRIORITY_IDENTITY = 100; // role / rules +export const PRIORITY_WORKSPACE = 80; // workspace map +export const PRIORITY_MEMORY = 70; // memory.md +export const PRIORITY_RAG = 60; // RAG context +export const PRIORITY_HISTORY = 50; // conversation history +export const PRIORITY_LEARNINGS = 40; // learnings +export const PRIORITY_WORKER_NEXT = 30; // worker next steps +export const PRIORITY_ANALYSIS = 20; // analysis context + +interface PromptSection { + name: string; + content: string; + priority: number; + maxChars?: number; +} + +/** + * Budget-aware prompt assembler that prioritizes sections by importance. + * + * Higher-priority sections are kept first; lower-priority sections are + * truncated or dropped when the total budget is exceeded. + */ +export class PromptAssembler { + private sections: PromptSection[] = []; + + /** + * Add a named section to the prompt. + * @param name - Human-readable section name (for logging) + * @param content - The text content of this section + * @param priority - Higher values = higher priority (kept first) + * @param maxChars - Optional per-section cap (content truncated to this before budget check) + */ + addSection(name: string, content: string, priority: number, maxChars?: number): void { + if (!content || content.trim().length === 0) return; + this.sections.push({ name, content, priority, maxChars }); + } + + /** + * Assemble all sections into a single string within the given budget. + * + * Sections are sorted by priority (highest first). Each section is + * included in full if it fits, truncated if partially fits, or dropped + * if no budget remains. Warnings are logged for truncated/dropped sections. + */ + assemble(totalBudget: number): string { + if (this.sections.length === 0) return ''; + + // Sort by priority descending (highest priority first) + const sorted = [...this.sections].sort((a, b) => b.priority - a.priority); + + const included: string[] = []; + let remaining = totalBudget; + const dropped: string[] = []; + const truncated: string[] = []; + + for (const section of sorted) { + // Apply per-section cap first + let content = section.content; + if (section.maxChars && content.length > section.maxChars) { + content = content.slice(0, section.maxChars); + truncated.push(`${section.name} (capped: ${section.content.length} → ${section.maxChars})`); + } + + if (remaining <= 0) { + dropped.push(`${section.name} (${content.length} chars)`); + continue; + } + + if (content.length <= remaining) { + // Fits entirely + included.push(content); + remaining -= content.length; + } else { + // Partial fit — truncate to remaining budget + included.push(content.slice(0, remaining)); + truncated.push(`${section.name} (budget: ${content.length} → ${remaining})`); + remaining = 0; + } + } + + if (truncated.length > 0) { + logger.warn({ truncated }, 'Prompt sections truncated'); + } + if (dropped.length > 0) { + logger.warn({ dropped }, 'Prompt sections dropped — budget exceeded'); + } + + const result = included.join('\n\n'); + logger.debug( + { + assembledChars: result.length, + totalBudget, + sectionsIncluded: included.length, + sectionsTruncated: truncated.length, + sectionsDropped: dropped.length, + }, + 'Prompt assembled', + ); + return result; + } + + /** Reset all sections (for reuse). */ + clear(): void { + this.sections = []; + } + + /** Returns the number of sections currently added. */ + get sectionCount(): number { + return this.sections.length; + } +} diff --git a/src/core/qr-store.ts b/src/core/qr-store.ts new file mode 100644 index 00000000..5e11b2fb --- /dev/null +++ b/src/core/qr-store.ts @@ -0,0 +1,14 @@ +/** + * Module-level QR code store — bridges WhatsApp QR events to the WebChat /qr endpoint. + * Used only in headless mode: WhatsApp connector calls setQrCode(), WebChat serves it. + */ + +let _latestQr: string | null = null; + +export function setQrCode(qr: string): void { + _latestQr = qr; +} + +export function getQrCode(): string | null { + return _latestQr; +} diff --git a/src/core/queue.ts b/src/core/queue.ts index c9500174..d32a61a7 100644 --- a/src/core/queue.ts +++ b/src/core/queue.ts @@ -2,6 +2,7 @@ import type { InboundMessage } from '../types/message.js'; import type { QueueConfig } from '../types/config.js'; import type { MetricsCollector } from './metrics.js'; import { ProviderError } from '../providers/claude-code/provider-error.js'; +import { AgentExhaustedError, classifyError } from './agent-runner.js'; import { createLogger } from './logger.js'; const logger = createLogger('queue'); @@ -10,6 +11,8 @@ interface QueueItem { message: InboundMessage; addedAt: Date; attempts: number; + /** 1 = quick-answer (highest), 2 = tool-use, 3 = complex-task (lowest) */ + priority: 1 | 2 | 3; } export interface DeadLetterItem { @@ -39,6 +42,20 @@ export class MessageQueue { private drainResolvers: (() => void)[] = []; private readonly dlq: DeadLetterItem[] = []; private readonly metrics?: MetricsCollector; + /** Rolling window of the last 10 message processing durations (ms). */ + private readonly recentProcessingTimes: number[] = []; + /** Called when a message must wait in queue — provides position and estimated wait. */ + private queuedHandler: + | ((message: InboundMessage, position: number, estimatedWaitMs: number) => void) + | null = null; + /** + * Called when a priority-1 message is enqueued for a sender whose previous message + * is still being processed. The Router uses this to trigger a checkpoint-handle-resume + * cycle so that session state is saved before the urgent message is handled. + */ + private urgentEnqueuedHandler: ((message: InboundMessage) => void) | null = null; + /** Called when a message is permanently failed and moved to the dead letter queue. */ + private deadLetterCallback?: (message: InboundMessage, error: string) => Promise; constructor(config: Partial = {}, metrics?: MetricsCollector) { this.config = { ...DEFAULT_CONFIG, ...config }; @@ -50,8 +67,52 @@ export class MessageQueue { this.handler = handler; } - /** Add a message to the queue */ - async enqueue(message: InboundMessage): Promise { + /** + * Register a callback invoked when a message is queued behind another in-flight message. + * Receives the queued message, its 1-based position in the waiting queue, and the + * estimated wait time in milliseconds based on recent processing durations. + */ + onQueued( + handler: (message: InboundMessage, position: number, estimatedWaitMs: number) => void, + ): void { + this.queuedHandler = handler; + } + + /** + * Register a callback invoked when a priority-1 (urgent) message is enqueued for a sender + * whose previous message is still in flight. + * + * Used by the Router to trigger a checkpoint-handle-resume cycle: the session is + * checkpointed before the urgent message is processed and restored afterwards, preserving + * the pre-interruption Master context for subsequent messages. + */ + onUrgentEnqueued(handler: (message: InboundMessage) => void): void { + this.urgentEnqueuedHandler = handler; + } + + /** Register a callback invoked when a message is permanently failed and moved to the dead letter queue. */ + onDeadLetter(cb: (message: InboundMessage, error: string) => Promise): void { + this.deadLetterCallback = cb; + } + + /** + * Rolling average of recent message processing times (ms). + * Returns 30 000 ms as a default when no history is available. + */ + get averageProcessingTimeMs(): number { + if (this.recentProcessingTimes.length === 0) return 30_000; + const sum = this.recentProcessingTimes.reduce((a, b) => a + b, 0); + return Math.round(sum / this.recentProcessingTimes.length); + } + + /** + * Add a message to the queue. + * + * @param message - The inbound message to queue. + * @param priority - Queue priority: 1 (quick-answer, highest) · 2 (tool-use, default) · 3 (complex-task, lowest). + * Quick-answer messages (priority 1) jump ahead of pending tool-use and complex-task messages. + */ + async enqueue(message: InboundMessage, priority: 1 | 2 | 3 = 2): Promise { const sender = message.sender; let queue = this.userQueues.get(sender); @@ -59,13 +120,41 @@ export class MessageQueue { queue = []; this.userQueues.set(sender, queue); } - queue.push({ message, addedAt: new Date(), attempts: 0 }); + + const item: QueueItem = { message, addedAt: new Date(), attempts: 0, priority }; + + // Insert in priority order — find the first existing item with lower priority (higher number) + // and insert before it so that higher-priority messages are processed first. + // Within the same priority, FIFO order is preserved (insert before any equal-priority item + // that was already in the queue at the time of insertion is NOT done — we only jump ahead of + // lower-priority items to keep same-priority ordering stable). + const insertAt = queue.findIndex((existing) => existing.priority > item.priority); + if (insertAt === -1) { + queue.push(item); + } else { + queue.splice(insertAt, 0, item); + } + this.metrics?.recordEnqueued(); - logger.debug({ messageId: message.id, sender, queueSize: queue.length }, 'Message enqueued'); + logger.debug( + { messageId: message.id, sender, queueSize: queue.length, priority }, + 'Message enqueued', + ); + + // Notify when a priority-1 message is enqueued for a sender whose current message is + // still in flight. The Router uses this to trigger a checkpoint-handle-resume cycle. + if (priority === 1 && this.activeUsers.has(sender) && this.urgentEnqueuedHandler) { + this.urgentEnqueuedHandler(message); + } if (!this.activeUsers.has(sender)) { await this.processNextForUser(sender); + } else if (this.queuedHandler) { + // Another message for this sender is already in-flight — notify about queue position. + const position = queue.length; // 1-based: how many messages are waiting (including this one) + const estimatedWaitMs = this.averageProcessingTimeMs * position; + this.queuedHandler(message, position, estimatedWaitMs); } } @@ -81,76 +170,117 @@ export class MessageQueue { /** Process the next message for a specific user */ private async processNextForUser(sender: string): Promise { - const queue = this.userQueues.get(sender); - if (!queue || queue.length === 0 || !this.handler) { - this.activeUsers.delete(sender); - if (queue?.length === 0) { - this.userQueues.delete(sender); + this.activeUsers.add(sender); + + while (true) { + const queue = this.userQueues.get(sender); + if (!queue || queue.length === 0 || !this.handler) { + this.activeUsers.delete(sender); + if (queue?.length === 0) { + this.userQueues.delete(sender); + } + this.resolveDrainIfIdle(); + return; } - this.resolveDrainIfIdle(); - return; - } - this.activeUsers.add(sender); - const item = queue.shift()!; + const item = queue.shift()!; + const processingStart = Date.now(); + + logger.info({ messageId: item.message.id, sender }, 'Processing message'); + + let lastError: unknown; + for (let attempt = 0; attempt <= this.config.maxRetries; attempt++) { + if (attempt > 0) { + this.metrics?.recordRetry(); + const delay = this.config.retryDelayMs * attempt; + logger.warn( + { messageId: item.message.id, attempt, delay }, + 'Retrying message after delay', + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } + + try { + await this.handler(item.message); + lastError = undefined; + break; + } catch (error) { + lastError = error; + item.attempts = attempt + 1; - logger.info({ messageId: item.message.id, sender }, 'Processing message'); + // Classify the error kind for logging and retry decisions. + // AgentExhaustedError carries attempt records with exit codes and stderr + // that classifyError() can inspect (timeout, rate-limit, auth, etc.). + // ProviderError has its own kind. Everything else is 'unknown'. + let errorKind: string; + if (error instanceof AgentExhaustedError) { + const lastAttempt = error.attempts[error.attempts.length - 1]; + errorKind = lastAttempt + ? classifyError(lastAttempt.stderr, lastAttempt.exitCode) + : 'unknown'; + } else if (error instanceof ProviderError) { + errorKind = error.kind; + } else { + errorKind = 'unknown'; + } - let lastError: unknown; - for (let attempt = 0; attempt <= this.config.maxRetries; attempt++) { - if (attempt > 0) { - this.metrics?.recordRetry(); - const delay = this.config.retryDelayMs * attempt; - logger.warn({ messageId: item.message.id, attempt, delay }, 'Retrying message after delay'); - await new Promise((resolve) => setTimeout(resolve, delay)); + const isPermanent = error instanceof ProviderError && error.kind === 'permanent'; + // Timeout errors are non-retryable — retrying a 180s timeout will just + // produce another 180s timeout, wasting compute and blocking the user. + const isTimeout = errorKind === 'timeout'; + logger.error( + { + messageId: item.message.id, + attempt: attempt + 1, + maxRetries: this.config.maxRetries, + errorKind, + error, + }, + isPermanent + ? 'Permanent error — skipping retries' + : isTimeout + ? 'Timeout error — skipping retries (retrying would timeout again)' + : 'Failed to process message', + ); + + if (isPermanent || isTimeout) break; + } + } + + // Record elapsed time (including any retries) for rolling average used by onQueued. + const elapsed = Date.now() - processingStart; + this.recentProcessingTimes.push(elapsed); + if (this.recentProcessingTimes.length > 10) { + this.recentProcessingTimes.shift(); } - try { - await this.handler(item.message); - lastError = undefined; - break; - } catch (error) { - lastError = error; - item.attempts = attempt + 1; + if (lastError !== undefined) { + const deadLetterItem: DeadLetterItem = { + message: item.message, + error: + lastError instanceof Error + ? lastError.message + : typeof lastError === 'string' + ? lastError + : 'Unknown error', + attempts: item.attempts, + failedAt: new Date(), + }; + this.dlq.push(deadLetterItem); + this.metrics?.recordDeadLettered(); + + try { + await this.deadLetterCallback?.(item.message, deadLetterItem.error); + } catch (cbErr) { + logger.error({ err: cbErr }, 'onDeadLetter callback failed'); + } - const isPermanent = error instanceof ProviderError && error.kind === 'permanent'; logger.error( - { - messageId: item.message.id, - attempt: attempt + 1, - maxRetries: this.config.maxRetries, - errorKind: error instanceof ProviderError ? error.kind : 'unknown', - error, - }, - isPermanent ? 'Permanent error — skipping retries' : 'Failed to process message', + { messageId: item.message.id, attempts: item.attempts, dlqSize: this.dlq.length }, + 'Message permanently failed — moved to dead letter queue', ); - - if (isPermanent) break; } } - - if (lastError !== undefined) { - const deadLetterItem: DeadLetterItem = { - message: item.message, - error: - lastError instanceof Error - ? lastError.message - : typeof lastError === 'string' - ? lastError - : 'Unknown error', - attempts: item.attempts, - failedAt: new Date(), - }; - this.dlq.push(deadLetterItem); - this.metrics?.recordDeadLettered(); - - logger.error( - { messageId: item.message.id, attempts: item.attempts, dlqSize: this.dlq.length }, - 'Message permanently failed — moved to dead letter queue', - ); - } - - await this.processNextForUser(sender); } private resolveDrainIfIdle(): void { @@ -193,4 +323,23 @@ export class MessageQueue { flushDeadLetters(): DeadLetterItem[] { return this.dlq.splice(0); } + + /** + * Returns a per-user snapshot of queued (waiting) messages. + * Only includes users that have at least one pending message. + * `estimatedWaitMs` is computed as `averageProcessingTimeMs × pending`. + */ + getQueueSnapshot(): Array<{ sender: string; pending: number; estimatedWaitMs: number }> { + const result: Array<{ sender: string; pending: number; estimatedWaitMs: number }> = []; + for (const [sender, items] of this.userQueues) { + if (items.length > 0) { + result.push({ + sender, + pending: items.length, + estimatedWaitMs: this.averageProcessingTimeMs * items.length, + }); + } + } + return result; + } } diff --git a/src/core/rate-limiter.ts b/src/core/rate-limiter.ts index 704c76ef..e090c707 100644 --- a/src/core/rate-limiter.ts +++ b/src/core/rate-limiter.ts @@ -3,12 +3,15 @@ import { createLogger } from './logger.js'; const logger = createLogger('rate-limiter'); +const CLEANUP_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes + export class RateLimiter { private windowMs: number; private maxMessages: number; private enabled: boolean; /** sender → timestamps of recent messages */ private readonly windows = new Map(); + private cleanupInterval: ReturnType | null = null; constructor(config: RateLimitConfig) { this.enabled = config.enabled; @@ -21,6 +24,30 @@ export class RateLimiter { 'Rate limiter initialized', ); } + + this.cleanupInterval = setInterval(() => { + this.cleanup(); + }, CLEANUP_INTERVAL_MS); + // Allow the process to exit even if this interval is still active + this.cleanupInterval.unref?.(); + } + + /** Remove stale window entries to prevent unbounded Map growth */ + private cleanup(): void { + const cutoff = Date.now() - 2 * this.windowMs; + for (const [sender, timestamps] of this.windows) { + if (timestamps.every((t) => t < cutoff)) { + this.windows.delete(sender); + } + } + } + + /** Stop the background cleanup timer. Call from Bridge.stop(). */ + dispose(): void { + if (this.cleanupInterval) { + clearInterval(this.cleanupInterval); + this.cleanupInterval = null; + } } /** diff --git a/src/core/router.ts b/src/core/router.ts index e423de97..ab872cbb 100644 --- a/src/core/router.ts +++ b/src/core/router.ts @@ -1,82 +1,1837 @@ import type { AIProvider, ProviderResult } from '../types/provider.js'; -import type { InboundMessage, OutboundMessage } from '../types/message.js'; +import type { InboundMessage, OutboundMessage, ProgressEvent } from '../types/message.js'; import type { Connector } from '../types/connector.js'; -import type { RouterConfig } from '../types/config.js'; +import type { RouterConfig, SecurityConfig } from '../types/config.js'; import type { AuditLogger } from './audit-logger.js'; import type { MetricsCollector } from './metrics.js'; import type { AgentOrchestrator } from './agent-orchestrator.js'; import type { MasterManager } from '../master/master-manager.js'; +import type { AuthService } from './auth.js'; +import type { EmailConfig } from '../types/config.js'; +import type { AppServer } from './app-server.js'; +import type { InteractionRelay, RelayMessage } from './interaction-relay.js'; +import type { SecretMatch } from './secret-scanner.js'; +import type { MemoryManager } from '../memory/index.js'; +import type { MessageQueue } from './queue.js'; +import type { RiskLevel, DeepPhase, DocumentFileFormat } from '../types/agent.js'; +import { PROFILE_RISK_MAP, BuiltInProfileNameSchema } from '../types/agent.js'; +import type { SkillManager } from '../master/skill-manager.js'; +import type { IntegrationHub } from '../integrations/hub.js'; +import type { WorkflowStore } from '../workflows/workflow-store.js'; +import type { WorkflowEngine } from '../workflows/engine.js'; +import type { WorkflowScheduler } from '../workflows/scheduler.js'; +import type { CredentialStore } from '../integrations/credential-store.js'; +import type { ParsedSpawnMarker } from '../master/spawn-parser.js'; +import { extractTaskSummaries } from '../master/spawn-parser.js'; +import type { PermissionRelay } from './permission-relay.js'; +import type { FileServer } from './file-server.js'; import { ProviderError } from '../providers/claude-code/provider-error.js'; +import { OutputMarkerProcessor } from './output-marker-processor.js'; +import { AgentRunner, estimateCost, DEFAULT_MAX_TURNS_TASK } from './agent-runner.js'; +import { FastPathResponder } from './fast-path-responder.js'; import { createLogger } from './logger.js'; +import { CommandHandlers } from './command-handlers.js'; +import type { CommandHandlerDeps } from './command-handlers.js'; + +// Re-export types that were moved to command-handlers.ts for backward compatibility +export type { + PendingConfirmation, + PendingSpawnEntry, + PendingEscalation, +} from './command-handlers.js'; const logger = createLogger('router'); -const PROGRESS_MESSAGES = [ - 'Still working on it...', - 'This is taking a moment — hang tight...', - 'Still processing your request...', - 'Almost there — still working...', -]; +// SEND/VOICE/SHARE/APP marker regexes and getMimeType moved to output-marker-processor.ts (OB-1284) + +/** Pattern matching [CONTINUE:batch-{id}] internal batch continuation messages */ +const CONTINUE_MARKER_RE = /^\[CONTINUE:batch-([^\]]+)\]$/; + +// formatDuration, makeProgressBar, escapeHtml moved to command-handlers.ts (OB-1283) +// getMimeType moved to output-marker-processor.ts (OB-1284) + +// PendingConfirmation, PendingSpawnEntry, PendingEscalation moved to command-handlers.ts (OB-1283) +// Re-exported above for backward compatibility. +import type { + PendingConfirmation, + PendingSpawnEntry, + PendingEscalation, +} from './command-handlers.js'; + +/** + * Message priority levels used for queue ordering. + * Lower number = higher priority (processed first). + * + * 1 = quick-answer — status, list, simple questions (no file changes) + * 2 = tool-use — generate, create, fix, single-file edits + * 3 = complex-task — implement, refactor, multi-step work + */ +export type MessagePriority = 1 | 2 | 3; + +/** + * Creative intent types — maps to the skill packs in src/master/skill-packs/. + * - diagram → diagram-maker skill pack + * - chart → chart-generator skill pack + * - design → web-designer or slide-designer skill pack + * - art → generative-art skill pack + * - brand → brand-assets skill pack + */ +export type CreativeIntent = 'diagram' | 'chart' | 'design' | 'art' | 'brand'; + +/** + * Classify a message to detect creative/visual output intent. + * + * Returns the creative intent type when the message is asking for a visual + * output (diagram, chart, web design, generative art, brand assets), or + * `null` if no creative intent is detected. Runs synchronously — no AI calls. + * + * Matching priority (highest to lowest): + * 1. Explicit format or tool keywords (mermaid, d3, plantuml, p5.js, etc.) + * 2. Output-type keywords (diagram, chart, logo, landing page, etc.) + * 3. Verb + visual noun combinations + */ +export function classifyCreativeIntent(content: string): CreativeIntent | null { + const lower = content.toLowerCase().trim(); + + // Diagram — explicit tool or output-type keywords + if ( + /\b(mermaid|plantuml|plant uml|d2 diagram|graphviz)\b/.test(lower) || + /\b(flowchart|flow chart|sequence diagram|er diagram|erd|class diagram|architecture diagram|network diagram|uml)\b/.test( + lower, + ) || + (/\bdiagram\b/.test(lower) && + /\b(create|generate|make|draw|build|produce|show|render)\b/.test(lower)) + ) + return 'diagram'; + + // Chart — explicit tool or visualization keywords + if ( + /\b(d3\.?js|chart\.?js|chartjs|d3 chart|recharts|vega)\b/.test(lower) || + /\b(bar chart|line chart|pie chart|scatter plot|histogram|area chart|bubble chart|heatmap|treemap)\b/.test( + lower, + ) || + (/\b(chart|graph|visualization|visualise|visualize)\b/.test(lower) && + /\b(create|generate|make|draw|build|produce|show|render|plot)\b/.test(lower)) + ) + return 'chart'; + + // Brand assets — logos, favicons, social media images + if ( + /\b(logo|favicon|brand asset|social media image|og image|twitter card|app icon)\b/.test( + lower, + ) && + /\b(create|generate|make|design|draw|build|produce)\b/.test(lower) + ) + return 'brand'; + + // Generative art — p5.js, algorithmic, creative coding + if ( + /\b(p5\.?js|processing|generative art|algorithmic art|creative coding|svg pattern|svg art)\b/.test( + lower, + ) || + (/\b(generative|algorithmic|procedural)\b/.test(lower) && + /\b(art|image|pattern|design|visual)\b/.test(lower)) + ) + return 'art'; + + // Design — web pages, landing pages, slides, HTML templates + if ( + /\b(landing page|web page|webpage|email template|html template|marketing page|hero section)\b/.test( + lower, + ) || + /\b(presentation slide|html slide|slide deck)\b/.test(lower) || + (/\b(design|redesign)\b/.test(lower) && + /\b(website|webpage|web page|ui|interface|layout|page)\b/.test(lower)) + ) + return 'design'; + + return null; +} + +/** + * Classify a message to detect document-generation intent. + * + * Returns the target file format when the message is asking for document + * generation (docx, pptx, xlsx, or pdf), or `null` if no document intent + * is detected. Runs synchronously — no AI calls. + * + * Matching priority (highest to lowest): + * 1. Explicit format extension or acronym in the message (.docx, pptx, etc.) + * 2. Document-type keywords (presentation, spreadsheet, report, …) + * 3. Generic document-creation verb + "document" noun → defaults to docx + */ +export function classifyDocumentIntent(content: string): DocumentFileFormat | null { + const lower = content.toLowerCase().trim(); + + // Explicit format keywords — highest confidence + if (lower.includes('.pptx') || /\bpptx\b/.test(lower)) return 'pptx'; + if (lower.includes('.xlsx') || /\bxlsx\b/.test(lower)) return 'xlsx'; + if (lower.includes('.docx') || /\bdocx\b/.test(lower)) return 'docx'; + if (lower.includes('.pdf') || /\bpdf\b/.test(lower)) return 'pdf'; + + // Presentation / slides + if (/\b(presentation|slide deck|slideshow|powerpoint|slides)\b/.test(lower)) return 'pptx'; + + // Spreadsheet / Excel + if (/\b(spreadsheet|excel|workbook)\b/.test(lower)) return 'xlsx'; + + // Report — maps to PDF (report-generator skill pack) + if ( + /\breport\b/.test(lower) && + /\b(generate|create|make|write|build|produce|draft)\b/.test(lower) + ) + return 'pdf'; + + // Word document — proposals, memos, letters, business documents + if (/\b(word document|word doc|proposal|memo|business document|cover letter)\b/.test(lower)) + return 'docx'; + + // Generic "write/create/draft a document" → default to docx + if (/\b(write|create|generate|make|draft)\b/.test(lower) && /\bdocument\b/.test(lower)) + return 'docx'; + + return null; +} + +/** + * Classify a message by priority using keyword heuristics. + * Returns 1 (quick-answer), 2 (tool-use), or 3 (complex-task). + * Runs synchronously — no AI calls, safe to call before enqueueing. + */ +export function classifyMessagePriority(content: string): MessagePriority { + const lower = content.toLowerCase().trim(); + + // Document-generation tasks — always complex (multi-step: plan → generate → write → deliver) + if (classifyDocumentIntent(lower) !== null) { + return 3; + } + + // Creative/visual tasks — always complex (render pipeline: generate → render → deliver) + if (classifyCreativeIntent(lower) !== null) { + return 3; + } + + // Complex-task keywords — multi-step work requiring planning and delegation + const complexKeywords = [ + 'implement', + 'build', + 'refactor', + 'develop', + 'set up', + 'setup', + 'redesign', + 'migrate', + 'overhaul', + ]; + if (complexKeywords.some((kw) => lower.includes(kw)) || /\barchitect\b/.test(lower)) { + return 3; + } + + // Compound action pattern — two verbs joined by "and" signal multi-step work + const actionVerbs = [ + 'review', + 'analyze', + 'audit', + 'check', + 'fix', + 'add', + 'update', + 'create', + 'remove', + 'test', + 'write', + 'optimize', + 'improve', + ]; + const matchedVerbs = actionVerbs.filter((v) => lower.includes(v)); + if (matchedVerbs.length >= 2 && lower.includes(' and ')) { + return 3; + } + + // Tool-use keywords — single-action file generation or targeted edits + const toolUseKeywords = ['generate', 'create', 'write', 'fix', 'update file', 'add to', 'make a']; + if (toolUseKeywords.some((kw) => lower.includes(kw))) { + return 2; + } + + // Quick-answer patterns — questions and lookups (no file changes needed) + const questionPatterns = [ + 'what is', + 'what are', + 'how does', + 'how do', + 'explain', + 'describe', + 'show me', + 'list all', + 'list the', + 'tell me', + 'status', + ]; + const isShortQuestion = lower.endsWith('?') && lower.length <= 80; + const hasQuestionKeyword = questionPatterns.some((qp) => lower.includes(qp)); + if (isShortQuestion || (hasQuestionKeyword && lower.length <= 120)) { + return 1; + } + + // Default: tool-use — most non-question messages require file operations + return 2; +} + +/** + * Parse approximate counts of file reads, file modifications, and commands run + * from a worker's stdout content. Uses heuristic regex patterns since the Claude + * CLI --print mode outputs plain text rather than structured tool-call logs. + * + * Counts are best-effort: they reflect what the AI described doing, not raw tool calls. + */ +function parseWorkerStats(content: string): { + filesRead: number; + filesModified: number; + commandsRun: number; +} { + // File modification: action verbs before a backtick-wrapped file path + const modifiedRe = + /\b(?:edit(?:ed|ing)?|writ(?:e|ing|ten)|creat(?:e|ed|ing)|modif(?:y|ied|ying)|updat(?:e|ed|ing)|add(?:ed|ing)\s+to|rewrit(?:e|ing|ten))\b[^`\n]{0,50}`[^`]+\.[a-zA-Z0-9]{1,6}`/gi; + + // File reads: action verbs indicating a file was read + const readRe = + /\b(?:read(?:ing)?|examin(?:e|ed|ing)|analyz(?:e|ed|ing)|check(?:ed|ing)?|look(?:ed|ing)\s+at|review(?:ed|ing)?|inspect(?:ed|ing)?|open(?:ed|ing)?|search(?:ed|ing)?\s+(?:in|through))\b[^`\n]{0,50}`[^`]+\.[a-zA-Z0-9]{1,6}`/gi; + + // Shell commands: backtick-wrapped strings starting with common CLI tools + const commandRe = + /`(?:npm|yarn|pnpm|npx|git|bash|sh|node|python3?|pip3?|cargo|go|make|docker|kubectl|curl|wget|tsc|eslint|prettier|vitest|jest)\s[^`]+`/gi; + + const filesModified = (content.match(modifiedRe) ?? []).length; + const filesReadRaw = (content.match(readRe) ?? []).length; + const filesRead = Math.max(0, filesReadRaw - filesModified); + const commandsRun = (content.match(commandRe) ?? []).length; + + return { filesRead, filesModified, commandsRun }; +} + +/** + * Returns true only when the Q&A pair is worth caching. + * Skips greetings, single-word acks, and error/refusal responses. (OB-1603) + */ +function isSubstantiveResponse(question: string, answer: string): boolean { + const trimmedAnswer = answer.trim(); + + // Too short to be useful + if (trimmedAnswer.length < 30) return false; + + // Too-short question is likely a greeting or ack, not a cacheable Q + if (question.trim().length < 5) return false; + + // Single-word / short acknowledgements + const shortAckRe = + /^(ok|okay|sure|got it|done|yes|no|hi|hello|hey|thanks|thank you|bye|goodbye|np|noted)[.!?\s]*$/i; + if (shortAckRe.test(trimmedAnswer)) return false; + + // Error / refusal patterns + const errorRe = + /^(sorry|i (can'?t|cannot|am unable|don'?t know)|i'?m not able|there (was|is) an error|an error occurred)/i; + if (errorRe.test(trimmedAnswer)) return false; + + return true; +} + +export class Router { + private readonly connectors = new Map(); + private readonly providers = new Map(); + private defaultProviderName: string; + private readonly auditLogger?: AuditLogger; + private readonly metrics?: MetricsCollector; + private orchestrator?: AgentOrchestrator; + private master?: MasterManager; + private auth?: AuthService; + private workspacePath?: string; + private emailConfig?: EmailConfig; + private fileServer?: FileServer; + private memory?: MemoryManager; + private queue?: MessageQueue; + private appServer?: AppServer; + private relay?: InteractionRelay; + private skillManager?: SkillManager; + private integrationHub?: IntegrationHub; + private credentialStore?: CredentialStore; + private workflowStore?: WorkflowStore; + private workflowEngine?: WorkflowEngine; + private workflowScheduler?: WorkflowScheduler; + /** Pending "stop all" confirmations — keyed by sender, value contains expiresAt timestamp. */ + private readonly pendingStopConfirmations = new Map(); + /** Pending high-risk spawn confirmations — keyed by sender, awaiting user "go" or "skip". */ + private readonly pendingSpawnConfirmations = new Map(); + /** Pending tool escalation requests — keyed by sender, queue of pending entries (FIFO). Each /allow pops the first. */ + private readonly pendingEscalations = new Map(); + /** Tracks senders who have already received a 50% reminder for the current escalation batch (OB-1640). */ + private readonly escalationReminderSent = new Set(); + /** Session-level tool grants — keyed by sender, value is the set of tool/profile names granted for this session. */ + private readonly sessionGrantedTools = new Map>(); + /** Security config — controls confirmation requirements for high-risk spawns. */ + private securityConfig?: SecurityConfig; + /** Sensitive files detected by the startup secret scanner. Populated via setVisibilityState(). */ + private detectedSecrets: readonly SecretMatch[] = []; + /** Workspace-relative patterns auto-excluded this session after secret scanning. */ + private sessionExcludePatterns: readonly string[] = []; + /** User-configured include patterns (workspace.include). */ + private workspaceInclude: readonly string[] = []; + /** User-configured exclude patterns (workspace.exclude). */ + private workspaceExclude: readonly string[] = []; + /** + * IDs of priority-1 messages that should trigger a checkpoint-handle-resume cycle. + * Populated by the `onUrgentEnqueued` queue callback; consumed in `route()`. + */ + private readonly urgentCycleMessageIds = new Set(); + /** Pool of short-lived read-only agents for quick-answer responses during Master processing. */ + private readonly fastPathResponder = new FastPathResponder(new AgentRunner()); + /** Extracted command handlers — delegates to CommandHandlers class (OB-1283). */ + private readonly commandHandlers: CommandHandlers; + /** Extracted output marker processor — delegates to OutputMarkerProcessor class (OB-1284). */ + private readonly outputMarkerProcessor: OutputMarkerProcessor; + /** Permission relay for interactive tool approval via messaging channels (OB-1499). */ + private permissionRelay?: PermissionRelay; + /** On-demand tunnel starter — wired in by Bridge.start() for remote channel APP delivery (OB-1633). */ + private ensureTunnelFn?: () => Promise; + + constructor( + defaultProvider: string, + private readonly routerConfig?: RouterConfig, + auditLogger?: AuditLogger, + metrics?: MetricsCollector, + ) { + this.defaultProviderName = defaultProvider; + this.auditLogger = auditLogger; + this.metrics = metrics; + + // Initialize output marker processor with deps that reference Router's mutable state + this.outputMarkerProcessor = new OutputMarkerProcessor({ + getWorkspacePath: () => this.workspacePath, + getEmailConfig: () => this.emailConfig, + getFileServer: () => this.fileServer, + getAppServer: () => this.appServer, + getRelay: () => this.relay, + getConnectors: () => this.connectors, + getAuth: () => this.auth, + getWorkflowStore: () => this.workflowStore, + getWorkflowEngine: () => this.workflowEngine, + getWorkflowScheduler: () => this.workflowScheduler, + getIntegrationHub: () => this.integrationHub, + ensureTunnel: () => (this.ensureTunnelFn ? this.ensureTunnelFn() : Promise.resolve(null)), + }); + + // Initialize command handlers with deps that reference Router's mutable state + const deps: CommandHandlerDeps = { + getMaster: () => this.master, + getMemory: () => this.memory, + getQueue: () => this.queue, + getAuth: () => this.auth, + getAppServer: () => this.appServer, + getSkillManager: () => this.skillManager, + getWorkspacePath: () => this.workspacePath, + getIntegrationHub: () => this.integrationHub, + getCredentialStore: () => this.credentialStore, + getWorkflowStore: () => this.workflowStore, + getWorkflowEngine: () => this.workflowEngine, + getConnectors: () => this.connectors, + getProviders: () => this.providers, + getSecurityConfig: () => this.securityConfig, + getPendingStopConfirmations: () => this.pendingStopConfirmations, + getSessionGrantedTools: () => this.sessionGrantedTools, + getDetectedSecrets: () => this.detectedSecrets, + getSessionExcludePatterns: () => this.sessionExcludePatterns, + getWorkspaceInclude: () => this.workspaceInclude, + getWorkspaceExclude: () => this.workspaceExclude, + takePendingSpawnConfirmation: (sender) => this.takePendingSpawnConfirmation(sender), + takePendingEscalation: (sender) => this.takePendingEscalation(sender), + takeAllPendingEscalations: (sender) => this.takeAllPendingEscalations(sender), + pendingEscalationCount: (sender) => this.pendingEscalationCount(sender), + route: (msg) => this.route(msg), + }; + this.commandHandlers = new CommandHandlers(deps); + } + + /** Set the agent orchestrator — when set, messages route through it instead of directly to a provider */ + setOrchestrator(orchestrator: AgentOrchestrator): void { + this.orchestrator = orchestrator; + logger.info('Router configured to use Agent Orchestrator'); + } + + /** Set the Master AI — when set, all messages route through it (priority over orchestrator/provider) */ + setMaster(master: MasterManager): void { + this.master = master; + logger.info('Router configured to use Master AI'); + } + + /** Set the SkillManager — enables the "/skills" command */ + setSkillManager(skillManager: SkillManager): void { + this.skillManager = skillManager; + logger.info('Router configured with SkillManager (/skills command enabled)'); + } + + /** Set the IntegrationHub — exposes connected integrations to command handlers */ + setIntegrationHub(hub: IntegrationHub): void { + this.integrationHub = hub; + logger.info('Router configured with IntegrationHub'); + } + + /** Set the CredentialStore — used by /connect to encrypt and persist integration credentials */ + setCredentialStore(store: CredentialStore): void { + this.credentialStore = store; + logger.info('Router configured with CredentialStore'); + } + + /** Set workflow components — enables [WORKFLOW:create] marker support */ + setWorkflowComponents( + store: WorkflowStore, + engine: WorkflowEngine, + scheduler: WorkflowScheduler, + ): void { + this.workflowStore = store; + this.workflowEngine = engine; + this.workflowScheduler = scheduler; + logger.info('Router configured with workflow engine (WORKFLOW markers enabled)'); + } + + /** Set the permission relay — enables interactive tool approval via messaging channels (OB-1499) */ + setPermissionRelay(relay: PermissionRelay): void { + this.permissionRelay = relay; + logger.info('Router configured with PermissionRelay (interactive tool approval enabled)'); + } + + /** Set the auth service — used to whitelist-check recipients in SEND markers */ + setAuth(auth: AuthService): void { + this.auth = auth; + } + + /** Set the workspace path — used to validate file paths in SHARE markers */ + setWorkspacePath(workspacePath: string): void { + this.workspacePath = workspacePath; + } + + /** Set the email config — enables [SHARE:email] marker support */ + setEmailConfig(config: EmailConfig): void { + this.emailConfig = config; + } + + /** Set the file server — enables [SHARE:FILE] marker support (creates shareable links) */ + setFileServer(server: FileServer): void { + this.fileServer = server; + } + + /** Set the MemoryManager — enables the "status" command and fast-path context chunks */ + setMemory(memory: MemoryManager): void { + this.memory = memory; + this.fastPathResponder.setMemory(memory); + logger.info('Router configured with MemoryManager (status command enabled)'); + } + + /** Set the MessageQueue — enables queue depth display in the "status" command */ + setQueue(queue: MessageQueue): void { + this.queue = queue; + // Register urgent-message callback: when a priority-1 message is enqueued while + // the sender already has a message in flight, mark it for checkpoint-handle-resume. + queue.onUrgentEnqueued((msg) => { + if (this.master) { + this.urgentCycleMessageIds.add(msg.id); + logger.info( + { messageId: msg.id }, + 'Urgent message detected — session will be checkpointed before handling', + ); + } + }); + } + + /** Set the AppServer — enables [APP:start] and [APP:stop] marker support */ + setAppServer(appServer: AppServer): void { + this.appServer = appServer; + logger.info('Router configured with AppServer (APP markers enabled)'); + } + + /** Set the auto-tunnel function — enables on-demand tunnel for remote channel APP delivery (OB-1633) */ + setEnsureTunnel(fn: () => Promise): void { + this.ensureTunnelFn = fn; + logger.info('Router configured with ensureTunnel (remote channel APP:start tunnel enabled)'); + } + + /** Set the InteractionRelay — routes app messages to Master as app-interaction InboundMessages */ + setInteractionRelay(relay: InteractionRelay): void { + this.relay = relay; + relay.onAppMessage((relayMsg) => this.handleAppInteraction(relayMsg)); + logger.info('Router configured with InteractionRelay (app interactions enabled)'); + } + + /** + * Handle a message received from a served app via InteractionRelay. + * Converts the relay message into an InboundMessage and passes it to Master AI. + * No-op when Master is not configured. + */ + private async handleAppInteraction(relayMsg: RelayMessage): Promise { + if (!this.master) { + logger.warn( + { appId: relayMsg.appId }, + 'App interaction received but no Master is set — ignoring', + ); + return; + } + + const msgId = relayMsg.id ?? `relay-${relayMsg.appId}-${Date.now()}`; + const dataStr = + relayMsg.data !== null && relayMsg.data !== undefined + ? JSON.stringify(relayMsg.data, null, 2) + : '(no data)'; + + const inboundMessage: InboundMessage = { + id: msgId, + source: 'interaction-relay', + sender: `app:${relayMsg.appId}`, + rawContent: dataStr, + content: `[App: ${relayMsg.appId}] ${relayMsg.type}\n${dataStr}`, + timestamp: relayMsg.timestamp ? new Date(relayMsg.timestamp) : new Date(), + metadata: { + type: 'app-interaction', + appId: relayMsg.appId, + data: relayMsg.data, + }, + }; + + logger.info( + { appId: relayMsg.appId, type: relayMsg.type, msgId }, + 'Routing app interaction to Master', + ); + + try { + await this.master.processMessage(inboundMessage); + } catch (err) { + logger.error( + { appId: relayMsg.appId, type: relayMsg.type, err }, + 'Error processing app interaction', + ); + } + } + + /** + * Inject a synthetic batch continuation message directly to Master AI. + * + * Bypasses all auth checks, rate limiting, and the message queue. Called by + * MasterManager (OB-1613) after each batch item completes to trigger processing + * of the next batch item. The `[CONTINUE:batch-{batchId}]` content is the internal + * marker that route() recognises and forwards to Master without user-facing output. + * + * @param batchId The active batch identifier. + * @param sender The original sender who initiated the batch (for message context). + */ + async routeBatchContinuation(batchId: string, sender: string): Promise { + if (!this.master) { + logger.warn({ batchId }, 'routeBatchContinuation: no Master configured — skipping'); + return; + } + + const syntheticMsg: InboundMessage = { + id: `batch-continue-${batchId}-${Date.now()}`, + source: 'internal-batch', + sender, + rawContent: `[CONTINUE:batch-${batchId}]`, + content: `[CONTINUE:batch-${batchId}]`, + timestamp: new Date(), + metadata: { internal: true, batchId, type: 'batch-continuation' }, + }; + + logger.info( + { batchId, sender, messageId: syntheticMsg.id }, + 'Injecting batch continuation — routing to Master without auth/rate limiting', + ); + + try { + await this.master.processMessage(syntheticMsg); + } catch (err) { + logger.error({ batchId, err }, 'Error processing batch continuation'); + // (OB-1616) Notify master of batch item failure so it can pause and alert the user. + const reason = err instanceof Error ? err.message : String(err); + await this.master.onBatchItemFailure(batchId, reason); + } + } + + /** Set the security config — controls confirmation requirements for high-risk spawns */ + setSecurityConfig(config: SecurityConfig): void { + this.securityConfig = config; + logger.info( + { confirmHighRisk: config.confirmHighRisk }, + 'Router configured with SecurityConfig', + ); + } + + /** + * Set workspace visibility state — populates data shown by the /scope command. + * Called by Bridge after startup secret scanning and config wiring are complete. + */ + setVisibilityState( + detectedSecrets: readonly SecretMatch[], + sessionExcludePatterns: readonly string[], + workspaceInclude: readonly string[], + workspaceExclude: readonly string[], + ): void { + this.detectedSecrets = detectedSecrets; + this.sessionExcludePatterns = sessionExcludePatterns; + this.workspaceInclude = workspaceInclude; + this.workspaceExclude = workspaceExclude; + logger.info( + { secretCount: detectedSecrets.length, sessionExcludeCount: sessionExcludePatterns.length }, + 'Router visibility state updated', + ); + } + + /** + * Check whether a given tool-profile name maps to a high or critical risk level. + * Unknown/custom profiles default to 'high' (conservative). + */ + private getProfileRisk(profileName: string): RiskLevel { + const parsed = BuiltInProfileNameSchema.safeParse(profileName); + if (parsed.success) { + return PROFILE_RISK_MAP[parsed.data]; + } + return 'high'; + } + + /** + * Intercept high-risk SPAWN markers before dispatch and request user confirmation. + * + * Called by MasterManager (via the router reference) before spawning workers. + * When `security.confirmHighRisk` is enabled and any marker has a high or critical + * risk profile, this method: + * 1. Sends a confirmation prompt to the user listing the tasks and risk level. + * 2. Stores the pending spawn entry keyed by sender. + * 3. Returns `true` to signal the caller should defer dispatch. + * + * When confirmation is not required (low/medium risk, or confirmHighRisk disabled), + * returns `false` so the caller proceeds immediately. + */ + public async requestSpawnConfirmation( + sender: string, + connector: Connector, + markers: ParsedSpawnMarker[], + message: InboundMessage, + ): Promise { + const trustLevel = this.securityConfig?.trustLevel ?? 'standard'; + + if (trustLevel === 'trusted') { + logger.debug({ sender }, 'Trusted mode — auto-approving worker spawn'); + return false; + } + + if (trustLevel === 'sandbox') { + const denyMsg: OutboundMessage = { + target: message.source, + recipient: sender, + content: + '⛔ Sandbox mode — high-risk operations are not available. Change trustLevel in config.json to enable.', + }; + await connector.sendMessage(denyMsg); + return true; + } + + if (!this.securityConfig?.confirmHighRisk) return false; + + // Check per-user consent preference — skip confirmation when the user has opted out + if (this.memory) { + const consentMode = await this.memory.getConsentMode(sender, message.source); + if (consentMode === 'auto-approve-all') { + logger.debug({ sender, consentMode }, 'Skipping spawn confirmation — auto-approve-all'); + return false; + } + if (consentMode === 'auto-approve-read') { + const allLowRisk = markers.every((m) => { + const risk = this.getProfileRisk(m.profile); + return risk === 'low'; + }); + if (allLowRisk) { + logger.debug( + { sender, consentMode }, + 'Skipping spawn confirmation — auto-approve-read with all low-risk profiles', + ); + return false; + } + } + if (consentMode === 'auto-approve-up-to-edit') { + const allMediumOrLower = markers.every((m) => { + const risk = this.getProfileRisk(m.profile); + return risk === 'low' || risk === 'medium'; + }); + if (allMediumOrLower) { + logger.debug( + { sender, consentMode }, + 'Skipping spawn confirmation — auto-approve-up-to-edit with all medium-or-lower-risk profiles', + ); + return false; + } + } + } + + const highRiskMarkers = markers.filter((m) => { + const risk = this.getProfileRisk(m.profile); + return risk === 'high' || risk === 'critical'; + }); + + if (highRiskMarkers.length === 0) return false; + + const hasCritical = highRiskMarkers.some((m) => this.getProfileRisk(m.profile) === 'critical'); + const riskLevel: RiskLevel = hasCritical ? 'critical' : 'high'; + const firstHighRiskMarker = highRiskMarkers[0]!; + const summaries = extractTaskSummaries(highRiskMarkers); + + const profileDisplay = firstHighRiskMarker.profile; + const riskDisplay = riskLevel.toUpperCase(); + const taskList = summaries.map((s, i) => `${i + 1}. ${s}`).join('\n'); + + // Aggregate cost estimate across all high-risk workers + let totalTurns = 0; + let totalCostUsd = 0; + for (const marker of highRiskMarkers) { + const maxTurns = marker.body.maxTurns ?? DEFAULT_MAX_TURNS_TASK; + const modelTier = marker.body.model ?? 'balanced'; + const est = estimateCost(marker.profile, maxTurns, modelTier); + totalTurns += est.estimatedTurns; + totalCostUsd += parseFloat(est.costString.slice(2)); // strip leading "~$" + } + const aggCostString = `~$${totalCostUsd.toFixed(2)}`; + const aggTimeMinutes = Math.ceil((totalTurns * 10) / 60); + const aggTimeString = aggTimeMinutes <= 1 ? '~1 min' : `~${aggTimeMinutes} min`; + const estimateText = `Estimated: ~${totalTurns} turns, ${aggCostString}, ${aggTimeString}`; + + const confirmText = + `⚠️ Confirmation required — ${highRiskMarkers.length} worker(s) with ${riskDisplay} risk` + + ` profile (${profileDisplay}):\n\n${taskList}\n\n${estimateText}\n\nReply "go" to proceed or "skip" to cancel.`; + + // Clear any superseded pending confirmation for this sender before creating a new one. + const existingConfirmation = this.pendingSpawnConfirmations.get(sender); + if (existingConfirmation) { + clearTimeout(existingConfirmation.timeoutHandle); + logger.debug( + { + sender, + profile: existingConfirmation.profile, + riskLevel: existingConfirmation.riskLevel, + }, + 'Cleared superseded spawn confirmation timer — new request overwrites pending one', + ); + } + + // Set a 60-second auto-cancel timeout. Stored in the entry so it can be cleared on reply. + const timeoutHandle = setTimeout(() => { + const stillPending = this.pendingSpawnConfirmations.get(sender); + if (!stillPending) return; + this.pendingSpawnConfirmations.delete(sender); + + const timeoutMsg: OutboundMessage = { + target: message.source, + recipient: sender, + content: '⏱ Confirmation timed out — spawn cancelled. Send your request again to retry.', + }; + connector.sendMessage(timeoutMsg).catch((err: unknown) => { + logger.warn({ err, sender }, 'Failed to send spawn confirmation timeout notification'); + }); + + logger.warn( + { sender, profile: profileDisplay, riskLevel }, + 'Spawn confirmation timed out — pending entry auto-cancelled', + ); + }, 60_000); + + this.pendingSpawnConfirmations.set(sender, { + markers, + message, + connector, + taskSummaries: summaries, + profile: firstHighRiskMarker.profile, + riskLevel, + timeoutHandle, + }); + + const confirmMsg: OutboundMessage = { + target: message.source, + recipient: sender, + content: confirmText, + }; + await connector.sendMessage(confirmMsg); + + logger.info( + { sender, profile: profileDisplay, riskLevel, markerCount: highRiskMarkers.length }, + 'High-risk SPAWN intercepted — confirmation prompt sent to user', + ); + + return true; + } + + /** + * Retrieve and remove the pending spawn confirmation entry for a sender. + * Clears the auto-cancel timeout so it does not fire after the user has replied. + * Returns `undefined` if no confirmation is pending for that sender. + * Used by the /confirm and /skip command handlers (OB-1388). + */ + public takePendingSpawnConfirmation(sender: string): PendingSpawnEntry | undefined { + const entry = this.pendingSpawnConfirmations.get(sender); + if (entry) { + clearTimeout(entry.timeoutHandle); + this.pendingSpawnConfirmations.delete(sender); + } + return entry; + } + + /** Check whether a pending spawn confirmation exists for a sender. */ + public hasPendingSpawnConfirmation(sender: string): boolean { + return this.pendingSpawnConfirmations.has(sender); + } + + /** + * Send a tool escalation prompt to the user and register a pending escalation. + * Called when a worker needs additional tool access beyond its current profile. + * + * Sends: "Worker {id} needs {tools} access for: {reason}. Reply '/allow {tool}' or + * '/allow {profile}' to grant, '/deny' to reject." + * + * Registers an auto-deny timeout (default 300s, configurable via `router.escalationTimeoutMs`) — + * cleared when the user replies with /allow or /deny via the respective command handlers (OB-1586, OB-1587). + */ + public async requestToolEscalation( + workerId: string, + requestedTools: string[], + currentProfile: string, + reason: string, + message: InboundMessage, + connector: Connector, + respawn?: (grantedTools: string[]) => Promise, + ): Promise { + const sender = message.sender; + const toolsList = requestedTools.join(', '); + + // Check consent mode — auto-approve escalations based on user trust level (OB-1601). + if (this.memory) { + const consentMode = await this.memory.getConsentMode(sender, message.source); + + // auto-approve-all: skip all escalation prompts — user trusts the system fully. + if (consentMode === 'auto-approve-all') { + logger.info( + { sender, workerId, requestedTools }, + 'Auto-approving tool escalation — auto-approve-all mode', + ); + await connector.sendMessage({ + target: message.source, + recipient: sender, + content: `✅ Auto-approved: *${toolsList}* for worker ${workerId}`, + }); + if (respawn) { + await respawn(requestedTools); + } + return; + } + + if (consentMode === 'auto-approve-up-to-edit') { + const allWithinEditLevel = requestedTools.every((tool) => { + const parsed = BuiltInProfileNameSchema.safeParse(tool); + if (parsed.success) { + const risk = PROFILE_RISK_MAP[parsed.data]; + // Auto-approve low and medium risk profiles (read-only, code-audit, code-edit) + return risk === 'low' || risk === 'medium'; + } + // Specific tool names (not profile names) are single-tool grants — treat as within edit level + return true; + }); + if (allWithinEditLevel) { + logger.info( + { sender, workerId, requestedTools }, + 'Auto-approving tool escalation — auto-approve-up-to-edit mode', + ); + await connector.sendMessage({ + target: message.source, + recipient: sender, + content: `✅ Auto-approved tool escalation for worker ${workerId}: *${toolsList}* (auto-approve-up-to-edit mode)`, + }); + if (respawn) { + await respawn(requestedTools); + } + return; + } + } + } + + // Get queue state BEFORE adding new entry to compute scaled timeout (OB-1639) + const existingQueue = this.pendingEscalations.get(sender) ?? []; + const pendingCount = existingQueue.length + 1; // includes the escalation being added + + // Scale timeout: base + 60s per additional pending escalation beyond the first, capped at 600s + const baseTimeoutMs = this.routerConfig?.escalationTimeoutMs ?? 300_000; + const scaledTimeoutMs = Math.min(baseTimeoutMs + (pendingCount - 1) * 60_000, 600_000); + const scaledTimeoutSec = Math.round(scaledTimeoutMs / 1000); + + // Schedule a 50% reminder once per batch — only for the first escalation in a new batch (OB-1640) + if (existingQueue.length === 0) { + const reminderDelayMs = Math.round(scaledTimeoutMs / 2); + setTimeout(() => { + // Skip if the reminder was already sent for this sender (defensive guard) + if (this.escalationReminderSent.has(sender)) return; + const currentQueue = this.pendingEscalations.get(sender); + if (!currentQueue || currentQueue.length === 0) return; + this.escalationReminderSent.add(sender); + const count = currentQueue.length; + const reminderMsg: OutboundMessage = { + target: message.source, + recipient: sender, + content: `⏰ Reminder: You have ${count} pending escalation request${count === 1 ? '' : 's'} — reply /allow, /allow all, or /deny.`, + }; + connector.sendMessage(reminderMsg).catch((err: unknown) => { + logger.warn({ err, sender }, 'Failed to send escalation reminder'); + }); + logger.info({ sender, count }, 'Escalation reminder sent at 50% timeout'); + }, reminderDelayMs); + } + + // Set auto-deny timeout — removes only this entry from the queue + const timeoutHandle = setTimeout(() => { + const queue = this.pendingEscalations.get(sender); + if (!queue) return; + const idx = queue.findIndex((e) => e.workerId === workerId); + if (idx === -1) return; + queue.splice(idx, 1); + if (queue.length === 0) { + this.pendingEscalations.delete(sender); + this.escalationReminderSent.delete(sender); + } + + const timeoutMsg: OutboundMessage = { + target: message.source, + recipient: sender, + content: `⏱ Escalation timed out — worker ${workerId} continuing with current profile (${currentProfile}).`, + }; + connector.sendMessage(timeoutMsg).catch((err: unknown) => { + logger.warn({ err, sender, workerId }, 'Failed to send escalation timeout notification'); + }); + + logger.warn( + { sender, workerId, requestedTools, currentProfile }, + 'Tool escalation timed out — auto-denied', + ); + + // Mark the worker as cancelled in the registry (OB-1644) + if (this.master) { + const registry = this.master.getWorkerRegistry(); + const workerRecord = registry.getWorker(workerId); + if ( + workerRecord && + workerRecord.status !== 'completed' && + workerRecord.status !== 'failed' + ) { + try { + registry.markCancelled(workerId, 'escalation-timeout'); + logger.info({ workerId }, 'Worker marked cancelled after escalation timeout'); + } catch (err) { + logger.warn( + { workerId, err }, + 'Failed to mark worker cancelled after escalation timeout', + ); + } + } + } + }, scaledTimeoutMs); + + const queueEntry: PendingEscalation = { + workerId, + requestedTools, + currentProfile, + reason, + message, + connector, + timeoutHandle, + respawn, + }; + existingQueue.push(queueEntry); + this.pendingEscalations.set(sender, existingQueue); + + // Build escalation prompt showing full queue state (OB-1635) + const queueCount = existingQueue.length; + let escalationText: string; + if (queueCount === 1) { + const allowExample = + requestedTools.length === 1 + ? `/allow ${requestedTools[0]}` + : `/allow ${requestedTools[0]} (or /allow code-edit)`; + escalationText = + `⚠️ Worker ${workerId} needs *${toolsList}* access for:\n${reason}\n\n` + + `Current profile: ${currentProfile}\n\n` + + `Reply '${allowExample}', '/allow all', or '/deny' to reject.\n` + + `Auto-deny in ${scaledTimeoutSec} seconds if no reply.`; + } else { + const workerLines = existingQueue + .map((e, i) => `(${i + 1}) ${e.workerId} needs ${e.requestedTools.join(', ')}`) + .join('\n'); + escalationText = + `⚠️ ${queueCount} workers requesting elevated access:\n${workerLines}\n\n` + + `Reply /allow for next, /allow all for all, or /deny to reject next.\n` + + `Auto-deny for first request in ${scaledTimeoutSec} seconds if no reply.`; + } + + await connector.sendMessage({ + target: message.source, + recipient: sender, + content: escalationText, + replyTo: message.id, + }); + + logger.info( + { sender, workerId, requestedTools, currentProfile }, + 'Tool escalation prompt sent to user', + ); + } + + /** + * Retrieve and remove the pending escalation entry for a sender. + * Clears the auto-deny timeout so it does not fire after the user has replied. + * Returns `undefined` if no escalation is pending for that sender. + * Used by the /allow and /deny command handlers (OB-1586, OB-1587). + */ + public takePendingEscalation(sender: string): PendingEscalation | undefined { + const queue = this.pendingEscalations.get(sender); + if (!queue || queue.length === 0) return undefined; + const entry = queue.shift()!; + clearTimeout(entry.timeoutHandle); + if (queue.length === 0) { + this.pendingEscalations.delete(sender); + this.escalationReminderSent.delete(sender); + } + return entry; + } + + /** Return the number of pending escalations queued for a sender. */ + public pendingEscalationCount(sender: string): number { + return this.pendingEscalations.get(sender)?.length ?? 0; + } + + /** + * Retrieve and remove ALL pending escalation entries for a sender. + * Clears all auto-deny timeouts. Returns an empty array if none are pending. + * Used by the /allow all command handler (OB-1632). + */ + public takeAllPendingEscalations(sender: string): PendingEscalation[] { + const queue = this.pendingEscalations.get(sender); + if (!queue || queue.length === 0) return []; + this.pendingEscalations.delete(sender); + this.escalationReminderSent.delete(sender); + for (const entry of queue) clearTimeout(entry.timeoutHandle); + return queue; + } + + /** Check whether a pending tool escalation exists for a sender. */ + public hasPendingEscalation(sender: string): boolean { + const queue = this.pendingEscalations.get(sender); + return !!queue && queue.length > 0; + } + + /** + * Return the set of tool/profile names granted for this session for a sender. + * Returns an empty set when no session grants exist for the sender. + * Used by workers and Master to check session-level tool access without DB lookup. + */ + public getSessionGrants(sender: string): ReadonlySet { + return this.sessionGrantedTools.get(sender) ?? new Set(); + } + + /** Register an active connector */ + addConnector(connector: Connector): void { + this.connectors.set(connector.name, connector); + } + + /** Look up a registered connector by source name. Returns undefined when not found. */ + getConnector(source: string): Connector | undefined { + return this.connectors.get(source); + } + + /** Register an active provider */ + addProvider(provider: AIProvider): void { + this.providers.set(provider.name, provider); + } + + /** + * Send a progress event to a specific connector (best-effort). + * Used by MasterManager to emit typed ProgressEvents to the right connector + * without going through the full routing flow. + * + * When the event is `worker-result`, also sends a plain-text execution summary + * to the user (OB-1391): "Worker completed (Ns, N turns): N files read, N files modified, N commands run". + */ + async sendProgress(source: string, recipient: string, event: ProgressEvent): Promise { + const connector = this.connectors.get(source); + if (!connector) return; + if (connector.sendProgress) { + try { + await connector.sendProgress(event, recipient); + } catch (err) { + logger.warn({ err, source, recipient }, 'sendProgress: failed to send progress event'); + } + } + + // Send execution summary after each worker completes (OB-1391) + if (event.type === 'worker-result') { + await this.sendWorkerExecutionSummary(connector, recipient, event); + } + + // Send phase transition message when a Deep Mode phase completes (OB-1415) + if (event.type === 'deep-phase' && event.status === 'completed') { + await this.sendDeepPhaseTransitionMessage(connector, recipient, event); + } + } + + /** + * Format and send a plain-text execution summary to the user after a worker completes. + * Format: "Worker [N/T] completed (Xs, N turns): N files read, N files modified, N commands run" + * Counts are parsed from the worker's stdout content using heuristic patterns (OB-1391). + */ + private async sendWorkerExecutionSummary( + connector: Connector, + recipient: string, + event: Extract, + ): Promise { + const durationSec = event.durationMs !== undefined ? Math.round(event.durationMs / 1000) : null; + const { filesRead, filesModified, commandsRun } = parseWorkerStats(event.content); + + const timePart = durationSec !== null ? `${durationSec}s` : '?s'; + const turnsPart = + event.turnsUsed !== undefined && event.turnsUsed > 0 + ? `${event.turnsUsed} turn${event.turnsUsed !== 1 ? 's' : ''}` + : null; + const durationStr = turnsPart ? `${timePart}, ${turnsPart}` : timePart; + + const status = event.success ? 'completed' : 'failed'; + const workerLabel = event.total > 1 ? `Worker ${event.workerIndex}/${event.total}` : 'Worker'; + + const summary = + `${workerLabel} ${status} (${durationStr}): ` + + `${filesRead} file${filesRead !== 1 ? 's' : ''} read, ` + + `${filesModified} file${filesModified !== 1 ? 's' : ''} modified, ` + + `${commandsRun} command${commandsRun !== 1 ? 's' : ''} run`; + + const msg: OutboundMessage = { + target: connector.name, + recipient, + content: summary, + }; + + try { + await connector.sendMessage(msg); + logger.debug( + { recipient, workerIndex: event.workerIndex, total: event.total, durationSec }, + 'Worker execution summary sent', + ); + } catch (err) { + logger.warn( + { err, recipient }, + 'sendWorkerExecutionSummary: failed to send execution summary', + ); + } + } + + /** + * Build and send a phase transition message to the user after a Deep Mode phase completes. + * + * Includes a completion header with item count, a brief snippet of the phase output, and + * per-phase guidance for the next available actions (/proceed, /focus N, /skip N, etc.). + * + * Called from sendProgress() when a deep-phase event with status 'completed' arrives (OB-1415). + */ + private async sendDeepPhaseTransitionMessage( + connector: Connector, + recipient: string, + event: Extract, + ): Promise { + const { sessionId, phase } = event; + + // Try to get the full phase result for item counting and an extended snippet + const fullResult = this.master + ?.getDeepModeManager() + .getPhaseResult(sessionId, phase as DeepPhase); + + const outputText = fullResult?.output ?? event.resultSummary ?? ''; + + // Count numbered list items (e.g. "1. Finding" or "1) Task") + const numberedCount = (outputText.match(/^\s*\d+[.)]\s/gm) ?? []).length; + const itemCount = numberedCount > 0 ? numberedCount : undefined; + + // Build a concise summary snippet (first 300 chars of the result) + const snippet = + outputText.length > 0 + ? outputText.slice(0, 300).trimEnd() + (outputText.length > 300 ? '…' : '') + : ''; + + // Per-phase completion header and tailored next-action guidance + const phaseMessages: Record = { + investigate: { + header: `*Investigation complete*${itemCount !== undefined ? ` — ${itemCount} finding${itemCount !== 1 ? 's' : ''} identified` : ''}.`, + guidance: + 'To dig deeper into a finding: `/focus N`\nTo proceed to the report phase: `/proceed`', + }, + report: { + header: `*Report ready*${itemCount !== undefined ? ` — ${itemCount} item${itemCount !== 1 ? 's' : ''}` : ''}.`, + guidance: + 'To focus on a specific finding: `/focus N`\nTo proceed to the plan phase: `/proceed`', + }, + plan: { + header: `*Plan ready*${itemCount !== undefined ? ` — ${itemCount} task${itemCount !== 1 ? 's' : ''}` : ''}.`, + guidance: 'To skip a task: `/skip N`\nTo execute the plan: `/proceed`', + }, + execute: { + header: '*Execution complete.*', + guidance: 'To run verification checks: `/proceed`', + }, + verify: { + header: '*Verification complete. Deep Mode session finished.*', + guidance: 'Send `/phase` to review the full results.', + }, + }; + + const phaseInfo = phaseMessages[phase]; + if (!phaseInfo) return; // Unknown phase — skip + + const parts: string[] = [phaseInfo.header]; + if (snippet) parts.push('', snippet); + parts.push('', phaseInfo.guidance); + + try { + await connector.sendMessage({ target: connector.name, recipient, content: parts.join('\n') }); + logger.debug({ recipient, sessionId, phase }, 'Deep Mode phase transition message sent'); + } catch (err) { + logger.warn( + { err, recipient, sessionId, phase }, + 'sendDeepPhaseTransitionMessage: failed to send', + ); + } + } + + /** + * Broadcast a progress event to all connected connectors (best-effort). + * Used during workspace exploration when there is no specific message sender. + */ + async broadcastProgress(event: ProgressEvent): Promise { + for (const [name, connector] of this.connectors) { + if (connector.sendProgress) { + try { + await connector.sendProgress(event, '__system__'); + } catch (err) { + logger.warn({ err, connector: name }, 'broadcastProgress: failed'); + } + } + } + } + + /** + * Send a message directly to a user on a specific connector (best-effort). + * Used by MasterManager to deliver progress updates during worker delegation + * without going through the full routing flow. + */ + async sendDirect( + source: string, + recipient: string, + content: string, + replyTo?: string, + ): Promise { + const connector = this.connectors.get(source); + if (!connector) { + logger.warn({ source }, 'sendDirect: connector not found'); + return; + } + const msg: OutboundMessage = { target: source, recipient, content, replyTo }; + try { + await connector.sendMessage(msg); + } catch (err) { + logger.warn({ err, source, recipient }, 'sendDirect: failed to send message'); + } + } + + /** Route an inbound message to the appropriate provider and send the response back */ + async route(message: InboundMessage): Promise { + // Validate routing target exists + // Priority: Master → Orchestrator → Direct Provider + if (!this.master && !this.orchestrator) { + const provider = this.providers.get(this.defaultProviderName); + if (!provider) { + logger.error({ provider: this.defaultProviderName }, 'Default provider not found'); + return; + } + } + + // Detect internal batch continuation messages — [CONTINUE:batch-{id}] — before connector + // lookup so that synthetic messages (source='internal-batch') work without a registered + // connector. These messages bypass all auth checks, rate limiting, and user-facing acks. + const continueMatch = CONTINUE_MARKER_RE.exec(message.content.trim()); + if (continueMatch !== null) { + const batchId = continueMatch[1]!; + logger.info( + { batchId, sender: message.sender, messageId: message.id }, + 'Internal batch continuation detected — routing to Master directly', + ); + if (this.master) { + try { + await this.master.processMessage(message); + } catch (err) { + logger.error({ batchId, err }, 'Error processing batch continuation'); + // (OB-1616) Notify master of batch item failure so it can pause and alert the user. + const reason = err instanceof Error ? err.message : String(err); + await this.master.onBatchItemFailure(batchId, reason); + } + } else { + logger.warn({ batchId }, 'Batch continuation received but no Master is set — ignoring'); + } + return; + } + + const connector = this.connectors.get(message.source); + if (!connector) { + logger.error({ source: message.source }, 'Source connector not found'); + return; + } + + // Handle permission relay responses — intercept YES/NO/ALLOW/DENY replies + // when a permission request is pending for this user (OB-1499) + if (this.permissionRelay?.hasPending(message.sender)) { + const consumed = this.permissionRelay.handleResponse(message.sender, message.content); + if (consumed) { + logger.info( + { sender: message.sender, content: message.content.trim() }, + 'Permission response consumed', + ); + return; + } + // Not a valid YES/NO — fall through to normal routing + } + + // Handle built-in "status" command — intercept before routing to Master AI + if (message.content.trim().toLowerCase() === 'status') { + await this.handleStatusCommand(message, connector); + return; + } + + // Handle /pause command — pause active batch (OB-1619). + if (/^\/pause$/i.test(message.content.trim())) { + if (this.master) { + const response = await this.master.handleBatchCommand( + 'pause', + message.sender, + message.source, + ); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: response, + }); + } else { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'No active batch.', + }); + } + return; + } + + // Handle /continue command — resume paused batch (OB-1620). + if (/^\/continue$/i.test(message.content.trim())) { + if (this.master) { + const response = await this.master.handleBatchCommand( + 'resume', + message.sender, + message.source, + ); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: response, + }); + } else { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'No active batch.', + }); + } + return; + } + + // Handle /batch (no args) — show batch status (OB-1621). + if (/^\/batch$/i.test(message.content.trim())) { + const response = this.master ? this.master.getBatchStatus() : 'No active batch.'; + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: response, + }); + return; + } + + // Handle batch control commands: /batch skip | /batch retry | /batch abort (OB-1616). + // These are intercepted before routing to Master so they take effect immediately. + const batchCmdMatch = /^\/batch\s+(skip|retry|abort)$/i.exec(message.content.trim()); + if (batchCmdMatch !== null) { + const action = batchCmdMatch[1]!.toLowerCase() as 'skip' | 'retry' | 'abort'; + if (this.master) { + const response = await this.master.handleBatchCommand( + action, + message.sender, + message.source, + ); + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: response, + }); + } else { + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: 'No active batch.', + }); + } + return; + } + + // Handle "confirm" for a pending stop-all confirmation — intercept before routing to Master AI + if (message.content.trim().toLowerCase() === 'confirm') { + const pending = this.pendingStopConfirmations.get(message.sender); + if (pending) { + await this.handleConfirmCommand(message, connector, pending); + return; + } + } + + // Handle "go" / "/confirm" for a pending high-risk spawn confirmation + if ( + message.content.trim().toLowerCase() === 'go' || + message.content.trim().toLowerCase() === '/confirm' + ) { + // "go" with no pending spawn but Deep Mode active → treat as /proceed (OB-1413) + if ( + message.content.trim().toLowerCase() === 'go' && + !this.pendingSpawnConfirmations.has(message.sender) && + this.master !== undefined && + this.master.getDeepModeManager().getActiveSessions().length > 0 + ) { + await this.handleProceedCommand(message, connector); + return; + } + await this.handleConfirmSpawnCommand(message, connector); + return; + } + + // Handle "skip" / "/skip" for a pending high-risk spawn cancellation + if ( + message.content.trim().toLowerCase() === 'skip' || + message.content.trim().toLowerCase() === '/skip' + ) { + await this.handleSkipSpawnCommand(message, connector); + return; + } + + // Handle built-in "stop" command — intercept before routing to Master AI + if (/^stop(\s+.*)?$/i.test(message.content.trim())) { + await this.handleStopCommand(message, connector); + return; + } + + // Handle built-in "explore" command — intercept before routing to Master AI + if (/^explore(\s+.*)?$/i.test(message.content.trim())) { + await this.handleExploreCommand(message, connector); + return; + } + + // Handle built-in "history" command — intercept before routing to Master AI + if (/^history(\s+.*)?$/i.test(message.content.trim())) { + await this.handleHistoryCommand(message, connector); + return; + } + + // Handle built-in "/audit" command — intercept before routing to Master AI + if (/^\/audit(\s+.*)?$/i.test(message.content.trim())) { + await this.handleAuditCommand(message, connector); + return; + } + + // Handle built-in "/deep" command — starts/toggles/configures Deep Mode (OB-1407) + if (/^\/deep(\s+.*)?$/i.test(message.content.trim())) { + await this.handleDeepCommand(message, connector); + return; + } + + // Handle built-in "/proceed" command — advances to next Deep Mode phase (OB-1408) + if (/^\/proceed(\s+.*)?$/i.test(message.content.trim())) { + await this.handleProceedCommand(message, connector); + return; + } + + // Handle built-in "/focus N" command — focused investigation on finding N (OB-1409) + if (/^\/focus(\s+.*)?$/i.test(message.content.trim())) { + await this.handleFocusCommand(message, connector); + return; + } + + // Handle built-in "/skip N" command — skip item N from Deep Mode plan (OB-1410) + if (/^\/skip\s+\d+/i.test(message.content.trim())) { + await this.handleSkipItemCommand(message, connector); + return; + } + + // Handle built-in "/phase" command — shows current Deep Mode phase and progress (OB-1411) + if (/^\/phase(\s+.*)?$/i.test(message.content.trim())) { + await this.handlePhaseCommand(message, connector); + return; + } + + // Handle built-in "/help" command — lists all available commands (OB-1430) + if (/^\/help$/i.test(message.content.trim())) { + await this.handleHelpCommand(message, connector); + return; + } + + // Handle built-in "/skills" command — list available skills with descriptions and usage counts (OB-1712) + if (/^\/skills$/i.test(message.content.trim())) { + await this.handleSkillsCommand(message, connector); + return; + } -export class Router { - private readonly connectors = new Map(); - private readonly providers = new Map(); - private defaultProviderName: string; - private readonly progressIntervalMs: number; - private readonly auditLogger?: AuditLogger; - private readonly metrics?: MetricsCollector; - private orchestrator?: AgentOrchestrator; - private master?: MasterManager; + // Handle built-in "/skill-packs" command — list available skill packs with descriptions (OB-1756) + if (/^\/skill-packs$/i.test(message.content.trim())) { + await this.handleSkillPacksCommand(message, connector); + return; + } - constructor( - defaultProvider: string, - config?: RouterConfig, - auditLogger?: AuditLogger, - metrics?: MetricsCollector, - ) { - this.defaultProviderName = defaultProvider; - this.progressIntervalMs = config?.progressIntervalMs ?? 15_000; - this.auditLogger = auditLogger; - this.metrics = metrics; - } + // Handle built-in "/apps" command — shows running app instances with URLs (OB-1450) + if (/^\/apps$/i.test(message.content.trim())) { + await this.handleAppsCommand(message, connector); + return; + } - /** Set the agent orchestrator — when set, messages route through it instead of directly to a provider */ - setOrchestrator(orchestrator: AgentOrchestrator): void { - this.orchestrator = orchestrator; - logger.info('Router configured to use Agent Orchestrator'); - } + // Handle built-in "/scope" command — shows visibility rules and detected secrets (OB-1472) + if (/^\/scope$/i.test(message.content.trim())) { + await this.handleScopeCommand(message, connector); + return; + } - /** Set the Master AI — when set, all messages route through it (priority over orchestrator/provider) */ - setMaster(master: MasterManager): void { - this.master = master; - logger.info('Router configured to use Master AI'); - } + // Handle built-in "/allow" command — grant pending tool escalation (OB-1586) + if (/^\/allow(\s+.*)?$/i.test(message.content.trim())) { + await this.handleAllowCommand(message, connector); + return; + } - /** Register an active connector */ - addConnector(connector: Connector): void { - this.connectors.set(connector.name, connector); - } + // Handle built-in "/deny all" command — reject all pending tool escalations (OB-1634) + if (/^\/deny\s+all$/i.test(message.content.trim())) { + await this.handleDenyAllCommand(message, connector); + return; + } - /** Register an active provider */ - addProvider(provider: AIProvider): void { - this.providers.set(provider.name, provider); - } + // Handle built-in "/deny" command — reject pending tool escalation (OB-1587) + if (/^\/deny$/i.test(message.content.trim())) { + await this.handleDenyCommand(message, connector); + return; + } - /** Route an inbound message to the appropriate provider and send the response back */ - async route(message: InboundMessage): Promise { - // Validate routing target exists - // Priority: Master → Orchestrator → Direct Provider - if (!this.master && !this.orchestrator) { - const provider = this.providers.get(this.defaultProviderName); - if (!provider) { - logger.error({ provider: this.defaultProviderName }, 'Default provider not found'); + // Handle built-in "/permissions" command — show current user's grants and consent mode (OB-1589) + if (/^\/permissions$/i.test(message.content.trim())) { + await this.handlePermissionsCommand(message, connector); + return; + } + + // Detect natural-language trust intent and convert to /trust command (OB-1575) + const trimmedContent = message.content.trim(); + if ( + /^(trust\s+(all|everything|auto|it)|auto[- ]?approve(\s+all)?|approve\s+(all|everything))$/i.test( + trimmedContent, + ) + ) { + logger.info('Natural language trust intent detected — routing to /trust auto'); + message = { ...message, content: '/trust auto' }; + } + + // Handle built-in "/trust" command — change consent mode (auto/edit/ask) + if (/^\/trust(\s+.*)?$/i.test(message.content.trim())) { + await this.handleTrustCommand(message, connector); + return; + } + + // Handle built-in "/whoami" command — show user's role, channel, allowed actions, daily cost, consent mode (OB-1720) + if (/^\/whoami$/i.test(message.content.trim())) { + await this.handleWhoamiCommand(message, connector); + return; + } + + // Handle built-in "/role " command — owner/admin only, sets role for another user (OB-1721) + if (/^\/role\b/i.test(message.content.trim())) { + await this.handleRoleCommand(message, connector); + return; + } + + // Handle built-in "/workers" command — list active workers with ID, status, profile, duration, PID (OB-1646) + if (/^\/workers$/i.test(message.content.trim())) { + await this.handleWorkersCommand(message, connector); + return; + } + + // Handle built-in "/kill " command — force-stop a stuck worker (OB-1646) + if (/^\/kill\s+\S+/i.test(message.content.trim())) { + await this.handleKillWorkerCommand(message, connector); + return; + } + + // Handle built-in "/stats" command — show exploration ROI (OB-1680) + if (/^\/stats$/i.test(message.content.trim())) { + await this.handleStatsCommand(message, connector); + return; + } + + // Handle built-in "/doctor" command — run health checks and send summary (OB-1693) + if (/^\/doctor$/i.test(message.content.trim())) { + await this.handleDoctorCommand(message, connector); + return; + } + + // Handle built-in "/doctypes" command — list all DocTypes (OB-1386) + if (/^\/doctypes$/i.test(message.content.trim())) { + await this.handleDoctypesCommand(message, connector); + return; + } + + // Handle built-in "/doctype " command — show DocType details (OB-1386) + if (/^\/doctype\b/i.test(message.content.trim())) { + await this.handleDoctypeCommand(message, connector); + return; + } + + // Handle built-in "/dt " command — DocType record CRUD (OB-1386) + if (/^\/dt\b/i.test(message.content.trim())) { + await this.handleDtCommand(message, connector); + return; + } + + // Accumulate standalone cURL messages for deferred /connect api (OB-1453) + // When a user sends cURL examples without /connect api, store them so they + // can be flushed when they say "done" or "connect". + if (/^curl\s/i.test(message.content.trim())) { + await this.commandHandlers.handleCurlAccumulationMessage(message, connector); + return; + } + + // Flush cURL buffer when user says "done" or "connect" after accumulating (OB-1453) + if ( + /^(?:done|connect)\s*$/i.test(message.content.trim()) && + this.commandHandlers.hasPendingCurls(message.sender) + ) { + await this.handleCurlBufferFlush(message, connector); + return; + } + + // Handle built-in "/connect []" command — credential collection (OB-1397) + if (/^\/connect(\s.*)?$/i.test(message.content.trim())) { + await this.handleConnectCommand(message, connector); + return; + } + + // Handle built-in "/integrations" command — list all registered integrations (OB-1398) + if (/^\/integrations\b/i.test(message.content.trim())) { + await this.handleIntegrationsCommand(message, connector); + return; + } + + // Handle built-in "/workflows" command — list/enable/disable/runs/delete (OB-1431) + if (/^\/workflows(\b.*)?$/i.test(message.content.trim())) { + await this.handleWorkflowsCommand(message, connector); + return; + } + + // Handle built-in "/process " command — extract document entities (OB-1349) + if (/^\/process(\s+.*)?$/i.test(message.content.trim())) { + await this.handleProcessCommand(message, connector); + return; + } + + // Handle built-in "/approve " command — owner/admin approves a pairing code (OB-1698) + if (/^\/approve\b/i.test(message.content.trim())) { + await this.handleApproveCommand(message, connector); + return; + } + + // Detect natural language model overrides — "use opus for task 1" / "use haiku for this" (OB-1412) + if ( + /\b(?:use|switch\s+to|change\s+to)\s+(?:\w+[-\s]?)?(opus|sonnet|haiku|fast|balanced|powerful)\b/i.test( + message.content.trim(), + ) && + this.master !== undefined && + this.master.getDeepModeManager().getActiveSessions().length > 0 + ) { + await this.handleModelOverrideCommand(message, connector); + return; + } + + // Natural language Deep Mode navigation — regex checked first (short-circuit before + // getDeepModeManager()) to avoid calling it on every message (OB-1413) + { + const trimmedNl = message.content.trim(); + + // "proceed" / "next" / "continue" → /proceed + if ( + /^(?:proceed|next|continue)\s*$/i.test(trimmedNl) && + this.master !== undefined && + this.master.getDeepModeManager().getActiveSessions().length > 0 + ) { + await this.handleProceedCommand(message, connector); + return; + } + + // "focus on #3", "dig into finding 3", "investigate 3", "look at item 3" → /focus 3 + const focusNlMatch = + /\b(?:focus\s+on|dig\s+into|investigate|look\s+at)\s+(?:finding\s+|item\s+|#)?(\d+)/i.exec( + trimmedNl, + ); + if ( + focusNlMatch !== null && + this.master !== undefined && + this.master.getDeepModeManager().getActiveSessions().length > 0 + ) { + await this.handleFocusCommand( + { ...message, content: `/focus ${focusNlMatch[1]}` }, + connector, + ); + return; + } + + // "skip item 2", "skip finding 2", "skip task 2", "skip 2" → /skip 2 + const skipNlMatch = /\bskip\s+(?:item\s+|finding\s+|task\s+|#)?(\d+)/i.exec(trimmedNl); + if ( + skipNlMatch !== null && + this.master !== undefined && + this.master.getDeepModeManager().getActiveSessions().length > 0 + ) { + await this.handleSkipItemCommand( + { ...message, content: `/skip ${skipNlMatch[1]}` }, + connector, + ); return; } } - const connector = this.connectors.get(message.source); - if (!connector) { - logger.error({ source: message.source }, 'Source connector not found'); + // Checkpoint-handle-resume cycle for urgent messages. + // When a priority-1 message was flagged by the queue (sender had a message in flight when + // this arrived), checkpoint session state before processing so that: + // 1. A crash during urgent handling is recoverable from the checkpoint. + // 2. After the urgent message completes, we can restore pre-interruption context. + const isUrgentCycle = this.urgentCycleMessageIds.has(message.id); + let sessionCheckpointed = false; + if (isUrgentCycle) { + this.urgentCycleMessageIds.delete(message.id); + if (this.master) { + await this.master.checkpointSession(); + sessionCheckpointed = true; + } + } + + // Fast-path: when Master is processing a complex task and a quick-answer message arrives, + // spawn a lightweight read-only agent to answer immediately without waiting for Master. + const messagePriority = classifyMessagePriority(message.content); + if (messagePriority === 1 && this.master?.getState() === 'processing') { + await this.runFastPath(message, connector); return; } @@ -92,7 +1847,14 @@ export class Router { 'Routing message', ); - // Send acknowledgment + // Notify the Electron parent process (if running as a forked child) so it can show + // notification badges without parsing log output. Guarded by OPENBRIDGE_ELECTRON so + // this is a no-op in tests, CLI runs, and other non-Electron contexts. + if (typeof process.send === 'function' && process.env['OPENBRIDGE_ELECTRON'] === '1') { + process.send({ type: 'message-received', sender: message.sender, channel: message.source }); + } + + // Send single acknowledgment (no cycling timer — progress events handle the rest) const ack: OutboundMessage = { target: message.source, recipient: message.sender, @@ -106,8 +1868,15 @@ export class Router { await connector.sendTypingIndicator(message.sender); } - // Start progress updates - const stopProgress = this.startProgressUpdates(connector, message); + // Inject attachment context so Master AI and workers know about attached files + if (message.attachments && message.attachments.length > 0) { + const attachmentLines = message.attachments.map((a) => { + const sizeKb = (a.sizeBytes / 1024).toFixed(1); + const namePart = a.filename ? ` (${a.filename})` : ''; + return `- **${a.type}**${namePart}: \`${a.filePath}\` — ${a.mimeType} — ${sizeKb} KB`; + }); + message.content = `${message.content}\n\n## Attachments\n${attachmentLines.join('\n')}`; + } // Process message — through Master, orchestrator, or directly via provider let result: ProviderResult; @@ -130,7 +1899,6 @@ export class Router { } } } catch (error) { - stopProgress(); const errorKind = error instanceof ProviderError ? error.kind : ('unknown' as const); this.metrics?.recordFailed(errorKind); if (error instanceof ProviderError) { @@ -156,52 +1924,375 @@ export class Router { error instanceof Error ? error.message : 'Unknown error', ); throw error; + } finally { + // Resume from checkpoint after urgent message handling (success or failure) — restores + // the pre-interruption Master context so subsequent messages continue with correct state. + if (sessionCheckpointed && this.master) { + try { + await this.master.resumeSession(); + } catch (resumeErr) { + logger.error({ err: resumeErr }, 'Failed to resume session after urgent cycle'); + } + } } - stopProgress(); this.metrics?.recordProcessed(Date.now() - startTime); + // Parse and dispatch output markers (SHARE, APP, SEND, VOICE) before sending main reply + const cleanedContent = await this.outputMarkerProcessor.processAll( + result.content, + connector, + message.sender, + message.id, + message.source, + ); + // Send result back const response: OutboundMessage = { target: message.source, recipient: message.sender, - content: result.content, + content: cleanedContent, replyTo: message.id, metadata: result.metadata, }; await connector.sendMessage(response); void this.auditLogger?.logOutbound(response); + // Cache Q&A pair after successful Master response (OB-1602) + // Only cache substantive responses — skip greetings, short acks, and errors (OB-1603) + if ( + useMaster && + this.memory?.qaCache && + isSubstantiveResponse(message.content, cleanedContent) + ) { + try { + this.memory.qaCache.store({ + question: message.content, + answer: cleanedContent, + confidence: 0.9, + }); + } catch { + logger.debug({ messageId: message.id }, 'QA cache store failed (non-critical)'); + } + } + logger.info({ messageId: message.id }, 'Message processed and response sent'); } - /** Start sending periodic progress updates, returns a stop function */ - private startProgressUpdates(connector: Connector, message: InboundMessage): () => void { - let tickCount = 0; - const timer = setInterval(() => { - const progressMsg = PROGRESS_MESSAGES[tickCount % PROGRESS_MESSAGES.length]!; - tickCount++; + // processSendMarkers moved to output-marker-processor.ts (OB-1284) - const update: OutboundMessage = { + // processShareMarkers moved to output-marker-processor.ts (OB-1284) + + // processVoiceMarkers moved to output-marker-processor.ts (OB-1284) + + // handleEmailShare moved to output-marker-processor.ts (OB-1284) + + // handleGitHubPagesShare moved to output-marker-processor.ts (OB-1284) + + // processAppMarkers moved to output-marker-processor.ts (OB-1284) + + /** + * Fast-path responder for quick-answer messages that arrive while Master is processing. + * + * Spawns a lightweight `claude --print` call with read-only tools (Read, Glob, Grep) + * and maxTurns=3. Injects a compact workspace context from the cached workspace map + * so the agent can answer questions about the project without waiting for Master. + * + * Falls back to a "Master is busy" message if the workspace path is not set or if + * the fast-path agent itself fails. + */ + private async runFastPath(message: InboundMessage, connector: Connector): Promise { + logger.info( + { sender: message.sender }, + 'Fast-path: Master is processing, routing quick-answer directly', + ); + + if (connector.sendTypingIndicator) { + await connector.sendTypingIndicator(message.sender); + } + + if (!this.workspacePath) { + await connector.sendMessage({ target: message.source, recipient: message.sender, - content: progressMsg, + content: 'The AI is busy processing a task. Please wait a moment before asking again.', replyTo: message.id, - }; - - connector.sendMessage(update).catch((err: unknown) => { - logger.warn({ err, messageId: message.id }, 'Failed to send progress update'); }); + return; + } - // Refresh typing indicator (best-effort) - if (connector.sendTypingIndicator) { - connector.sendTypingIndicator(message.sender).catch(() => { - // best-effort - }); + const workspaceContext = await this.buildWorkspaceContext(); + const reply = await this.fastPathResponder.answer({ + question: message.content, + workspacePath: this.workspacePath, + workspaceContext: workspaceContext || undefined, + }); + + await connector.sendMessage({ + target: message.source, + recipient: message.sender, + content: reply, + replyTo: message.id, + }); + logger.info({ sender: message.sender }, 'Fast-path response delivered'); + } + + /** + * Build a compact workspace context string from the cached workspace map. + * Used by the fast-path responder to give the lightweight agent project awareness. + * Returns an empty string if no map is available. + */ + private async buildWorkspaceContext(): Promise { + if (!this.master) return ''; + try { + const map = await this.master.getWorkspaceMap(); + if (!map) return ''; + + const lines: string[] = []; + lines.push(`Project: ${map.projectName} (${map.projectType})`); + if (map.frameworks.length > 0) { + lines.push(`Frameworks: ${map.frameworks.join(', ')}`); + } + if (map.summary) { + lines.push(`Summary: ${map.summary}`); + } + const commandEntries = Object.entries(map.commands); + if (commandEntries.length > 0) { + lines.push(`Commands: ${commandEntries.map(([k, v]) => `${k}: ${v}`).join(', ')}`); + } + if (map.keyFiles.length > 0) { + const filesSummary = map.keyFiles + .slice(0, 10) + .map((f) => ` ${f.path}: ${f.purpose}`) + .join('\n'); + lines.push(`Key files:\n${filesSummary}`); } - }, this.progressIntervalMs); + return lines.join('\n'); + } catch { + return ''; + } + } + + // --------------------------------------------------------------------------- + // Command handlers — delegated to CommandHandlers (OB-1283, OB-F159) + // --------------------------------------------------------------------------- + + private async handleStatusCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleStatusCommand(message, connector); + } + + private async handleConfirmCommand( + message: InboundMessage, + connector: Connector, + pending: PendingConfirmation, + ): Promise { + return this.commandHandlers.handleConfirmCommand(message, connector, pending); + } + + private async handleConfirmSpawnCommand( + message: InboundMessage, + connector: Connector, + ): Promise { + return this.commandHandlers.handleConfirmSpawnCommand(message, connector); + } + + private async handleSkipSpawnCommand( + message: InboundMessage, + connector: Connector, + ): Promise { + return this.commandHandlers.handleSkipSpawnCommand(message, connector); + } + + private async handleAllowCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleAllowCommand(message, connector); + } + + private async handleAllowAllCommand( + message: InboundMessage, + connector: Connector, + ): Promise { + return this.commandHandlers.handleAllowAllCommand(message, connector); + } + + private async handleDenyCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleDenyCommand(message, connector); + } + + private async handleDenyAllCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleDenyAllCommand(message, connector); + } + + private async handlePermissionsCommand( + message: InboundMessage, + connector: Connector, + ): Promise { + return this.commandHandlers.handlePermissionsCommand(message, connector); + } + + private async handleTrustCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleTrustCommand(message, connector); + } + + private async handleWhoamiCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleWhoamiCommand(message, connector); + } + + private async handleRoleCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleRoleCommand(message, connector); + } + + private async handleApproveCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleApproveCommand(message, connector); + } + + private async handleStopCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleStopCommand(message, connector); + } + + private async handleExploreCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleExploreCommand(message, connector); + } + + private async handleExploreStatusSubcommand( + message: InboundMessage, + connector: Connector, + ): Promise { + return this.commandHandlers.handleExploreStatusSubcommand(message, connector); + } + + private async handleAuditCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleAuditCommand(message, connector); + } + + private async handleDeepCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleDeepCommand(message, connector); + } + + private async handleProceedCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleProceedCommand(message, connector); + } + + private async handleFocusCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleFocusCommand(message, connector); + } + + private async handleSkipItemCommand( + message: InboundMessage, + connector: Connector, + ): Promise { + return this.commandHandlers.handleSkipItemCommand(message, connector); + } + + private async handlePhaseCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handlePhaseCommand(message, connector); + } + + private async handleModelOverrideCommand( + message: InboundMessage, + connector: Connector, + ): Promise { + return this.commandHandlers.handleModelOverrideCommand(message, connector); + } + + private async handleHistoryCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleHistoryCommand(message, connector); + } + + private async handleAppsCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleAppsCommand(message, connector); + } + + private async handleScopeCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleScopeCommand(message, connector); + } + + private async handleSkillsCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleSkillsCommand(message, connector); + } + + private async handleSkillPacksCommand( + message: InboundMessage, + connector: Connector, + ): Promise { + return this.commandHandlers.handleSkillPacksCommand(message, connector); + } + + private async handleWorkersCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleWorkersCommand(message, connector); + } + + private async handleKillWorkerCommand( + message: InboundMessage, + connector: Connector, + ): Promise { + return this.commandHandlers.handleKillWorkerCommand(message, connector); + } + + private async handleStatsCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleStatsCommand(message, connector); + } + + private async handleDoctorCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleDoctorCommand(message, connector); + } + + private async handleProcessCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleProcessCommand(message, connector); + } + + private async handleConnectCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleConnectCommand(message, connector); + } + + /** + * handleCurlBufferFlush — flush accumulated cURL commands and trigger /connect api (OB-1453). + * Called when the user says "done" or "connect" after sending standalone cURL messages. + */ + private async handleCurlBufferFlush( + message: InboundMessage, + connector: Connector, + ): Promise { + const joinedCurls = this.commandHandlers.flushCurlBuffer(message.sender); + // Synthesize a /connect api message with the accumulated cURLs as input + const synthetic: InboundMessage = { + ...message, + content: `/connect api ${joinedCurls}`, + rawContent: `/connect api ${joinedCurls}`, + }; + return this.commandHandlers.handleConnectCommand(synthetic, connector); + } + + private async handleIntegrationsCommand( + message: InboundMessage, + connector: Connector, + ): Promise { + return this.commandHandlers.handleIntegrationsCommand(message, connector); + } + + private async handleWorkflowsCommand( + message: InboundMessage, + connector: Connector, + ): Promise { + return this.commandHandlers.handleWorkflowsCommand(message, connector); + } + + private async handleDoctypesCommand( + message: InboundMessage, + connector: Connector, + ): Promise { + return this.commandHandlers.handleDoctypesCommand(message, connector); + } + + private async handleDoctypeCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleDoctypeCommand(message, connector); + } + + private async handleDtCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleDtCommand(message, connector); + } - return () => clearInterval(timer); + private async handleHelpCommand(message: InboundMessage, connector: Connector): Promise { + return this.commandHandlers.handleHelpCommand(message, connector); } /** Drain a streaming provider response, returning the final ProviderResult */ diff --git a/src/core/secret-scanner.ts b/src/core/secret-scanner.ts new file mode 100644 index 00000000..4507beb9 --- /dev/null +++ b/src/core/secret-scanner.ts @@ -0,0 +1,145 @@ +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; +import { createLogger } from './logger.js'; + +const logger = createLogger('secret-scanner'); + +export type SecretSeverity = 'critical' | 'high' | 'medium'; + +export interface SecretMatch { + /** Absolute path to the file */ + path: string; + /** The sensitive pattern that matched */ + pattern: string; + /** How sensitive the file is likely to be */ + severity: SecretSeverity; +} + +interface SensitivePattern { + pattern: string; + severity: SecretSeverity; +} + +/** + * Environment-file basenames that are documentation / templates, not real secrets. + * These are matched case-insensitively against the file basename and skipped + * before any SENSITIVE_PATTERNS check. + */ +export const SENSITIVE_FILE_EXCEPTIONS: ReadonlySet = new Set([ + '.env.example', + '.env.sample', + '.env.template', + '.env.test', + '.env.defaults', +]); + +/** + * Scans a workspace root (1 level deep) for files whose names match sensitive patterns. + * Name-check only — file contents are never read. + * + * Patterns are matched against the file's basename using minimatch-style glob semantics: + * a leading '*' matches any prefix, trailing '*' matches any suffix. + * The full list of patterns is defined in SENSITIVE_PATTERNS below and extended in OB-1468. + */ +export class SecretScanner { + private readonly patterns: readonly SensitivePattern[]; + private readonly configExceptionPatterns: readonly string[]; + + constructor(patterns?: SensitivePattern[], configExceptionPatterns?: string[]) { + this.patterns = patterns ?? SENSITIVE_PATTERNS; + this.configExceptionPatterns = configExceptionPatterns ?? []; + } + + /** + * Scan the workspace root directory (1 level deep) for files whose names match + * any sensitive pattern. + * + * @param workspacePath Absolute path to the workspace root. + * @returns Array of matches, one per detected sensitive file. + */ + public async scanWorkspace(workspacePath: string): Promise { + const matches: SecretMatch[] = []; + + let entries: { name: string; isFile(): boolean }[]; + try { + entries = await fs.readdir(workspacePath, { withFileTypes: true }); + } catch (err) { + logger.warn({ err, workspacePath }, 'SecretScanner: cannot read workspace directory'); + return matches; + } + + for (const entry of entries) { + // Only inspect files (skip directories and other special entries at root level) + if (!entry.isFile()) continue; + + const basename = entry.name; + const filePath = path.join(workspacePath, basename); + + // Skip well-known documentation / template env files — not real secrets + if (SENSITIVE_FILE_EXCEPTIONS.has(basename.toLowerCase())) continue; + + // Skip user-configured exception patterns from config.security.sensitiveFileExceptions + if (this.configExceptionPatterns.some((pat) => matchesPattern(basename, pat))) continue; + + for (const { pattern, severity } of this.patterns) { + if (matchesPattern(basename, pattern)) { + matches.push({ path: filePath, pattern, severity }); + // Report first matching pattern per file — avoid duplicate entries + break; + } + } + } + + logger.debug( + { workspacePath, scanned: entries.length, detected: matches.length }, + 'SecretScanner: scan complete', + ); + + return matches; + } +} + +/** + * Match a filename against a glob-style pattern. + * Supports '*' wildcards anywhere in the pattern (any number of them). + * A plain string pattern is an exact match against the basename. + * Matching is case-insensitive. + */ +function matchesPattern(basename: string, pattern: string): boolean { + const lower = basename.toLowerCase(); + const pat = pattern.toLowerCase(); + + // Convert glob pattern to a regex: escape regex specials, then replace '*' with '.*' + const regexStr = pat + .replace(/[.+^${}()|[\]\\]/g, '\\$&') // escape regex special chars (preserves *) + .replace(/\*/g, '.*'); // glob '*' → regex '.*' + + return new RegExp(`^${regexStr}$`).test(lower); +} + +/** + * Default sensitive file patterns. + * Severity tiers: + * critical — private keys, certificates, keystores (exposure = immediate compromise) + * high — credential files, password databases, environment secrets + * medium — config files that may contain secrets + */ +export const SENSITIVE_PATTERNS: SensitivePattern[] = [ + // --- Environment files (high) --- + { pattern: '.env', severity: 'high' }, + { pattern: '.env.*', severity: 'high' }, + + // --- Private keys and certificates (critical) --- + { pattern: '*.pem', severity: 'critical' }, + { pattern: '*.key', severity: 'critical' }, + { pattern: '*.p12', severity: 'critical' }, + { pattern: '*.pfx', severity: 'critical' }, + { pattern: 'id*rsa*', severity: 'critical' }, // SSH RSA keys (id_rsa, id_rsa.pub, etc.) + { pattern: 'id_ed25519*', severity: 'critical' }, // SSH Ed25519 keys + { pattern: '*.jks', severity: 'critical' }, // Java KeyStore + + // --- Credential and secret files (high) --- + { pattern: 'service-account*.json', severity: 'high' }, // GCP service accounts + { pattern: 'credentials*.json', severity: 'high' }, // OAuth / API credentials + { pattern: '*.kdbx', severity: 'high' }, // KeePass password database +]; diff --git a/src/core/static/openbridge-client.js b/src/core/static/openbridge-client.js new file mode 100644 index 00000000..c32b613a --- /dev/null +++ b/src/core/static/openbridge-client.js @@ -0,0 +1,283 @@ +/** + * openbridge-client.js — Browser SDK for OpenBridge served apps. + * + * Auto-injected into apps served by the AppServer. + * Provides bidirectional communication between the app and Master AI via the + * InteractionRelay WebSocket server (default port 3099). + * + * Usage: + * openbridge.submit({ action: 'search', query: 'hello' }); + * openbridge.onUpdate(function(data) { console.log('update:', data); }); + * openbridge.request({ action: 'lookup', id: 42 }).then(function(result) { + * console.log('response:', result); + * }); + * + * URL detection order: + * 1. window.OPENBRIDGE_RELAY_URL — explicit override + * 2. — meta tag + * 3. ws://:3099 — same host, relay port + * 4. ws://localhost:3099 — local fallback + * + * Token detection order (appended as ?token=): + * 1. window.OPENBRIDGE_AUTH_TOKEN — explicit override + * 2. — meta tag + */ +(function (global) { + 'use strict'; + + var RELAY_PORT = 3099; + var RECONNECT_BASE_MS = 1000; + var RECONNECT_MAX_MS = 30000; + var RECONNECT_FACTOR = 2; + var REQUEST_TIMEOUT_MS = 30000; + + // ── Helpers ──────────────────────────────────────────────────────────────── + + /** Generate a UUID v4 without Node.js crypto (browser-compatible). */ + function generateId() { + return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) { + var r = (Math.random() * 16) | 0; + var v = c === 'x' ? r : (r & 0x3) | 0x8; + return v.toString(16); + }); + } + + // ── URL detection ────────────────────────────────────────────────────────── + + function detectRelayUrl() { + // 1. Explicit window variable override + if (global.OPENBRIDGE_RELAY_URL) { + return String(global.OPENBRIDGE_RELAY_URL); + } + + // 2. Meta tag: + if (typeof document !== 'undefined') { + var meta = document.querySelector('meta[name="openbridge-relay"]'); + if (meta) { + var content = meta.getAttribute('content'); + if (content) return content; + } + } + + // 3. Same host as the page, relay port + if (typeof location !== 'undefined' && location.hostname) { + return 'ws://' + location.hostname + ':' + RELAY_PORT; + } + + // 4. Local fallback + return 'ws://localhost:' + RELAY_PORT; + } + + /** Detect the per-app auth token to authenticate with the relay. */ + function detectAuthToken() { + // 1. Explicit window variable override + if (global.OPENBRIDGE_AUTH_TOKEN) { + return String(global.OPENBRIDGE_AUTH_TOKEN); + } + + // 2. Meta tag: + if (typeof document !== 'undefined') { + var meta = document.querySelector('meta[name="openbridge-auth-token"]'); + if (meta) { + var content = meta.getAttribute('content'); + if (content) return content; + } + } + + return null; + } + + /** Append ?token= to a WebSocket URL when a token is available. */ + function buildRelayUrl(baseUrl) { + var token = detectAuthToken(); + if (!token) return baseUrl; + var sep = baseUrl.indexOf('?') === -1 ? '?' : '&'; + return baseUrl + sep + 'token=' + encodeURIComponent(token); + } + + // ── Client class ─────────────────────────────────────────────────────────── + + function OpenBridgeClient() { + this._relayUrl = detectRelayUrl(); + this._handlers = []; + this._pendingMessages = []; + this._pendingRequests = {}; // { [requestId]: { resolve, reject, timer } } + this._ws = null; + this._reconnectDelay = RECONNECT_BASE_MS; + this._stopped = false; + this._connect(); + } + + OpenBridgeClient.prototype._connect = function () { + if (this._stopped) return; + var self = this; + + try { + var ws = new WebSocket(buildRelayUrl(self._relayUrl)); + self._ws = ws; + + ws.onopen = function () { + self._reconnectDelay = RECONNECT_BASE_MS; + // Flush messages that arrived before connection was ready + while (self._pendingMessages.length > 0) { + var pending = self._pendingMessages.shift(); + self._sendRaw(pending); + } + }; + + ws.onmessage = function (event) { + var message; + try { + message = JSON.parse(event.data); + } catch (_e) { + return; + } + + // Resolve pending request() calls that match by requestId + if (message && message.requestId && self._pendingRequests[message.requestId]) { + var pending = self._pendingRequests[message.requestId]; + delete self._pendingRequests[message.requestId]; + clearTimeout(pending.timer); + pending.resolve(message.data !== undefined ? message.data : message); + // Fall through — also dispatch to onUpdate handlers + } + + var payload = message && message.data !== undefined ? message.data : message; + for (var i = 0; i < self._handlers.length; i++) { + try { + self._handlers[i](payload, message); + } catch (handlerErr) { + console.error('[openbridge] onUpdate handler threw', handlerErr); + } + } + }; + + ws.onclose = function () { + self._ws = null; + if (!self._stopped) { + setTimeout(function () { + self._reconnectDelay = Math.min( + self._reconnectDelay * RECONNECT_FACTOR, + RECONNECT_MAX_MS, + ); + self._connect(); + }, self._reconnectDelay); + } + }; + + ws.onerror = function (_err) { + // onclose fires immediately after onerror — reconnect is handled there + console.warn('[openbridge] WebSocket error — will reconnect'); + }; + } catch (e) { + console.error('[openbridge] Failed to open WebSocket', e); + } + }; + + OpenBridgeClient.prototype._sendRaw = function (payload) { + if (this._ws && this._ws.readyState === 1 /* OPEN */) { + this._ws.send(JSON.stringify(payload)); + return true; + } + return false; + }; + + /** + * Send data to Master AI via the relay. + * If the connection is not yet open the message is queued and sent on connect. + * + * @param {*} data - Any JSON-serialisable value. + */ + OpenBridgeClient.prototype.submit = function (data) { + var payload = { + type: 'submit', + data: data, + timestamp: new Date().toISOString(), + }; + if (!this._sendRaw(payload)) { + this._pendingMessages.push(payload); + } + }; + + /** + * Register a callback invoked whenever Master AI sends an update. + * + * @param {function} callback - Called with (data, rawMessage). + * @returns {function} Call the returned function to unsubscribe. + */ + OpenBridgeClient.prototype.onUpdate = function (callback) { + if (typeof callback !== 'function') { + throw new TypeError('[openbridge] onUpdate expects a function'); + } + this._handlers.push(callback); + var self = this; + return function unsubscribe() { + var idx = self._handlers.indexOf(callback); + if (idx !== -1) self._handlers.splice(idx, 1); + }; + }; + + /** + * Send a request to Master AI and wait for a matching response. + * The relay matches responses by requestId. Rejects if no response arrives + * within the timeout window. + * + * @param {*} data - Any JSON-serialisable value. + * @param {number} [timeout] - Timeout in ms (default: 30000). + * @returns {Promise<*>} Resolves with the response data from Master AI. + */ + OpenBridgeClient.prototype.request = function (data, timeout) { + var self = this; + var requestId = generateId(); + var timeoutMs = typeof timeout === 'number' ? timeout : REQUEST_TIMEOUT_MS; + + return new Promise(function (resolve, reject) { + var timer = setTimeout(function () { + if (self._pendingRequests[requestId]) { + delete self._pendingRequests[requestId]; + reject(new Error('[openbridge] request timed out after ' + timeoutMs + 'ms')); + } + }, timeoutMs); + + self._pendingRequests[requestId] = { resolve: resolve, reject: reject, timer: timer }; + + var payload = { + type: 'request', + requestId: requestId, + data: data, + timestamp: new Date().toISOString(), + }; + + if (!self._sendRaw(payload)) { + self._pendingMessages.push(payload); + } + }); + }; + + /** + * Disconnect from the relay and stop all reconnect attempts. + * Outstanding request() promises are rejected. + */ + OpenBridgeClient.prototype.stop = function () { + this._stopped = true; + if (this._ws) { + this._ws.close(); + this._ws = null; + } + this._handlers = []; + this._pendingMessages = []; + + // Reject any in-flight request() calls + var ids = Object.keys(this._pendingRequests); + for (var i = 0; i < ids.length; i++) { + var pending = this._pendingRequests[ids[i]]; + clearTimeout(pending.timer); + pending.reject(new Error('[openbridge] connection stopped')); + } + this._pendingRequests = {}; + }; + + // ── Expose global ────────────────────────────────────────────────────────── + + global.openbridge = new OpenBridgeClient(); +})(typeof window !== 'undefined' ? window : this); diff --git a/src/core/tunnel-manager.ts b/src/core/tunnel-manager.ts new file mode 100644 index 00000000..f2e2195c --- /dev/null +++ b/src/core/tunnel-manager.ts @@ -0,0 +1,286 @@ +/** + * TunnelManager — manages a tunnel process that exposes a local port to the internet. + * + * Usage: + * const manager = new TunnelManager('cloudflared'); + * const url = await manager.start(3001); + * // ... distribute url to users ... + * await manager.stop(); + * + * Tunnel adapters are registered via TunnelManager.registerAdapter(). Each tool + * (cloudflared, ngrok) provides its own adapter implementing buildArgs() and parseUrl(). + * If no adapter is registered for the given tool name, start() will reject. + */ + +import { spawn, type ChildProcess } from 'node:child_process'; +import { createLogger } from './logger.js'; + +const logger = createLogger('tunnel-manager'); + +/** Optional configuration for a tunnel session */ +export interface TunnelConfig { + /** Preferred subdomain (if supported by the tool) */ + subdomain?: string; + /** Auth token (required for ngrok when using reserved domains) */ + authToken?: string; +} + +/** + * Strategy interface for a specific tunnel tool. + * Implement this interface and call TunnelManager.registerAdapter() to add support + * for a new tunnel provider (e.g. cloudflared, ngrok, localtunnel). + */ +export interface TunnelAdapter { + /** Tool name matching DiscoveredTool.name (e.g. 'cloudflared', 'ngrok') */ + readonly toolName: string; + /** Build CLI arguments for spawning the tunnel process */ + buildArgs(port: number, config?: TunnelConfig): string[]; + /** + * Attempt to extract a public URL from one line of stdout/stderr output. + * Returns the URL string if found, or null if this line does not contain a URL. + */ + parseUrl(line: string): string | null; + /** + * Optional: fetch the public URL via a local API endpoint. + * Used as a parallel detection path alongside stdout parsing. + * Returns the URL string if found, or null if not yet available. + * TunnelManager will poll this every second after process starts. + */ + fetchUrl?(port: number): Promise; +} + +type TunnelState = 'idle' | 'starting' | 'active' | 'stopped'; + +/** Timeout waiting for the tunnel URL to appear in output (ms) */ +const URL_DETECT_TIMEOUT_MS = 30_000; + +export class TunnelManager { + private readonly toolName: string; + private readonly config: TunnelConfig | undefined; + private child: ChildProcess | null = null; + private state: TunnelState = 'idle'; + private publicUrl: string | null = null; + private adapter: TunnelAdapter | null; + private exitHandler: (() => void) | null = null; + private sigintHandler: (() => void) | null = null; + + private static readonly registry = new Map(); + + /** + * Register a TunnelAdapter for a specific tool name. + * Called by adapter modules (e.g. cloudflared-adapter.ts) at module load time. + */ + static registerAdapter(adapter: TunnelAdapter): void { + TunnelManager.registry.set(adapter.toolName, adapter); + logger.debug({ toolName: adapter.toolName }, 'Registered tunnel adapter'); + } + + /** + * @param toolName - The name of the detected tunnel tool (e.g. 'cloudflared', 'ngrok') + * @param config - Optional configuration (subdomain, authToken) + */ + constructor(toolName: string, config?: TunnelConfig) { + this.toolName = toolName; + this.config = config; + this.adapter = TunnelManager.registry.get(toolName) ?? null; + } + + /** + * Start the tunnel on the given local port and wait for the public URL. + * Resolves with the public URL when the tunnel is ready. + * Rejects if no adapter is registered, the process exits early, or URL detection times out. + */ + async start(port: number): Promise { + if (this.state === 'active' && this.publicUrl !== null) { + return this.publicUrl; + } + + if (this.state === 'starting') { + throw new Error('TunnelManager.start() called while already starting'); + } + + if (!this.adapter) { + throw new Error( + `No tunnel adapter registered for tool: "${this.toolName}". ` + + `Register one via TunnelManager.registerAdapter() before calling start().`, + ); + } + + const args = this.adapter.buildArgs(port, this.config); + logger.info({ toolName: this.toolName, port, args }, 'Starting tunnel'); + + this.state = 'starting'; + this.publicUrl = null; + + // Register exit handlers so the tunnel process is killed even on abnormal exit + this.exitHandler = (): void => { + this.stop(); + }; + this.sigintHandler = (): void => { + this.stop(); + }; + process.on('exit', this.exitHandler); + process.on('SIGINT', this.sigintHandler); + + return new Promise((resolve, reject) => { + const child = spawn(this.toolName, args, { + stdio: ['ignore', 'pipe', 'pipe'], + detached: false, + }); + + this.child = child; + let resolved = false; + let apiPollHandle: ReturnType | null = null; + + const cleanup = (): void => { + if (apiPollHandle !== null) { + clearInterval(apiPollHandle); + apiPollHandle = null; + } + }; + + const timeoutHandle = setTimeout(() => { + if (!resolved) { + resolved = true; + cleanup(); + this.state = 'stopped'; + this.child = null; + child.kill('SIGKILL'); + reject( + new Error( + `Tunnel URL not detected within ${URL_DETECT_TIMEOUT_MS / 1000}s ` + + `for tool: "${this.toolName}"`, + ), + ); + } + }, URL_DETECT_TIMEOUT_MS); + + const onOutput = (data: Buffer): void => { + if (resolved) return; + const text = data.toString('utf-8'); + for (const line of text.split('\n')) { + const url = this.adapter!.parseUrl(line.trim()); + if (url) { + clearTimeout(timeoutHandle); + cleanup(); + resolved = true; + this.publicUrl = url; + this.state = 'active'; + logger.info({ toolName: this.toolName, url }, 'Tunnel active'); + resolve(url); + return; + } + } + }; + + child.stdout?.on('data', onOutput); + child.stderr?.on('data', onOutput); + + // If the adapter supports API-based URL detection, poll it in parallel + if (this.adapter!.fetchUrl) { + const adapterFetchUrl = this.adapter!.fetchUrl.bind(this.adapter); + // Give the process 1s to start before polling + const startPolling = (): void => { + apiPollHandle = setInterval(() => { + if (resolved) { + cleanup(); + return; + } + adapterFetchUrl(port) + .then((url) => { + if (url !== null && !resolved) { + clearTimeout(timeoutHandle); + cleanup(); + resolved = true; + this.publicUrl = url; + this.state = 'active'; + logger.info({ toolName: this.toolName, url }, 'Tunnel active (via API)'); + resolve(url); + } + }) + .catch(() => { + // API not ready yet — continue polling + }); + }, 1_000); + }; + setTimeout(startPolling, 1_000); + } + + child.on('error', (err) => { + clearTimeout(timeoutHandle); + if (!resolved) { + resolved = true; + cleanup(); + this.state = 'stopped'; + this.child = null; + logger.error({ toolName: this.toolName, err }, 'Tunnel process error'); + reject(err); + } + }); + + child.on('exit', (code, signal) => { + clearTimeout(timeoutHandle); + cleanup(); + const wasActive = this.state === 'active'; + this.state = 'stopped'; + this.child = null; + + if (!resolved) { + resolved = true; + reject( + new Error( + `Tunnel process exited before URL was detected ` + + `(code=${String(code)}, signal=${String(signal)})`, + ), + ); + } else if (wasActive) { + logger.warn( + { toolName: this.toolName, code, signal }, + 'Tunnel process exited unexpectedly after URL was established', + ); + this.publicUrl = null; + } + }); + }); + } + + /** + * Stop the tunnel process. + * Safe to call even if the tunnel is not running. + */ + stop(): void { + // Always clean up process exit handlers when stop() is called + if (this.exitHandler !== null) { + process.removeListener('exit', this.exitHandler); + this.exitHandler = null; + } + if (this.sigintHandler !== null) { + process.removeListener('SIGINT', this.sigintHandler); + this.sigintHandler = null; + } + + if (this.child === null || this.state === 'idle' || this.state === 'stopped') { + return; + } + + logger.info({ toolName: this.toolName }, 'Stopping tunnel'); + this.child.kill('SIGTERM'); + this.child = null; + this.state = 'stopped'; + this.publicUrl = null; + } + + /** + * Returns the public URL if the tunnel is active, or null if not running. + */ + getUrl(): string | null { + return this.publicUrl; + } + + /** + * Returns true if the tunnel is running and a public URL is available. + */ + isActive(): boolean { + return this.state === 'active' && this.publicUrl !== null; + } +} diff --git a/src/core/voice-transcriber.ts b/src/core/voice-transcriber.ts new file mode 100644 index 00000000..fa2707b3 --- /dev/null +++ b/src/core/voice-transcriber.ts @@ -0,0 +1,341 @@ +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { access as fsAccess, unlink, readFile } from 'node:fs/promises'; +import { tmpdir, homedir } from 'node:os'; +import { basename, join } from 'node:path'; +import { createLogger } from './logger.js'; + +const execFileAsync = promisify(execFile); + +const logger = createLogger('voice-transcriber'); + +export type TranscriptionBackend = 'api' | 'cli' | 'none'; + +export interface TranscriptionResult { + text: string; + backend: TranscriptionBackend; + durationMs: number; +} + +/** Fallback message shown when no transcription backend is available. */ +export const TRANSCRIPTION_FALLBACK_MESSAGE = + '[Voice message — set OPENAI_API_KEY or install whisper for transcription]'; + +/** Whisper CLI binary names to search for, in priority order. */ +const WHISPER_BINARY_NAMES = ['whisper', 'whisper-cli']; + +/** + * Locate a whisper CLI binary on PATH. + * Searches for both `whisper` (pip install) and `whisper-cli` (brew install whisper-cpp). + * Returns the full path if found, null otherwise. + */ +export async function findWhisper(): Promise { + for (const name of WHISPER_BINARY_NAMES) { + try { + const { stdout } = await execFileAsync('which', [name]); + const path = stdout.trim(); + if (path) return path; + } catch { + // not found, try next + } + } + return null; +} + +let _cachedBackend: TranscriptionBackend | undefined; +let _cachedWhisperPath: string | null = null; +let _cachedApiToken: string | null = null; + +/** + * Resolve an OpenAI API key for Whisper API calls. + * + * Priority: + * 1. `OPENAI_API_KEY` env var (explicit API key) + * 2. `~/.codex/auth.json` → `OPENAI_API_KEY` field (Codex API key login) + * + * Note: Codex OAuth tokens (access_token from `codex login` with ChatGPT) are NOT + * usable with the Whisper API — they lack the required scopes. Only real API keys + * (sk-...) work with the audio transcription endpoint. + * + * Returns the API key string or null if no credentials are available. + */ +export async function resolveOpenAIToken(): Promise { + // 1. Env var takes priority + const envKey = process.env['OPENAI_API_KEY']; + if (envKey) return envKey; + + // 2. Read from Codex auth file — only use real API keys, not OAuth tokens + try { + const authPath = join(homedir(), '.codex', 'auth.json'); + const raw = await readFile(authPath, 'utf-8'); + const auth = JSON.parse(raw) as Record; + + // Only use OPENAI_API_KEY if it's a real key string (not null) + if (auth['OPENAI_API_KEY'] && typeof auth['OPENAI_API_KEY'] === 'string') { + return auth['OPENAI_API_KEY']; + } + } catch { + // auth.json doesn't exist or is unreadable — not an error + } + + return null; +} + +/** + * Detect and cache the best available transcription backend for this process. + * + * Priority: + * 1. OpenAI Whisper API (env var OPENAI_API_KEY or Codex-stored API key) → 'api' + * 2. Local whisper binary (whisper or whisper-cli on PATH) → 'cli' + * 3. Neither → 'none' + * + * Result is cached after the first call — detection runs exactly once per process. + */ +export async function detectAvailableBackend(): Promise { + if (_cachedBackend !== undefined) return _cachedBackend; + + let backend: TranscriptionBackend; + + // Check for a real API key (env var or Codex-stored) + _cachedApiToken = await resolveOpenAIToken(); + if (_cachedApiToken) { + backend = 'api'; + } else { + // No API key — check for local whisper binary + _cachedWhisperPath = await findWhisper(); + backend = _cachedWhisperPath ? 'cli' : 'none'; + } + + logger.info({ backend }, 'Transcription backend detected'); + _cachedBackend = backend; + return backend; +} + +/** + * Reset the cached backend, whisper path, and API token. + * @internal For testing only — do not call in production code. + */ +export function _resetCachedBackend(): void { + _cachedBackend = undefined; + _cachedWhisperPath = null; + _cachedApiToken = null; +} + +/** + * Transcribe an audio file using the OpenAI Whisper API. + * + * Uses the resolved API token (env var, Codex OAuth, or Codex API key). + * Uses Node.js native fetch() + FormData + Blob (Node >= 22, no extra deps). + * Returns null if no credentials are available, the request fails, or the response + * is non-2xx. Errors are logged via Pino but never thrown. + * + * @param audioPath - Absolute path to the audio file. + * @returns Transcription text, or null if unavailable. + */ +export async function transcribeViaApi(audioPath: string): Promise { + const token = _cachedApiToken ?? (await resolveOpenAIToken()); + if (!token) return null; + + try { + const audioData = await readFile(audioPath); + const blob = new Blob([audioData]); + const filename = basename(audioPath); + + const form = new FormData(); + form.append('file', blob, filename); + form.append('model', 'whisper-1'); + + const response = await fetch('https://api.openai.com/v1/audio/transcriptions', { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + }, + body: form, + }); + + if (!response.ok) { + logger.warn({ status: response.status }, 'Whisper API returned non-2xx response'); + return null; + } + + const json = (await response.json()) as { text?: string }; + return json.text ?? null; + } catch (err) { + logger.warn({ err }, 'Whisper API transcription failed'); + return null; + } +} + +/** Default model search paths for whisper-cli (Homebrew whisper-cpp). */ +const WHISPER_MODEL_PATHS = [ + join(homedir(), '.local', 'share', 'whisper-models', 'ggml-base.en.bin'), + join(homedir(), '.local', 'share', 'whisper-models', 'ggml-base.bin'), + join(homedir(), '.local', 'share', 'whisper-models', 'ggml-small.en.bin'), + '/opt/homebrew/share/whisper-cpp/models/ggml-base.en.bin', + '/usr/local/share/whisper-cpp/models/ggml-base.en.bin', +]; + +/** + * Find a GGML model file for whisper-cli. + * Returns the first existing path from the search list, or null. + */ +async function findWhisperModel(): Promise { + for (const p of WHISPER_MODEL_PATHS) { + try { + await fsAccess(p); + return p; + } catch { + // not found, try next + } + } + return null; +} + +/** + * Detect if a whisper binary is the Homebrew whisper-cpp variant (whisper-cli) + * vs the Python whisper package. + */ +function isWhisperCpp(whisperPath: string): boolean { + return basename(whisperPath) === 'whisper-cli'; +} + +/** Audio formats that whisper-cli can read directly (without ffmpeg conversion). */ +const WHISPER_CPP_NATIVE_FORMATS = new Set(['.wav', '.mp3', '.flac']); + +/** + * Convert an audio file to 16 kHz mono WAV using ffmpeg. + * Required for whisper-cli which cannot reliably read OGG/Opus files. + * Returns the path to the WAV file, or null if ffmpeg is not available / conversion fails. + */ +async function convertToWav(audioPath: string): Promise { + const wavPath = audioPath.replace(/\.[^.]+$/, '') + '-converted.wav'; + try { + await execFileAsync( + 'ffmpeg', + ['-y', '-i', audioPath, '-ar', '16000', '-ac', '1', '-c:a', 'pcm_s16le', wavPath], + { timeout: 30_000 }, + ); + return wavPath; + } catch { + logger.warn({ audioPath }, 'ffmpeg conversion failed or ffmpeg not installed'); + return null; + } +} + +/** + * Transcribe an audio file using the local Whisper CLI binary. + * Handles both `whisper` (Python/pip) and `whisper-cli` (Homebrew whisper-cpp). + * For whisper-cli, non-WAV files are converted via ffmpeg first. + * + * @param whisperPath - Absolute path to the whisper binary. + * @param audioPath - Absolute path to the audio file. + * @returns Transcription text, or null on failure. + */ +async function transcribeViaCli(whisperPath: string, audioPath: string): Promise { + let convertedPath: string | null = null; + try { + const outputDir = tmpdir(); + + if (isWhisperCpp(whisperPath)) { + const modelPath = await findWhisperModel(); + if (!modelPath) { + logger.warn( + 'No GGML model file found for whisper-cli — download one from https://huggingface.co/ggerganov/whisper.cpp', + ); + return null; + } + + // whisper-cli only reliably reads WAV/MP3/FLAC — convert OGG/Opus via ffmpeg + const ext = audioPath.substring(audioPath.lastIndexOf('.')).toLowerCase(); + let inputPath = audioPath; + if (!WHISPER_CPP_NATIVE_FORMATS.has(ext)) { + convertedPath = await convertToWav(audioPath); + if (!convertedPath) return null; + inputPath = convertedPath; + } + + const outputBasename = basename(audioPath).replace(/\.[^.]+$/, ''); + await execFileAsync( + whisperPath, + [ + '--model', + modelPath, + '--file', + inputPath, + '--output-txt', + '--output-file', + join(outputDir, outputBasename), + '--no-prints', + ], + { timeout: 120_000 }, + ); + } else { + // Python whisper: handles all formats natively + await execFileAsync( + whisperPath, + [audioPath, '--output-format', 'txt', '--output-dir', outputDir], + { timeout: 120_000 }, + ); + } + + const inputBasename = basename(audioPath).replace(/\.[^.]+$/, ''); + const txtPath = join(outputDir, `${inputBasename}.txt`); + const text = await readFile(txtPath, 'utf-8').catch(() => ''); + await unlink(txtPath).catch(() => {}); + return text.trim() || null; + } catch (err) { + logger.warn({ err }, 'Voice transcription failed'); + return null; + } finally { + // Clean up converted WAV file if we created one + if (convertedPath) { + await unlink(convertedPath).catch(() => {}); + } + } +} + +/** + * Transcribe an audio file using the best available backend. + * + * Fallback chain: OpenAI Whisper API → local Whisper CLI → null. + * If the API backend is detected but fails for a given file, falls through to CLI. + * + * @param audioPath - Absolute path to the audio file (ogg, oga, wav, mp3, etc.) + * @returns TranscriptionResult with text, backend used, and duration — or null if + * no backend is available or all attempts fail. + */ +export async function transcribeAudio(audioPath: string): Promise { + const startMs = Date.now(); + const backend = await detectAvailableBackend(); + + if (backend === 'api') { + const text = await transcribeViaApi(audioPath); + if (text !== null) { + return { text, backend: 'api', durationMs: Date.now() - startMs }; + } + // API failed — fall through to CLI + const whisperPath = await findWhisper(); + if (whisperPath) { + const cliText = await transcribeViaCli(whisperPath, audioPath); + if (cliText !== null) { + return { text: cliText, backend: 'cli', durationMs: Date.now() - startMs }; + } + } + return null; + } + + if (backend === 'cli') { + // Use path cached during detection to avoid a redundant `which` call + const whisperPath = _cachedWhisperPath ?? (await findWhisper()); + if (whisperPath) { + const cliText = await transcribeViaCli(whisperPath, audioPath); + if (cliText !== null) { + return { text: cliText, backend: 'cli', durationMs: Date.now() - startMs }; + } + } + return null; + } + + // backend === 'none' + return null; +} diff --git a/src/core/workspace-manager.ts b/src/core/workspace-manager.ts new file mode 100644 index 00000000..5cb018f5 --- /dev/null +++ b/src/core/workspace-manager.ts @@ -0,0 +1,375 @@ +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { createLogger } from './logger.js'; +import { DEFAULT_EXCLUDE_PATTERNS } from '../types/config.js'; + +const execFileAsync = promisify(execFile); +const logger = createLogger('workspace-manager'); + +const DEFAULT_PULL_INTERVAL_SECONDS = 300; // 5 minutes + +export interface WorkspaceManagerOptions { + /** Polling interval in seconds for remote workspace auto-pull (default: 300) */ + pullIntervalSeconds?: number; + /** Callback invoked when git pull detects new commits — use to trigger re-exploration */ + onChangesDetected?: () => void | Promise; +} + +/** + * Manages workspace setup for both local and remote (git URL) workspaces. + * + * For remote workspaces (https:// or git@ URLs): + * - Clones the repo to ~/.openbridge/workspaces/{repo-name}/ on init + * - Polls for upstream changes at a configurable interval + * - Calls onChangesDetected when new commits are pulled + * + * For local workspaces: + * - Returns the path as-is; no polling + */ +export class WorkspaceManager { + private readonly originalPath: string; + private localPath: string; + private readonly options: WorkspaceManagerOptions; + private pollTimer: ReturnType | null = null; + private lastCommitHash: string | null = null; + + constructor(workspacePath: string, options: WorkspaceManagerOptions = {}) { + this.originalPath = workspacePath; + this.localPath = workspacePath; + this.options = options; + } + + /** Returns true if the given path is a remote git URL */ + public static isRemoteUrl(workspacePath: string): boolean { + return workspacePath.startsWith('https://') || workspacePath.startsWith('git@'); + } + + /** Returns the resolved local workspace path (after possible clone) */ + public getLocalPath(): string { + return this.localPath; + } + + /** + * Initialize the workspace: + * - Remote URL → clone to ~/.openbridge/workspaces/{repo-name}/ (or pull if already cloned) + * - Local path → use as-is + * + * @returns The local filesystem path to the workspace + */ + public async init(): Promise { + if (!WorkspaceManager.isRemoteUrl(this.originalPath)) { + this.localPath = this.originalPath; + return this.localPath; + } + + const repoName = WorkspaceManager.extractRepoName(this.originalPath); + const workspacesDir = path.join(os.homedir(), '.openbridge', 'workspaces'); + this.localPath = path.join(workspacesDir, repoName); + + await fs.mkdir(workspacesDir, { recursive: true }); + + const alreadyCloned = await this.isGitRepo(this.localPath); + if (alreadyCloned) { + logger.info( + { localPath: this.localPath }, + 'Remote workspace already cloned — pulling latest', + ); + await this.pull(); + } else { + logger.info( + { url: this.originalPath, localPath: this.localPath }, + 'Cloning remote workspace...', + ); + await this.clone(); + logger.info({ localPath: this.localPath }, 'Clone complete'); + } + + this.lastCommitHash = await this.getHeadCommit(); + logger.info( + { localPath: this.localPath, commit: this.lastCommitHash?.slice(0, 8) }, + 'Remote workspace ready', + ); + + return this.localPath; + } + + /** + * Start polling for upstream changes. + * No-op for local workspaces. + * The poll timer is unref'd so it won't prevent process exit. + */ + public startPolling(): void { + if (!WorkspaceManager.isRemoteUrl(this.originalPath)) { + return; + } + + const intervalMs = (this.options.pullIntervalSeconds ?? DEFAULT_PULL_INTERVAL_SECONDS) * 1000; + + logger.info({ intervalSeconds: intervalMs / 1000 }, 'Starting remote workspace polling'); + + this.pollTimer = setInterval(() => { + void this.checkForRemoteChanges(); + }, intervalMs); + + // Unref so the timer doesn't prevent clean process exit + this.pollTimer.unref(); + } + + /** Set or replace the callback invoked when remote changes are detected */ + public setOnChangesDetected(callback: () => void | Promise): void { + this.options.onChangesDetected = callback; + } + + /** Stop the polling timer */ + public stopPolling(): void { + if (this.pollTimer) { + clearInterval(this.pollTimer); + this.pollTimer = null; + logger.debug('Remote workspace polling stopped'); + } + } + + /** Pull latest commits and fire onChangesDetected if HEAD moved */ + private async checkForRemoteChanges(): Promise { + try { + logger.debug({ localPath: this.localPath }, 'Checking remote workspace for changes...'); + const beforeHash = this.lastCommitHash ?? (await this.getHeadCommit()); + + await this.pull(); + + const afterHash = await this.getHeadCommit(); + + if (afterHash && afterHash !== beforeHash) { + this.lastCommitHash = afterHash; + logger.info( + { before: beforeHash?.slice(0, 8), after: afterHash.slice(0, 8) }, + 'Remote workspace updated — triggering re-exploration', + ); + if (this.options.onChangesDetected) { + await this.options.onChangesDetected(); + } + } else { + logger.debug('No remote changes detected'); + this.lastCommitHash = afterHash ?? beforeHash; + } + } catch (error) { + logger.warn({ err: error }, 'Remote workspace pull check failed — will retry next interval'); + } + } + + /** Run `git clone` for a fresh checkout */ + private async clone(): Promise { + await execFileAsync('git', ['clone', '--', this.originalPath, this.localPath]); + } + + /** Run `git pull --ff-only` in the local clone */ + private async pull(): Promise { + await execFileAsync('git', ['pull', '--ff-only'], { cwd: this.localPath }); + } + + /** Return HEAD commit hash, or null if unavailable */ + private async getHeadCommit(): Promise { + try { + const { stdout } = await execFileAsync('git', ['rev-parse', 'HEAD'], { + cwd: this.localPath, + }); + return stdout.trim(); + } catch { + return null; + } + } + + /** Return true if `dirPath` is inside a git work tree */ + private async isGitRepo(dirPath: string): Promise { + try { + await fs.access(dirPath); + await execFileAsync('git', ['rev-parse', '--is-inside-work-tree'], { cwd: dirPath }); + return true; + } catch { + return false; + } + } + + /** + * Derive a safe directory name from a git URL. + * + * Examples: + * https://github.com/user/my-repo.git → my-repo + * git@github.com:user/my-repo.git → my-repo + * https://github.com/user/my-repo → my-repo + */ + public static extractRepoName(url: string): string { + const normalized = url.replace(/\.git$/, ''); + const lastSlash = normalized.lastIndexOf('/'); + const lastColon = normalized.lastIndexOf(':'); + const start = Math.max(lastSlash, lastColon) + 1; + const name = normalized.slice(start); + // Keep only safe filesystem characters + return name.replace(/[^a-zA-Z0-9\-_.]/g, '_') || 'workspace'; + } +} + +// ── File visibility helpers ───────────────────────────────────────────────── + +/** + * Convert a glob pattern to a RegExp. + * Supports * (single-segment wildcard) and ** (multi-segment wildcard). + */ +function globToRegex(pattern: string): RegExp { + let regexStr = ''; + let i = 0; + while (i < pattern.length) { + const ch = pattern[i]!; + if (ch === '*') { + if (pattern[i + 1] === '*') { + // ** matches any sequence of characters including path separators + regexStr += '.*'; + i += 2; + // Consume optional trailing '/' after ** + if (pattern[i] === '/') i++; + } else { + // * matches any character except '/' + regexStr += '[^/]*'; + i++; + } + } else if (ch === '?') { + regexStr += '[^/]'; + i++; + } else if ('.+^${}()|[\\]'.includes(ch)) { + regexStr += '\\' + ch; + i++; + } else { + regexStr += ch; + i++; + } + } + return new RegExp('^' + regexStr + '$'); +} + +/** + * Check if a relative file path matches a single glob pattern. + * + * Semantics (gitignore-like): + * - Pattern ending with '/' → directory pattern; matches any file inside that + * directory at any depth (e.g. `node_modules/` hides all node_modules trees). + * - Pattern without '/' → matches against the file's basename at any depth + * (e.g. `*.pem` hides all .pem files regardless of location). + * - Pattern with '/' → matched against the full relative path from the + * workspace root (e.g. `src/**` matches everything under src/). + */ +function matchesGlob(relativePath: string, pattern: string): boolean { + const normalizedPath = relativePath.replace(/\\/g, '/'); + + // Directory pattern — match any file inside that directory at any depth + if (pattern.endsWith('/')) { + const dirName = pattern.slice(0, -1); + if (!dirName.includes('/')) { + // No path separator in dir name → match at any depth + const segments = normalizedPath.split('/'); + const dirRegex = globToRegex(dirName); + return segments.some((seg, idx) => idx < segments.length - 1 && dirRegex.test(seg)); + } + // Has separator → match from workspace root + return normalizedPath === dirName || normalizedPath.startsWith(dirName + '/'); + } + + // No '/' in pattern → match against basename only + if (!pattern.includes('/')) { + const basename = normalizedPath.includes('/') + ? normalizedPath.slice(normalizedPath.lastIndexOf('/') + 1) + : normalizedPath; + return globToRegex(pattern).test(basename); + } + + // Pattern with '/' → match against full relative path + return globToRegex(pattern).test(normalizedPath); +} + +/** + * Check whether a file should be visible to the AI based on workspace visibility rules. + * + * Algorithm: + * 1. Normalize `workspacePath` to an absolute path via `path.resolve()` — eliminates + * any relative components before the path is used as a scope anchor. + * 2. Resolve `filePath` to an absolute path using the normalized workspace root. + * 3. Resolve symlinks via `fs.realpath()` so that symlinks pointing outside the + * workspace are treated as out-of-scope (prevents symlink escape attacks). + * 4. Compute the relative path from the real workspace root to the real file. + * If the relative path escapes the workspace (starts with "..") → NOT visible. + * This also blocks path-traversal inputs like `../../etc/passwd`. + * 5. Combine DEFAULT_EXCLUDE_PATTERNS with `config.workspace?.exclude`. + * 6. If the resolved relative path matches any exclude pattern → NOT visible (exclude takes priority). + * 7. If `config.workspace?.include` is set and non-empty: + * - File must match at least one include pattern to be visible. + * 8. Otherwise → visible. + * + * @param filePath Absolute or workspace-relative file path. + * @param config Object with `workspacePath` and optional `workspace` include/exclude arrays. + */ +export async function isFileVisible( + filePath: string, + config: { + workspacePath: string; + workspace?: { include?: string[]; exclude?: string[] }; + }, +): Promise { + // Normalize workspace root to an absolute path first — eliminates any relative + // components (e.g. "../secret") so that all subsequent path operations have a + // stable, canonical anchor and path.relative() comparisons are reliable. + const workspaceRoot = path.resolve(config.workspacePath); + + // Resolve to absolute path — path.resolve handles both relative and absolute + // filePaths. An absolute filePath (potential path-traversal attempt such as + // ../../etc/passwd) is still caught by the scope check below. + const absFile = path.resolve(workspaceRoot, filePath); + + // Resolve symlinks — prevents symlink escape to files outside the workspace. + // Fall back to the unresolved path if the file does not exist yet. + let realFile: string; + try { + realFile = await fs.realpath(absFile); + } catch { + realFile = absFile; + } + + // Resolve the workspace root symlinks for an accurate containment check. + let realWorkspace: string; + try { + realWorkspace = await fs.realpath(workspaceRoot); + } catch { + realWorkspace = workspaceRoot; + } + + // Compute relative path from the resolved workspace root to the resolved file. + const relative = path.relative(realWorkspace, realFile); + + // Symlink escape guard — if the real path is outside the workspace, reject. + if (relative.startsWith('..') || path.isAbsolute(relative)) { + return false; + } + + // Build combined exclude list: defaults first, then user overrides + const excludePatterns: readonly string[] = [ + ...DEFAULT_EXCLUDE_PATTERNS, + ...(config.workspace?.exclude ?? []), + ]; + + // Exclude takes priority — any match makes the file invisible + for (const pattern of excludePatterns) { + if (matchesGlob(relative, pattern)) { + return false; + } + } + + // If an include list is specified, file must match at least one pattern + const includePatterns = config.workspace?.include; + if (includePatterns && includePatterns.length > 0) { + return includePatterns.some((pattern) => matchesGlob(relative, pattern)); + } + + // No include restriction — file is visible + return true; +} diff --git a/src/discovery/index.ts b/src/discovery/index.ts index b9bd8e1f..168207ba 100644 --- a/src/discovery/index.ts +++ b/src/discovery/index.ts @@ -1,4 +1,4 @@ -import { scanForCLITools, selectMaster } from './tool-scanner.js'; +import { scanForCLITools, scanForTunnelTools, selectMaster, detectDocker } from './tool-scanner.js'; import { scanVSCodeExtensions } from './vscode-scanner.js'; import type { ScanResult } from '../types/discovery.js'; import { createLogger } from '../core/logger.js'; @@ -25,6 +25,21 @@ export async function scanForAITools(): Promise { const vscodeExtensions = await scanVSCodeExtensions(); logger.info({ count: vscodeExtensions.length }, 'VS Code extension scan complete'); + // Scan for tunnel tools (synchronous) + const tunnelTools = scanForTunnelTools(); + logger.info({ count: tunnelTools.length }, 'Tunnel tool scan complete'); + + // Detect Docker availability (synchronous) + const dockerStatus = detectDocker(); + logger.info( + { + installed: dockerStatus.installed, + daemonRunning: dockerStatus.daemonRunning, + version: dockerStatus.version, + }, + 'Docker detection complete', + ); + // Select master from CLI tools (VS Code extensions cannot be Master) const master = selectMaster(cliTools); @@ -35,11 +50,13 @@ export async function scanForAITools(): Promise { } const timestamp = new Date().toISOString(); - const totalDiscovered = cliTools.length + vscodeExtensions.length; + const totalDiscovered = cliTools.length + vscodeExtensions.length + tunnelTools.length; const result: ScanResult = { cliTools, vscodeExtensions, + tunnelTools, + dockerStatus, master, timestamp, totalDiscovered, @@ -50,6 +67,7 @@ export async function scanForAITools(): Promise { totalDiscovered, cliTools: cliTools.length, vscodeExtensions: vscodeExtensions.length, + tunnelTools: tunnelTools.length, master: master?.name ?? 'none', }, 'AI tool discovery complete', @@ -59,5 +77,5 @@ export async function scanForAITools(): Promise { } // Re-export individual scanners for advanced use cases -export { scanForCLITools, selectMaster } from './tool-scanner.js'; +export { scanForCLITools, scanForTunnelTools, selectMaster, detectDocker } from './tool-scanner.js'; export { scanVSCodeExtensions } from './vscode-scanner.js'; diff --git a/src/discovery/tool-scanner.ts b/src/discovery/tool-scanner.ts index 6241af77..ed31096c 100644 --- a/src/discovery/tool-scanner.ts +++ b/src/discovery/tool-scanner.ts @@ -1,6 +1,6 @@ import { execSync } from 'node:child_process'; import { createLogger } from '../core/logger.js'; -import type { DiscoveredTool } from '../types/discovery.js'; +import type { DiscoveredTool, DockerStatus } from '../types/discovery.js'; const logger = createLogger('tool-scanner'); @@ -64,6 +64,33 @@ const KNOWN_TOOLS: ToolDefinition[] = [ }, ]; +const TUNNEL_TOOLS: ToolDefinition[] = [ + { + name: 'cloudflared', + command: 'cloudflared', + versionFlag: '--version', + versionPattern: /(\d+\.\d+\.\d+)/, + capabilities: ['tunnel'], + priority: 30, + }, + { + name: 'ngrok', + command: 'ngrok', + versionFlag: '--version', + versionPattern: /(\d+\.\d+\.\d+)/, + capabilities: ['tunnel'], + priority: 25, + }, + { + name: 'localtunnel', + command: 'lt', + versionFlag: '--version', + versionPattern: /(\d+\.\d+\.\d+)/, + capabilities: ['tunnel'], + priority: 20, + }, +]; + /** * Check if a CLI tool is available on the system PATH */ @@ -158,6 +185,87 @@ export function scanForCLITools(): DiscoveredTool[] { return discovered; } +/** + * Scan for tunnel tools (cloudflared, ngrok, localtunnel) on the system PATH + */ +export function scanForTunnelTools(): DiscoveredTool[] { + logger.info('Scanning for tunnel tools'); + + const discovered: DiscoveredTool[] = []; + + for (const tool of TUNNEL_TOOLS) { + logger.debug({ tool: tool.name }, 'Checking for tunnel tool'); + + const available = isCommandAvailable(tool.command); + + if (!available) { + logger.debug({ tool: tool.name }, 'Tunnel tool not found on PATH'); + continue; + } + + const path = getCommandPath(tool.command); + + if (!path) { + logger.debug({ tool: tool.name }, 'Could not determine tunnel tool path'); + continue; + } + + const version = getToolVersion(tool.command, tool.versionFlag, tool.versionPattern); + + discovered.push({ + name: tool.name, + path, + version, + capabilities: tool.capabilities, + role: 'none', + available: true, + }); + + logger.info({ tool: tool.name, path, version }, 'Discovered tunnel tool'); + } + + return discovered; +} + +/** + * Detect Docker availability on this machine. + * + * Three possible states: + * - Not installed: `docker` binary not in PATH + * - Installed, no daemon: binary found but `docker info` fails + * - Available: binary found AND daemon is running + */ +export function detectDocker(): DockerStatus { + // Step 1: check if docker binary is in PATH + const installed = isCommandAvailable('docker'); + + if (!installed) { + logger.debug('Docker not found in PATH'); + return { installed: false, daemonRunning: false, available: false, version: null }; + } + + // Step 2: get Docker client version + const rawVersion = getToolVersion('docker', '--version', /Docker version (\d+\.\d+\.\d+)/); + const version = rawVersion === 'unknown' ? null : rawVersion; + + // Step 3: verify daemon is running by calling `docker info` + let daemonRunning = false; + try { + execSync('docker info', { stdio: 'pipe', timeout: 10_000 }); + daemonRunning = true; + } catch { + logger.warn('Docker is installed but the daemon is not running'); + } + + const available = daemonRunning; + + if (available) { + logger.info({ version }, 'Docker is available'); + } + + return { installed, daemonRunning, available, version }; +} + /** * Select the master AI tool from discovered tools * diff --git a/src/index.ts b/src/index.ts index e4dae068..7ac80336 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,19 +1,64 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { readFile } from 'node:fs/promises'; -import { Bridge, loadConfig, resolveConfigPath, createLogger, isV2Config } from './core/index.js'; +import { createRequire } from 'node:module'; +import { + Bridge, + loadConfig, + resolveConfigPath, + createLogger, + setLogLevel, + isV2Config, +} from './core/index.js'; +import { injectDevConnectors } from './core/config.js'; +import { WorkspaceManager } from './core/workspace-manager.js'; import { V2ConfigSchema } from './types/config.js'; - // whatsapp-web.js / puppeteer registers multiple exit handlers — raise the limit to avoid the warning process.setMaxListeners(20); import { registerBuiltInConnectors } from './connectors/index.js'; import { registerBuiltInProviders } from './providers/index.js'; import { scanForAITools } from './discovery/index.js'; import { MasterManager } from './master/index.js'; +import { createAdapterRegistry } from './core/adapter-registry.js'; +import { McpRegistry } from './core/mcp-registry.js'; import type { V2Config } from './types/config.js'; +import { runHealthCheck } from './core/health.js'; + +interface PackageJson { + version: string; +} +const _require = createRequire(import.meta.url); +const _pkg = _require('../package.json') as PackageJson; +const OPENBRIDGE_VERSION = _pkg.version; const logger = createLogger('main'); +/** Emit trust-level startup warnings. Exported for unit testing. */ +export function logTrustLevelAtStartup( + log: { warn: (msg: string) => void; info: (msg: string) => void }, + trustLevel: string, +): void { + if (trustLevel === 'trusted') { + log.warn('Running in TRUSTED mode — all agents have full access within workspace'); + } else if (trustLevel === 'sandbox') { + log.info('Running in SANDBOX mode — agents are read-only'); + } +} + +// Module-level flag prevents double-shutdown when SIGINT and SIGTERM arrive together +let shutdownInProgress = false; + +// Safety net: log unhandled rejections so they don't disappear silently +process.on('unhandledRejection', (reason: unknown) => { + logger.error({ reason }, 'Unhandled promise rejection'); +}); + +// Safety net: log and exit on uncaught exceptions — the process is in an unknown state +process.on('uncaughtException', (error: Error) => { + logger.fatal({ err: error }, 'Uncaught exception — exiting'); + process.exit(1); +}); + /** * V0 startup flow (legacy) * - Load config @@ -25,6 +70,8 @@ async function startV0Flow(configPath: string): Promise { logger.info('Starting V0 flow (legacy mode)'); const config = await loadConfig(); + setLogLevel(process.env['LOG_LEVEL'] ?? config.logLevel); + injectDevConnectors(config); const bridge = new Bridge(config, { configPath }); // Register built-in plugins (manual fallback) @@ -38,7 +85,22 @@ async function startV0Flow(configPath: string): Promise { await bridge.start(); - logger.info('OpenBridge (V0) is running. Press Ctrl+C to stop.'); + const connectorNames = bridge.getActiveConnectorNames(); + if (process.env['OPENBRIDGE_HEADLESS'] === 'true') { + process.stdout.write( + JSON.stringify({ + event: 'ready', + version: OPENBRIDGE_VERSION, + mode: 'v0', + connectors: connectorNames, + }) + '\n', + ); + } else { + process.stdout.write( + `OpenBridge v${OPENBRIDGE_VERSION} | Connectors: ${connectorNames.join(', ') || 'none'}\n`, + ); + logger.info('OpenBridge (V0) is running. Press Ctrl+C to stop.'); + } return bridge; } @@ -53,14 +115,101 @@ async function startV0Flow(configPath: string): Promise { * - Launch Master AI * - Explore workspace autonomously */ -async function startV2Flow(configPath: string, v2Config: V2Config): Promise { +async function startV2Flow( + configPath: string, + v2Config: V2Config, +): Promise<{ bridge: Bridge; workspaceManager: WorkspaceManager }> { logger.info('Starting V2 flow (autonomous AI bridge)'); + // Step 0: Resolve workspace path — clone remote repo if needed + const workspaceManager = new WorkspaceManager(v2Config.workspacePath, { + pullIntervalSeconds: v2Config.workspace?.pullInterval, + }); + const resolvedWorkspacePath = await workspaceManager.init(); + if (WorkspaceManager.isRemoteUrl(v2Config.workspacePath)) { + logger.info({ localPath: resolvedWorkspacePath }, 'Using cloned remote workspace'); + } + // Step 1: Discover AI tools on the machine logger.info('Discovering AI tools...'); const scanResult = await scanForAITools(); - if (!scanResult.master) { + // --verbose flag enables full tool selection trace logs + const isVerbose = process.argv.includes('--verbose'); + + // Step 1a: Exclude tools if configured — removes them from discovery results entirely + const excludeTools = v2Config.master?.excludeTools; + if (excludeTools && excludeTools.length > 0) { + const excludeSet = new Set(excludeTools.map((t) => t.toLowerCase())); + + // Log reason for each tool that will be excluded + if (isVerbose) { + for (const tool of scanResult.cliTools) { + if (excludeSet.has(tool.name.toLowerCase())) { + logger.info(`${tool.name} excluded: listed in config.excludeTools`); + } + } + } + + const before = scanResult.cliTools.length; + scanResult.cliTools = scanResult.cliTools.filter( + (tool) => !excludeSet.has(tool.name.toLowerCase()), + ); + if (isVerbose) { + logger.info( + { excludeTools, removed: before - scanResult.cliTools.length }, + 'Excluded tools from discovery', + ); + } + + // Re-select master if the current master was excluded + if (scanResult.master && excludeSet.has(scanResult.master.name.toLowerCase())) { + scanResult.master = scanResult.cliTools[0] ?? null; + if (isVerbose && scanResult.master) { + logger.info( + { newMaster: scanResult.master.name }, + 'Previous master was excluded — auto-selected next available tool', + ); + } + } + } + + // Step 1b: Apply master tool override if configured + let selectedMaster = scanResult.master; + // Capture the post-exclusion auto-selected master to detect redundant overrides + const autoSelectedMaster = scanResult.master; + + if (v2Config.master?.tool) { + if (isVerbose) { + logger.info({ override: v2Config.master.tool }, 'Master tool override specified in config'); + } + + // Try to find the tool in discovered tools by name or path + const overrideTool = scanResult.cliTools.find( + (tool) => + tool.name === v2Config.master?.tool || + tool.path === v2Config.master?.tool || + tool.path.endsWith(`/${v2Config.master?.tool}`), + ); + + if (overrideTool) { + selectedMaster = overrideTool; + // Only log the override selection if it actually changed from the auto-selected master + if (isVerbose && autoSelectedMaster?.name !== overrideTool.name) { + logger.info( + { tool: overrideTool.name }, + 'Using overridden Master tool from discovered tools', + ); + } + } else { + logger.warn( + { requested: v2Config.master.tool }, + 'Overridden tool not found in discovered tools — falling back to auto-detected Master', + ); + } + } + + if (!selectedMaster) { logger.error('No Master AI tool found. V2 flow requires at least one CLI AI tool.'); logger.error( 'Install Claude Code CLI (https://claude.ai/download) or another supported AI tool.', @@ -68,19 +217,71 @@ async function startV2Flow(configPath: string, v2Config: V2Config): Promise = { + claude: 'claude-code', + 'claude-code': 'claude-code', + codex: 'codex', + }; + const masterProviderName = MASTER_PROVIDER_MAP[selectedMaster.name]; + if (masterProviderName === undefined) { + throw new Error( + `No AI provider available for master tool '${selectedMaster.name}'. ` + + `Supported options: 'claude' (requires Claude Code CLI) or 'codex' (requires Codex CLI — authenticate via 'codex login' or OPENAI_API_KEY).`, + ); + } + + // Single summary log — always shown regardless of verbosity + { + const summaryParts: string[] = []; + if (excludeTools?.length) { + summaryParts.push(`${excludeTools.join(', ')} excluded per config.excludeTools`); + } + // Only note the override if it actually changed the selection from auto-detected + if ( + v2Config.master?.tool && + selectedMaster.name === v2Config.master.tool && + autoSelectedMaster?.name !== v2Config.master.tool + ) { + summaryParts.push('override: config.master.tool'); + } + const suffix = summaryParts.length ? ` (${summaryParts.join('; ')})` : ''; + logger.info(`Master AI: ${selectedMaster.name}${suffix}`); + } + if (isVerbose) { + logger.info( + { + master: selectedMaster.name, + provider: masterProviderName, + totalTools: scanResult.cliTools.length + scanResult.vscodeExtensions.length, + cliTools: scanResult.cliTools.length, + vscodeExtensions: scanResult.vscodeExtensions.length, + }, + 'AI tool discovery complete', + ); + } // Step 2: Load config and create bridge const config = await loadConfig(); - const bridge = new Bridge(config, { configPath }); + setLogLevel(process.env['LOG_LEVEL'] ?? config.logLevel); + injectDevConnectors(config); + + // Create MCP registry from config — only when MCP is enabled and servers are configured + const mcpServers = v2Config.mcp?.enabled !== false ? (v2Config.mcp?.servers ?? []) : []; + const mcpRegistry = new McpRegistry(configPath, mcpServers); + + const bridge = new Bridge(config, { + configPath, + workspacePath: resolvedWorkspacePath, + mcpRegistry, + securityConfig: v2Config.security, + workspaceInclude: v2Config.workspace?.include, + workspaceExclude: v2Config.workspace?.exclude, + discoveredTools: scanResult.cliTools, + }); // Register built-in plugins const registry = bridge.getRegistry(); @@ -92,32 +293,149 @@ async function startV2Flow(configPath: string, v2Config: V2Config): Promise { + logger.info('Remote workspace changes detected — scheduling re-exploration'); + void masterManager.reExplore().catch((err: unknown) => { + logger.error({ err }, 'Re-exploration after remote pull failed'); + }); }); // Wire Master into the bridge router (must happen before start) bridge.setMaster(masterManager); - // Step 4: Start bridge (connectors only — Master handles AI routing) + // Wire email config if provided + if (v2Config.email) { + bridge.setEmailConfig(v2Config.email); + logger.info('Email config wired into bridge'); + } + + // Wire AppServer with resource limits from config + { + const { AppServer } = await import('./core/app-server.js'); + const appsConfig = v2Config.apps; + const appServer = new AppServer({ + maxConcurrent: appsConfig?.maxConcurrent, + maxMemoryMB: appsConfig?.maxMemoryMB, + idleTimeoutMs: + appsConfig?.idleTimeoutMinutes !== undefined + ? appsConfig.idleTimeoutMinutes * 60 * 1000 + : undefined, + }); + await appServer.scanUsedPorts(); + bridge.setAppServer(appServer); + logger.info( + { + maxConcurrent: appServer.maxConcurrent, + maxMemoryMB: appServer.maxMemoryMB, + idleTimeoutMinutes: appsConfig?.idleTimeoutMinutes ?? 30, + }, + 'AppServer wired into bridge', + ); + } + + // Step 4: Start bridge first — this initializes MemoryManager (SQLite) via memory.init(). + // MasterManager holds a reference to the MemoryManager but must not use it until init() completes. + // Running bridge.start() first eliminates the race condition where MasterManager reads from + // the DB before it is open (this.db would be null, causing 'MemoryManager not initialised' errors). await bridge.start(); - // Step 5: Start exploration (runs in background — does NOT block the bridge) - masterManager.start().catch((error) => { + // Re-sync MasterManager's memory reference after bridge.start() — + // if MemoryManager.init() failed, bridge.start() sets bridge.memory to null, + // but MasterManager still holds the old (uninitialised) reference. + masterManager.memory = bridge.getMemory(); + + // Inform MasterManager which connectors are active — included in the Master system prompt + // so the Master knows which SHARE targets are available (e.g. whatsapp, telegram, console). + masterManager.setActiveConnectorNames(bridge.getActiveConnectorNames()); + + // Inform MasterManager of the file server port — included in the Master system prompt + // so the Master knows it can reference generated files via http://localhost:/shared/. + const fileServerPort = bridge.getFileServerPort(); + if (fileServerPort !== null) { + masterManager.setFileServerPort(fileServerPort); + } + + // Inform MasterManager of the tunnel public URL — included in the Master system prompt + // so the Master knows generated files are publicly accessible (not just on localhost). + const tunnelUrl = bridge.getTunnelUrl(); + if (tunnelUrl !== null) { + masterManager.setTunnelUrl(tunnelUrl); + } + + // Wire MCP servers into the health endpoint (runs after bridge.start() initialises the health server) + if (v2Config.mcp?.enabled !== false && (v2Config.mcp?.servers ?? []).length > 0) { + bridge.setMcpServers(v2Config.mcp?.servers ?? []); + } + + // Step 5: Start Master AI in the background — bridge is already serving messages. + // This loads workspace-map.json and transitions from 'idle' to 'ready'. + // masterManager.start() can take minutes (workspace exploration) so we don't await it. + masterManager.start().catch((error: unknown) => { logger.error( { err: error }, 'Master AI exploration failed — bridge continues running without workspace context', ); }); - logger.info('OpenBridge (V2) is running. Master AI is exploring workspace...'); - logger.info('Press Ctrl+C to stop.'); + // Step 6: Start remote workspace polling (no-op for local workspaces) + workspaceManager.startPolling(); + + const connectorNames = bridge.getActiveConnectorNames(); + const webChatInfo = bridge.getWebChatInfo(); + if (process.env['OPENBRIDGE_HEADLESS'] === 'true') { + const payload: Record = { + event: 'ready', + version: OPENBRIDGE_VERSION, + mode: 'v2', + master: selectedMaster.name, + connectors: connectorNames, + }; + if (webChatInfo) { + payload['webChatUrl'] = webChatInfo.url; + payload['webChatToken'] = webChatInfo.token; + } + process.stdout.write(JSON.stringify(payload) + '\n'); + } else { + process.stdout.write( + `OpenBridge v${OPENBRIDGE_VERSION} | Master: ${selectedMaster.name} | Connectors: ${connectorNames.join(', ') || 'none'}\n`, + ); + if (webChatInfo) { + process.stdout.write(`WebChat URL: ${webChatInfo.url}\n`); + process.stdout.write(`WebChat token: ${webChatInfo.token}\n`); + // QR code for phone scanning is displayed by the WebChat connector + // during initialize() using the LAN URL (or tunnel URL if active). + } + logger.info('OpenBridge (V2) is running. Master AI is exploring workspace...'); + logger.info('Press Ctrl+C to stop.'); + } - return bridge; + return { bridge, workspaceManager }; } /** @@ -134,18 +452,30 @@ async function detectConfigVersion(configPath: string): Promise<'v0' | 'v2'> { return 'v0'; } catch (error) { - logger.error({ err: error }, 'Failed to detect config version'); + // ENOENT: rethrow without logging — main() will show an actionable message + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + logger.error({ err: error }, 'Failed to detect config version'); + } throw error; } } async function main(): Promise { - logger.info('OpenBridge starting...'); + // Detect headless mode: --headless CLI flag or OPENBRIDGE_HEADLESS env var + const isHeadless = + process.argv.includes('--headless') || process.env['OPENBRIDGE_HEADLESS'] === 'true'; + if (isHeadless) { + process.env['OPENBRIDGE_HEADLESS'] = 'true'; + } + + logger.info({ headless: isHeadless }, 'OpenBridge starting...'); let bridge: Bridge | null = null; + let workspaceManager: WorkspaceManager | null = null; + let configPath: string | undefined; try { - const configPath = resolveConfigPath(); + configPath = resolveConfigPath(); // Detect config version const version = await detectConfigVersion(configPath); @@ -155,27 +485,70 @@ async function main(): Promise { const raw = await readFile(configPath, 'utf-8'); const parsed: unknown = JSON.parse(raw); const v2Config = V2ConfigSchema.parse(parsed); - bridge = await startV2Flow(configPath, v2Config); + const result = await startV2Flow(configPath, v2Config); + bridge = result.bridge; + workspaceManager = result.workspaceManager; } else { // V0 flow: legacy mode (backward compatibility) bridge = await startV0Flow(configPath); } - // Graceful shutdown + // Graceful shutdown — guarded against concurrent SIGINT + SIGTERM const shutdown = async (): Promise => { + if (shutdownInProgress) { + logger.warn('Shutdown already in progress — ignoring duplicate signal'); + return; + } + shutdownInProgress = true; + console.log('\nShutting down gracefully... please wait'); logger.info('Shutting down...'); + workspaceManager?.stopPolling(); if (bridge) { - await bridge.stop(); + await Promise.race([ + bridge.stop(), + new Promise((_, reject) => + setTimeout(() => reject(new Error('Shutdown timeout')), 10_000), + ), + ]).catch((err: unknown) => { + if (err instanceof Error && err.message === 'Shutdown timeout') { + console.error('Shutdown timeout exceeded (10s) — forcing exit'); + process.exit(1); + } + throw err; + }); } process.exit(0); }; process.on('SIGINT', () => void shutdown()); process.on('SIGTERM', () => void shutdown()); + // SIGHUP: reload is handled by ConfigWatcher (file-change events) — ignore gracefully + process.on('SIGHUP', () => { + logger.info('SIGHUP received — config hot-reload is file-driven, ignoring signal'); + }); } catch (error) { - logger.fatal({ err: error }, 'Failed to start OpenBridge'); + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + const resolvedPath = configPath ?? 'config.json'; + logger.error( + `Config file not found: ${resolvedPath}. Create one by running: npx openbridge init`, + ); + } else { + logger.fatal({ err: error }, 'Failed to start OpenBridge'); + } process.exit(1); } } -void main(); +// Handle --version flag for the packaged binary entry point +if (process.argv.includes('--version') || process.argv.includes('-v')) { + process.stdout.write(OPENBRIDGE_VERSION + '\n'); + // Defer exit to let Pino's sonic-boom stream finish initialization + setTimeout(() => process.exit(0), 50); +} else if (process.argv.includes('--health')) { + // Handle --health flag for the packaged binary entry point + const result = runHealthCheck(); + process.stdout.write(JSON.stringify(result) + '\n'); + setTimeout(() => process.exit(result.passed ? 0 : 1), 50); +} else { + void main(); +} diff --git a/src/integrations/adapters/.gitkeep b/src/integrations/adapters/.gitkeep new file mode 100644 index 00000000..5a89614a --- /dev/null +++ b/src/integrations/adapters/.gitkeep @@ -0,0 +1,2 @@ +# Placeholder to ensure adapters/ directory is tracked by git +# Adapter implementations (Stripe, Google Drive, OpenAPI, Email, Database, etc.) go here. diff --git a/src/integrations/adapters/database-adapter.ts b/src/integrations/adapters/database-adapter.ts new file mode 100644 index 00000000..2d2b6fac --- /dev/null +++ b/src/integrations/adapters/database-adapter.ts @@ -0,0 +1,317 @@ +import type { Pool, PoolConfig } from 'pg'; +import { createLogger } from '../../core/logger.js'; +import type { + BusinessIntegration, + HealthStatus, + IntegrationCapability, + IntegrationConfig, +} from '../../types/integration.js'; + +const logger = createLogger('database-adapter'); + +/** + * PostgreSQL database integration adapter (read-only). + * + * Capabilities: + * - query: Execute a read-only SQL SELECT statement + * - list_tables: List all tables in the public schema + * - describe_table: Describe columns of a table + * - count_rows: Count rows in a table + * + * Credentials expected (from config.options): + * - connectionString: PostgreSQL connection URI + * e.g. "postgresql://user:pass@host:5432/dbname" + * OR individual fields: + * - host, port, database, user, password, ssl (boolean) + * + * IMPORTANT: execute() only allows read operations. + * INSERT/UPDATE/DELETE/DDL require explicit human approval via the approval relay. + */ +export class DatabaseAdapter implements BusinessIntegration { + readonly name = 'database'; + readonly type = 'database' as const; + + private pool: Pool | null = null; + + async initialize(config: IntegrationConfig): Promise { + const { Pool } = await import('pg'); + const opts = config.options; + const connectionString = opts['connectionString'] as string | undefined; + + let poolConfig: PoolConfig; + + if (connectionString) { + if (typeof connectionString !== 'string') { + throw new Error('Database adapter: connectionString must be a string'); + } + poolConfig = { connectionString, max: 5, idleTimeoutMillis: 30_000 }; + } else { + const host = opts['host'] as string | undefined; + const database = opts['database'] as string | undefined; + const user = opts['user'] as string | undefined; + const password = opts['password'] as string | undefined; + + if (!host) throw new Error('Database adapter: host is required in config.options'); + if (!database) throw new Error('Database adapter: database is required in config.options'); + if (!user) throw new Error('Database adapter: user is required in config.options'); + if (!password) throw new Error('Database adapter: password is required in config.options'); + + poolConfig = { + host, + port: (opts['port'] as number | undefined) ?? 5432, + database, + user, + password, + ssl: (opts['ssl'] as boolean | undefined) ?? false, + max: 5, + idleTimeoutMillis: 30_000, + }; + } + + this.pool = new Pool(poolConfig); + + // Verify connectivity + const client = await this.pool.connect(); + try { + await client.query('SELECT 1'); + } finally { + client.release(); + } + + logger.info('Database adapter initialized'); + } + + async healthCheck(): Promise { + const checkedAt = new Date().toISOString(); + + if (!this.pool) { + return { status: 'unhealthy', message: 'Not initialized', checkedAt, details: {} }; + } + + try { + const result = await this.pool.query<{ now: string }>('SELECT NOW() AS now'); + return { + status: 'healthy', + message: 'Database connection OK', + checkedAt, + details: { serverTime: result.rows[0]?.now ?? '' }, + }; + } catch (err) { + return { + status: 'unhealthy', + message: err instanceof Error ? err.message : String(err), + checkedAt, + details: {}, + }; + } + } + + async shutdown(): Promise { + if (this.pool) { + await this.pool.end(); + this.pool = null; + } + logger.info('Database adapter shut down'); + } + + describeCapabilities(): IntegrationCapability[] { + return [ + { + name: 'query', + description: + 'Execute a read-only SQL SELECT statement. Params: sql (string — must be a SELECT query), params (array, optional — parameterized query values). Returns rows array and rowCount.', + category: 'read', + requiresApproval: false, + }, + { + name: 'list_tables', + description: + 'List all tables in the connected database. Params: schema (string, optional — defaults to "public"). Returns array of { tableName, schema, rowEstimate }.', + category: 'read', + requiresApproval: false, + }, + { + name: 'describe_table', + description: + 'Describe the columns and types of a table. Params: table (string — table name), schema (string, optional — defaults to "public"). Returns array of { columnName, dataType, isNullable, columnDefault }.', + category: 'read', + requiresApproval: false, + }, + { + name: 'count_rows', + description: + 'Count the number of rows in a table. Params: table (string — table name), schema (string, optional — defaults to "public"), where (string, optional — WHERE clause without the WHERE keyword). Returns { count }.', + category: 'read', + requiresApproval: false, + }, + ]; + } + + async query(operation: string, params: Record): Promise { + this.assertInitialized(); + + switch (operation) { + case 'query': + return await this.runQuery(params); + case 'list_tables': + return await this.listTables(params); + case 'describe_table': + return await this.describeTable(params); + case 'count_rows': + return await this.countRows(params); + default: + throw new Error(`Unknown query operation: ${operation}`); + } + } + + async execute(operation: string, params: Record): Promise { + // Only read operations are allowed without explicit approval. + // Write operations (INSERT/UPDATE/DELETE/DDL) must go through the approval relay. + const readOnlyOps = new Set(['query', 'list_tables', 'describe_table', 'count_rows']); + if (readOnlyOps.has(operation)) { + return await this.query(operation, params); + } + throw new Error( + `Operation "${operation}" requires human approval via the approval relay. ` + + `Only read operations (query, list_tables, describe_table, count_rows) are permitted directly.`, + ); + } + + // ── Private: operations ───────────────────────────────────────── + + private async runQuery( + params: Record, + ): Promise<{ rows: Record[]; rowCount: number }> { + const sql = params['sql'] as string | undefined; + const queryParams = (params['params'] as unknown[] | undefined) ?? []; + + if (!sql || typeof sql !== 'string') { + throw new Error('query: sql parameter is required'); + } + + // Enforce read-only: only allow SELECT statements + const normalised = sql.trimStart().toUpperCase(); + if (!normalised.startsWith('SELECT') && !normalised.startsWith('WITH')) { + throw new Error( + 'query: only SELECT (and WITH ... SELECT) statements are permitted. ' + + 'Use the approval relay for INSERT/UPDATE/DELETE/DDL.', + ); + } + + const result = await this.pool!.query(sql, queryParams); + return { + rows: result.rows as Record[], + rowCount: result.rowCount ?? result.rows.length, + }; + } + + private async listTables( + params: Record, + ): Promise<{ tables: Array<{ tableName: string; schema: string; rowEstimate: number }> }> { + const schema = (params['schema'] as string | undefined) ?? 'public'; + + const result = await this.pool!.query<{ + table_name: string; + table_schema: string; + row_estimate: string; + }>( + `SELECT + t.table_name, + t.table_schema, + COALESCE(s.n_live_tup, 0) AS row_estimate + FROM information_schema.tables t + LEFT JOIN pg_stat_user_tables s + ON s.schemaname = t.table_schema AND s.relname = t.table_name + WHERE t.table_type = 'BASE TABLE' + AND t.table_schema = $1 + ORDER BY t.table_name`, + [schema], + ); + + return { + tables: result.rows.map((r) => ({ + tableName: r.table_name, + schema: r.table_schema, + rowEstimate: parseInt(r.row_estimate, 10), + })), + }; + } + + private async describeTable(params: Record): Promise<{ + columns: Array<{ + columnName: string; + dataType: string; + isNullable: boolean; + columnDefault: string | null; + }>; + }> { + const table = params['table'] as string | undefined; + const schema = (params['schema'] as string | undefined) ?? 'public'; + + if (!table || typeof table !== 'string') { + throw new Error('describe_table: table parameter is required'); + } + + const result = await this.pool!.query<{ + column_name: string; + data_type: string; + is_nullable: string; + column_default: string | null; + }>( + `SELECT column_name, data_type, is_nullable, column_default + FROM information_schema.columns + WHERE table_schema = $1 AND table_name = $2 + ORDER BY ordinal_position`, + [schema, table], + ); + + return { + columns: result.rows.map((r) => ({ + columnName: r.column_name, + dataType: r.data_type, + isNullable: r.is_nullable === 'YES', + columnDefault: r.column_default, + })), + }; + } + + private async countRows(params: Record): Promise<{ count: number }> { + const table = params['table'] as string | undefined; + const schema = (params['schema'] as string | undefined) ?? 'public'; + const where = params['where'] as string | undefined; + + if (!table || typeof table !== 'string') { + throw new Error('count_rows: table parameter is required'); + } + + // Validate table and schema identifiers to prevent SQL injection + validateIdentifier(table); + validateIdentifier(schema); + + const whereClause = where ? ` WHERE ${where}` : ''; + const sql = `SELECT COUNT(*) AS count FROM "${schema}"."${table}"${whereClause}`; + + const result = await this.pool!.query<{ count: string }>(sql); + return { count: parseInt(result.rows[0]?.count ?? '0', 10) }; + } + + private assertInitialized(): void { + if (!this.pool) { + throw new Error('Database adapter not initialized — call initialize() first'); + } + } +} + +// ── Utility ────────────────────────────────────────────────────── + +/** + * Validates that an identifier (table/schema name) contains only safe characters. + * Prevents SQL injection through identifier interpolation. + */ +function validateIdentifier(identifier: string): void { + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(identifier)) { + throw new Error( + `Invalid identifier "${identifier}" — only alphanumeric characters and underscores are allowed`, + ); + } +} diff --git a/src/integrations/adapters/dropbox-adapter.ts b/src/integrations/adapters/dropbox-adapter.ts new file mode 100644 index 00000000..df413c25 --- /dev/null +++ b/src/integrations/adapters/dropbox-adapter.ts @@ -0,0 +1,277 @@ +import { Dropbox } from 'dropbox'; +import { createLogger } from '../../core/logger.js'; +import type { + BusinessIntegration, + HealthStatus, + IntegrationCapability, + IntegrationConfig, +} from '../../types/integration.js'; + +const logger = createLogger('dropbox-adapter'); + +/** + * Dropbox integration adapter. + * + * Capabilities: + * - upload_file: Upload a local file to Dropbox + * - download_file: Download a file from Dropbox to a local path + * - list_files: List files and folders in a Dropbox path + * - create_shared_link: Create a public shared link for a file + * + * Credentials expected (from credential store via config.options): + * - accessToken: Dropbox OAuth2 access token + */ +export class DropboxAdapter implements BusinessIntegration { + readonly name = 'dropbox'; + readonly type = 'storage' as const; + + private dbx: Dropbox | null = null; + + async initialize(config: IntegrationConfig): Promise { + const opts = config.options; + const accessToken = opts['accessToken'] as string | undefined; + + if (!accessToken || typeof accessToken !== 'string') { + throw new Error('Dropbox adapter requires an accessToken in config.options'); + } + + this.dbx = new Dropbox({ accessToken }); + + // Verify credentials work + try { + await this.dbx.usersGetCurrentAccount(); + } catch (err) { + this.dbx = null; + throw new Error( + `Dropbox initialization failed: ${err instanceof Error ? err.message : String(err)}`, + ); + } + + logger.info('Dropbox adapter initialized'); + } + + async healthCheck(): Promise { + const checkedAt = new Date().toISOString(); + + if (!this.dbx) { + return { status: 'unhealthy', message: 'Not initialized', checkedAt, details: {} }; + } + + try { + const res = await this.dbx.usersGetCurrentAccount(); + return { + status: 'healthy', + message: 'Dropbox API reachable', + checkedAt, + details: { + accountId: res.result.account_id, + displayName: res.result.name.display_name, + email: res.result.email, + }, + }; + } catch (err) { + return { + status: 'unhealthy', + message: err instanceof Error ? err.message : String(err), + checkedAt, + details: {}, + }; + } + } + + // eslint-disable-next-line @typescript-eslint/require-await + async shutdown(): Promise { + this.dbx = null; + logger.info('Dropbox adapter shut down'); + } + + describeCapabilities(): IntegrationCapability[] { + return [ + { + name: 'upload_file', + description: + 'Upload a local file to Dropbox. Params: filePath (string, local path), dropboxPath (string, destination path in Dropbox, e.g. "/folder/file.txt"), overwrite (boolean, default false).', + category: 'write', + requiresApproval: true, + }, + { + name: 'download_file', + description: + 'Download a file from Dropbox to a local path. Params: dropboxPath (string, Dropbox file path), destPath (string, local destination path).', + category: 'read', + requiresApproval: false, + }, + { + name: 'list_files', + description: + 'List files and folders in a Dropbox path. Params: path (string, Dropbox folder path — use "" for root), recursive (boolean, default false).', + category: 'read', + requiresApproval: false, + }, + { + name: 'create_shared_link', + description: + 'Create a publicly accessible shared link for a file or folder. Params: dropboxPath (string, Dropbox file/folder path).', + category: 'write', + requiresApproval: true, + }, + ]; + } + + async query(operation: string, params: Record): Promise { + if (!this.dbx) { + throw new Error('Dropbox adapter not initialized — call initialize() first'); + } + + switch (operation) { + case 'list_files': + return await this.listFiles(params); + case 'download_file': + return await this.downloadFile(params); + default: + throw new Error(`Unknown query operation: ${operation}`); + } + } + + async execute(operation: string, params: Record): Promise { + if (!this.dbx) { + throw new Error('Dropbox adapter not initialized — call initialize() first'); + } + + switch (operation) { + case 'upload_file': + return await this.uploadFile(params); + case 'create_shared_link': + return await this.createSharedLink(params); + default: + throw new Error(`Unknown execute operation: ${operation}`); + } + } + + // ── Private helpers ──────────────────────────────────────────── + + private async uploadFile( + params: Record, + ): Promise<{ id: string; name: string; path: string; size: number }> { + const { readFile } = await import('node:fs/promises'); + const nodePath = await import('node:path'); + + const filePath = params['filePath'] as string; + if (!filePath || typeof filePath !== 'string') { + throw new Error('filePath is required'); + } + + const dropboxPath = params['dropboxPath'] as string | undefined; + const resolvedDropboxPath = dropboxPath ?? `/${nodePath.basename(filePath)}`; + const overwrite = (params['overwrite'] as boolean) ?? false; + + const contents = await readFile(filePath); + + const res = await this.dbx!.filesUpload({ + path: resolvedDropboxPath, + contents, + mode: overwrite ? { '.tag': 'overwrite' } : { '.tag': 'add' }, + autorename: !overwrite, + }); + + logger.info( + { dropboxPath: res.result.path_display, size: res.result.size }, + 'File uploaded to Dropbox', + ); + return { + id: res.result.id, + name: res.result.name, + path: res.result.path_display ?? resolvedDropboxPath, + size: res.result.size, + }; + } + + private async downloadFile( + params: Record, + ): Promise<{ dropboxPath: string; destPath: string; size: number }> { + const { writeFile } = await import('node:fs/promises'); + + const dropboxPath = params['dropboxPath'] as string; + const destPath = params['destPath'] as string; + + if (!dropboxPath || typeof dropboxPath !== 'string') { + throw new Error('dropboxPath is required'); + } + if (!destPath || typeof destPath !== 'string') { + throw new Error('destPath is required'); + } + + const res = await this.dbx!.filesDownload({ path: dropboxPath }); + + // The Dropbox SDK returns file content in `fileBinary` when running in Node.js + const fileContent = (res.result as unknown as Record)['fileBinary'] as Buffer; + + await writeFile(destPath, fileContent); + + logger.info( + { dropboxPath, destPath, size: fileContent.length }, + 'File downloaded from Dropbox', + ); + return { dropboxPath, destPath, size: fileContent.length }; + } + + private async listFiles( + params: Record, + ): Promise<{ entries: Array>; hasMore: boolean }> { + const path = (params['path'] as string) ?? ''; + const recursive = (params['recursive'] as boolean) ?? false; + + const res = await this.dbx!.filesListFolder({ path, recursive }); + + const entries = res.result.entries.map((entry) => ({ + tag: entry['.tag'], + name: entry.name, + path: entry.path_display ?? entry.path_lower, + ...(entry['.tag'] === 'file' + ? { + size: (entry as { size?: number }).size, + clientModified: (entry as { client_modified?: string }).client_modified, + serverModified: (entry as { server_modified?: string }).server_modified, + } + : {}), + })); + + return { entries, hasMore: res.result.has_more }; + } + + private async createSharedLink( + params: Record, + ): Promise<{ url: string; dropboxPath: string }> { + const dropboxPath = params['dropboxPath'] as string; + if (!dropboxPath || typeof dropboxPath !== 'string') { + throw new Error('dropboxPath is required'); + } + + try { + const res = await this.dbx!.sharingCreateSharedLinkWithSettings({ path: dropboxPath }); + logger.info({ dropboxPath, url: res.result.url }, 'Shared link created for Dropbox file'); + return { url: res.result.url, dropboxPath }; + } catch (err) { + // If a shared link already exists, retrieve it + const errObj = err as Record; + if ( + errObj['error'] && + typeof errObj['error'] === 'object' && + (errObj['error'] as Record)['.tag'] === 'shared_link_already_exists' + ) { + const existing = (errObj['error'] as Record)[ + 'shared_link_already_exists' + ] as Record | undefined; + const existingUrl = + existing && + typeof existing === 'object' && + (existing['metadata'] as Record | undefined)?.['url']; + if (typeof existingUrl === 'string') { + logger.info({ dropboxPath, url: existingUrl }, 'Returning existing shared link'); + return { url: existingUrl, dropboxPath }; + } + } + throw err; + } + } +} diff --git a/src/integrations/adapters/email-adapter.ts b/src/integrations/adapters/email-adapter.ts new file mode 100644 index 00000000..7f255ad1 --- /dev/null +++ b/src/integrations/adapters/email-adapter.ts @@ -0,0 +1,721 @@ +import nodemailer from 'nodemailer'; +import type { gmail_v1 } from 'googleapis'; +import { createLogger } from '../../core/logger.js'; +import type { + BusinessIntegration, + HealthStatus, + IntegrationCapability, + IntegrationConfig, +} from '../../types/integration.js'; + +const logger = createLogger('email-adapter'); + +interface EmailMessage { + id: string; + from: string; + to: string | string[]; + subject: string; + date: string; + snippet: string; + hasAttachments: boolean; +} + +interface EmailAttachment { + filename: string; + mimeType: string; + size: number; + data?: string; // base64-encoded content +} + +/** + * Email integration adapter. + * + * Capabilities: + * - send_email: Send an email via SMTP (nodemailer) + * - read_emails: Read recent emails from Gmail (oauth2) or IMAP + * - search_emails: Search emails by query + * - get_attachments: Download attachments from an email + * + * Credentials expected (from config.options): + * - Auth type "oauth2" (Gmail): + * clientId: OAuth2 client ID + * clientSecret: OAuth2 client secret + * refreshToken: OAuth2 refresh token + * userEmail: Gmail address (used as sender + mailbox owner) + * - Auth type "smtp" (generic SMTP + IMAP): + * smtpHost: SMTP server hostname + * smtpPort: SMTP port (default 587) + * imapHost: IMAP server hostname + * imapPort: IMAP port (default 993) + * user: Email username + * pass: Email password + * from: From address (defaults to user if not specified) + */ +export class EmailAdapter implements BusinessIntegration { + readonly name = 'email'; + readonly type = 'communication' as const; + + private authType: 'oauth2' | 'smtp' = 'smtp'; + private config: IntegrationConfig | null = null; + + // Gmail-specific + private gmailClient: gmail_v1.Gmail | null = null; + private gmailUserId = 'me'; + + // SMTP transporter (used for both auth types) + private transporter: nodemailer.Transporter | null = null; + + async initialize(config: IntegrationConfig): Promise { + this.config = config; + const opts = config.options; + this.authType = ((opts['authType'] as string) ?? 'smtp') as 'oauth2' | 'smtp'; + + if (this.authType === 'oauth2') { + await this.initializeGmail(opts); + } else { + await this.initializeSmtp(opts); + } + + logger.info({ authType: this.authType }, 'Email adapter initialized'); + } + + async healthCheck(): Promise { + const checkedAt = new Date().toISOString(); + + if (this.authType === 'oauth2') { + if (!this.gmailClient) { + return { status: 'unhealthy', message: 'Not initialized', checkedAt, details: {} }; + } + try { + const res = await this.gmailClient.users.getProfile({ userId: this.gmailUserId }); + return { + status: 'healthy', + message: 'Gmail API reachable', + checkedAt, + details: { + email: res.data.emailAddress ?? 'unknown', + totalMessages: res.data.messagesTotal ?? 0, + }, + }; + } catch (err) { + return { + status: 'unhealthy', + message: err instanceof Error ? err.message : String(err), + checkedAt, + details: {}, + }; + } + } else { + if (!this.transporter) { + return { status: 'unhealthy', message: 'Not initialized', checkedAt, details: {} }; + } + try { + await this.transporter.verify(); + return { + status: 'healthy', + message: 'SMTP connection verified', + checkedAt, + details: {}, + }; + } catch (err) { + return { + status: 'unhealthy', + message: err instanceof Error ? err.message : String(err), + checkedAt, + details: {}, + }; + } + } + } + + // eslint-disable-next-line @typescript-eslint/require-await + async shutdown(): Promise { + this.gmailClient = null; + if (this.transporter) { + this.transporter.close(); + this.transporter = null; + } + this.config = null; + logger.info('Email adapter shut down'); + } + + describeCapabilities(): IntegrationCapability[] { + return [ + { + name: 'send_email', + description: + 'Send an email. Params: to (string, recipient address), subject (string), body (string, plain text), html (string, optional HTML body).', + category: 'write', + requiresApproval: true, + }, + { + name: 'read_emails', + description: + 'Read recent emails. Params: folder (string, optional — e.g. "INBOX", default "INBOX"), limit (number, optional — default 20, max 50).', + category: 'read', + requiresApproval: false, + }, + { + name: 'search_emails', + description: + 'Search emails by query. Params: query (string — Gmail query syntax for oauth2, or plain search term for IMAP), limit (number, optional — default 20, max 50).', + category: 'read', + requiresApproval: false, + }, + { + name: 'get_attachments', + description: + 'Get attachments from a specific email. Params: messageId (string — email ID from read_emails/search_emails), includeData (boolean, optional — whether to include base64 attachment data, default false).', + category: 'read', + requiresApproval: false, + }, + ]; + } + + async query(operation: string, params: Record): Promise { + this.assertInitialized(); + + switch (operation) { + case 'read_emails': + return await this.readEmails(params); + case 'search_emails': + return await this.searchEmails(params); + case 'get_attachments': + return await this.getAttachments(params); + default: + throw new Error(`Unknown query operation: ${operation}`); + } + } + + async execute(operation: string, params: Record): Promise { + this.assertInitialized(); + + switch (operation) { + case 'send_email': + return await this.sendEmail(params); + default: + throw new Error(`Unknown execute operation: ${operation}`); + } + } + + // ── Private: initialization ───────────────────────────────────── + + private async initializeGmail(opts: Record): Promise { + const { google } = await import('googleapis'); + + const clientId = opts['clientId'] as string | undefined; + const clientSecret = opts['clientSecret'] as string | undefined; + const refreshToken = opts['refreshToken'] as string | undefined; + const userEmail = opts['userEmail'] as string | undefined; + + if (!clientId || typeof clientId !== 'string') { + throw new Error('Email adapter (oauth2) requires clientId in config.options'); + } + if (!clientSecret || typeof clientSecret !== 'string') { + throw new Error('Email adapter (oauth2) requires clientSecret in config.options'); + } + if (!refreshToken || typeof refreshToken !== 'string') { + throw new Error('Email adapter (oauth2) requires refreshToken in config.options'); + } + + const auth = new google.auth.OAuth2(clientId, clientSecret); + auth.setCredentials({ refresh_token: refreshToken }); + + this.gmailClient = google.gmail({ version: 'v1', auth }); + if (userEmail) { + this.gmailUserId = userEmail; + } + + // Verify access + try { + await this.gmailClient.users.getProfile({ userId: this.gmailUserId }); + } catch (err) { + this.gmailClient = null; + throw new Error( + `Gmail initialization failed: ${err instanceof Error ? err.message : String(err)}`, + ); + } + + // Set up nodemailer OAuth2 transporter for sending + this.transporter = nodemailer.createTransport({ + service: 'gmail', + auth: { + type: 'OAuth2', + user: userEmail ?? this.gmailUserId, + clientId, + clientSecret, + refreshToken, + }, + }); + } + + private async initializeSmtp(opts: Record): Promise { + const smtpHost = opts['smtpHost'] as string | undefined; + const smtpPort = (opts['smtpPort'] as number | undefined) ?? 587; + const user = opts['user'] as string | undefined; + const pass = opts['pass'] as string | undefined; + + if (!smtpHost || typeof smtpHost !== 'string') { + throw new Error('Email adapter (smtp) requires smtpHost in config.options'); + } + if (!user || typeof user !== 'string') { + throw new Error('Email adapter (smtp) requires user in config.options'); + } + if (!pass || typeof pass !== 'string') { + throw new Error('Email adapter (smtp) requires pass in config.options'); + } + + this.transporter = nodemailer.createTransport({ + host: smtpHost, + port: smtpPort, + secure: smtpPort === 465, + auth: { user, pass }, + }); + + await this.transporter.verify(); + } + + // ── Private: operations ───────────────────────────────────────── + + private async sendEmail( + params: Record, + ): Promise<{ success: boolean; messageId: string }> { + const to = params['to'] as string; + const subject = params['subject'] as string; + const body = params['body'] as string; + const html = params['html'] as string | undefined; + + if (!to || typeof to !== 'string') throw new Error('to is required'); + if (!subject || typeof subject !== 'string') throw new Error('subject is required'); + if (!body || typeof body !== 'string') throw new Error('body is required'); + + const opts = this.config?.options ?? {}; + const from = + this.authType === 'oauth2' + ? ((opts['userEmail'] as string | undefined) ?? 'me') + : ((opts['from'] as string | undefined) ?? (opts['user'] as string | undefined) ?? ''); + + const result = (await this.transporter!.sendMail({ + from, + to, + subject, + text: body, + ...(html ? { html } : {}), + })) as { messageId?: string }; + + logger.info({ to, subject, messageId: result.messageId }, 'Email sent'); + return { success: true, messageId: result.messageId ?? '' }; + } + + private async readEmails( + params: Record, + ): Promise<{ messages: EmailMessage[]; total: number }> { + const limit = Math.min((params['limit'] as number | undefined) ?? 20, 50); + + if (this.authType === 'oauth2') { + return await this.gmailReadEmails('', limit); + } else { + const folder = (params['folder'] as string | undefined) ?? 'INBOX'; + return await this.imapReadEmails(folder, '', limit); + } + } + + private async searchEmails( + params: Record, + ): Promise<{ messages: EmailMessage[]; total: number }> { + const query = (params['query'] as string | undefined) ?? ''; + const limit = Math.min((params['limit'] as number | undefined) ?? 20, 50); + + if (this.authType === 'oauth2') { + return await this.gmailReadEmails(query, limit); + } else { + const folder = (params['folder'] as string | undefined) ?? 'INBOX'; + return await this.imapReadEmails(folder, query, limit); + } + } + + private async getAttachments( + params: Record, + ): Promise<{ messageId: string; attachments: EmailAttachment[] }> { + const messageId = params['messageId'] as string; + const includeData = (params['includeData'] as boolean | undefined) ?? false; + + if (!messageId || typeof messageId !== 'string') throw new Error('messageId is required'); + + if (this.authType === 'oauth2') { + return await this.gmailGetAttachments(messageId, includeData); + } else { + return await this.imapGetAttachments(messageId, includeData); + } + } + + // ── Gmail helpers ─────────────────────────────────────────────── + + private async gmailReadEmails( + query: string, + limit: number, + ): Promise<{ messages: EmailMessage[]; total: number }> { + const listRes = await this.gmailClient!.users.messages.list({ + userId: this.gmailUserId, + q: query || undefined, + maxResults: limit, + }); + + const messageList = listRes.data.messages ?? []; + const total = listRes.data.resultSizeEstimate ?? messageList.length; + + const messages: EmailMessage[] = await Promise.all( + messageList.map(async (m) => { + const msg = await this.gmailClient!.users.messages.get({ + userId: this.gmailUserId, + id: m.id ?? '', + format: 'metadata', + metadataHeaders: ['From', 'To', 'Subject', 'Date'], + }); + + const headers = msg.data.payload?.headers ?? []; + const header = (name: string) => + headers.find((h) => h.name?.toLowerCase() === name.toLowerCase())?.value ?? ''; + + const hasAttachments = (msg.data.payload?.parts ?? []).some( + (p) => p.filename && p.filename.length > 0, + ); + + return { + id: m.id ?? '', + from: header('From'), + to: header('To'), + subject: header('Subject'), + date: header('Date'), + snippet: msg.data.snippet ?? '', + hasAttachments, + }; + }), + ); + + return { messages, total }; + } + + private async gmailGetAttachments( + messageId: string, + includeData: boolean, + ): Promise<{ messageId: string; attachments: EmailAttachment[] }> { + const msg = await this.gmailClient!.users.messages.get({ + userId: this.gmailUserId, + id: messageId, + format: 'full', + }); + + const parts = msg.data.payload?.parts ?? []; + const attachments: EmailAttachment[] = []; + + for (const part of parts) { + if (!part.filename || part.filename.length === 0) continue; + + const att: EmailAttachment = { + filename: part.filename, + mimeType: part.mimeType ?? 'application/octet-stream', + size: part.body?.size ?? 0, + }; + + if (includeData && part.body?.attachmentId) { + const attRes = await this.gmailClient!.users.messages.attachments.get({ + userId: this.gmailUserId, + messageId, + id: part.body.attachmentId, + }); + att.data = attRes.data.data ?? undefined; + } else if (includeData && part.body?.data) { + att.data = part.body.data; + } + + attachments.push(att); + } + + return { messageId, attachments }; + } + + // ── IMAP helpers ──────────────────────────────────────────────── + + private async imapReadEmails( + folder: string, + query: string, + limit: number, + ): Promise<{ messages: EmailMessage[]; total: number }> { + const opts = this.config?.options ?? {}; + const imapHost = opts['imapHost'] as string | undefined; + const imapPort = (opts['imapPort'] as number | undefined) ?? 993; + const user = opts['user'] as string; + const pass = opts['pass'] as string; + + if (!imapHost) { + throw new Error('imapHost is required in config.options for IMAP email reading'); + } + + const Imap = (await import('imap')).default; + + return new Promise((resolve, reject) => { + const imap = new Imap({ + user, + password: pass, + host: imapHost, + port: imapPort, + tls: imapPort === 993, + tlsOptions: { rejectUnauthorized: false }, + }); + + const messages: EmailMessage[] = []; + + imap.once('ready', () => { + imap.openBox(folder, true, (err, box) => { + if (err) { + imap.end(); + reject(err); + return; + } + + const total = box.messages.total; + if (total === 0) { + imap.end(); + resolve({ messages: [], total: 0 }); + return; + } + + // Build search criteria + const criteria: Array = query ? ['ALL', ['TEXT', query]] : ['ALL']; + + imap.search(criteria, (searchErr, uids) => { + if (searchErr) { + imap.end(); + reject(searchErr); + return; + } + + if (!uids || uids.length === 0) { + imap.end(); + resolve({ messages: [], total: 0 }); + return; + } + + // Take last N messages + const selectedUids = uids.slice(-limit); + + const fetch = imap.fetch(selectedUids, { + bodies: 'HEADER.FIELDS (FROM TO SUBJECT DATE)', + struct: true, + }); + + fetch.on('message', (msg, seqno) => { + let headerData = ''; + const msgId = String(seqno); + + msg.on('body', (stream) => { + stream.on('data', (chunk: Buffer) => { + headerData += chunk.toString('utf8'); + }); + }); + + msg.once('attributes', (attrs: { struct?: Array> }) => { + const struct = attrs.struct; + const hasAttachments = struct + ? struct.some( + (p) => + p['disposition'] && + (p['disposition'] as Record)['type']?.toLowerCase() === + 'attachment', + ) + : false; + + msg.once('end', () => { + const parsed = parseImapHeaders(headerData); + messages.push({ + id: msgId, + from: parsed['from'] ?? '', + to: parsed['to'] ?? '', + subject: parsed['subject'] ?? '', + date: parsed['date'] ?? '', + snippet: '', + hasAttachments, + }); + }); + }); + }); + + fetch.once('error', (fetchErr) => { + imap.end(); + reject(fetchErr); + }); + + fetch.once('end', () => { + imap.end(); + resolve({ messages, total: uids.length }); + }); + }); + }); + }); + + imap.once('error', reject); + imap.connect(); + }); + } + + private async imapGetAttachments( + messageId: string, + includeData: boolean, + ): Promise<{ messageId: string; attachments: EmailAttachment[] }> { + const opts = this.config?.options ?? {}; + const imapHost = opts['imapHost'] as string | undefined; + const imapPort = (opts['imapPort'] as number | undefined) ?? 993; + const user = opts['user'] as string; + const pass = opts['pass'] as string; + const folder = (opts['defaultFolder'] as string | undefined) ?? 'INBOX'; + + if (!imapHost) { + throw new Error('imapHost is required in config.options for IMAP attachment retrieval'); + } + + const Imap = (await import('imap')).default; + const seqno = parseInt(messageId, 10); + + if (isNaN(seqno)) { + throw new Error(`Invalid messageId for IMAP: ${messageId} — must be a sequence number`); + } + + return new Promise((resolve, reject) => { + const imap = new Imap({ + user, + password: pass, + host: imapHost, + port: imapPort, + tls: imapPort === 993, + tlsOptions: { rejectUnauthorized: false }, + }); + + const attachments: EmailAttachment[] = []; + + imap.once('ready', () => { + imap.openBox(folder, true, (err) => { + if (err) { + imap.end(); + reject(err); + return; + } + + const fetch = imap.fetch([seqno], { bodies: '', struct: true }); + + fetch.on('message', (msg) => { + msg.once('attributes', (attrs: { struct?: ImapStruct[] }) => { + const parts = flattenImapStruct(attrs.struct ?? []); + + for (const part of parts) { + if (part.disposition?.type?.toLowerCase() !== 'attachment' || !part.partID) { + continue; + } + + const att: EmailAttachment = { + filename: + part.disposition.params?.['filename'] ?? part.params?.['name'] ?? 'attachment', + mimeType: `${part.type}/${part.subtype}`, + size: part.size ?? 0, + }; + + if (includeData) { + // We need to fetch the part body separately + const partFetch = imap.fetch([seqno], { bodies: [part.partID] }); + partFetch.on('message', (partMsg) => { + let data = Buffer.alloc(0); + partMsg.on('body', (stream) => { + stream.on('data', (chunk: Buffer) => { + data = Buffer.concat([data, chunk]); + }); + stream.once('end', () => { + att.data = data.toString('base64'); + }); + }); + }); + } + + attachments.push(att); + } + + msg.once('end', () => { + // small delay to allow part fetches to complete + setTimeout(() => { + imap.end(); + resolve({ messageId, attachments }); + }, 200); + }); + }); + }); + + fetch.once('error', (fetchErr) => { + imap.end(); + reject(fetchErr); + }); + }); + }); + + imap.once('error', reject); + imap.connect(); + }); + } + + private assertInitialized(): void { + if (this.authType === 'oauth2' && !this.gmailClient) { + throw new Error('Email adapter not initialized — call initialize() first'); + } + if (this.authType === 'smtp' && !this.transporter) { + throw new Error('Email adapter not initialized — call initialize() first'); + } + } +} + +// ── Utility helpers ───────────────────────────────────────────── + +/** Parse simple IMAP header block into a key-value map */ +function parseImapHeaders(raw: string): Record { + const result: Record = {}; + const lines = raw.split(/\r?\n/); + let currentKey = ''; + + for (const line of lines) { + if (/^\s/.test(line) && currentKey) { + // Continuation line + result[currentKey] = (result[currentKey] ?? '') + ' ' + line.trim(); + } else { + const colonIdx = line.indexOf(':'); + if (colonIdx > 0) { + currentKey = line.slice(0, colonIdx).toLowerCase().trim(); + result[currentKey] = line.slice(colonIdx + 1).trim(); + } + } + } + + return result; +} + +/** Minimal interface for IMAP struct parts */ +interface ImapStruct { + type?: string; + subtype?: string; + partID?: string; + size?: number; + disposition?: { + type?: string; + params?: Record; + }; + params?: Record; +} + +/** Flatten nested IMAP struct into a list of leaf parts */ +function flattenImapStruct(struct: ImapStruct[] | ImapStruct): ImapStruct[] { + const result: ImapStruct[] = []; + if (Array.isArray(struct)) { + for (const item of struct) { + result.push(...flattenImapStruct(item)); + } + } else if (struct && typeof struct === 'object') { + result.push(struct); + } + return result; +} diff --git a/src/integrations/adapters/google-calendar-adapter.ts b/src/integrations/adapters/google-calendar-adapter.ts new file mode 100644 index 00000000..4cfcbb8c --- /dev/null +++ b/src/integrations/adapters/google-calendar-adapter.ts @@ -0,0 +1,373 @@ +import { google, type calendar_v3 } from 'googleapis'; +import { createLogger } from '../../core/logger.js'; +import type { + BusinessIntegration, + HealthStatus, + IntegrationCapability, + IntegrationConfig, +} from '../../types/integration.js'; + +const logger = createLogger('google-calendar-adapter'); + +/** + * Google Calendar integration adapter. + * + * Capabilities: + * - create_event: Create a calendar event + * - list_events: List upcoming events + * - update_event: Update an existing event + * - delete_event: Delete an event + * - check_availability: Check free/busy status for a time range + * + * Credentials expected (from credential store): + * - Auth type "oauth2": + * clientId: OAuth2 client ID + * clientSecret: OAuth2 client secret + * refreshToken: OAuth2 refresh token (obtained via consent flow) + */ +export class GoogleCalendarAdapter implements BusinessIntegration { + readonly name = 'google-calendar'; + readonly type = 'calendar' as const; + + private calendar: calendar_v3.Calendar | null = null; + private calendarId = 'primary'; + + async initialize(config: IntegrationConfig): Promise { + const opts = config.options; + + const clientId = opts['clientId'] as string | undefined; + const clientSecret = opts['clientSecret'] as string | undefined; + const refreshToken = opts['refreshToken'] as string | undefined; + + if (!clientId || typeof clientId !== 'string') { + throw new Error('Google Calendar adapter requires clientId in config.options'); + } + if (!clientSecret || typeof clientSecret !== 'string') { + throw new Error('Google Calendar adapter requires clientSecret in config.options'); + } + if (!refreshToken || typeof refreshToken !== 'string') { + throw new Error('Google Calendar adapter requires refreshToken in config.options'); + } + + const auth = new google.auth.OAuth2(clientId, clientSecret); + auth.setCredentials({ refresh_token: refreshToken }); + this.calendar = google.calendar({ version: 'v3', auth }); + + if (opts['calendarId'] && typeof opts['calendarId'] === 'string') { + this.calendarId = opts['calendarId']; + } + + // Verify credentials work + try { + await this.calendar.calendars.get({ calendarId: this.calendarId }); + } catch (err) { + this.calendar = null; + throw new Error( + `Google Calendar initialization failed: ${err instanceof Error ? err.message : String(err)}`, + ); + } + + logger.info({ calendarId: this.calendarId }, 'Google Calendar adapter initialized'); + } + + async healthCheck(): Promise { + const checkedAt = new Date().toISOString(); + + if (!this.calendar) { + return { status: 'unhealthy', message: 'Not initialized', checkedAt, details: {} }; + } + + try { + const res = await this.calendar.calendars.get({ calendarId: this.calendarId }); + return { + status: 'healthy', + message: 'Google Calendar API reachable', + checkedAt, + details: { + calendarId: this.calendarId, + calendarSummary: res.data.summary ?? 'unknown', + timeZone: res.data.timeZone ?? 'unknown', + }, + }; + } catch (err) { + return { + status: 'unhealthy', + message: err instanceof Error ? err.message : String(err), + checkedAt, + details: {}, + }; + } + } + + // eslint-disable-next-line @typescript-eslint/require-await + async shutdown(): Promise { + this.calendar = null; + logger.info('Google Calendar adapter shut down'); + } + + describeCapabilities(): IntegrationCapability[] { + return [ + { + name: 'create_event', + description: + 'Create a calendar event. Params: summary (string), start (string, ISO 8601 datetime), end (string, ISO 8601 datetime), description (string, optional), attendees (string[], optional — list of email addresses), location (string, optional), timeZone (string, optional, e.g. "America/New_York").', + category: 'write', + requiresApproval: true, + }, + { + name: 'list_events', + description: + 'List upcoming calendar events. Params: timeMin (string, ISO 8601 datetime, default now), timeMax (string, ISO 8601 datetime, optional), maxResults (number, default 10), query (string, optional free-text search).', + category: 'read', + requiresApproval: false, + }, + { + name: 'update_event', + description: + 'Update an existing calendar event. Params: eventId (string), summary (string, optional), start (string, ISO 8601, optional), end (string, ISO 8601, optional), description (string, optional), attendees (string[], optional), location (string, optional).', + category: 'write', + requiresApproval: true, + }, + { + name: 'delete_event', + description: 'Delete a calendar event by ID. Params: eventId (string).', + category: 'write', + requiresApproval: true, + }, + { + name: 'check_availability', + description: + 'Check free/busy availability for one or more calendars. Params: timeMin (string, ISO 8601), timeMax (string, ISO 8601), calendarIds (string[], optional — defaults to the configured calendar).', + category: 'read', + requiresApproval: false, + }, + ]; + } + + async query(operation: string, params: Record): Promise { + if (!this.calendar) { + throw new Error('Google Calendar adapter not initialized — call initialize() first'); + } + + switch (operation) { + case 'list_events': + return await this.listEvents(params); + case 'check_availability': + return await this.checkAvailability(params); + default: + throw new Error(`Unknown query operation: ${operation}`); + } + } + + async execute(operation: string, params: Record): Promise { + if (!this.calendar) { + throw new Error('Google Calendar adapter not initialized — call initialize() first'); + } + + switch (operation) { + case 'create_event': + return await this.createEvent(params); + case 'update_event': + return await this.updateEvent(params); + case 'delete_event': + return await this.deleteEvent(params); + default: + throw new Error(`Unknown execute operation: ${operation}`); + } + } + + // ── Private helpers ──────────────────────────────────────────── + + private async createEvent( + params: Record, + ): Promise<{ eventId: string; summary: string; htmlLink: string | null }> { + const summary = params['summary'] as string; + const start = params['start'] as string; + const end = params['end'] as string; + + if (!summary || typeof summary !== 'string') { + throw new Error('summary is required'); + } + if (!start || typeof start !== 'string') { + throw new Error('start is required (ISO 8601 datetime)'); + } + if (!end || typeof end !== 'string') { + throw new Error('end is required (ISO 8601 datetime)'); + } + + const timeZone = (params['timeZone'] as string) ?? 'UTC'; + const description = params['description'] as string | undefined; + const location = params['location'] as string | undefined; + const attendeeEmails = params['attendees'] as string[] | undefined; + + const event: calendar_v3.Schema$Event = { + summary, + start: { dateTime: start, timeZone }, + end: { dateTime: end, timeZone }, + }; + + if (description) event.description = description; + if (location) event.location = location; + if (attendeeEmails && attendeeEmails.length > 0) { + event.attendees = attendeeEmails.map((email) => ({ email })); + } + + const res = await this.calendar!.events.insert({ + calendarId: this.calendarId, + requestBody: event, + }); + + logger.info({ eventId: res.data.id, summary }, 'Calendar event created'); + return { + eventId: res.data.id ?? '', + summary: res.data.summary ?? summary, + htmlLink: res.data.htmlLink ?? null, + }; + } + + private async listEvents( + params: Record, + ): Promise<{ events: Array>; nextPageToken: string | null }> { + const timeMin = (params['timeMin'] as string) ?? new Date().toISOString(); + const timeMax = params['timeMax'] as string | undefined; + const maxResults = Math.min((params['maxResults'] as number) ?? 10, 100); + const query = params['query'] as string | undefined; + + const res = await this.calendar!.events.list({ + calendarId: this.calendarId, + timeMin, + ...(timeMax ? { timeMax } : {}), + maxResults, + ...(query ? { q: query } : {}), + singleEvents: true, + orderBy: 'startTime', + }); + + const events = (res.data.items ?? []).map((e) => ({ + id: e.id, + summary: e.summary, + start: e.start?.dateTime ?? e.start?.date, + end: e.end?.dateTime ?? e.end?.date, + description: e.description, + location: e.location, + attendees: (e.attendees ?? []).map((a) => ({ + email: a.email, + responseStatus: a.responseStatus, + })), + htmlLink: e.htmlLink, + status: e.status, + })); + + return { + events, + nextPageToken: res.data.nextPageToken ?? null, + }; + } + + private async updateEvent( + params: Record, + ): Promise<{ eventId: string; summary: string; htmlLink: string | null }> { + const eventId = params['eventId'] as string; + if (!eventId || typeof eventId !== 'string') { + throw new Error('eventId is required'); + } + + // Fetch current event to patch only provided fields + const current = await this.calendar!.events.get({ + calendarId: this.calendarId, + eventId, + }); + + const patch: calendar_v3.Schema$Event = {}; + + if (params['summary'] && typeof params['summary'] === 'string') { + patch.summary = params['summary']; + } + if (params['description'] !== undefined) { + patch.description = params['description'] as string; + } + if (params['location'] !== undefined) { + patch.location = params['location'] as string; + } + + const timeZone = (params['timeZone'] as string) ?? current.data.start?.timeZone ?? 'UTC'; + if (params['start'] && typeof params['start'] === 'string') { + patch.start = { dateTime: params['start'], timeZone }; + } + if (params['end'] && typeof params['end'] === 'string') { + patch.end = { dateTime: params['end'], timeZone }; + } + + const attendeeEmails = params['attendees'] as string[] | undefined; + if (attendeeEmails) { + patch.attendees = attendeeEmails.map((email) => ({ email })); + } + + const res = await this.calendar!.events.patch({ + calendarId: this.calendarId, + eventId, + requestBody: patch, + }); + + logger.info({ eventId, summary: res.data.summary }, 'Calendar event updated'); + return { + eventId: res.data.id ?? eventId, + summary: res.data.summary ?? '', + htmlLink: res.data.htmlLink ?? null, + }; + } + + private async deleteEvent( + params: Record, + ): Promise<{ eventId: string; deleted: boolean }> { + const eventId = params['eventId'] as string; + if (!eventId || typeof eventId !== 'string') { + throw new Error('eventId is required'); + } + + await this.calendar!.events.delete({ calendarId: this.calendarId, eventId }); + + logger.info({ eventId }, 'Calendar event deleted'); + return { eventId, deleted: true }; + } + + private async checkAvailability( + params: Record, + ): Promise<{ timeMin: string; timeMax: string; busy: Array<{ start: string; end: string }> }> { + const timeMin = params['timeMin'] as string; + const timeMax = params['timeMax'] as string; + + if (!timeMin || typeof timeMin !== 'string') { + throw new Error('timeMin is required (ISO 8601 datetime)'); + } + if (!timeMax || typeof timeMax !== 'string') { + throw new Error('timeMax is required (ISO 8601 datetime)'); + } + + const calendarIds = (params['calendarIds'] as string[] | undefined) ?? [this.calendarId]; + + const res = await this.calendar!.freebusy.query({ + requestBody: { + timeMin, + timeMax, + items: calendarIds.map((id) => ({ id })), + }, + }); + + const allBusy: Array<{ start: string; end: string }> = []; + const calendars = res.data.calendars ?? {}; + for (const cal of Object.values(calendars)) { + for (const slot of cal.busy ?? []) { + if (slot.start && slot.end) { + allBusy.push({ start: slot.start, end: slot.end }); + } + } + } + + // Sort by start time + allBusy.sort((a, b) => a.start.localeCompare(b.start)); + + logger.info({ timeMin, timeMax, busySlots: allBusy.length }, 'Availability checked'); + return { timeMin, timeMax, busy: allBusy }; + } +} diff --git a/src/integrations/adapters/google-drive-adapter.ts b/src/integrations/adapters/google-drive-adapter.ts new file mode 100644 index 00000000..938621c6 --- /dev/null +++ b/src/integrations/adapters/google-drive-adapter.ts @@ -0,0 +1,373 @@ +import { google, type drive_v3 } from 'googleapis'; +import { createLogger } from '../../core/logger.js'; +import type { + BusinessIntegration, + HealthStatus, + IntegrationCapability, + IntegrationConfig, +} from '../../types/integration.js'; + +const logger = createLogger('google-drive-adapter'); + +/** + * Google Drive integration adapter. + * + * Capabilities: + * - upload_file: Upload a file to Google Drive + * - list_files: List files in a folder or root + * - download_file: Download a file by ID + * - create_folder: Create a folder + * - share_file: Share a file with a user or make it public + * + * Credentials expected (from credential store): + * - Auth type "apiKey": + * apiKey: Google API key (limited to public files) + * - Auth type "oauth2": + * clientId: OAuth2 client ID + * clientSecret: OAuth2 client secret + * refreshToken: OAuth2 refresh token (obtained via consent flow) + */ +export class GoogleDriveAdapter implements BusinessIntegration { + readonly name = 'google-drive'; + readonly type = 'storage' as const; + + private drive: drive_v3.Drive | null = null; + + async initialize(config: IntegrationConfig): Promise { + const opts = config.options; + const authType = (opts['authType'] as string) ?? 'oauth2'; + + if (authType === 'apiKey') { + const apiKey = opts['apiKey'] as string | undefined; + if (!apiKey || typeof apiKey !== 'string') { + throw new Error( + 'Google Drive adapter requires an apiKey in config.options when authType is "apiKey"', + ); + } + this.drive = google.drive({ version: 'v3', auth: apiKey }); + } else { + // OAuth2 + const clientId = opts['clientId'] as string | undefined; + const clientSecret = opts['clientSecret'] as string | undefined; + const refreshToken = opts['refreshToken'] as string | undefined; + + if (!clientId || typeof clientId !== 'string') { + throw new Error('Google Drive adapter requires clientId in config.options'); + } + if (!clientSecret || typeof clientSecret !== 'string') { + throw new Error('Google Drive adapter requires clientSecret in config.options'); + } + if (!refreshToken || typeof refreshToken !== 'string') { + throw new Error('Google Drive adapter requires refreshToken in config.options'); + } + + const auth = new google.auth.OAuth2(clientId, clientSecret); + auth.setCredentials({ refresh_token: refreshToken }); + this.drive = google.drive({ version: 'v3', auth }); + } + + // Verify credentials work + try { + await this.drive.about.get({ fields: 'user' }); + } catch (err) { + this.drive = null; + throw new Error( + `Google Drive initialization failed: ${err instanceof Error ? err.message : String(err)}`, + ); + } + + logger.info('Google Drive adapter initialized'); + } + + async healthCheck(): Promise { + const checkedAt = new Date().toISOString(); + + if (!this.drive) { + return { status: 'unhealthy', message: 'Not initialized', checkedAt, details: {} }; + } + + try { + const res = await this.drive.about.get({ fields: 'user,storageQuota' }); + const user = res.data.user; + const quota = res.data.storageQuota; + return { + status: 'healthy', + message: 'Google Drive API reachable', + checkedAt, + details: { + userEmail: user?.emailAddress ?? 'unknown', + ...(quota + ? { + storageUsed: quota.usage ?? '0', + storageLimit: quota.limit ?? 'unlimited', + } + : {}), + }, + }; + } catch (err) { + return { + status: 'unhealthy', + message: err instanceof Error ? err.message : String(err), + checkedAt, + details: {}, + }; + } + } + + // eslint-disable-next-line @typescript-eslint/require-await + async shutdown(): Promise { + this.drive = null; + logger.info('Google Drive adapter shut down'); + } + + describeCapabilities(): IntegrationCapability[] { + return [ + { + name: 'upload_file', + description: + 'Upload a file to Google Drive. Params: filePath (string, local path), name (string, optional filename), folderId (string, optional parent folder ID), mimeType (string, optional).', + category: 'write', + requiresApproval: true, + }, + { + name: 'list_files', + description: + 'List files in Google Drive. Params: folderId (string, optional — defaults to root), query (string, optional Drive search query), pageSize (number, default 20).', + category: 'read', + requiresApproval: false, + }, + { + name: 'download_file', + description: + 'Download a file from Google Drive by ID. Params: fileId (string), destPath (string, local destination path).', + category: 'read', + requiresApproval: false, + }, + { + name: 'create_folder', + description: + 'Create a folder in Google Drive. Params: name (string), parentId (string, optional parent folder ID).', + category: 'write', + requiresApproval: true, + }, + { + name: 'share_file', + description: + 'Share a file or folder. Params: fileId (string), email (string, optional — share with specific user), role (string, "reader"|"writer"|"commenter", default "reader"), type (string, "user"|"anyone", default "user"). If type is "anyone", creates a public link.', + category: 'write', + requiresApproval: true, + }, + ]; + } + + async query(operation: string, params: Record): Promise { + if (!this.drive) { + throw new Error('Google Drive adapter not initialized — call initialize() first'); + } + + switch (operation) { + case 'list_files': + return await this.listFiles(params); + case 'download_file': + return await this.downloadFile(params); + default: + throw new Error(`Unknown query operation: ${operation}`); + } + } + + async execute(operation: string, params: Record): Promise { + if (!this.drive) { + throw new Error('Google Drive adapter not initialized — call initialize() first'); + } + + switch (operation) { + case 'upload_file': + return await this.uploadFile(params); + case 'create_folder': + return await this.createFolder(params); + case 'share_file': + return await this.shareFile(params); + default: + throw new Error(`Unknown execute operation: ${operation}`); + } + } + + // ── Private helpers ──────────────────────────────────────────── + + private async uploadFile( + params: Record, + ): Promise<{ fileId: string; name: string; webViewLink: string | null }> { + const { createReadStream } = await import('node:fs'); + const path = await import('node:path'); + + const filePath = params['filePath'] as string; + if (!filePath || typeof filePath !== 'string') { + throw new Error('filePath is required'); + } + + const name = (params['name'] as string) ?? path.basename(filePath); + const folderId = params['folderId'] as string | undefined; + const mimeType = params['mimeType'] as string | undefined; + + const fileMetadata: drive_v3.Schema$File = { name }; + if (folderId) { + fileMetadata.parents = [folderId]; + } + + const res = await this.drive!.files.create({ + requestBody: fileMetadata, + media: { + mimeType: mimeType ?? 'application/octet-stream', + body: createReadStream(filePath), + }, + fields: 'id,name,webViewLink', + }); + + logger.info({ fileId: res.data.id, name: res.data.name }, 'File uploaded to Google Drive'); + return { + fileId: res.data.id ?? '', + name: res.data.name ?? name, + webViewLink: res.data.webViewLink ?? null, + }; + } + + private async listFiles( + params: Record, + ): Promise<{ files: Array>; nextPageToken: string | null }> { + const folderId = params['folderId'] as string | undefined; + const userQuery = params['query'] as string | undefined; + const pageSize = Math.min((params['pageSize'] as number) ?? 20, 100); + + const queryParts: string[] = []; + if (folderId) { + queryParts.push(`'${folderId}' in parents`); + } + if (userQuery) { + queryParts.push(userQuery); + } + queryParts.push('trashed = false'); + + const res = await this.drive!.files.list({ + q: queryParts.join(' and '), + pageSize, + fields: 'nextPageToken,files(id,name,mimeType,size,modifiedTime,webViewLink)', + }); + + return { + files: (res.data.files ?? []).map((f) => ({ + id: f.id, + name: f.name, + mimeType: f.mimeType, + size: f.size, + modifiedTime: f.modifiedTime, + webViewLink: f.webViewLink, + })), + nextPageToken: res.data.nextPageToken ?? null, + }; + } + + private async downloadFile( + params: Record, + ): Promise<{ fileId: string; destPath: string; size: number }> { + const { createWriteStream } = await import('node:fs'); + const { pipeline } = await import('node:stream/promises'); + const { stat } = await import('node:fs/promises'); + const { Readable } = await import('node:stream'); + + const fileId = params['fileId'] as string; + const destPath = params['destPath'] as string; + + if (!fileId || typeof fileId !== 'string') { + throw new Error('fileId is required'); + } + if (!destPath || typeof destPath !== 'string') { + throw new Error('destPath is required'); + } + + const res = await this.drive!.files.get({ fileId, alt: 'media' }, { responseType: 'stream' }); + + const stream = res.data as unknown as NodeJS.ReadableStream; + await pipeline(Readable.from(stream as AsyncIterable), createWriteStream(destPath)); + + const fileStat = await stat(destPath); + logger.info({ fileId, destPath, size: fileStat.size }, 'File downloaded from Google Drive'); + return { fileId, destPath, size: fileStat.size }; + } + + private async createFolder( + params: Record, + ): Promise<{ folderId: string; name: string; webViewLink: string | null }> { + const name = params['name'] as string; + if (!name || typeof name !== 'string') { + throw new Error('name is required'); + } + + const parentId = params['parentId'] as string | undefined; + + const fileMetadata: drive_v3.Schema$File = { + name, + mimeType: 'application/vnd.google-apps.folder', + }; + if (parentId) { + fileMetadata.parents = [parentId]; + } + + const res = await this.drive!.files.create({ + requestBody: fileMetadata, + fields: 'id,name,webViewLink', + }); + + logger.info({ folderId: res.data.id, name: res.data.name }, 'Folder created in Google Drive'); + return { + folderId: res.data.id ?? '', + name: res.data.name ?? name, + webViewLink: res.data.webViewLink ?? null, + }; + } + + private async shareFile( + params: Record, + ): Promise<{ permissionId: string; webViewLink: string | null }> { + const fileId = params['fileId'] as string; + if (!fileId || typeof fileId !== 'string') { + throw new Error('fileId is required'); + } + + const shareType = (params['type'] as string) ?? 'user'; + const role = (params['role'] as string) ?? 'reader'; + const email = params['email'] as string | undefined; + + if (shareType === 'user' && (!email || typeof email !== 'string')) { + throw new Error('email is required when type is "user"'); + } + + const permission: drive_v3.Schema$Permission = { + role, + type: shareType, + }; + if (shareType === 'user' && email) { + permission.emailAddress = email; + } + + const res = await this.drive!.permissions.create({ + fileId, + requestBody: permission, + fields: 'id', + }); + + // Fetch updated file to get the web view link + const fileRes = await this.drive!.files.get({ + fileId, + fields: 'webViewLink', + }); + + logger.info( + { fileId, permissionId: res.data.id, shareType, role }, + 'File shared on Google Drive', + ); + return { + permissionId: res.data.id ?? '', + webViewLink: fileRes.data.webViewLink ?? null, + }; + } +} diff --git a/src/integrations/adapters/google-sheets-adapter.ts b/src/integrations/adapters/google-sheets-adapter.ts new file mode 100644 index 00000000..81cb479e --- /dev/null +++ b/src/integrations/adapters/google-sheets-adapter.ts @@ -0,0 +1,341 @@ +import { google, type sheets_v4 } from 'googleapis'; +import { createLogger } from '../../core/logger.js'; +import type { + BusinessIntegration, + HealthStatus, + IntegrationCapability, + IntegrationConfig, +} from '../../types/integration.js'; + +const logger = createLogger('google-sheets-adapter'); + +/** + * Google Sheets integration adapter. + * + * Capabilities: + * - read_sheet: Read rows from a sheet (tab) in a spreadsheet + * - write_rows: Append or overwrite rows in a sheet + * - create_sheet: Add a new sheet (tab) to an existing spreadsheet + * - list_sheets: List all sheets (tabs) in a spreadsheet + * + * Credentials expected (from credential store): + * - Auth type "apiKey": + * apiKey: Google API key (limited to public spreadsheets) + * - Auth type "oauth2": + * clientId: OAuth2 client ID + * clientSecret: OAuth2 client secret + * refreshToken: OAuth2 refresh token (obtained via consent flow) + */ +export class GoogleSheetsAdapter implements BusinessIntegration { + readonly name = 'google-sheets'; + readonly type = 'storage' as const; + + private sheets: sheets_v4.Sheets | null = null; + + // eslint-disable-next-line @typescript-eslint/require-await + async initialize(config: IntegrationConfig): Promise { + const opts = config.options; + const authType = (opts['authType'] as string) ?? 'oauth2'; + + if (authType === 'apiKey') { + const apiKey = opts['apiKey'] as string | undefined; + if (!apiKey || typeof apiKey !== 'string') { + throw new Error( + 'Google Sheets adapter requires an apiKey in config.options when authType is "apiKey"', + ); + } + this.sheets = google.sheets({ version: 'v4', auth: apiKey }); + } else { + // OAuth2 + const clientId = opts['clientId'] as string | undefined; + const clientSecret = opts['clientSecret'] as string | undefined; + const refreshToken = opts['refreshToken'] as string | undefined; + + if (!clientId || typeof clientId !== 'string') { + throw new Error('Google Sheets adapter requires clientId in config.options'); + } + if (!clientSecret || typeof clientSecret !== 'string') { + throw new Error('Google Sheets adapter requires clientSecret in config.options'); + } + if (!refreshToken || typeof refreshToken !== 'string') { + throw new Error('Google Sheets adapter requires refreshToken in config.options'); + } + + const auth = new google.auth.OAuth2(clientId, clientSecret); + auth.setCredentials({ refresh_token: refreshToken }); + this.sheets = google.sheets({ version: 'v4', auth }); + } + + // Verify credentials work by fetching spreadsheet metadata for a minimal call + // We just ensure the auth is usable; actual validation happens at query time. + logger.info('Google Sheets adapter initialized'); + } + + // eslint-disable-next-line @typescript-eslint/require-await + async healthCheck(): Promise { + const checkedAt = new Date().toISOString(); + + if (!this.sheets) { + return { status: 'unhealthy', message: 'Not initialized', checkedAt, details: {} }; + } + + return { + status: 'healthy', + message: 'Google Sheets adapter ready', + checkedAt, + details: {}, + }; + } + + // eslint-disable-next-line @typescript-eslint/require-await + async shutdown(): Promise { + this.sheets = null; + logger.info('Google Sheets adapter shut down'); + } + + describeCapabilities(): IntegrationCapability[] { + return [ + { + name: 'read_sheet', + description: + 'Read rows from a Google Sheets tab. Params: spreadsheetId (string), sheetName (string, optional — defaults to first sheet), range (string, optional A1 notation e.g. "A1:Z100"), includeHeaders (boolean, default true).', + category: 'read', + requiresApproval: false, + }, + { + name: 'write_rows', + description: + 'Write rows to a Google Sheets tab. Params: spreadsheetId (string), sheetName (string), rows (array of arrays — each inner array is one row of cell values), startCell (string, optional A1 cell to start writing, default "A1"), valueInputOption (string, "RAW"|"USER_ENTERED", default "USER_ENTERED").', + category: 'write', + requiresApproval: true, + }, + { + name: 'create_sheet', + description: + 'Add a new sheet (tab) to an existing Google Spreadsheet. Params: spreadsheetId (string), title (string — name for the new sheet), index (number, optional — position among tabs, 0-based).', + category: 'write', + requiresApproval: true, + }, + { + name: 'list_sheets', + description: + 'List all sheets (tabs) in a Google Spreadsheet. Params: spreadsheetId (string). Returns array of { sheetId, title, index, rowCount, columnCount }.', + category: 'read', + requiresApproval: false, + }, + ]; + } + + async query(operation: string, params: Record): Promise { + if (!this.sheets) { + throw new Error('Google Sheets adapter not initialized — call initialize() first'); + } + + switch (operation) { + case 'read_sheet': + return await this.readSheet(params); + case 'list_sheets': + return await this.listSheets(params); + default: + throw new Error(`Unknown query operation: ${operation}`); + } + } + + async execute(operation: string, params: Record): Promise { + if (!this.sheets) { + throw new Error('Google Sheets adapter not initialized — call initialize() first'); + } + + switch (operation) { + case 'write_rows': + return await this.writeRows(params); + case 'create_sheet': + return await this.createSheet(params); + default: + throw new Error(`Unknown execute operation: ${operation}`); + } + } + + // ── Private helpers ──────────────────────────────────────────── + + private async readSheet(params: Record): Promise<{ + headers: string[] | null; + rows: unknown[][]; + totalRows: number; + sheetName: string; + range: string; + }> { + const spreadsheetId = params['spreadsheetId'] as string; + if (!spreadsheetId || typeof spreadsheetId !== 'string') { + throw new Error('spreadsheetId is required'); + } + + const sheetName = params['sheetName'] as string | undefined; + const rangeParam = params['range'] as string | undefined; + const includeHeaders = params['includeHeaders'] !== false; + + // Build range notation: "SheetName!A1:Z" or just "A1:Z" for default sheet + const rangeNotation = sheetName + ? `${sheetName}${rangeParam ? `!${rangeParam}` : ''}` + : (rangeParam ?? ''); + + const res = await this.sheets!.spreadsheets.values.get({ + spreadsheetId, + range: rangeNotation || undefined, + majorDimension: 'ROWS', + }); + + const values = res.data.values ?? []; + const actualRange = res.data.range ?? rangeNotation; + const actualSheet = sheetName ?? actualRange.split('!')[0] ?? 'Sheet1'; + + if (values.length === 0) { + return { headers: null, rows: [], totalRows: 0, sheetName: actualSheet, range: actualRange }; + } + + if (includeHeaders) { + const headers = (values[0] as string[]).map((h) => String(h ?? '')); + const rows = values.slice(1); + logger.info( + { spreadsheetId, sheetName: actualSheet, rowCount: rows.length }, + 'Sheet data read', + ); + return { headers, rows, totalRows: rows.length, sheetName: actualSheet, range: actualRange }; + } + + logger.info( + { spreadsheetId, sheetName: actualSheet, rowCount: values.length }, + 'Sheet data read', + ); + return { + headers: null, + rows: values, + totalRows: values.length, + sheetName: actualSheet, + range: actualRange, + }; + } + + private async writeRows(params: Record): Promise<{ + spreadsheetId: string; + updatedRange: string; + updatedRows: number; + updatedCells: number; + }> { + const spreadsheetId = params['spreadsheetId'] as string; + if (!spreadsheetId || typeof spreadsheetId !== 'string') { + throw new Error('spreadsheetId is required'); + } + + const sheetName = params['sheetName'] as string | undefined; + const rows = params['rows'] as unknown[][]; + if (!Array.isArray(rows) || rows.length === 0) { + throw new Error('rows must be a non-empty array'); + } + + const startCell = (params['startCell'] as string) ?? 'A1'; + const valueInputOption = (params['valueInputOption'] as string) ?? 'USER_ENTERED'; + + const range = sheetName ? `${sheetName}!${startCell}` : startCell; + + const res = await this.sheets!.spreadsheets.values.update({ + spreadsheetId, + range, + valueInputOption, + requestBody: { values: rows }, + }); + + logger.info( + { + spreadsheetId, + updatedRange: res.data.updatedRange, + updatedRows: res.data.updatedRows, + }, + 'Rows written to sheet', + ); + + return { + spreadsheetId, + updatedRange: res.data.updatedRange ?? range, + updatedRows: res.data.updatedRows ?? rows.length, + updatedCells: res.data.updatedCells ?? 0, + }; + } + + private async createSheet(params: Record): Promise<{ + spreadsheetId: string; + sheetId: number; + title: string; + index: number; + }> { + const spreadsheetId = params['spreadsheetId'] as string; + if (!spreadsheetId || typeof spreadsheetId !== 'string') { + throw new Error('spreadsheetId is required'); + } + + const title = params['title'] as string; + if (!title || typeof title !== 'string') { + throw new Error('title is required'); + } + + const index = params['index'] as number | undefined; + + const addSheetRequest: sheets_v4.Schema$AddSheetRequest = { + properties: { title, ...(typeof index === 'number' ? { index } : {}) }, + }; + + const res = await this.sheets!.spreadsheets.batchUpdate({ + spreadsheetId, + requestBody: { requests: [{ addSheet: addSheetRequest }] }, + }); + + const addedProps = res.data.replies?.[0]?.addSheet?.properties; + logger.info( + { spreadsheetId, title, sheetId: addedProps?.sheetId }, + 'New sheet created in spreadsheet', + ); + + return { + spreadsheetId, + sheetId: addedProps?.sheetId ?? 0, + title: addedProps?.title ?? title, + index: addedProps?.index ?? 0, + }; + } + + private async listSheets(params: Record): Promise<{ + spreadsheetId: string; + sheets: Array<{ + sheetId: number; + title: string; + index: number; + rowCount: number; + columnCount: number; + }>; + }> { + const spreadsheetId = params['spreadsheetId'] as string; + if (!spreadsheetId || typeof spreadsheetId !== 'string') { + throw new Error('spreadsheetId is required'); + } + + const res = await this.sheets!.spreadsheets.get({ + spreadsheetId, + fields: 'sheets.properties', + }); + + const sheets = (res.data.sheets ?? []).map((s) => { + const p = s.properties ?? {}; + const grid = p.gridProperties ?? {}; + return { + sheetId: p.sheetId ?? 0, + title: p.title ?? '', + index: p.index ?? 0, + rowCount: grid.rowCount ?? 0, + columnCount: grid.columnCount ?? 0, + }; + }); + + logger.info({ spreadsheetId, sheetCount: sheets.length }, 'Sheets listed'); + return { spreadsheetId, sheets }; + } +} diff --git a/src/integrations/adapters/openapi-adapter.ts b/src/integrations/adapters/openapi-adapter.ts new file mode 100644 index 00000000..06b0e8e7 --- /dev/null +++ b/src/integrations/adapters/openapi-adapter.ts @@ -0,0 +1,870 @@ +import SwaggerParser from '@apidevtools/swagger-parser'; +import type { OpenAPI, OpenAPIV3 } from 'openapi-types'; +import { z, type ZodTypeAny } from 'zod/v3'; +import { createLogger } from '../../core/logger.js'; +import type { + BusinessIntegration, + HealthStatus, + HealthStatusState, + IntegrationCapability, + IntegrationConfig, + RoleConfig, +} from '../../types/integration.js'; +import { postmanToOpenAPI, type PostmanCollection } from '../parsers/postman-parser.js'; +import { curlsToOpenAPI, splitCurlCommands } from '../parsers/curl-parser.js'; +import type Database from 'better-sqlite3'; + +const logger = createLogger('openapi-adapter'); + +// ── Input format detection ──────────────────────────────────────────────────── + +/** Supported input formats for /connect api . */ +export type InputFormat = 'openapi' | 'postman' | 'curl' | 'url' | 'unknown'; + +/** + * Detect the format of a raw user-provided API input string. + * + * Rules (evaluated in order): + * 1. `curl` — input starts with `curl ` (trimmed) or first non-empty line does + * 2. `url` — input is a single HTTP/HTTPS URL (no spaces, no newlines after trim) + * 3. `postman` — JSON with a top-level `info.schema` containing "postman", + * or a top-level `collection.info.schema` field + * 4. `openapi` — JSON/YAML with a top-level `openapi` or `swagger` field + * 5. `unknown` — everything else + * + * @param input Raw string from the user (cURL command, JSON, YAML, URL, etc.) + * @returns Detected format identifier + */ +export function detectInputFormat(input: string): InputFormat { + const trimmed = input.trim(); + if (!trimmed) return 'unknown'; + + // 1. cURL detection — first non-empty line starts with "curl " + const firstLine = trimmed.split('\n').find((l) => l.trim().length > 0) ?? ''; + if (/^curl\s+/i.test(firstLine.trim())) { + return 'curl'; + } + + // 2. URL detection — single-token HTTP/HTTPS string + if (/^https?:\/\/\S+$/i.test(trimmed)) { + return 'url'; + } + + // 3. JSON-based detection (Postman or OpenAPI) + if (trimmed.startsWith('{') || trimmed.startsWith('[')) { + try { + const parsed: unknown = JSON.parse(trimmed); + if (parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)) { + const obj = parsed as Record; + + // Postman Collection v2.x: { info: { schema: "...postman..." }, item: [...] } + const info = obj['info']; + if ( + info !== null && + typeof info === 'object' && + !Array.isArray(info) && + typeof (info as Record)['schema'] === 'string' && + String((info as Record)['schema']) + .toLowerCase() + .includes('postman') + ) { + return 'postman'; + } + + // Postman wrapped export: { collection: { info: { schema: "...postman..." } } } + const collection = obj['collection']; + if (collection !== null && typeof collection === 'object' && !Array.isArray(collection)) { + const colInfo = (collection as Record)['info']; + if ( + colInfo !== null && + typeof colInfo === 'object' && + !Array.isArray(colInfo) && + typeof (colInfo as Record)['schema'] === 'string' && + String((colInfo as Record)['schema']) + .toLowerCase() + .includes('postman') + ) { + return 'postman'; + } + } + + // OpenAPI 3.x: { openapi: "3.x.x", ... } + if (typeof obj['openapi'] === 'string') { + return 'openapi'; + } + + // Swagger 2.x: { swagger: "2.0", ... } + if (typeof obj['swagger'] === 'string') { + return 'openapi'; + } + } + } catch { + // Not valid JSON — fall through to YAML check + } + } + + // 4. YAML-based OpenAPI detection (openapi: or swagger: at the start of a line) + if (/^openapi\s*:/m.test(trimmed) || /^swagger\s*:/m.test(trimmed)) { + return 'openapi'; + } + + return 'unknown'; +} + +/** + * Parse a raw API input string into an OpenAPI document. + * + * Routes based on `detectInputFormat()`: + * - `openapi` — parsed directly via swagger-parser + * - `url` — fetched and parsed as OpenAPI spec + * - `postman` — requires postman-parser (OB-1446, not yet implemented) + * - `curl` — requires curl-parser (OB-1447, not yet implemented) + * - `unknown` — throws an error + * + * @param input Raw user input + * @returns Validated OpenAPI document + * @throws Error if format is unsupported or parsing fails + */ +export async function parseInputToOpenAPI(input: string): Promise { + const format = detectInputFormat(input); + + switch (format) { + case 'openapi': { + const trimmed = input.trim(); + if (trimmed.startsWith('{')) { + const parsed: unknown = JSON.parse(trimmed); + return await SwaggerParser.validate(parsed as OpenAPI.Document); + } + // YAML — swagger-parser accepts YAML strings via validate(string) + return await SwaggerParser.validate(trimmed); + } + + case 'url': { + return await SwaggerParser.validate(input.trim()); + } + + case 'postman': { + const parsed: unknown = JSON.parse(input.trim()); + const collection = parsed as PostmanCollection; + const openApiDoc = postmanToOpenAPI(collection); + return await SwaggerParser.validate(openApiDoc as OpenAPI.Document); + } + + case 'curl': { + const commands = splitCurlCommands(input); + if (commands.length === 0) { + throw new Error('No valid cURL commands found in the input.'); + } + const curlSpec = curlsToOpenAPI(commands); + return await SwaggerParser.validate(curlSpec as OpenAPI.Document); + } + + default: + throw new Error( + 'Unrecognised API input format. Provide a Swagger/OpenAPI JSON or YAML, ' + + 'a Postman collection JSON, one or more cURL commands, or a URL to an OpenAPI spec.', + ); + } +} + +// ── Role-based capability tagging ──────────────────────────────────────────── + +/** + * Tag API capabilities with roles based on path prefix matching and persist + * the tags in the `integration_capabilities` SQLite table. + * + * @param db Open SQLite database with the integration_capabilities table + * @param integration Name of the integration (e.g. "my-api") + * @param capabilities Array of capabilities produced by the adapter + * @param roleConfig Map of role name → path prefix, e.g. `{ seller: '/supplier', driver: '/delivery' }` + */ +export function tagCapabilitiesByRole( + db: Database.Database, + integration: string, + capabilities: IntegrationCapability[], + roleConfig: RoleConfig, +): void { + const now = new Date().toISOString(); + const upsert = db.prepare(` + INSERT OR IGNORE INTO integration_capabilities + (integration_name, capability_name, role, tagged_at) + VALUES (?, ?, ?, ?) + `); + + db.transaction((): void => { + for (const cap of capabilities) { + // Each capability's path is embedded in the name — but we need the raw + // path. Capabilities from OpenAPIAdapter are ResolvedCapability instances + // that carry a `path` property. For plain IntegrationCapability objects + // we fall back to matching against the capability name. + const pathOrName = + 'path' in cap && typeof (cap as Record)['path'] === 'string' + ? ((cap as Record)['path'] as string) + : cap.name; + + for (const [role, prefix] of Object.entries(roleConfig)) { + if (pathOrName.startsWith(prefix)) { + upsert.run(integration, cap.name, role, now); + } + } + } + })(); + + logger.info( + { integration, roles: Object.keys(roleConfig), capabilities: capabilities.length }, + 'Role tags applied to capabilities', + ); +} + +/** + * Retrieve the set of roles assigned to a specific capability. + * + * @param db Open SQLite database + * @param integration Integration name + * @param capability Capability name + * @returns Array of role strings assigned to this capability + */ +export function getCapabilityRoles( + db: Database.Database, + integration: string, + capability: string, +): string[] { + const rows = db + .prepare( + `SELECT role FROM integration_capabilities + WHERE integration_name = ? AND capability_name = ?`, + ) + .all(integration, capability) as { role: string }[]; + return rows.map((r) => r.role); +} + +/** Resolved capability generated from an OpenAPI path+method. */ +interface ResolvedCapability extends IntegrationCapability { + /** HTTP method (GET, POST, PUT, DELETE, PATCH) */ + method: string; + /** Full path template, e.g. "/pets/{petId}" */ + path: string; + /** Zod schema for validating call params */ + paramSchema: ZodTypeAny; +} + +/** + * Universal REST API connector that auto-generates capabilities from an + * OpenAPI / Swagger specification. + * + * On `initialize()`: + * - Parses and validates the spec via swagger-parser + * - Auto-generates capabilities from paths + methods + * - Builds Zod parameter schemas from OpenAPI parameter definitions + * + * `query()` handles read (GET) operations. + * `execute()` handles write (POST/PUT/DELETE/PATCH) operations. + * Both make HTTP calls with proper auth headers. + * + * Credentials expected (from credential store): + * - specUrl OR specJson: URL to OpenAPI spec or inline JSON string + * - baseUrl (optional): Override the server URL from the spec + * - authType (optional): "bearer" | "apiKey" | "basic" | "none" + * - authToken (optional): Bearer token, API key value, or "user:pass" for basic + * - authHeader (optional): Custom header name for apiKey auth (default "Authorization") + */ +export class OpenAPIAdapter implements BusinessIntegration { + readonly name: string; + readonly type = 'api' as const; + + private capabilities: ResolvedCapability[] = []; + private baseUrl = ''; + private authHeaders: Record = {}; + private initialized = false; + private db: Database.Database | null = null; + private healthEndpoint: string | null = null; + private lastHealthStatus: HealthStatusState = 'unknown'; + private healthAlertHandler: ((integration: string, status: HealthStatus) => void) | null = null; + + constructor(name = 'openapi') { + this.name = name; + } + + /** + * Optionally wire in a SQLite database so that role tags can be stored and + * retrieved from the `integration_capabilities` table, and health history + * is persisted to the `integration_health_log` table. + */ + setDatabase(db: Database.Database): void { + this.db = db; + } + + /** + * Set a callback invoked when this adapter transitions to unhealthy state. + * Called at most once per transition (healthy/unknown → unhealthy). + */ + setHealthAlertHandler(handler: (integration: string, status: HealthStatus) => void): void { + this.healthAlertHandler = handler; + } + + async initialize(config: IntegrationConfig): Promise { + const opts = config.options; + + // Parse the OpenAPI spec + const specUrl = opts['specUrl'] as string | undefined; + const specJson = opts['specJson'] as string | undefined; + + if (!specUrl && !specJson) { + throw new Error('OpenAPI adapter requires specUrl or specJson in config.options'); + } + + let api: OpenAPI.Document; + try { + if (specUrl) { + api = await SwaggerParser.validate(specUrl); + } else { + const parsed: unknown = JSON.parse(specJson!); + api = await SwaggerParser.validate(parsed as OpenAPI.Document); + } + } catch (err) { + throw new Error( + `OpenAPI spec validation failed: ${err instanceof Error ? err.message : String(err)}`, + ); + } + + // Resolve base URL + const specBaseUrl = this.extractBaseUrl(api); + this.baseUrl = (opts['baseUrl'] as string) ?? specBaseUrl; + if (!this.baseUrl) { + throw new Error('No server URL found in spec and no baseUrl provided'); + } + // Strip trailing slash + this.baseUrl = this.baseUrl.replace(/\/+$/, ''); + + // Build auth headers + this.authHeaders = this.buildAuthHeaders(opts); + + // Generate capabilities from paths + this.capabilities = this.generateCapabilities(api); + + // Detect health endpoint (explicit config > well-known spec path > first GET path) + this.healthEndpoint = this.detectHealthEndpoint(opts, api); + + this.initialized = true; + logger.info( + { + name: this.name, + baseUrl: this.baseUrl, + capabilities: this.capabilities.length, + healthEndpoint: this.healthEndpoint, + }, + 'OpenAPI adapter initialized', + ); + } + + async healthCheck(): Promise { + const checkedAt = new Date().toISOString(); + + if (!this.initialized) { + const result: HealthStatus = { + status: 'unhealthy', + message: 'Not initialized', + checkedAt, + details: {}, + }; + this.storeHealthLog(result, null, null, null); + return result; + } + + if (!this.healthEndpoint) { + // No probeable endpoint — report basic availability + const result: HealthStatus = { + status: 'healthy', + message: `OpenAPI adapter ready (${this.capabilities.length} capabilities)`, + checkedAt, + details: { baseUrl: this.baseUrl, capabilityCount: this.capabilities.length }, + }; + this.storeHealthLog(result, null, null, null); + this.lastHealthStatus = 'healthy'; + return result; + } + + // Probe the health endpoint + const start = Date.now(); + let httpStatus: number | null = null; + let resultStatus: HealthStatusState = 'unhealthy'; + let message = ''; + + try { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), 10_000); + try { + const response = await fetch(this.healthEndpoint, { + method: 'GET', + headers: { ...this.authHeaders }, + signal: controller.signal, + }); + httpStatus = response.status; + if (response.ok) { + resultStatus = 'healthy'; + message = `Health check passed (HTTP ${httpStatus})`; + } else if (httpStatus >= 500) { + resultStatus = 'unhealthy'; + message = `Health check failed (HTTP ${httpStatus})`; + } else { + resultStatus = 'degraded'; + message = `Health check returned HTTP ${httpStatus}`; + } + } finally { + clearTimeout(timeoutId); + } + } catch (err) { + resultStatus = 'unhealthy'; + message = `Health check failed: ${err instanceof Error ? err.message : String(err)}`; + } + + const latencyMs = Date.now() - start; + const result: HealthStatus = { + status: resultStatus, + message, + checkedAt, + details: { + baseUrl: this.baseUrl, + healthEndpoint: this.healthEndpoint, + httpStatus, + latencyMs, + capabilityCount: this.capabilities.length, + }, + }; + + this.storeHealthLog(result, this.healthEndpoint, httpStatus, latencyMs); + + // Alert on healthy/unknown → unhealthy transition + if (resultStatus === 'unhealthy' && this.lastHealthStatus !== 'unhealthy') { + this.healthAlertHandler?.(this.name, result); + } + this.lastHealthStatus = resultStatus; + + return result; + } + + /** + * Return the last known health check result for this adapter from the DB. + * Returns null if no health log entries exist or no DB is wired in. + */ + getLastHealthLog(): { + status: string; + message: string | null; + checkedAt: string; + latencyMs: number | null; + } | null { + if (!this.db) return null; + try { + return this.db + .prepare( + `SELECT status, message, checked_at AS checkedAt, latency_ms AS latencyMs + FROM integration_health_log + WHERE integration_name = ? + ORDER BY checked_at DESC LIMIT 1`, + ) + .get(this.name) as { + status: string; + message: string | null; + checkedAt: string; + latencyMs: number | null; + } | null; + } catch { + return null; + } + } + + // eslint-disable-next-line @typescript-eslint/require-await + async shutdown(): Promise { + this.capabilities = []; + this.baseUrl = ''; + this.authHeaders = {}; + this.initialized = false; + logger.info({ name: this.name }, 'OpenAPI adapter shut down'); + } + + /** + * Describe the capabilities exposed by this adapter. + * + * @param role Optional role filter. When provided, only capabilities that + * have been tagged for that role (via `tagCapabilitiesByRole()`) + * are returned. When omitted, all capabilities are returned. + */ + describeCapabilities(role?: string): IntegrationCapability[] { + const all = this.capabilities.map(({ method: _m, path: _p, paramSchema: _s, ...cap }) => cap); + + if (!role || !this.db) { + return all; + } + + const rows = this.db + .prepare( + `SELECT capability_name FROM integration_capabilities + WHERE integration_name = ? AND role = ?`, + ) + .all(this.name, role) as { capability_name: string }[]; + + const allowed = new Set(rows.map((r) => r.capability_name)); + return all.filter((c) => allowed.has(c.name)); + } + + /** + * Tag capabilities with roles based on path prefix patterns and persist + * the mappings to the SQLite database. + * + * Requires a database to be wired in via `setDatabase()`. + * + * @param roleConfig Map of role name → path prefix + * @throws Error if no database has been wired in + */ + tagCapabilitiesByRole(roleConfig: RoleConfig): void { + if (!this.db) { + throw new Error('OpenAPIAdapter: call setDatabase() before tagCapabilitiesByRole()'); + } + tagCapabilitiesByRole(this.db, this.name, this.capabilities, roleConfig); + } + + async query(operation: string, params: Record): Promise { + const cap = this.findCapability(operation, 'read'); + return await this.makeRequest(cap, params); + } + + async execute(operation: string, params: Record): Promise { + const cap = this.findCapability(operation, 'write'); + return await this.makeRequest(cap, params); + } + + // ── Internal helpers ───────────────────────────────────────────── + + /** Detect the health-check endpoint from spec paths or explicit config. */ + private detectHealthEndpoint( + opts: Record, + api: OpenAPI.Document, + ): string | null { + // 1. Explicit override in config options + const configured = opts['healthEndpoint'] as string | undefined; + if (configured) { + const path = configured.startsWith('/') ? configured : `/${configured}`; + return `${this.baseUrl}${path}`; + } + + // 2. Well-known health endpoint paths in the spec + const v3 = api as OpenAPIV3.Document; + const paths = v3.paths ?? {}; + const wellKnown = ['/health', '/healthz', '/status', '/ping', '/ready', '/liveness']; + for (const hp of wellKnown) { + if (paths[hp]?.get) return `${this.baseUrl}${hp}`; + } + + // 3. Fall back to first GET path (avoid parameterised paths — use literal ones first) + let firstGet: string | null = null; + for (const [pathTemplate, pathItem] of Object.entries(paths)) { + if (!pathItem?.get) continue; + if (!pathTemplate.includes('{')) { + return `${this.baseUrl}${pathTemplate}`; + } + firstGet ??= pathTemplate; + } + + // Last resort: use first GET path even if parameterised (replace params with "probe") + if (firstGet) { + const resolved = firstGet.replace(/\{[^}]+\}/g, 'probe'); + return `${this.baseUrl}${resolved}`; + } + + return null; + } + + /** Write a health check result to the `integration_health_log` table. */ + private storeHealthLog( + status: HealthStatus, + endpointUrl: string | null, + httpStatus: number | null, + latencyMs: number | null, + ): void { + if (!this.db) return; + try { + this.db + .prepare( + `INSERT INTO integration_health_log + (integration_name, status, message, endpoint_url, http_status, latency_ms, checked_at) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + ) + .run( + this.name, + status.status, + status.message ?? null, + endpointUrl, + httpStatus, + latencyMs, + status.checkedAt, + ); + } catch (err) { + logger.warn({ err }, 'Failed to store health log'); + } + } + + private findCapability( + operation: string, + expectedCategory: 'read' | 'write', + ): ResolvedCapability { + if (!this.initialized) { + throw new Error('OpenAPI adapter not initialized — call initialize() first'); + } + + const cap = this.capabilities.find((c) => c.name === operation); + if (!cap) { + throw new Error(`Unknown operation: ${operation}`); + } + + if (cap.category !== expectedCategory) { + const correctMethod = expectedCategory === 'read' ? 'query()' : 'execute()'; + throw new Error( + `Operation "${operation}" is a ${cap.category} operation — use ${correctMethod}`, + ); + } + + return cap; + } + + private async makeRequest( + cap: ResolvedCapability, + params: Record, + ): Promise { + // Validate params + const validated: unknown = cap.paramSchema.parse(params); + const validatedParams = ( + validated !== null && typeof validated === 'object' ? validated : {} + ) as Record; + + // Build URL — replace path parameters + let url = `${this.baseUrl}${cap.path}`; + const queryParams: Record = {}; + let body: unknown = undefined; + + // Separate path params, query params, and body + for (const [key, value] of Object.entries(validatedParams)) { + const placeholder = `{${key}}`; + if (url.includes(placeholder)) { + url = url.replace(placeholder, encodeURIComponent(`${value as string | number | boolean}`)); + } else if (cap.method === 'GET' || cap.method === 'DELETE') { + if (value !== undefined && value !== null) { + queryParams[key] = `${value as string | number | boolean}`; + } + } + } + + // For non-GET methods, put remaining non-path params in body + if (cap.method !== 'GET' && cap.method !== 'DELETE') { + const bodyParams: Record = {}; + for (const [key, value] of Object.entries(validatedParams)) { + if (!url.includes(encodeURIComponent(String(value)))) { + bodyParams[key] = value; + } + } + if (Object.keys(bodyParams).length > 0) { + body = bodyParams; + } + } + + // Append query string + const qs = new URLSearchParams(queryParams).toString(); + if (qs) { + url += `?${qs}`; + } + + // Make the HTTP request + const fetchOptions: RequestInit = { + method: cap.method, + headers: { + ...this.authHeaders, + ...(body !== undefined ? { 'Content-Type': 'application/json' } : {}), + }, + }; + if (body !== undefined) { + fetchOptions.body = JSON.stringify(body); + } + + logger.debug({ method: cap.method, url }, 'OpenAPI request'); + + const response = await fetch(url, fetchOptions); + + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`HTTP ${response.status} ${response.statusText}: ${text}`); + } + + const contentType = response.headers.get('content-type') ?? ''; + if (contentType.includes('application/json')) { + return await response.json(); + } + return await response.text(); + } + + private extractBaseUrl(api: OpenAPI.Document): string { + // OpenAPI 3.x + const v3 = api as OpenAPIV3.Document; + if (v3.servers && v3.servers.length > 0 && v3.servers[0]) { + return v3.servers[0].url; + } + + // Swagger 2.x + const v2 = api as { host?: string; basePath?: string; schemes?: string[] }; + if (v2.host) { + const scheme = v2.schemes?.[0] ?? 'https'; + const basePath = v2.basePath ?? ''; + return `${scheme}://${v2.host}${basePath}`; + } + + return ''; + } + + private buildAuthHeaders(opts: Record): Record { + const authType = (opts['authType'] as string) ?? 'none'; + const authToken = opts['authToken'] as string | undefined; + + if (authType === 'none' || !authToken) { + return {}; + } + + switch (authType) { + case 'bearer': + return { Authorization: `Bearer ${authToken}` }; + case 'apiKey': { + const headerName = (opts['authHeader'] as string) ?? 'Authorization'; + return { [headerName]: authToken }; + } + case 'basic': { + const encoded = Buffer.from(authToken).toString('base64'); + return { Authorization: `Basic ${encoded}` }; + } + default: + logger.warn({ authType }, 'Unknown auth type — no auth headers set'); + return {}; + } + } + + private generateCapabilities(api: OpenAPI.Document): ResolvedCapability[] { + const v3 = api as OpenAPIV3.Document; + const paths = v3.paths; + if (!paths) return []; + + const caps: ResolvedCapability[] = []; + + for (const [pathTemplate, pathItem] of Object.entries(paths)) { + if (!pathItem) continue; + + const methods: Array<[string, OpenAPIV3.OperationObject | undefined]> = [ + ['GET', pathItem.get], + ['POST', pathItem.post], + ['PUT', pathItem.put], + ['DELETE', pathItem.delete], + ['PATCH', pathItem.patch], + ]; + + for (const [method, operation] of methods) { + if (!operation) continue; + + const opName = this.generateOperationName(method, pathTemplate, operation); + const isRead = method === 'GET'; + const paramSchema = this.buildParamSchema(operation, pathItem); + + caps.push({ + name: opName, + description: operation.summary ?? operation.description ?? `${method} ${pathTemplate}`, + category: isRead ? 'read' : 'write', + requiresApproval: !isRead, + method, + path: pathTemplate, + paramSchema, + }); + } + } + + return caps; + } + + private generateOperationName( + method: string, + pathTemplate: string, + operation: OpenAPIV3.OperationObject, + ): string { + // Use operationId if available + if (operation.operationId) { + return operation.operationId; + } + + // Generate from method + path: GET /pets/{petId} → get_pets_by_petId + const segments = pathTemplate + .split('/') + .filter(Boolean) + .map((s) => { + if (s.startsWith('{') && s.endsWith('}')) { + return `by_${s.slice(1, -1)}`; + } + return s.replace(/[^a-zA-Z0-9]/g, '_'); + }); + + return `${method.toLowerCase()}_${segments.join('_')}`; + } + + private buildParamSchema( + operation: OpenAPIV3.OperationObject, + pathItem: OpenAPIV3.PathItemObject, + ): ZodTypeAny { + const shape: Record = {}; + + // Merge path-level and operation-level parameters + const allParams = [ + ...((pathItem.parameters ?? []) as OpenAPIV3.ParameterObject[]), + ...((operation.parameters ?? []) as OpenAPIV3.ParameterObject[]), + ]; + + for (const param of allParams) { + if (!param.name) continue; + const zodType = this.openApiSchemaToZod(param.schema as OpenAPIV3.SchemaObject | undefined); + shape[param.name] = param.required ? zodType : zodType.optional(); + } + + // Add request body properties (for POST/PUT/PATCH) + if (operation.requestBody) { + const reqBody = operation.requestBody as OpenAPIV3.RequestBodyObject; + const jsonContent = reqBody.content?.['application/json']; + if (jsonContent?.schema) { + const bodySchema = jsonContent.schema as OpenAPIV3.SchemaObject; + if (bodySchema.properties) { + const requiredFields = new Set(bodySchema.required ?? []); + for (const [propName, propSchema] of Object.entries(bodySchema.properties)) { + const zodType = this.openApiSchemaToZod(propSchema as OpenAPIV3.SchemaObject); + shape[propName] = requiredFields.has(propName) ? zodType : zodType.optional(); + } + } + } + } + + if (Object.keys(shape).length === 0) { + return z.object({}).passthrough(); + } + + return z.object(shape).passthrough(); + } + + private openApiSchemaToZod(schema: OpenAPIV3.SchemaObject | undefined): ZodTypeAny { + if (!schema) return z.unknown(); + + switch (schema.type) { + case 'string': + return z.string(); + case 'integer': + case 'number': + return z.number(); + case 'boolean': + return z.boolean(); + case 'array': + return z.array(this.openApiSchemaToZod(schema.items as OpenAPIV3.SchemaObject | undefined)); + case 'object': + return z.record(z.unknown()); + default: + return z.unknown(); + } + } +} diff --git a/src/integrations/adapters/stripe-adapter.ts b/src/integrations/adapters/stripe-adapter.ts new file mode 100644 index 00000000..28697348 --- /dev/null +++ b/src/integrations/adapters/stripe-adapter.ts @@ -0,0 +1,498 @@ +import Stripe from 'stripe'; +import { createLogger } from '../../core/logger.js'; +import type { + BusinessIntegration, + EventHandler, + HealthStatus, + IntegrationCapability, + IntegrationConfig, +} from '../../types/integration.js'; +import type { WebhookRouter } from '../webhook-router.js'; + +const logger = createLogger('stripe-adapter'); + +/** + * Details extracted from a `payment_intent.succeeded` event. + * Passed to the registered `PaymentSucceededHandler` so that external + * code can update DocType records and notify owners without coupling + * the adapter to the intelligence layer. + */ +export interface PaymentSucceededDetails { + /** Stripe PaymentIntent ID (e.g. "pi_xxx") */ + paymentIntentId: string; + /** Stripe PaymentLink ID if the intent originated from a payment link, else null */ + paymentLinkId: string | null; + /** Amount charged in the smallest currency unit (e.g. cents for USD) */ + amount: number; + /** ISO currency code in lowercase (e.g. "usd") */ + currency: string; + /** PaymentIntent metadata attached at creation time */ + metadata: Record; +} + +/** + * Callback invoked when a `payment_intent.succeeded` webhook event is received + * and its signature has been verified. + * + * Implementors should: + * 1. Find the matching DocType record via `paymentLinkId` or `metadata` + * 2. Execute the "paid" state transition + * 3. Notify the record owner via the messaging channel + */ +export type PaymentSucceededHandler = (details: PaymentSucceededDetails) => Promise; + +/** + * Stripe payment integration adapter. + * + * Capabilities: + * - create_payment_link: Create a Stripe Payment Link + * - create_invoice: Create and optionally finalize a Stripe invoice + * - list_payments: List recent payment intents + * - get_balance: Retrieve account balance + * + * Credentials expected (from credential store): + * - apiKey: Stripe secret key (sk_test_... or sk_live_...) + * - webhookSecret (optional): Webhook endpoint signing secret (whsec_...) + */ +export class StripeAdapter implements BusinessIntegration { + readonly name = 'stripe'; + readonly type = 'payment' as const; + + private client: Stripe | null = null; + private webhookSecret: string | null = null; + private eventHandlers = new Map(); + private webhookEndpointId: string | null = null; + private webhookRouter: WebhookRouter | null = null; + private paymentSucceededHandler: PaymentSucceededHandler | null = null; + + async initialize(config: IntegrationConfig): Promise { + const opts = config.options; + const apiKey = opts['apiKey'] as string | undefined; + + if (!apiKey || typeof apiKey !== 'string') { + throw new Error('Stripe adapter requires an apiKey in config.options'); + } + + this.client = new Stripe(apiKey); + this.webhookSecret = (opts['webhookSecret'] as string) ?? null; + + // Verify the key works + try { + await this.client.balance.retrieve(); + } catch (err) { + this.client = null; + throw new Error( + `Stripe initialization failed: ${err instanceof Error ? err.message : String(err)}`, + ); + } + + logger.info('Stripe adapter initialized'); + } + + async healthCheck(): Promise { + const checkedAt = new Date().toISOString(); + + if (!this.client) { + return { status: 'unhealthy', message: 'Not initialized', checkedAt, details: {} }; + } + + try { + const balance = await this.client.balance.retrieve(); + return { + status: 'healthy', + message: 'Stripe API reachable', + checkedAt, + details: { availableCurrencies: balance.available.map((b) => b.currency) }, + }; + } catch (err) { + return { + status: 'unhealthy', + message: err instanceof Error ? err.message : String(err), + checkedAt, + details: {}, + }; + } + } + + // eslint-disable-next-line @typescript-eslint/require-await + async shutdown(): Promise { + this.client = null; + this.webhookSecret = null; + this.webhookEndpointId = null; + this.eventHandlers.clear(); + logger.info('Stripe adapter shut down'); + } + + // ── Webhook registration ──────────────────────────────────────── + + /** + * Inject a WebhookRouter so that `registerWebhook()` can create a route + * for `payment_intent.succeeded` events. + */ + setWebhookRouter(router: WebhookRouter): void { + this.webhookRouter = router; + } + + /** + * Register a callback that is invoked after a `payment_intent.succeeded` + * event has been verified. External callers should use this to update + * DocType records and notify owners. + */ + setPaymentSucceededHandler(fn: PaymentSucceededHandler): void { + this.paymentSucceededHandler = fn; + } + + /** + * Create a Stripe webhook endpoint pointing at `endpointUrl` and register + * the `payment_intent.succeeded` handler in the WebhookRouter. + * + * The Stripe-assigned endpoint ID is stored so that `unregisterWebhook()` + * can delete it later. + * + * @param endpointUrl - Publicly reachable URL Stripe should POST to, + * e.g. "https://example.com/webhook/stripe/payment_intent.succeeded" + */ + async registerWebhook(endpointUrl: string): Promise { + if (!this.client) { + throw new Error('Stripe adapter not initialized — call initialize() first'); + } + + // Create the webhook endpoint on Stripe's side + const endpoint = await this.client.webhookEndpoints.create({ + url: endpointUrl, + enabled_events: ['payment_intent.succeeded'], + }); + + this.webhookEndpointId = endpoint.id; + logger.info({ endpointId: endpoint.id, url: endpointUrl }, 'Stripe webhook endpoint created'); + + // Wire up the route in the WebhookRouter + if (this.webhookRouter) { + this.webhookRouter.registerIntegration(this); + this.webhookRouter.registerRoute('stripe', 'payment_intent.succeeded', async (payload) => { + await this.handlePaymentSucceededPayload(payload); + }); + logger.info('Stripe payment_intent.succeeded route registered in WebhookRouter'); + } + + // Subscribe to internal dispatched events as well + this.subscribe('payment_intent.succeeded', async (event) => { + await this.handlePaymentSucceededPayload(event); + }); + } + + /** + * Delete the Stripe webhook endpoint that was created via `registerWebhook()`. + * No-op if no endpoint has been registered. + */ + async unregisterWebhook(): Promise { + if (!this.client || !this.webhookEndpointId) { + return; + } + + try { + await this.client.webhookEndpoints.del(this.webhookEndpointId); + logger.info({ endpointId: this.webhookEndpointId }, 'Stripe webhook endpoint deleted'); + } catch (err) { + logger.warn( + { endpointId: this.webhookEndpointId, err }, + 'Failed to delete Stripe webhook endpoint', + ); + } + + this.webhookEndpointId = null; + } + + /** + * Verify a raw Stripe webhook HTTP request and dispatch the event. + * + * Call this from your HTTP handler when the path matches the Stripe webhook + * URL. The `stripeSignature` value should come from the `Stripe-Signature` + * request header. + * + * @param rawBody - Raw (unparsed) request body string + * @param stripeSignature - Value of the `Stripe-Signature` header + */ + async handleStripeWebhook(rawBody: string, stripeSignature: string): Promise { + const event = this.verifyWebhookSignature(rawBody, stripeSignature); + await this.dispatchWebhookEvent(event); + } + + describeCapabilities(): IntegrationCapability[] { + return [ + { + name: 'create_payment_link', + description: + 'Create a Stripe Payment Link. Params: amount (number, cents), currency (string, e.g. "usd"), description (string).', + category: 'write', + requiresApproval: true, + }, + { + name: 'create_invoice', + description: + 'Create a Stripe invoice for a customer. Params: customerId (string), items (array of {description, amount, currency}), autoFinalize (boolean, default true).', + category: 'write', + requiresApproval: true, + }, + { + name: 'list_payments', + description: + 'List recent payment intents. Params: limit (number, default 10), status (string, optional).', + category: 'read', + requiresApproval: false, + }, + { + name: 'get_balance', + description: 'Retrieve the current Stripe account balance. No params required.', + category: 'read', + requiresApproval: false, + }, + ]; + } + + async query(operation: string, params: Record): Promise { + if (!this.client) { + throw new Error('Stripe adapter not initialized — call initialize() first'); + } + + switch (operation) { + case 'list_payments': + return await this.listPayments(params); + case 'get_balance': + return await this.getBalance(); + default: + throw new Error(`Unknown query operation: ${operation}`); + } + } + + async execute(operation: string, params: Record): Promise { + if (!this.client) { + throw new Error('Stripe adapter not initialized — call initialize() first'); + } + + switch (operation) { + case 'create_payment_link': + return await this.createPaymentLink(params); + case 'create_invoice': + return await this.createInvoice(params); + default: + throw new Error(`Unknown execute operation: ${operation}`); + } + } + + // ── Event subscription (for webhook handling) ────────────────── + + subscribe(event: string, handler: EventHandler): void { + const handlers = this.eventHandlers.get(event) ?? []; + handlers.push(handler); + this.eventHandlers.set(event, handlers); + logger.debug({ event }, 'Stripe event handler subscribed'); + } + + /** + * Verify a Stripe webhook signature and construct the event. + * Called by the webhook router when a POST hits /webhook/stripe/:event. + * + * @param rawBody - Raw request body as a string + * @param signature - Stripe-Signature header value + * @returns The verified Stripe event object + */ + verifyWebhookSignature(rawBody: string, signature: string): Stripe.Event { + if (!this.webhookSecret) { + throw new Error('Webhook secret not configured — cannot verify signature'); + } + + return Stripe.webhooks.constructEvent(rawBody, signature, this.webhookSecret); + } + + /** + * Dispatch a verified webhook event to all subscribed handlers. + */ + async dispatchWebhookEvent(event: Stripe.Event): Promise { + const handlers = this.eventHandlers.get(event.type); + if (!handlers || handlers.length === 0) { + logger.debug({ eventType: event.type }, 'No handlers for Stripe event'); + return; + } + + for (const handler of handlers) { + try { + await handler(event as unknown as Record); + } catch (err) { + logger.error({ eventType: event.type, err }, 'Stripe event handler error'); + } + } + } + + // ── Internal webhook handlers ────────────────────────────────── + + /** + * Core handler for `payment_intent.succeeded` payloads. + * Extracts payment details and calls the injected `PaymentSucceededHandler` + * so that external code can update DocType records and notify owners. + */ + private async handlePaymentSucceededPayload(payload: Record): Promise { + // Stripe webhook payload structure: + // { type, data: { object: PaymentIntent } } + // The WebhookRouter dispatches the full event object as payload. + const dataObj = + payload['data'] !== undefined && + typeof payload['data'] === 'object' && + payload['data'] !== null + ? (payload['data'] as Record) + : payload; + + const pi = + dataObj['object'] !== undefined && + typeof dataObj['object'] === 'object' && + dataObj['object'] !== null + ? (dataObj['object'] as Record) + : dataObj; + + const paymentIntentId = typeof pi['id'] === 'string' ? pi['id'] : ''; + const paymentLinkId = typeof pi['payment_link'] === 'string' ? pi['payment_link'] : null; + const amount = typeof pi['amount'] === 'number' ? pi['amount'] : 0; + const currency = typeof pi['currency'] === 'string' ? pi['currency'] : 'usd'; + const metadata = + pi['metadata'] !== null && typeof pi['metadata'] === 'object' + ? (pi['metadata'] as Record) + : {}; + + logger.info( + { paymentIntentId, paymentLinkId, amount, currency }, + 'payment_intent.succeeded received', + ); + + if (!this.paymentSucceededHandler) { + logger.debug('No PaymentSucceededHandler registered — skipping DocType transition'); + return; + } + + try { + await this.paymentSucceededHandler({ + paymentIntentId, + paymentLinkId, + amount, + currency, + metadata, + }); + } catch (err) { + logger.error({ paymentIntentId, err }, 'PaymentSucceededHandler threw an error'); + } + } + + // ── Private helpers ──────────────────────────────────────────── + + private async createPaymentLink( + params: Record, + ): Promise<{ url: string; id: string }> { + const amount = params['amount'] as number; + const currency = (params['currency'] as string) ?? 'usd'; + const description = (params['description'] as string) ?? 'Payment'; + + if (typeof amount !== 'number' || amount <= 0) { + throw new Error('amount must be a positive number (in cents)'); + } + + const link = await this.client!.paymentLinks.create({ + line_items: [ + { + price_data: { + currency, + product_data: { name: description }, + unit_amount: Math.round(amount), + }, + quantity: 1, + }, + ], + }); + + logger.info({ linkId: link.id, url: link.url }, 'Payment link created'); + return { url: link.url, id: link.id }; + } + + private async createInvoice( + params: Record, + ): Promise<{ invoiceId: string; invoiceUrl: string | null; status: string }> { + const customerId = params['customerId'] as string; + const items = params['items'] as Array<{ + description: string; + amount: number; + currency?: string; + }>; + const autoFinalize = params['autoFinalize'] !== false; + + if (!customerId || typeof customerId !== 'string') { + throw new Error('customerId is required'); + } + if (!Array.isArray(items) || items.length === 0) { + throw new Error('items must be a non-empty array'); + } + + const invoice = await this.client!.invoices.create({ + customer: customerId, + auto_advance: autoFinalize, + }); + + // Add line items + for (const item of items) { + await this.client!.invoiceItems.create({ + customer: customerId, + invoice: invoice.id, + amount: Math.round(item.amount), + currency: item.currency ?? 'usd', + description: item.description, + }); + } + + // Finalize if requested + let finalInvoice = invoice; + if (autoFinalize) { + finalInvoice = await this.client!.invoices.finalizeInvoice(invoice.id); + } + + logger.info({ invoiceId: finalInvoice.id }, 'Invoice created'); + return { + invoiceId: finalInvoice.id, + invoiceUrl: finalInvoice.hosted_invoice_url ?? null, + status: finalInvoice.status ?? 'draft', + }; + } + + private async listPayments( + params: Record, + ): Promise<{ payments: Array>; hasMore: boolean }> { + const limit = Math.min((params['limit'] as number) ?? 10, 100); + const listParams: Stripe.PaymentIntentListParams = { limit }; + + if (params['status'] && typeof params['status'] === 'string') { + listParams.expand = undefined; // clear any defaults + } + + const result = await this.client!.paymentIntents.list(listParams); + + return { + payments: result.data.map((pi) => ({ + id: pi.id, + amount: pi.amount, + currency: pi.currency, + status: pi.status, + description: pi.description, + created: new Date(pi.created * 1000).toISOString(), + })), + hasMore: result.has_more, + }; + } + + private async getBalance(): Promise<{ + available: Array<{ amount: number; currency: string }>; + pending: Array<{ amount: number; currency: string }>; + }> { + const balance = await this.client!.balance.retrieve(); + + return { + available: balance.available.map((b) => ({ amount: b.amount, currency: b.currency })), + pending: balance.pending.map((b) => ({ amount: b.amount, currency: b.currency })), + }; + } +} diff --git a/src/integrations/credential-store.ts b/src/integrations/credential-store.ts new file mode 100644 index 00000000..37a41cee --- /dev/null +++ b/src/integrations/credential-store.ts @@ -0,0 +1,157 @@ +import { createCipheriv, createDecipheriv, randomBytes } from 'node:crypto'; +import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import type Database from 'better-sqlite3'; + +const ALGORITHM = 'aes-256-gcm'; +const IV_LENGTH = 12; // 12 bytes for GCM +const KEY_LENGTH = 32; // 256 bits + +/** Result of encrypting credential data. All values are hex-encoded. */ +export interface EncryptedCredential { + encrypted: string; + iv: string; + authTag: string; +} + +/** + * Manages AES-256-GCM encryption of integration credentials at rest. + * + * On first use, generates a 32-byte random key and writes it to + * `.openbridge/secrets.key` with chmod 600. All credential data is + * encrypted before storage and decrypted only on demand. + */ +export class CredentialStore { + private key: Buffer | null = null; + private readonly workspacePath: string; + + constructor(workspacePath: string) { + this.workspacePath = workspacePath; + } + + /** Returns the path to the secrets key file. */ + getSecretsKeyPath(): string { + return join(this.workspacePath, '.openbridge', 'secrets.key'); + } + + /** + * Loads the encryption key from disk, or generates a new one if it + * does not exist. The key file is written with mode 0o600 (owner-only). + */ + loadOrGenerateKey(): Buffer { + if (this.key) return this.key; + + const keyPath = this.getSecretsKeyPath(); + + if (existsSync(keyPath)) { + this.key = readFileSync(keyPath); + if (this.key.length !== KEY_LENGTH) { + throw new Error( + `Invalid secrets.key length: expected ${KEY_LENGTH}, got ${this.key.length}`, + ); + } + return this.key; + } + + // Generate new key + mkdirSync(dirname(keyPath), { recursive: true }); + const newKey = randomBytes(KEY_LENGTH); + writeFileSync(keyPath, newKey, { mode: 0o600 }); + + // Ensure permissions even if the file existed with different mode + try { + chmodSync(keyPath, 0o600); + } catch { + // chmod may fail on Windows — non-fatal + } + + this.key = newKey; + return this.key; + } + + /** + * Encrypts a credential data object using AES-256-GCM. + * Returns hex-encoded ciphertext, IV, and auth tag. + */ + encryptCredential(data: Record): EncryptedCredential { + const key = this.loadOrGenerateKey(); + const iv = randomBytes(IV_LENGTH); + const cipher = createCipheriv(ALGORITHM, key, iv); + + const plaintext = JSON.stringify(data); + let encrypted = cipher.update(plaintext, 'utf8', 'hex'); + encrypted += cipher.final('hex'); + + return { + encrypted, + iv: iv.toString('hex'), + authTag: cipher.getAuthTag().toString('hex'), + }; + } + + /** + * Decrypts a credential using AES-256-GCM. + * All inputs are hex-encoded strings. + */ + decryptCredential(encrypted: string, iv: string, authTag: string): Record { + const key = this.loadOrGenerateKey(); + const decipher = createDecipheriv(ALGORITHM, key, Buffer.from(iv, 'hex')); + decipher.setAuthTag(Buffer.from(authTag, 'hex')); + + let decrypted = decipher.update(encrypted, 'hex', 'utf8'); + decrypted += decipher.final('utf8'); + + return JSON.parse(decrypted) as Record; + } + + // ── SQLite credential CRUD ────────────────────────────────────── + + /** Store an encrypted credential for an integration. Upserts by integration name. */ + storeCredential( + db: Database.Database, + integrationName: string, + data: Record, + ): void { + const { encrypted, iv, authTag } = this.encryptCredential(data); + const now = new Date().toISOString(); + + db.prepare( + `INSERT INTO integration_credentials (integration_name, encrypted, iv, auth_tag, health_status, created_at, updated_at) + VALUES (?, ?, ?, ?, 'unknown', ?, ?) + ON CONFLICT(integration_name) DO UPDATE SET + encrypted = excluded.encrypted, + iv = excluded.iv, + auth_tag = excluded.auth_tag, + updated_at = excluded.updated_at`, + ).run(integrationName, encrypted, iv, authTag, now, now); + } + + /** Retrieve and decrypt a credential for an integration. Returns null if not found. */ + getCredential(db: Database.Database, integrationName: string): Record | null { + const row = db + .prepare( + 'SELECT encrypted, iv, auth_tag FROM integration_credentials WHERE integration_name = ?', + ) + .get(integrationName) as { encrypted: string; iv: string; auth_tag: string } | undefined; + + if (!row) return null; + + return this.decryptCredential(row.encrypted, row.iv, row.auth_tag); + } + + /** Delete a credential for an integration. */ + deleteCredential(db: Database.Database, integrationName: string): boolean { + const result = db + .prepare('DELETE FROM integration_credentials WHERE integration_name = ?') + .run(integrationName); + return result.changes > 0; + } + + /** Update the health status for a credential record. */ + updateHealthStatus(db: Database.Database, integrationName: string, healthStatus: string): void { + const now = new Date().toISOString(); + db.prepare( + 'UPDATE integration_credentials SET health_status = ?, updated_at = ? WHERE integration_name = ?', + ).run(healthStatus, now, integrationName); + } +} diff --git a/src/integrations/event-bridge.ts b/src/integrations/event-bridge.ts new file mode 100644 index 00000000..b78b63f5 --- /dev/null +++ b/src/integrations/event-bridge.ts @@ -0,0 +1,587 @@ +import { EventEmitter } from 'node:events'; +import { createLogger } from '../core/logger.js'; +import type { WebhookRouter } from './webhook-router.js'; + +const logger = createLogger('event-bridge'); + +// ── Types ──────────────────────────────────────────────────────── + +/** Supported event source types */ +export type EventSourceType = 'websocket' | 'sse' | 'polling' | 'webhook'; + +/** Configuration for an event source */ +export interface EventSourceConfig { + /** Unique identifier for this event source */ + id: string; + /** Type of event source */ + type: EventSourceType; + /** URL to connect to (WebSocket, SSE, or polling endpoint) */ + url: string; + /** Event patterns to listen for (glob-style, e.g. "order.*") */ + events: string[]; + /** Authentication configuration */ + auth?: EventSourceAuth; + /** Polling interval in milliseconds (only for 'polling' type, default 30000) */ + pollingInterval?: number; + /** Integration name this source belongs to */ + integration: string; +} + +/** Authentication for event sources */ +export interface EventSourceAuth { + type: 'bearer' | 'basic' | 'header' | 'query'; + /** Token value (for bearer), password (for basic), header/query value */ + value: string; + /** Username (for basic auth) or header/query param name */ + key?: string; +} + +/** A normalized event emitted by any source */ +export interface BridgeEvent { + /** Auto-generated event ID */ + id: string; + /** Source ID that produced this event */ + sourceId: string; + /** Integration name */ + integration: string; + /** Event name/type */ + event: string; + /** Event payload */ + payload: Record; + /** When the event was received (ISO 8601) */ + receivedAt: string; +} + +/** Callback for event notifications */ +export type EventNotificationHandler = (event: BridgeEvent) => void | Promise; + +/** Internal state for an active event source */ +interface ActiveSource { + config: EventSourceConfig; + /** Cleanup function to stop this source */ + cleanup: () => void; + /** Whether the source is currently connected/active */ + active: boolean; +} + +// ── EventBridge ────────────────────────────────────────────────── + +/** + * Generic real-time event bridge. + * + * Supports multiple event source types: + * 1. WebSocket — persistent connection, receives JSON messages + * 2. Server-Sent Events (SSE) — HTTP streaming, receives named events + * 3. Polling — periodic HTTP GET, diffs for new data + * 4. Webhook — delegates to WebhookRouter (already exists) + * + * On event: match to event pattern, normalize payload, route to handlers. + */ +export class EventBridge extends EventEmitter { + private readonly sources = new Map(); + private readonly handlers = new Map(); + private readonly webhookRouter: WebhookRouter | null; + private eventCounter = 0; + + constructor(webhookRouter?: WebhookRouter) { + super(); + this.webhookRouter = webhookRouter ?? null; + } + + /** + * Register an event source. Starts listening immediately. + */ + addSource(config: EventSourceConfig): void { + if (this.sources.has(config.id)) { + logger.warn( + { sourceId: config.id }, + 'Event source already registered — removing old one first', + ); + this.removeSource(config.id); + } + + logger.info( + { sourceId: config.id, type: config.type, integration: config.integration }, + 'Adding event source', + ); + + let cleanup: () => void; + + switch (config.type) { + case 'websocket': + cleanup = this.startWebSocket(config); + break; + case 'sse': + cleanup = this.startSSE(config); + break; + case 'polling': + cleanup = this.startPolling(config); + break; + case 'webhook': + cleanup = this.startWebhook(config); + break; + default: + throw new Error(`Unsupported event source type: ${config.type as string}`); + } + + this.sources.set(config.id, { config, cleanup, active: true }); + } + + /** + * Remove and stop an event source. + */ + removeSource(id: string): void { + const source = this.sources.get(id); + if (!source) return; + + source.cleanup(); + source.active = false; + this.sources.delete(id); + logger.info({ sourceId: id }, 'Event source removed'); + } + + /** + * Register a handler for events matching a pattern. + * Pattern supports glob-style: "order.*" matches "order.created", "order.updated". + * Use "*" to match all events. + */ + onEvent(pattern: string, handler: EventNotificationHandler): void { + const handlers = this.handlers.get(pattern) ?? []; + handlers.push(handler); + this.handlers.set(pattern, handlers); + logger.debug({ pattern }, 'Event handler registered'); + } + + /** + * Remove a handler for a specific pattern. + */ + offEvent(pattern: string, handler: EventNotificationHandler): void { + const handlers = this.handlers.get(pattern); + if (!handlers) return; + const idx = handlers.indexOf(handler); + if (idx !== -1) handlers.splice(idx, 1); + if (handlers.length === 0) this.handlers.delete(pattern); + } + + /** + * List all registered event sources with their status. + */ + listSources(): Array<{ + id: string; + type: EventSourceType; + integration: string; + active: boolean; + }> { + return [...this.sources.values()].map((s) => ({ + id: s.config.id, + type: s.config.type, + integration: s.config.integration, + active: s.active, + })); + } + + /** + * Shut down all event sources. + */ + shutdown(): void { + logger.info({ count: this.sources.size }, 'Shutting down event bridge'); + for (const [id] of this.sources) { + this.removeSource(id); + } + } + + // ── Internal: event dispatch ───────────────────────────────── + + private generateEventId(): string { + return `evt_${Date.now()}_${++this.eventCounter}`; + } + + /** + * Dispatch a raw event through the bridge. Matches against registered + * patterns and invokes handlers. + */ + private async dispatchEvent( + sourceId: string, + integration: string, + event: string, + payload: Record, + ): Promise { + const bridgeEvent: BridgeEvent = { + id: this.generateEventId(), + sourceId, + integration, + event, + payload, + receivedAt: new Date().toISOString(), + }; + + logger.debug({ eventId: bridgeEvent.id, integration, event }, 'Dispatching event'); + + // Emit on the EventEmitter for generic listeners + this.emit('event', bridgeEvent); + + // Match against registered patterns + const matchedHandlers: EventNotificationHandler[] = []; + for (const [pattern, handlers] of this.handlers) { + if ( + this.matchPattern(pattern, event) || + this.matchPattern(pattern, `${integration}.${event}`) + ) { + matchedHandlers.push(...handlers); + } + } + + // Invoke all matched handlers + for (const handler of matchedHandlers) { + try { + await handler(bridgeEvent); + } catch (err) { + logger.error({ eventId: bridgeEvent.id, err }, 'Event handler threw an error'); + } + } + } + + /** + * Simple glob-style pattern matching. + * Supports: "*" (match everything), "prefix.*" (match prefix), exact match. + */ + private matchPattern(pattern: string, value: string): boolean { + if (pattern === '*') return true; + if (pattern === value) return true; + if (pattern.endsWith('.*')) { + const prefix = pattern.slice(0, -2); + return value.startsWith(prefix + '.'); + } + if (pattern.endsWith('*')) { + const prefix = pattern.slice(0, -1); + return value.startsWith(prefix); + } + return false; + } + + // ── Internal: build auth headers ───────────────────────────── + + private buildAuthHeaders(auth?: EventSourceAuth): Record { + if (!auth) return {}; + switch (auth.type) { + case 'bearer': + return { Authorization: `Bearer ${auth.value}` }; + case 'basic': { + const encoded = Buffer.from(`${auth.key ?? ''}:${auth.value}`).toString('base64'); + return { Authorization: `Basic ${encoded}` }; + } + case 'header': + return auth.key ? { [auth.key]: auth.value } : {}; + default: + return {}; + } + } + + private buildAuthUrl(url: string, auth?: EventSourceAuth): string { + if (!auth || auth.type !== 'query' || !auth.key) return url; + const separator = url.includes('?') ? '&' : '?'; + return `${url}${separator}${encodeURIComponent(auth.key)}=${encodeURIComponent(auth.value)}`; + } + + // ── Internal: WebSocket source ─────────────────────────────── + + private startWebSocket(config: EventSourceConfig): () => void { + let socket: WebSocket | null = null; + let closed = false; + let reconnectTimer: ReturnType | null = null; + + const connect = (): void => { + if (closed) return; + + try { + const finalUrl = this.buildAuthUrl(config.url, config.auth); + + // Node 22+ has native WebSocket global + const ws = new WebSocket(finalUrl); + socket = ws; + + ws.onopen = (): void => { + logger.info({ sourceId: config.id }, 'WebSocket connected'); + const source = this.sources.get(config.id); + if (source) source.active = true; + }; + + ws.onmessage = (msg: MessageEvent): void => { + try { + const data = typeof msg.data === 'string' ? msg.data : String(msg.data); + let parsed: unknown; + try { + parsed = JSON.parse(data); + } catch { + parsed = { raw: data }; + } + const payload = + parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed) + ? (parsed as Record) + : { data: parsed }; + + const eventName = + (typeof payload['event'] === 'string' ? payload['event'] : undefined) ?? + (typeof payload['type'] === 'string' ? payload['type'] : undefined) ?? + 'message'; + + if (this.shouldHandle(config, eventName)) { + void this.dispatchEvent(config.id, config.integration, eventName, payload); + } + } catch (err) { + logger.error({ sourceId: config.id, err }, 'Error processing WebSocket message'); + } + }; + + ws.onclose = (): void => { + logger.info({ sourceId: config.id }, 'WebSocket disconnected'); + const source = this.sources.get(config.id); + if (source) source.active = false; + if (!closed) { + reconnectTimer = setTimeout(connect, 5000); + } + }; + + ws.onerror = (): void => { + logger.error({ sourceId: config.id }, 'WebSocket error'); + }; + } catch (err) { + logger.error({ sourceId: config.id, err }, 'Failed to create WebSocket connection'); + if (!closed) { + reconnectTimer = setTimeout(connect, 5000); + } + } + }; + + connect(); + + return () => { + closed = true; + if (reconnectTimer) clearTimeout(reconnectTimer); + if (socket) { + try { + socket.close(); + } catch { + // ignore + } + } + }; + } + + // ── Internal: SSE source ───────────────────────────────────── + + private startSSE(config: EventSourceConfig): () => void { + let abortController: AbortController | null = null; + let closed = false; + let reconnectTimer: ReturnType | null = null; + + const connect = (): void => { + if (closed) return; + + abortController = new AbortController(); + const headers = this.buildAuthHeaders(config.auth); + const url = this.buildAuthUrl(config.url, config.auth); + + fetch(url, { + headers: { ...headers, Accept: 'text/event-stream' }, + signal: abortController.signal, + }) + .then(async (response) => { + if (!response.ok || !response.body) { + logger.error({ sourceId: config.id, status: response.status }, 'SSE connection failed'); + if (!closed) { + reconnectTimer = setTimeout(connect, 5000); + } + return; + } + + logger.info({ sourceId: config.id }, 'SSE connected'); + const source = this.sources.get(config.id); + if (source) source.active = true; + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + let currentEvent = 'message'; + let currentData = ''; + + try { + while (!closed) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split('\n'); + buffer = lines.pop() ?? ''; + + for (const line of lines) { + if (line.startsWith('event:')) { + currentEvent = line.slice(6).trim(); + } else if (line.startsWith('data:')) { + currentData += (currentData ? '\n' : '') + line.slice(5).trim(); + } else if (line === '') { + // Empty line = end of event + if (currentData) { + let payload: Record; + try { + const parsed: unknown = JSON.parse(currentData); + payload = + parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed) + ? (parsed as Record) + : { data: parsed }; + } catch { + payload = { data: currentData }; + } + + if (this.shouldHandle(config, currentEvent)) { + void this.dispatchEvent(config.id, config.integration, currentEvent, payload); + } + } + currentEvent = 'message'; + currentData = ''; + } + } + } + } catch (err) { + if (!closed) { + logger.error({ sourceId: config.id, err }, 'SSE stream error'); + } + } + + const src = this.sources.get(config.id); + if (src) src.active = false; + if (!closed) { + reconnectTimer = setTimeout(connect, 5000); + } + }) + .catch((err: unknown) => { + if (!closed) { + logger.error({ sourceId: config.id, err }, 'SSE fetch error'); + reconnectTimer = setTimeout(connect, 5000); + } + }); + }; + + connect(); + + return () => { + closed = true; + if (reconnectTimer) clearTimeout(reconnectTimer); + if (abortController) abortController.abort(); + }; + } + + // ── Internal: Polling source ───────────────────────────────── + + private startPolling(config: EventSourceConfig): () => void { + const interval = config.pollingInterval ?? 30_000; + let lastData: string | null = null; + let timer: ReturnType | null = null; + let closed = false; + + const poll = async (): Promise => { + if (closed) return; + + try { + const headers = this.buildAuthHeaders(config.auth); + const url = this.buildAuthUrl(config.url, config.auth); + const response = await fetch(url, { headers }); + + if (!response.ok) { + logger.warn({ sourceId: config.id, status: response.status }, 'Polling request failed'); + return; + } + + const text = await response.text(); + + // Only dispatch if data changed + if (text !== lastData) { + const isFirst = lastData === null; + lastData = text; + + // Skip first poll (baseline) — only dispatch on changes + if (isFirst) { + logger.debug({ sourceId: config.id }, 'Polling baseline captured'); + return; + } + + let payload: Record; + try { + const parsed: unknown = JSON.parse(text); + payload = + parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed) + ? (parsed as Record) + : { data: parsed }; + } catch { + payload = { data: text }; + } + + void this.dispatchEvent(config.id, config.integration, 'data_changed', payload); + } + } catch (err) { + logger.error({ sourceId: config.id, err }, 'Polling error'); + } + }; + + // Initial poll + void poll(); + timer = setInterval(() => void poll(), interval); + + const source = this.sources.get(config.id); + if (source) source.active = true; + + return () => { + closed = true; + if (timer) clearInterval(timer); + }; + } + + // ── Internal: Webhook source (delegates to WebhookRouter) ──── + + private startWebhook(config: EventSourceConfig): () => void { + if (!this.webhookRouter) { + logger.warn( + { sourceId: config.id }, + 'Webhook source requires a WebhookRouter — event source will not receive events', + ); + return () => {}; + } + + // Register a route for each event pattern in the webhook router + for (const event of config.events) { + // Webhook events use exact names (not glob patterns) + if (event.includes('*')) { + logger.warn( + { sourceId: config.id, event }, + 'Webhook events do not support glob patterns — use exact event names', + ); + continue; + } + + this.webhookRouter.registerRoute(config.integration, event, async (payload) => { + await this.dispatchEvent(config.id, config.integration, event, payload); + }); + } + + const source = this.sources.get(config.id); + if (source) source.active = true; + + return () => { + if (this.webhookRouter) { + void this.webhookRouter.deregisterIntegration(config.integration); + } + }; + } + + // ── Internal: event filtering ──────────────────────────────── + + /** + * Check if an event from a source should be handled based on the + * configured event patterns. + */ + private shouldHandle(config: EventSourceConfig, eventName: string): boolean { + // If no event filters, handle everything + if (config.events.length === 0) return true; + return config.events.some((pattern) => this.matchPattern(pattern, eventName)); + } +} diff --git a/src/integrations/hub.ts b/src/integrations/hub.ts new file mode 100644 index 00000000..bb950190 --- /dev/null +++ b/src/integrations/hub.ts @@ -0,0 +1,140 @@ +import type Database from 'better-sqlite3'; +import type { + BusinessIntegration, + HealthStatus, + IntegrationConfig, + IntegrationInfo, +} from '../types/integration.js'; +import type { CredentialStore } from './credential-store.js'; + +interface IntegrationEntry { + integration: BusinessIntegration; + connected: boolean; + healthStatus: HealthStatus['status']; + credentialStore?: CredentialStore; +} + +/** + * IntegrationHub registry and lifecycle manager. + * Manages registration, initialization, health checks, and shutdown of business integrations. + */ +export class IntegrationHub { + private integrations = new Map(); + private db: Database.Database | null = null; + private credentialStore: CredentialStore | null = null; + private healthAlertHandler: ((name: string, status: HealthStatus) => void) | null = null; + + /** + * Optionally wire in a SQLite DB and CredentialStore for health status persistence. + */ + setDatabase(db: Database.Database, credentialStore: CredentialStore): void { + this.db = db; + this.credentialStore = credentialStore; + } + + /** + * Set a callback invoked when any integration transitions to unhealthy state. + * Called once per healthy/unknown → unhealthy transition. + */ + setHealthAlertHandler(handler: (name: string, status: HealthStatus) => void): void { + this.healthAlertHandler = handler; + } + + /** Register an integration adapter. Must be called before initialize(). */ + register(integration: BusinessIntegration): void { + this.integrations.set(integration.name, { + integration, + connected: false, + healthStatus: 'unknown', + }); + } + + /** + * Get a registered integration by name. + * Throws if the integration is not registered. + */ + get(name: string): BusinessIntegration { + const entry = this.integrations.get(name); + if (!entry) { + throw new Error(`Integration not found: ${name}`); + } + return entry.integration; + } + + /** List summary info for all registered integrations. */ + list(): IntegrationInfo[] { + return Array.from(this.integrations.entries()).map(([name, entry]) => ({ + name, + type: entry.integration.type, + connected: entry.connected, + healthStatus: entry.healthStatus, + capabilityCount: entry.integration.describeCapabilities().length, + })); + } + + /** + * Initialize an integration by name. + * Calls the integration's initialize() method, then updates health_status to 'healthy' + * in the credentials table on success. + */ + async initialize(name: string, config: IntegrationConfig): Promise { + const entry = this.integrations.get(name); + if (!entry) { + throw new Error(`Integration not found: ${name}`); + } + + await entry.integration.initialize(config); + entry.connected = true; + entry.healthStatus = 'healthy'; + + if (this.db && this.credentialStore) { + this.credentialStore.updateHealthStatus(this.db, name, 'healthy'); + } + } + + /** + * Run a health check on a named integration. + * Updates health_status in the credentials table after the check. + * Calls the health alert handler if the integration transitions to unhealthy. + */ + async healthCheck(name: string): Promise { + const entry = this.integrations.get(name); + if (!entry) { + throw new Error(`Integration not found: ${name}`); + } + + const previousStatus = entry.healthStatus; + const result = await entry.integration.healthCheck(); + entry.healthStatus = result.status; + + if (this.db && this.credentialStore) { + this.credentialStore.updateHealthStatus(this.db, name, result.status); + } + + // Alert on transition to unhealthy + if ( + result.status === 'unhealthy' && + previousStatus !== 'unhealthy' && + this.healthAlertHandler + ) { + this.healthAlertHandler(name, result); + } + + return result; + } + + /** Return the last known health status for an integration, or undefined if not registered. */ + getHealthStatus(name: string): HealthStatus['status'] | undefined { + return this.integrations.get(name)?.healthStatus; + } + + /** Shutdown all registered integrations. */ + async shutdown(): Promise { + const shutdowns = Array.from(this.integrations.values()).map((entry) => + entry.integration.shutdown().catch(() => { + // Ignore individual shutdown errors — best-effort + }), + ); + await Promise.all(shutdowns); + } +} diff --git a/src/integrations/index.ts b/src/integrations/index.ts new file mode 100644 index 00000000..97a98879 --- /dev/null +++ b/src/integrations/index.ts @@ -0,0 +1,36 @@ +/** + * Integration Hub Module + * Manages connections to external business services (Stripe, Google Drive, databases, etc.) + * + * Exports: + * - IntegrationHub: Registry and lifecycle manager for business integrations + * - credential-store: AES-256-GCM encryption for credentials at rest + * - webhook-router: Incoming webhook dispatcher for integration events + * + * Adapters (in src/integrations/adapters/): + * - stripe-adapter: Stripe payment integration + * - google-drive-adapter: Google Drive file storage + * - google-sheets-adapter: Google Sheets read/write sync + * - openapi-adapter: Universal REST API connector + * - email-adapter: Email (SMTP send + IMAP/Gmail read) + * - database-adapter: PostgreSQL/MySQL read-only connections + * - dropbox-adapter: Dropbox file storage + * - google-calendar-adapter: Google Calendar scheduling + */ + +export { IntegrationHub } from './hub.js'; +export { CredentialStore } from './credential-store.js'; +export type { EncryptedCredential } from './credential-store.js'; +export { WebhookRouter } from './webhook-router.js'; +export type { WebhookHandler } from './webhook-router.js'; +export { StripeAdapter } from './adapters/stripe-adapter.js'; +export { EventBridge } from './event-bridge.js'; +export type { + EventSourceConfig, + EventSourceType, + EventSourceAuth, + BridgeEvent, + EventNotificationHandler, +} from './event-bridge.js'; +export { generateSkillPack } from './skill-pack-generator.js'; +export { generateDefaultWorkflows } from './workflow-templates.js'; diff --git a/src/integrations/parsers/curl-parser.ts b/src/integrations/parsers/curl-parser.ts new file mode 100644 index 00000000..5590e327 --- /dev/null +++ b/src/integrations/parsers/curl-parser.ts @@ -0,0 +1,597 @@ +import type { OpenAPIV3 } from 'openapi-types'; +import { createLogger } from '../../core/logger.js'; + +const logger = createLogger('curl-parser'); + +// ── Parsed cURL representation ─────────────────────────────────────────────── + +/** Parsed representation of a single cURL command. */ +interface ParsedCurl { + method: string; + url: string; + headers: Record; + body?: string; + basicAuth?: { user: string; password: string }; + cookies?: string; +} + +// ── Public API ─────────────────────────────────────────────────────────────── + +/** + * Convert one or more cURL commands to an OpenAPI 3.0 specification. + * + * Supported cURL flags: + * - `-X`, `--request` — HTTP method + * - `-H`, `--header` — Headers (e.g. `-H "Authorization: Bearer ..."`) + * - `-d`, `--data`, `--data-raw`, `--data-binary` — Request body + * - `-u`, `--user` — Basic auth (user:password) + * - `-b`, `--cookie` — Cookies + * - Multi-line cURL with backslash continuation is joined before parsing + * + * @param curls Array of raw cURL command strings + * @returns OpenAPI 3.0 document + */ +export function curlsToOpenAPI(curls: string[]): OpenAPIV3.Document { + const parsed = curls.map(parseSingleCurl); + + // Derive common base URL + const baseUrl = deriveBaseUrl(parsed); + + const spec: OpenAPIV3.Document = { + openapi: '3.0.3', + info: { + title: 'Converted from cURL', + version: '1.0.0', + }, + paths: {}, + }; + + if (baseUrl) { + spec.servers = [{ url: baseUrl }]; + } + + // Track security schemes we need to add + let hasBearerAuth = false; + let hasBasicAuth = false; + let hasApiKey = false; + let apiKeyName = ''; + let apiKeyIn: 'header' | 'query' = 'header'; + + for (const curl of parsed) { + const { path, queryParams } = extractPath(curl.url, baseUrl); + if (!path) continue; + + const method = curl.method.toLowerCase(); + + const paths = spec.paths as Record>; + if (!paths[path]) paths[path] = {}; + // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion -- paths[path] is set above but TS can't narrow indexed access + const pathItem = paths[path] as Record; + + // Don't overwrite if same method already exists on this path + if (pathItem[method]) continue; + + const operation: OpenAPIV3.OperationObject = { + summary: `${curl.method} ${path}`, + operationId: generateOperationId(method, path), + responses: { + '200': { description: 'Successful response' }, + }, + }; + + // Parameters: path variables + const parameters: OpenAPIV3.ParameterObject[] = []; + const pathVarMatches = path.matchAll(/\{(\w+)\}/g); + for (const m of pathVarMatches) { + parameters.push({ + name: m[1]!, + in: 'path', + required: true, + schema: { type: 'string' }, + }); + } + + // Parameters: query params + for (const [name] of queryParams) { + parameters.push({ + name, + in: 'query', + schema: { type: 'string' }, + }); + } + + // Parameters: custom headers (skip common ones) + const skipHeaders = new Set([ + 'content-type', + 'accept', + 'authorization', + 'user-agent', + 'host', + 'connection', + 'cache-control', + 'cookie', + ]); + for (const [name] of Object.entries(curl.headers)) { + if (skipHeaders.has(name.toLowerCase())) continue; + parameters.push({ + name, + in: 'header', + schema: { type: 'string' }, + }); + } + + if (parameters.length > 0) { + operation.parameters = parameters; + } + + // Request body + if (curl.body && ['post', 'put', 'patch'].includes(method)) { + const contentType = findHeaderValue(curl.headers, 'content-type'); + operation.requestBody = buildRequestBody(curl.body, contentType); + } + + // Security detection + const security: Record[] = []; + const authHeader = findHeaderValue(curl.headers, 'authorization'); + if (authHeader) { + if (authHeader.toLowerCase().startsWith('bearer ')) { + hasBearerAuth = true; + security.push({ bearerAuth: [] }); + } else if (authHeader.toLowerCase().startsWith('basic ')) { + hasBasicAuth = true; + security.push({ basicAuth: [] }); + } + } + + // API key header detection (common patterns) + for (const [name, value] of Object.entries(curl.headers)) { + const lower = name.toLowerCase(); + if (lower.includes('api-key') || lower.includes('apikey') || lower.includes('x-api-key')) { + if (value) { + hasApiKey = true; + apiKeyName = name; + apiKeyIn = 'header'; + security.push({ apiKeyAuth: [] }); + break; + } + } + } + + // Basic auth from -u flag + if (curl.basicAuth) { + hasBasicAuth = true; + security.push({ basicAuth: [] }); + } + + if (security.length > 0) { + operation.security = security; + } + + pathItem[method] = operation; + } + + // Add security schemes + const securitySchemes: Record = {}; + if (hasBearerAuth) { + securitySchemes['bearerAuth'] = { type: 'http', scheme: 'bearer' }; + } + if (hasBasicAuth) { + securitySchemes['basicAuth'] = { type: 'http', scheme: 'basic' }; + } + if (hasApiKey) { + securitySchemes['apiKeyAuth'] = { + type: 'apiKey', + name: apiKeyName, + in: apiKeyIn, + }; + } + if (Object.keys(securitySchemes).length > 0) { + spec.components = { securitySchemes }; + } + + logger.info( + { curlCount: curls.length, pathCount: Object.keys(spec.paths ?? {}).length }, + 'cURL commands converted to OpenAPI', + ); + + return spec; +} + +/** + * Split a raw input string containing one or more cURL commands into individual commands. + * Handles backslash line continuation and separates commands by lines starting with `curl `. + */ +export function splitCurlCommands(input: string): string[] { + // Join backslash-continued lines first + const joined = input.replace(/\\\s*\n\s*/g, ' '); + + // Split by lines that start with `curl ` (case-insensitive) + const commands: string[] = []; + const lines = joined.split('\n'); + let current = ''; + + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) continue; + + if (/^curl\s+/i.test(trimmed)) { + if (current) { + commands.push(current.trim()); + } + current = trimmed; + } else if (current) { + // Continuation of previous command (shouldn't happen after join, but safety) + current += ' ' + trimmed; + } + } + + if (current) { + commands.push(current.trim()); + } + + return commands; +} + +// ── Internal helpers ───────────────────────────────────────────────────────── + +/** + * Parse a single cURL command string into a structured representation. + * Handles common cURL flags: -X, -H, -d, --data-raw, -u, -b. + */ +function parseSingleCurl(raw: string): ParsedCurl { + // Join backslash-continued lines + const input = raw.replace(/\\\s*\n\s*/g, ' ').trim(); + + // Tokenize respecting quoted strings + const tokens = tokenize(input); + + let method = ''; + const headers: Record = {}; + let body: string | undefined; + let basicAuth: { user: string; password: string } | undefined; + let cookies: string | undefined; + let url = ''; + + let i = 0; + while (i < tokens.length) { + const token = tokens[i]!; + + if (token === 'curl') { + i++; + continue; + } + + // Method: -X or --request + if (token === '-X' || token === '--request') { + method = (tokens[++i] ?? 'GET').toUpperCase(); + i++; + continue; + } + + // Header: -H or --header + if (token === '-H' || token === '--header') { + const headerStr = tokens[++i] ?? ''; + const colonIdx = headerStr.indexOf(':'); + if (colonIdx > 0) { + const name = headerStr.slice(0, colonIdx).trim(); + const value = headerStr.slice(colonIdx + 1).trim(); + headers[name] = value; + } + i++; + continue; + } + + // Body: -d, --data, --data-raw, --data-binary + if ( + token === '-d' || + token === '--data' || + token === '--data-raw' || + token === '--data-binary' + ) { + body = tokens[++i] ?? ''; + i++; + continue; + } + + // Basic auth: -u or --user + if (token === '-u' || token === '--user') { + const authStr = tokens[++i] ?? ''; + const colonIdx = authStr.indexOf(':'); + if (colonIdx > 0) { + basicAuth = { + user: authStr.slice(0, colonIdx), + password: authStr.slice(colonIdx + 1), + }; + } + i++; + continue; + } + + // Cookies: -b or --cookie + if (token === '-b' || token === '--cookie') { + cookies = tokens[++i] ?? ''; + i++; + continue; + } + + // Skip flags we don't care about that take a value + if ( + token === '-o' || + token === '--output' || + token === '-A' || + token === '--user-agent' || + token === '--connect-timeout' || + token === '--max-time' || + token === '-e' || + token === '--referer' + ) { + i += 2; // skip flag + value + continue; + } + + // Skip boolean flags + if ( + token === '-s' || + token === '--silent' || + token === '-S' || + token === '--show-error' || + token === '-v' || + token === '--verbose' || + token === '-k' || + token === '--insecure' || + token === '-L' || + token === '--location' || + token === '-i' || + token === '--include' || + token === '--compressed' + ) { + i++; + continue; + } + + // Skip unknown flags (single dash + letter combos like -sS) + if (token.startsWith('-') && !token.startsWith('http')) { + // If it looks like a combined short flag (e.g. -sS), skip it + if (/^-[a-zA-Z]+$/.test(token)) { + i++; + continue; + } + // Unknown long flag — might take a value, skip conservatively + i++; + continue; + } + + // Anything else that looks like a URL + if (token.startsWith('http://') || token.startsWith('https://') || token.includes('://')) { + url = token; + } else if (!url && token.includes('/') && !token.startsWith('-')) { + // Relative URL or domain/path + url = token; + } + + i++; + } + + // Infer method from body presence + if (!method) { + method = body ? 'POST' : 'GET'; + } + + return { method, url, headers, body, basicAuth, cookies }; +} + +/** + * Tokenize a shell-like string, respecting single and double quotes. + * Strips surrounding quotes from tokens. + */ +function tokenize(input: string): string[] { + const tokens: string[] = []; + let i = 0; + const len = input.length; + + while (i < len) { + // Skip whitespace + while (i < len && /\s/.test(input[i]!)) i++; + if (i >= len) break; + + const ch = input[i]!; + + if (ch === '"' || ch === "'") { + // Quoted string + const quote = ch; + i++; // skip opening quote + let token = ''; + while (i < len && input[i] !== quote) { + if (input[i] === '\\' && quote === '"' && i + 1 < len) { + // Escape handling for double quotes + i++; + token += input[i]; + } else { + token += input[i]; + } + i++; + } + if (i < len) i++; // skip closing quote + tokens.push(token); + } else { + // Unquoted token + let token = ''; + while (i < len && !/\s/.test(input[i]!)) { + token += input[i]; + i++; + } + tokens.push(token); + } + } + + return tokens; +} + +/** Derive the common base URL from a set of parsed cURL commands. */ +function deriveBaseUrl(curls: ParsedCurl[]): string { + const urls = curls.map((c) => c.url).filter(Boolean); + if (urls.length === 0) return ''; + + try { + const parsedUrls = urls.map((u) => new URL(u)); + // Use the origin of the first URL + const origin = parsedUrls[0]!.origin; + + // Check if all share the same origin + const allSameOrigin = parsedUrls.every((u) => u.origin === origin); + if (allSameOrigin) return origin; + + // Fall back to first URL's origin + return origin; + } catch { + // If URLs can't be parsed (relative, etc.), try to extract protocol+host manually + const match = urls[0]!.match(/^(https?:\/\/[^/]+)/); + return match?.[1] ?? ''; + } +} + +/** Extract path and query parameters from a URL, relative to a base URL. */ +function extractPath( + rawUrl: string, + baseUrl: string, +): { path: string; queryParams: URLSearchParams } { + if (!rawUrl) return { path: '/', queryParams: new URLSearchParams() }; + + try { + const parsed = new URL(rawUrl); + let path = parsed.pathname || '/'; + + // Normalize path: remove trailing slash except for root + if (path.length > 1 && path.endsWith('/')) { + path = path.slice(0, -1); + } + + return { path, queryParams: parsed.searchParams }; + } catch { + // Relative URL — strip base if present + let path = rawUrl; + if (baseUrl && path.startsWith(baseUrl)) { + path = path.slice(baseUrl.length); + } + if (!path.startsWith('/')) path = '/' + path; + + // Split query string + const qIdx = path.indexOf('?'); + if (qIdx >= 0) { + const query = new URLSearchParams(path.slice(qIdx + 1)); + return { path: path.slice(0, qIdx), queryParams: query }; + } + + return { path, queryParams: new URLSearchParams() }; + } +} + +/** Build an OpenAPI request body from raw body text and content type. */ +function buildRequestBody(body: string, contentType?: string): OpenAPIV3.RequestBodyObject { + const isJson = contentType?.includes('application/json') || looksLikeJson(body); + + if (isJson) { + return { + content: { + 'application/json': { + schema: inferJsonSchema(body), + }, + }, + }; + } + + if (contentType?.includes('application/x-www-form-urlencoded')) { + const params = new URLSearchParams(body); + const properties: Record = {}; + for (const [key] of params) { + properties[key] = { type: 'string' }; + } + return { + content: { + 'application/x-www-form-urlencoded': { + schema: { type: 'object', properties }, + }, + }, + }; + } + + // Default: text body + return { + content: { + [contentType ?? 'text/plain']: { + schema: { type: 'string' }, + }, + }, + }; +} + +/** Generate a unique operation ID from method + path. */ +function generateOperationId(method: string, path: string): string { + const segments = path + .split('/') + .filter(Boolean) + .map((s) => { + if (s.startsWith('{') && s.endsWith('}')) { + return `by_${s.slice(1, -1)}`; + } + return s.replace(/[^a-zA-Z0-9]/g, '_'); + }); + + return `${method}_${segments.join('_')}`; +} + +/** Find a header value by name (case-insensitive). */ +function findHeaderValue(headers: Record, name: string): string | undefined { + const lower = name.toLowerCase(); + for (const [key, value] of Object.entries(headers)) { + if (key.toLowerCase() === lower) return value; + } + return undefined; +} + +/** Check if a string looks like JSON. */ +function looksLikeJson(str: string | undefined): boolean { + if (!str) return false; + const trimmed = str.trim(); + return ( + (trimmed.startsWith('{') && trimmed.endsWith('}')) || + (trimmed.startsWith('[') && trimmed.endsWith(']')) + ); +} + +/** Infer an OpenAPI schema from a JSON example string. */ +function inferJsonSchema(raw: string | undefined): OpenAPIV3.SchemaObject { + if (!raw) return { type: 'object' }; + + try { + const parsed: unknown = JSON.parse(raw.trim()); + return inferSchemaFromValue(parsed); + } catch { + return { type: 'object' }; + } +} + +/** Recursively infer an OpenAPI schema from a JavaScript value. */ +function inferSchemaFromValue(value: unknown): OpenAPIV3.SchemaObject { + if (value === null || value === undefined) return {}; + if (typeof value === 'string') return { type: 'string' }; + if (typeof value === 'number') { + return Number.isInteger(value) ? { type: 'integer' } : { type: 'number' }; + } + if (typeof value === 'boolean') return { type: 'boolean' }; + + if (Array.isArray(value)) { + const items = value.length > 0 ? inferSchemaFromValue(value[0]) : {}; + return { type: 'array', items }; + } + + if (typeof value === 'object') { + const properties: Record = {}; + for (const [k, v] of Object.entries(value as Record)) { + properties[k] = inferSchemaFromValue(v); + } + return { type: 'object', properties }; + } + + return {}; +} diff --git a/src/integrations/parsers/doc-parser.ts b/src/integrations/parsers/doc-parser.ts new file mode 100644 index 00000000..4a9bd5c2 --- /dev/null +++ b/src/integrations/parsers/doc-parser.ts @@ -0,0 +1,410 @@ +import type { OpenAPIV3 } from 'openapi-types'; +import { AgentRunner, TOOLS_READ_ONLY } from '../../core/agent-runner.js'; +import { createLogger } from '../../core/logger.js'; + +const logger = createLogger('doc-parser'); + +// ── Extraction prompt ──────────────────────────────────────────────────────── + +const EXTRACTION_PROMPT = `You are an API documentation parser. Extract ALL API endpoints from the documentation below. + +For each endpoint, return a JSON object with these fields: +- method: HTTP method (GET, POST, PUT, PATCH, DELETE) +- path: URL path template (e.g. "/users/{id}") +- summary: Brief description of what the endpoint does +- description: Longer description if available +- parameters: Array of { name, in ("path"|"query"|"header"), required (boolean), type ("string"|"number"|"integer"|"boolean"), description } +- requestBody: { contentType, properties: { [name]: { type, description, required } } } or null +- responseDescription: Brief description of the response +- auth: Authentication requirement if mentioned (e.g. "Bearer token", "API key", "Basic auth") or null +- tags: Array of category/group names if the docs organize endpoints into sections + +IMPORTANT: +- Return ONLY a JSON array of endpoint objects. No markdown, no explanations. +- If the documentation mentions a base URL, include it as the first element: { "baseUrl": "https://..." } +- Use OpenAPI path parameter syntax: {paramName} not :paramName +- Infer types from examples when not explicitly stated +- If no endpoints are found, return an empty array: [] + +DOCUMENTATION: +`; + +// ── Types ──────────────────────────────────────────────────────────────────── + +/** Endpoint extracted by the AI worker. */ +interface ExtractedEndpoint { + method: string; + path: string; + summary?: string; + description?: string; + parameters?: Array<{ + name: string; + in: 'path' | 'query' | 'header'; + required?: boolean; + type?: string; + description?: string; + }>; + requestBody?: { + contentType?: string; + properties?: Record< + string, + { + type?: string; + description?: string; + required?: boolean; + } + >; + } | null; + responseDescription?: string; + auth?: string | null; + tags?: string[]; +} + +/** Base URL element that may appear as the first element in the AI output. */ +interface BaseUrlEntry { + baseUrl: string; +} + +// ── Public API ─────────────────────────────────────────────────────────────── + +/** + * AI-powered API endpoint extraction from arbitrary documentation. + * + * Spawns a `read-only` worker with the documentation content and a structured + * extraction prompt. The worker analyses the text (plain text, Markdown, HTML, + * or pre-extracted PDF text) and returns a JSON array of endpoints. This + * function parses that output into an OpenAPI 3.0 document. + * + * @param docContent Raw documentation text (Markdown, HTML, plain text, etc.) + * @param workspacePath Working directory for the AI worker (defaults to cwd) + * @returns OpenAPI 3.0 document derived from the documentation + * @throws Error if the AI worker fails or returns no parseable endpoints + */ +export async function docsToOpenAPI( + docContent: string, + workspacePath?: string, +): Promise { + if (!docContent.trim()) { + throw new Error('Empty documentation content provided'); + } + + // Truncate extremely long docs to avoid exceeding context limits + const maxDocLength = 100_000; + const truncated = + docContent.length > maxDocLength + ? docContent.slice(0, maxDocLength) + '\n\n[... documentation truncated ...]' + : docContent; + + const prompt = EXTRACTION_PROMPT + truncated; + + const runner = new AgentRunner(); + const result = await runner.spawn({ + prompt, + workspacePath: workspacePath ?? process.cwd(), + model: 'sonnet', + allowedTools: [...TOOLS_READ_ONLY], + maxTurns: 1, + timeout: 60_000, + retries: 1, + retryDelay: 5_000, + }); + + if (result.exitCode !== 0) { + logger.error( + { exitCode: result.exitCode, stderr: result.stderr.slice(0, 500) }, + 'AI worker failed to extract API endpoints from documentation', + ); + throw new Error( + `AI worker exited with code ${result.exitCode}: ${result.stderr.slice(0, 200)}`, + ); + } + + const output = result.stdout.trim(); + if (!output) { + throw new Error('AI worker returned empty output — no endpoints extracted'); + } + + // Parse the JSON array from the worker output + const endpoints = parseWorkerOutput(output); + + if (endpoints.length === 0) { + logger.warn('AI worker found no API endpoints in the documentation'); + } + + const spec = endpointsToOpenAPI(endpoints); + + logger.info( + { pathCount: Object.keys(spec.paths ?? {}).length }, + 'Documentation parsed into OpenAPI spec', + ); + + return spec; +} + +// ── Internal helpers ───────────────────────────────────────────────────────── + +/** + * Parse the AI worker's stdout into an array of extracted endpoints. + * Handles JSON wrapped in markdown code fences or with surrounding text. + */ +function parseWorkerOutput(output: string): Array { + // Try direct JSON parse first + const directResult = tryParseJson(output); + if (directResult) return directResult; + + // Try extracting from markdown code fences + const fenceMatch = output.match(/```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/); + if (fenceMatch?.[1]) { + const fenceResult = tryParseJson(fenceMatch[1]); + if (fenceResult) return fenceResult; + } + + // Try finding the first JSON array in the output + const arrayMatch = output.match(/\[[\s\S]*\]/); + if (arrayMatch) { + const arrayResult = tryParseJson(arrayMatch[0]); + if (arrayResult) return arrayResult; + } + + logger.warn({ outputSnippet: output.slice(0, 300) }, 'Could not parse worker output as JSON'); + return []; +} + +/** Attempt to parse a string as a JSON array of endpoints. */ +function tryParseJson(str: string): Array | null { + try { + const parsed: unknown = JSON.parse(str.trim()); + if (Array.isArray(parsed)) { + return parsed as Array; + } + return null; + } catch { + return null; + } +} + +/** Convert extracted endpoints into an OpenAPI 3.0 document. */ +function endpointsToOpenAPI(entries: Array): OpenAPIV3.Document { + const spec: OpenAPIV3.Document = { + openapi: '3.0.3', + info: { + title: 'Extracted from documentation', + version: '1.0.0', + }, + paths: {}, + }; + + const tags = new Set(); + let hasBearerAuth = false; + let hasBasicAuth = false; + let hasApiKey = false; + + for (const entry of entries) { + // Handle base URL entry + if ('baseUrl' in entry && typeof entry.baseUrl === 'string') { + spec.servers = [{ url: entry.baseUrl }]; + continue; + } + + const endpoint = entry as ExtractedEndpoint; + if (!endpoint.method || !endpoint.path) continue; + + const method = endpoint.method.toLowerCase(); + const path = normalizePath(endpoint.path); + + const paths = spec.paths as Record>; + if (!paths[path]) paths[path] = {}; + const pathItem = paths[path]; + + // Skip if method already exists on this path + if (pathItem[method]) continue; + + const operation: OpenAPIV3.OperationObject = { + summary: endpoint.summary ?? `${endpoint.method} ${path}`, + responses: { + '200': { + description: endpoint.responseDescription ?? 'Successful response', + }, + }, + }; + + if (endpoint.description) { + operation.description = endpoint.description; + } + + operation.operationId = generateOperationId(method, path); + + // Tags + if (endpoint.tags && endpoint.tags.length > 0) { + operation.tags = endpoint.tags; + for (const tag of endpoint.tags) tags.add(tag); + } + + // Parameters + const parameters: OpenAPIV3.ParameterObject[] = []; + + // Extract path parameters from the path template + const pathVarMatches = path.matchAll(/\{(\w+)\}/g); + const declaredPathParams = new Set(); + for (const m of pathVarMatches) { + declaredPathParams.add(m[1]!); + } + + if (endpoint.parameters) { + for (const param of endpoint.parameters) { + parameters.push({ + name: param.name, + in: param.in, + required: param.in === 'path' ? true : (param.required ?? false), + schema: mapTypeToSchema(param.type), + ...(param.description ? { description: param.description } : {}), + }); + if (param.in === 'path') declaredPathParams.delete(param.name); + } + } + + // Add any path params from the template that weren't declared + for (const paramName of declaredPathParams) { + parameters.push({ + name: paramName, + in: 'path', + required: true, + schema: { type: 'string' }, + }); + } + + if (parameters.length > 0) { + operation.parameters = parameters; + } + + // Request body + if ( + endpoint.requestBody?.properties && + Object.keys(endpoint.requestBody.properties).length > 0 + ) { + const contentType = endpoint.requestBody.contentType ?? 'application/json'; + const properties: Record = {}; + const required: string[] = []; + + for (const [name, prop] of Object.entries(endpoint.requestBody.properties)) { + properties[name] = { + ...mapTypeToSchema(prop.type), + ...(prop.description ? { description: prop.description } : {}), + }; + if (prop.required) required.push(name); + } + + operation.requestBody = { + content: { + [contentType]: { + schema: { + type: 'object', + properties, + ...(required.length > 0 ? { required } : {}), + }, + }, + }, + }; + } + + // Auth detection + if (endpoint.auth) { + const authLower = endpoint.auth.toLowerCase(); + const security: Record[] = []; + + if (authLower.includes('bearer')) { + hasBearerAuth = true; + security.push({ bearerAuth: [] }); + } else if (authLower.includes('basic')) { + hasBasicAuth = true; + security.push({ basicAuth: [] }); + } else if (authLower.includes('api') && authLower.includes('key')) { + hasApiKey = true; + security.push({ apiKeyAuth: [] }); + } + + if (security.length > 0) { + operation.security = security; + } + } + + pathItem[method] = operation; + } + + // Add tags + if (tags.size > 0) { + spec.tags = [...tags].map((name) => ({ name })); + } + + // Add security schemes + const securitySchemes: Record = {}; + if (hasBearerAuth) { + securitySchemes['bearerAuth'] = { type: 'http', scheme: 'bearer' }; + } + if (hasBasicAuth) { + securitySchemes['basicAuth'] = { type: 'http', scheme: 'basic' }; + } + if (hasApiKey) { + securitySchemes['apiKeyAuth'] = { type: 'apiKey', name: 'X-API-Key', in: 'header' }; + } + if (Object.keys(securitySchemes).length > 0) { + spec.components = { securitySchemes }; + } + + return spec; +} + +/** Normalize a path to ensure it starts with / and uses {param} syntax. */ +function normalizePath(path: string): string { + let normalized = path.trim(); + + // Convert :param to {param} + normalized = normalized.replace(/:(\w+)/g, '{$1}'); + + // Ensure leading slash + if (!normalized.startsWith('/')) { + // Strip protocol+host if present + const withoutProtocol = normalized.replace(/^https?:\/\/[^/]*/, ''); + normalized = withoutProtocol.startsWith('/') ? withoutProtocol : '/' + withoutProtocol; + } + + // Remove trailing slash (except root) + if (normalized.length > 1 && normalized.endsWith('/')) { + normalized = normalized.slice(0, -1); + } + + // Remove query string if present + const qIdx = normalized.indexOf('?'); + if (qIdx >= 0) { + normalized = normalized.slice(0, qIdx); + } + + return normalized; +} + +/** Map a loose type string to an OpenAPI SchemaObject. */ +function mapTypeToSchema(type: string | undefined): OpenAPIV3.SchemaObject { + if (!type) return { type: 'string' }; + const lower = type.toLowerCase(); + if (lower === 'integer' || lower === 'int') return { type: 'integer' }; + if (lower === 'number' || lower === 'float' || lower === 'double') return { type: 'number' }; + if (lower === 'boolean' || lower === 'bool') return { type: 'boolean' }; + if (lower === 'array') return { type: 'array', items: {} }; + if (lower === 'object') return { type: 'object' }; + return { type: 'string' }; +} + +/** Generate a unique operation ID from method + path. */ +function generateOperationId(method: string, path: string): string { + const segments = path + .split('/') + .filter(Boolean) + .map((s) => { + if (s.startsWith('{') && s.endsWith('}')) { + return `by_${s.slice(1, -1)}`; + } + return s.replace(/[^a-zA-Z0-9]/g, '_'); + }); + + return `${method}_${segments.join('_')}`; +} diff --git a/src/integrations/parsers/postman-parser.ts b/src/integrations/parsers/postman-parser.ts new file mode 100644 index 00000000..7bb68202 --- /dev/null +++ b/src/integrations/parsers/postman-parser.ts @@ -0,0 +1,597 @@ +import type { OpenAPIV3 } from 'openapi-types'; +import { createLogger } from '../../core/logger.js'; + +const logger = createLogger('postman-parser'); + +// ── Postman Collection v2.1 types ──────────────────────────────────────────── + +/** Postman key-value pair (used for headers, query params, form data). */ +interface PostmanKeyValue { + key: string; + value?: string; + description?: string; + disabled?: boolean; + type?: string; +} + +/** Postman URL object. */ +interface PostmanUrl { + raw?: string; + protocol?: string; + host?: string[]; + path?: string[]; + query?: PostmanKeyValue[]; + variable?: PostmanKeyValue[]; +} + +/** Postman request body. */ +interface PostmanBody { + mode?: 'raw' | 'urlencoded' | 'formdata' | 'file' | 'graphql'; + raw?: string; + urlencoded?: PostmanKeyValue[]; + formdata?: PostmanKeyValue[]; + options?: { raw?: { language?: string } }; +} + +/** Postman request object. */ +interface PostmanRequest { + method?: string; + header?: PostmanKeyValue[]; + body?: PostmanBody; + url?: PostmanUrl | string; + description?: string; + auth?: PostmanAuth; +} + +/** Postman response example. */ +interface PostmanResponse { + name?: string; + status?: string; + code?: number; + header?: PostmanKeyValue[]; + body?: string; + _postman_previewlanguage?: string; +} + +/** Postman auth definition. */ +interface PostmanAuth { + type?: string; + bearer?: PostmanKeyValue[]; + apikey?: PostmanKeyValue[]; + basic?: PostmanKeyValue[]; +} + +/** Postman collection item (request or folder). */ +interface PostmanItem { + name?: string; + description?: string; + request?: PostmanRequest; + response?: PostmanResponse[]; + item?: PostmanItem[]; + auth?: PostmanAuth; +} + +/** Postman variable definition. */ +interface PostmanVariable { + key?: string; + value?: string; + description?: string; + type?: string; +} + +/** Postman Collection v2.1 top-level structure. */ +export interface PostmanCollection { + info?: { + name?: string; + description?: string; + schema?: string; + version?: string; + }; + item?: PostmanItem[]; + variable?: PostmanVariable[]; + auth?: PostmanAuth; +} + +/** Variable values provided by the user for {{placeholder}} substitution. */ +export type VariableMap = Record; + +// ── Public API ─────────────────────────────────────────────────────────────── + +/** + * Convert a Postman Collection v2.1 JSON object to an OpenAPI 3.0 specification. + * + * - Extracts requests (method, URL, headers, body) from items and nested folders. + * - Maps folder structure to OpenAPI tags. + * - Converts auth settings to OpenAPI security schemes. + * - Converts example responses to OpenAPI response definitions. + * - Replaces `{{variable}}` placeholders with values from `variables` map; + * unresolved variables are left as-is in the path (e.g. `{baseUrl}`). + * + * @param collection Parsed Postman Collection v2.1 JSON + * @param variables Optional map of variable values for `{{placeholder}}` substitution + * @returns OpenAPI 3.0 document ready for swagger-parser validation + */ +export function postmanToOpenAPI( + collection: PostmanCollection, + variables?: VariableMap, +): OpenAPIV3.Document { + // Build variable map from collection-level variables + user overrides + const vars: VariableMap = {}; + if (collection.variable) { + for (const v of collection.variable) { + if (v.key) { + vars[v.key] = v.value ?? ''; + } + } + } + if (variables) { + Object.assign(vars, variables); + } + + const title = collection.info?.name ?? 'Converted from Postman'; + const description = collection.info?.description ?? ''; + + const spec: OpenAPIV3.Document = { + openapi: '3.0.3', + info: { + title, + description, + version: collection.info?.version ?? '1.0.0', + }, + paths: {}, + }; + + // Collect tags from folder structure + const tags = new Set(); + + // Process items recursively + const items = collection.item ?? []; + for (const item of items) { + processItem(item, [], spec, vars, tags, collection.auth); + } + + // Add tags to spec + if (tags.size > 0) { + spec.tags = [...tags].map((name) => ({ name })); + } + + // Add security schemes from collection-level auth + if (collection.auth) { + const scheme = convertAuth(collection.auth); + if (scheme) { + spec.components = { + securitySchemes: { [scheme.name]: scheme.scheme }, + }; + spec.security = [{ [scheme.name]: [] }]; + } + } + + // Derive server URL from variables (e.g. {{baseUrl}}) + if (vars['baseUrl']) { + spec.servers = [{ url: vars['baseUrl'] }]; + } + + logger.info( + { title, pathCount: Object.keys(spec.paths ?? {}).length, tagCount: tags.size }, + 'Postman collection converted to OpenAPI', + ); + + return spec; +} + +// ── Internal helpers ───────────────────────────────────────────────────────── + +/** + * Recursively process a Postman item (folder or request). + * Folders become tags; requests become path operations. + */ +function processItem( + item: PostmanItem, + folderPath: string[], + spec: OpenAPIV3.Document, + vars: VariableMap, + tags: Set, + collectionAuth?: PostmanAuth, +): void { + // Folder: has nested items + if (item.item && item.item.length > 0) { + const folderName = item.name ?? 'default'; + tags.add(folderName); + for (const child of item.item) { + processItem(child, [...folderPath, folderName], spec, vars, tags, collectionAuth); + } + return; + } + + // Request item + if (!item.request) return; + + const request = item.request; + const method = (request.method ?? 'GET').toLowerCase(); + const url = resolveUrl(request.url, vars); + const path = url.path; + + if (!path) return; + + // Initialize path item if needed + if (!spec.paths) spec.paths = {}; + if (!spec.paths[path]) spec.paths[path] = {}; + + const pathItem = spec.paths[path] as OpenAPIV3.PathItemObject; + + // Build the operation + const operation: OpenAPIV3.OperationObject = { + summary: item.name ?? `${method.toUpperCase()} ${path}`, + responses: {}, + }; + + // Description + const desc = item.description ?? request.description; + if (desc) { + operation.description = typeof desc === 'string' ? desc : ''; + } + + // Tags from folder hierarchy + if (folderPath.length > 0) { + operation.tags = [folderPath[folderPath.length - 1]!]; + } + + // Generate operationId + operation.operationId = generateOperationId(method, path); + + // Parameters: path variables + const parameters: OpenAPIV3.ParameterObject[] = []; + for (const v of url.pathVariables) { + parameters.push({ + name: v.key, + in: 'path', + required: true, + schema: { type: 'string' }, + ...(v.description ? { description: v.description } : {}), + }); + } + + // Parameters: query params + for (const q of url.queryParams) { + if (q.disabled) continue; + parameters.push({ + name: q.key, + in: 'query', + schema: { type: 'string' }, + ...(q.description ? { description: q.description } : {}), + }); + } + + // Parameters: headers (skip common ones) + const skipHeaders = new Set([ + 'content-type', + 'accept', + 'authorization', + 'user-agent', + 'host', + 'connection', + 'cache-control', + ]); + if (request.header) { + for (const h of request.header) { + if (h.disabled) continue; + if (skipHeaders.has(h.key.toLowerCase())) continue; + parameters.push({ + name: h.key, + in: 'header', + schema: { type: 'string' }, + ...(h.description ? { description: h.description } : {}), + }); + } + } + + if (parameters.length > 0) { + operation.parameters = parameters; + } + + // Request body + if (request.body && ['post', 'put', 'patch'].includes(method)) { + operation.requestBody = convertRequestBody(request.body); + } + + // Responses from examples + if (item.response && item.response.length > 0) { + operation.responses = convertResponses(item.response); + } else { + operation.responses = { + '200': { description: 'Successful response' }, + }; + } + + // Auth at item level + const auth = request.auth ?? item.auth; + if (auth && auth !== collectionAuth) { + const scheme = convertAuth(auth); + if (scheme) { + if (!spec.components) spec.components = {}; + if (!spec.components.securitySchemes) spec.components.securitySchemes = {}; + spec.components.securitySchemes[scheme.name] = scheme.scheme; + operation.security = [{ [scheme.name]: [] }]; + } + } + + // Set the operation on the path item + (pathItem as Record)[method] = operation; +} + +/** Resolved URL with path template, path variables, and query params. */ +interface ResolvedUrl { + path: string; + pathVariables: PostmanKeyValue[]; + queryParams: PostmanKeyValue[]; +} + +/** + * Resolve a Postman URL (string or object) into an OpenAPI path template. + * Replaces `{{var}}` with variable values or converts to `{var}` path params. + * Strips protocol and host to produce a path-only template. + */ +function resolveUrl(url: PostmanUrl | string | undefined, vars: VariableMap): ResolvedUrl { + if (!url) return { path: '/', pathVariables: [], queryParams: [] }; + + if (typeof url === 'string') { + const resolved = substituteVariables(url, vars); + const pathOnly = extractPathFromUrl(resolved); + return { path: pathOnly || '/', pathVariables: [], queryParams: [] }; + } + + // Object URL — build path from parts + const pathSegments = url.path ?? []; + const resolvedSegments = pathSegments.map((seg) => { + // Path variable like :id → {id} + if (seg.startsWith(':')) { + return `{${seg.slice(1)}}`; + } + return substituteVariables(seg, vars); + }); + + let path = '/' + resolvedSegments.join('/'); + // Clean up double slashes + path = path.replace(/\/+/g, '/'); + // Remove trailing slash (except root) + if (path.length > 1 && path.endsWith('/')) { + path = path.slice(0, -1); + } + + const pathVariables: PostmanKeyValue[] = url.variable ?? []; + const queryParams: PostmanKeyValue[] = url.query ?? []; + + return { path, pathVariables, queryParams }; +} + +/** Replace `{{variable}}` placeholders with values from the variable map. */ +function substituteVariables(input: string, vars: VariableMap): string { + return input.replace(/\{\{(\w+)\}\}/g, (_match, name: string) => { + if (vars[name] !== undefined) { + return vars[name]; + } + // Leave as OpenAPI path parameter syntax + return `{${name}}`; + }); +} + +/** Extract the path portion from a full URL string. */ +function extractPathFromUrl(url: string): string { + // Handle {{baseUrl}}/path → already substituted or left as {baseUrl}/path + // Remove protocol + host + const withoutProtocol = url.replace(/^https?:\/\/[^/]*/, ''); + if (withoutProtocol.startsWith('/') || withoutProtocol === '') { + return withoutProtocol || '/'; + } + // Might be a relative path starting with {baseUrl} + // Strip the first segment if it looks like a variable + if (withoutProtocol.startsWith('{')) { + const slashIdx = withoutProtocol.indexOf('/'); + if (slashIdx >= 0) { + return withoutProtocol.slice(slashIdx); + } + return '/'; + } + return '/' + withoutProtocol; +} + +/** Convert a Postman request body to an OpenAPI RequestBody object. */ +function convertRequestBody(body: PostmanBody): OpenAPIV3.RequestBodyObject { + switch (body.mode) { + case 'raw': { + const isJson = body.options?.raw?.language === 'json' || looksLikeJson(body.raw); + const mediaType = isJson ? 'application/json' : 'text/plain'; + + const schema = isJson ? inferJsonSchema(body.raw) : { type: 'string' as const }; + + return { + content: { + [mediaType]: { schema }, + }, + }; + } + + case 'urlencoded': { + const properties: Record = {}; + for (const kv of body.urlencoded ?? []) { + if (kv.disabled) continue; + properties[kv.key] = { + type: 'string', + ...(kv.description ? { description: kv.description } : {}), + }; + } + return { + content: { + 'application/x-www-form-urlencoded': { + schema: { type: 'object', properties }, + }, + }, + }; + } + + case 'formdata': { + const properties: Record = {}; + for (const kv of body.formdata ?? []) { + if (kv.disabled) continue; + if (kv.type === 'file') { + properties[kv.key] = { type: 'string', format: 'binary' }; + } else { + properties[kv.key] = { + type: 'string', + ...(kv.description ? { description: kv.description } : {}), + }; + } + } + return { + content: { + 'multipart/form-data': { + schema: { type: 'object', properties }, + }, + }, + }; + } + + default: + return { + content: { + 'application/octet-stream': { + schema: { type: 'string', format: 'binary' }, + }, + }, + }; + } +} + +/** Convert Postman example responses to OpenAPI responses. */ +function convertResponses(responses: PostmanResponse[]): Record { + const result: Record = {}; + + for (const resp of responses) { + const code = String(resp.code ?? 200); + const description = resp.name ?? resp.status ?? 'Response'; + + const responseObj: OpenAPIV3.ResponseObject = { description }; + + if (resp.body) { + const isJson = resp._postman_previewlanguage === 'json' || looksLikeJson(resp.body); + const mediaType = isJson ? 'application/json' : 'text/plain'; + const schema = isJson ? inferJsonSchema(resp.body) : { type: 'string' as const }; + + responseObj.content = { [mediaType]: { schema } }; + } + + // Don't overwrite if we already have a response for this code + // (first example wins) + if (!result[code]) { + result[code] = responseObj; + } + } + + return result; +} + +/** Convert Postman auth to an OpenAPI security scheme. */ +function convertAuth( + auth: PostmanAuth, +): { name: string; scheme: OpenAPIV3.SecuritySchemeObject } | null { + switch (auth.type) { + case 'bearer': + return { + name: 'bearerAuth', + scheme: { + type: 'http', + scheme: 'bearer', + }, + }; + + case 'basic': + return { + name: 'basicAuth', + scheme: { + type: 'http', + scheme: 'basic', + }, + }; + + case 'apikey': { + const keyEntry = auth.apikey?.find((k) => k.key === 'key'); + const inEntry = auth.apikey?.find((k) => k.key === 'in'); + return { + name: 'apiKeyAuth', + scheme: { + type: 'apiKey', + name: keyEntry?.value ?? 'api_key', + in: (inEntry?.value as 'header' | 'query') ?? 'header', + }, + }; + } + + default: + return null; + } +} + +/** Generate a unique operation ID from method + path. */ +function generateOperationId(method: string, path: string): string { + const segments = path + .split('/') + .filter(Boolean) + .map((s) => { + if (s.startsWith('{') && s.endsWith('}')) { + return `by_${s.slice(1, -1)}`; + } + return s.replace(/[^a-zA-Z0-9]/g, '_'); + }); + + return `${method}_${segments.join('_')}`; +} + +/** Check if a string looks like JSON. */ +function looksLikeJson(str: string | undefined): boolean { + if (!str) return false; + const trimmed = str.trim(); + return ( + (trimmed.startsWith('{') && trimmed.endsWith('}')) || + (trimmed.startsWith('[') && trimmed.endsWith(']')) + ); +} + +/** Infer an OpenAPI schema from a JSON example string. */ +function inferJsonSchema(raw: string | undefined): OpenAPIV3.SchemaObject { + if (!raw) return { type: 'object' }; + + try { + const parsed: unknown = JSON.parse(raw.trim()); + return inferSchemaFromValue(parsed); + } catch { + return { type: 'object' }; + } +} + +/** Recursively infer an OpenAPI schema from a JavaScript value. */ +function inferSchemaFromValue(value: unknown): OpenAPIV3.SchemaObject { + if (value === null || value === undefined) { + return {}; + } + + if (typeof value === 'string') return { type: 'string' }; + if (typeof value === 'number') { + return Number.isInteger(value) ? { type: 'integer' } : { type: 'number' }; + } + if (typeof value === 'boolean') return { type: 'boolean' }; + + if (Array.isArray(value)) { + const items = value.length > 0 ? inferSchemaFromValue(value[0]) : {}; + return { type: 'array', items }; + } + + if (typeof value === 'object') { + const properties: Record = {}; + for (const [k, v] of Object.entries(value as Record)) { + properties[k] = inferSchemaFromValue(v); + } + return { type: 'object', properties }; + } + + return {}; +} diff --git a/src/integrations/skill-pack-generator.ts b/src/integrations/skill-pack-generator.ts new file mode 100644 index 00000000..9a341853 --- /dev/null +++ b/src/integrations/skill-pack-generator.ts @@ -0,0 +1,238 @@ +import path from 'path'; +import fs from 'fs/promises'; +import type { OpenAPI, OpenAPIV3 } from 'openapi-types'; + +import { createLogger } from '../core/logger.js'; +import type { AgentRunner } from '../core/agent-runner.js'; +import { TOOLS_READ_ONLY } from '../core/agent-runner.js'; +import { parseSkillPackMd, skillPackToMarkdown } from '../master/skill-pack-loader.js'; +import { SkillPackSchema } from '../types/agent.js'; + +const logger = createLogger('skill-pack-generator'); + +// ── Helpers ────────────────────────────────────────────────────────────────── + +/** + * Extract a short API name from an OpenAPI document. + * Falls back to 'api' if no title is found. + */ +function extractApiName(spec: OpenAPI.Document): string { + const doc = spec as OpenAPIV3.Document; + const title = doc.info?.title; + if (!title) return 'api'; + return ( + title + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, '') + .slice(0, 40) || 'api' + ); +} + +/** + * Summarise an OpenAPI spec into a compact text representation suitable for + * inclusion in an AI prompt. Keeps only the essentials: title, base URL, + * auth schemes, and endpoint list with parameters. + */ +function summariseSpec(spec: OpenAPI.Document): string { + const doc = spec as OpenAPIV3.Document; + const lines: string[] = []; + + // Title + version + if (doc.info) { + lines.push(`API: ${doc.info.title ?? 'Unknown'} v${doc.info.version ?? '?'}`); + if (doc.info.description) { + lines.push(`Description: ${doc.info.description.slice(0, 300)}`); + } + } + + // Base URL + if (doc.servers && doc.servers.length > 0) { + lines.push(`Base URL: ${doc.servers[0]!.url}`); + } + + // Auth schemes + const securitySchemes = doc.components?.securitySchemes; + if (securitySchemes) { + const schemes = Object.entries(securitySchemes) + .map(([name, s]) => { + const scheme = s as OpenAPIV3.SecuritySchemeObject; + return `${name} (${scheme.type})`; + }) + .join(', '); + lines.push(`Auth: ${schemes}`); + } + + // Endpoints (method + path + summary) + lines.push(''); + lines.push('Endpoints:'); + const paths = doc.paths ?? {}; + let endpointCount = 0; + for (const [urlPath, pathItem] of Object.entries(paths)) { + if (!pathItem) continue; + const methods = ['get', 'post', 'put', 'patch', 'delete'] as const; + for (const method of methods) { + const operation = (pathItem as Record)[method] as + | OpenAPIV3.OperationObject + | undefined; + if (!operation) continue; + endpointCount++; + const summary = operation.summary ?? operation.description?.slice(0, 80) ?? ''; + lines.push(` ${method.toUpperCase()} ${urlPath}${summary ? ' — ' + summary : ''}`); + } + } + + if (endpointCount === 0) { + lines.push(' (no endpoints found)'); + } + + return lines.join('\n'); +} + +// ── Generation prompt ──────────────────────────────────────────────────────── + +function buildGenerationPrompt(specSummary: string, context?: string): string { + const contextSection = context ? `\n\nAdditional context from the user:\n${context}\n` : ''; + + return `You are a skill pack author for OpenBridge, an AI bridge system that connects messaging channels to AI agents. Your job is to create a SKILLPACK.md that teaches the Master AI how to use this API naturally through conversation. + +Here is the API specification: + +${specSummary}${contextSection} + +Create a skill pack (.md) that teaches the Master AI how to use this API naturally. Include: +1. Capability descriptions — what the API can do, expressed in plain language +2. Natural language command examples — how a user would ask for things via chat (e.g. "list my orders", "create a new customer") +3. Common workflows — multi-step sequences (e.g. "create product → set price → publish") +4. Error handling guidance — what to do when auth fails, rate limits hit, required fields missing +5. Parameter mapping — how to translate conversational requests to API parameters + +The SKILLPACK.md must follow this exact format: + +# + + + +## Tool Profile +full-access + +## When to Use + + +## Required Tools +- Bash(curl:*) + +## Tags +- +- +- api-integration + +## Prompt Extension + + +## Example Tasks +- "" +- "" +- "" + +## Constraints +- Never expose API keys or auth tokens in responses +- Always validate required parameters before making API calls +- Handle rate limiting gracefully with retry guidance + +Rules: +- Name must be a lowercase slug (hyphens only, no spaces, no special characters) +- Name should reflect the API (e.g. "stripe-api", "github-api", "my-crm-api") +- Prompt Extension must be at least 200 words and actionable +- Include at least 5 natural language command examples in the Prompt Extension +- Include at least 2 multi-step workflows in the Prompt Extension + +Respond with ONLY the SKILLPACK.md content. No explanations, no code fences.`; +} + +// ── Main export ────────────────────────────────────────────────────────────── + +/** + * AI-generates a skill pack from a parsed OpenAPI specification. + * + * Spawns a worker with the parsed API spec summary and asks the AI to create + * a SKILLPACK.md that teaches the Master AI how to use the API naturally. + * The generated pack is validated, saved to `.openbridge/skill-packs/{api-name}.md`, + * and can be picked up by the skill-pack-loader on next load. + * + * @param spec - Parsed OpenAPI document (from swagger-parser or any parser). + * @param context - Optional user-provided context about how they use the API. + * @returns Path to the saved skill pack file on success, null on failure. + */ +export async function generateSkillPack( + spec: OpenAPI.Document, + workspacePath: string, + agentRunner: AgentRunner, + context?: string, +): Promise { + const apiName = extractApiName(spec); + logger.info({ apiName }, 'Generating skill pack from API spec'); + + const specSummary = summariseSpec(spec); + const prompt = buildGenerationPrompt(specSummary, context); + + let result; + try { + result = await agentRunner.spawn({ + prompt, + workspacePath, + model: 'sonnet', + allowedTools: [...TOOLS_READ_ONLY], + maxTurns: 3, + retries: 1, + }); + } catch (err) { + logger.warn({ err, apiName }, 'Skill pack generation worker failed'); + return null; + } + + if (result.exitCode !== 0 || !result.stdout.trim()) { + logger.warn( + { apiName, exitCode: result.exitCode }, + 'Skill pack generation worker returned empty or failed output', + ); + return null; + } + + // Strip markdown code fences if the worker wrapped the output + const rawContent = result.stdout.trim(); + const fenceMatch = rawContent.match(/^```(?:\w+)?\n([\s\S]*?)\n```$/); + const content = fenceMatch?.[1] ?? rawContent; + + const partial = parseSkillPackMd(content); + const parsed = SkillPackSchema.safeParse({ ...partial, isUserDefined: true }); + + if (!parsed.success) { + logger.warn( + { apiName, issues: parsed.error.issues }, + 'Generated skill pack failed Zod validation', + ); + return null; + } + + // Persist to .openbridge/skill-packs/.md + const skillPacksDir = path.join(workspacePath, '.openbridge', 'skill-packs'); + const filePath = path.join(skillPacksDir, `${apiName}.md`); + + try { + await fs.mkdir(skillPacksDir, { recursive: true }); + await fs.writeFile(filePath, skillPackToMarkdown(parsed.data), 'utf-8'); + logger.info({ name: parsed.data.name, apiName, file: filePath }, 'Generated skill pack saved'); + } catch (err) { + logger.warn({ err, apiName }, 'Failed to save generated skill pack to disk'); + return null; + } + + return filePath; +} diff --git a/src/integrations/webhook-router.ts b/src/integrations/webhook-router.ts new file mode 100644 index 00000000..3ff285ce --- /dev/null +++ b/src/integrations/webhook-router.ts @@ -0,0 +1,223 @@ +import type { IncomingMessage, ServerResponse } from 'node:http'; +import { createLogger } from '../core/logger.js'; +import type { BusinessIntegration } from '../types/integration.js'; + +const logger = createLogger('webhook-router'); + +export type WebhookHandler = (payload: Record) => Promise; + +/** Registry key for a webhook handler: `/` */ +type RouteKey = string; + +/** Entry stored for a registered route */ +interface RouteEntry { + handler: WebhookHandler; + integration: string; + event: string; +} + +/** + * WebhookRouter — incoming webhook dispatcher for business integration events. + * + * Usage: + * const router = new WebhookRouter(); + * router.registerRoute('stripe', 'payment.succeeded', async (payload) => { ... }); + * // Handle a raw HTTP request (call from FileServer or a standalone HTTP server): + * await router.handleHttpRequest(req, res); + */ +export class WebhookRouter { + private readonly routes = new Map(); + + /** Map from integration name → registered integration (for signature verification) */ + private readonly integrations = new Map(); + + private static routeKey(integration: string, event: string): RouteKey { + return `${integration}/${event}`; + } + + /** + * Register an integration adapter so the router can call `subscribe` and + * verify signatures (if supported). + */ + registerIntegration(integration: BusinessIntegration): void { + this.integrations.set(integration.name, integration); + logger.debug({ integration: integration.name }, 'Integration registered with webhook router'); + } + + /** + * Register a webhook handler for a specific integration/event pair. + * If an integration adapter with a `subscribe` method is already registered, + * this wires it up as well. + */ + registerRoute(integration: string, event: string, handler: WebhookHandler): void { + const key = WebhookRouter.routeKey(integration, event); + this.routes.set(key, { handler, integration, event }); + logger.info({ integration, event }, 'Webhook route registered'); + + // Wire the integration's subscribe() if available + const adapter = this.integrations.get(integration); + if (adapter?.subscribe) { + adapter.subscribe(event, async (eventPayload) => { + await this.dispatch(integration, event, eventPayload); + }); + } + } + + /** + * Deregister all routes for a specific integration and call + * `unregisterWebhook()` on the adapter if present. + */ + async deregisterIntegration(integrationName: string): Promise { + for (const key of this.routes.keys()) { + if (key.startsWith(`${integrationName}/`)) { + this.routes.delete(key); + } + } + + const adapter = this.integrations.get(integrationName); + if (adapter?.unregisterWebhook) { + await adapter.unregisterWebhook().catch((err: unknown) => { + logger.warn({ integration: integrationName, err }, 'unregisterWebhook failed'); + }); + } + + this.integrations.delete(integrationName); + logger.info({ integration: integrationName }, 'Webhook routes deregistered'); + } + + /** + * Dispatch a parsed payload to all registered handlers for the given + * integration/event pair. Logs the event regardless of whether a handler + * exists. + */ + async dispatch( + integration: string, + event: string, + payload: Record, + ): Promise { + const key = WebhookRouter.routeKey(integration, event); + logger.info({ integration, event }, 'Webhook event received'); + + const entry = this.routes.get(key); + if (!entry) { + logger.warn({ integration, event }, 'No handler registered for webhook event — ignored'); + return; + } + + try { + await entry.handler(payload); + logger.debug({ integration, event }, 'Webhook handler completed'); + } catch (err) { + logger.error({ integration, event, err }, 'Webhook handler threw an error'); + throw err; + } + } + + /** + * Verify a webhook signature for a given integration. + * + * This is a best-effort, integration-agnostic check. Concrete adapters that + * need cryptographic verification (e.g. Stripe's `stripe.webhooks.constructEvent`) + * should override this by calling `router.registerSignatureVerifier(name, fn)`. + * + * Returns `true` when no verifier is registered (allow-by-default) so that + * integrations without signatures still work. + */ + verifySignature(integration: string, signature: string, payload: string): boolean { + const verifier = this.signatureVerifiers.get(integration); + if (!verifier) { + return true; // no verifier registered — allow + } + try { + return verifier(signature, payload); + } catch (err) { + logger.warn({ integration, err }, 'Signature verification threw — treating as invalid'); + return false; + } + } + + /** Map from integration name → custom signature verifier function */ + private readonly signatureVerifiers = new Map< + string, + (signature: string, payload: string) => boolean + >(); + + /** Register a custom signature verifier for an integration. */ + registerSignatureVerifier( + integration: string, + verifier: (signature: string, payload: string) => boolean, + ): void { + this.signatureVerifiers.set(integration, verifier); + } + + /** + * Handle an incoming HTTP request for a webhook endpoint. + * + * Expected URL pattern: `POST /webhook/:integration/:event` + * + * Returns `true` when the request was handled (caller should not write a + * further response), `false` when the URL did not match the webhook pattern. + */ + async handleHttpRequest(req: IncomingMessage, res: ServerResponse): Promise { + const url = req.url ?? ''; + const match = /^\/webhook\/([^/]+)\/([^/?]+)/.exec(url); + if (!match) return false; + if (req.method !== 'POST') { + res.writeHead(405, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Method not allowed' })); + return true; + } + + const integration = decodeURIComponent(match[1] ?? ''); + const event = decodeURIComponent(match[2] ?? ''); + + // Collect request body + let rawBody = ''; + try { + rawBody = await new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + req.on('data', (chunk: Buffer) => chunks.push(chunk)); + req.on('end', () => resolve(Buffer.concat(chunks).toString('utf-8'))); + req.on('error', reject); + }); + } catch (err) { + logger.error({ integration, event, err }, 'Failed to read webhook request body'); + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Failed to read request body' })); + return true; + } + + // Signature verification + const signature = (req.headers['x-webhook-signature'] as string | undefined) ?? ''; + if (!this.verifySignature(integration, signature, rawBody)) { + logger.warn({ integration, event }, 'Webhook signature verification failed'); + res.writeHead(401, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid signature' })); + return true; + } + + // Parse payload + let payload: Record; + try { + const parsed: unknown = JSON.parse(rawBody); + payload = + parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed) + ? (parsed as Record) + : { data: parsed }; + } catch { + payload = { raw: rawBody }; + } + + // Dispatch (errors are caught internally and logged; we always return 200) + try { + await this.dispatch(integration, event, payload); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + } catch { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Handler error' })); + } + + return true; + } +} diff --git a/src/integrations/workflow-templates.ts b/src/integrations/workflow-templates.ts new file mode 100644 index 00000000..3d1fd2a9 --- /dev/null +++ b/src/integrations/workflow-templates.ts @@ -0,0 +1,211 @@ +/** + * workflow-templates.ts — Pre-built workflow suggestions from OpenAPI specs (OB-1454). + * + * Analyses a parsed OpenAPI document and returns up to three workflow suggestions: + * 1. Notification on new records — if a POST endpoint exists. + * 2. Daily summary — if a list GET endpoint exists (no path params). + * 3. Status change alerts — if a PATCH/PUT endpoint with a status field exists. + * + * Returned workflows have status 'draft'; they become 'active' once the user approves them. + */ + +import { randomUUID } from 'node:crypto'; +import type { OpenAPI, OpenAPIV3 } from 'openapi-types'; +import type { Workflow, WorkflowTrigger, WorkflowStep } from '../types/workflow.js'; + +// ── Public API ──────────────────────────────────────────────────────────────── + +/** + * Analyse `spec` and produce up to three pre-built workflow suggestions. + * + * The returned `Workflow` objects use status `'draft'` so that they are not + * executed until the user explicitly approves them (via the conversation UI or + * the `/approve-workflow` command). + */ +export function generateDefaultWorkflows(spec: OpenAPI.Document): Workflow[] { + const doc = spec as OpenAPIV3.Document; + const paths = doc.paths ?? {}; + const now = new Date().toISOString(); + + let firstPostPath: string | null = null; + let firstGetListPath: string | null = null; + let statusChangePath: string | null = null; + let statusChangeMethod = ''; + + for (const [urlPath, pathItem] of Object.entries(paths)) { + if (!pathItem) continue; + const pi = pathItem as Record; + + // POST endpoint → candidate for "notify on new record" + if (pi['post'] && !firstPostPath) { + firstPostPath = urlPath; + } + + // Collection GET endpoint (no path param = list endpoint) + if (pi['get'] && !firstGetListPath && !urlPath.includes('{')) { + firstGetListPath = urlPath; + } + + // PATCH/PUT with a status-like field → candidate for "status change alert" + for (const method of ['patch', 'put'] as const) { + const op = pi[method] as OpenAPIV3.OperationObject | undefined; + if (op && !statusChangePath && hasStatusField(op, doc)) { + statusChangePath = urlPath; + statusChangeMethod = method.toUpperCase(); + } + } + } + + const workflows: Workflow[] = []; + + // ── Workflow 1: notification on new records ─────────────────────────────── + if (firstPostPath) { + const trigger: WorkflowTrigger = { + type: 'webhook', + event: 'record.created', + }; + const steps: WorkflowStep[] = [ + { + id: randomUUID(), + name: 'Notify on new record', + type: 'send', + config: { + template: `New record created via POST ${firstPostPath}: {{data}}`, + }, + sort_order: 0, + continue_on_error: false, + }, + ]; + workflows.push({ + id: 'notify-on-new-record', + name: 'Notify on new records', + description: + `Send a notification when a new record is created via POST ${firstPostPath}. ` + + 'Connect a webhook from your API to receive live events.', + trigger, + steps, + status: 'draft', + run_count: 0, + error_count: 0, + created_at: now, + }); + } + + // ── Workflow 2: daily summary ───────────────────────────────────────────── + if (firstGetListPath) { + const trigger: WorkflowTrigger = { + type: 'schedule', + cron: '0 9 * * *', + timezone: 'UTC', + }; + const steps: WorkflowStep[] = [ + { + id: randomUUID(), + name: `Fetch records from ${firstGetListPath}`, + type: 'query', + config: { operation: `GET ${firstGetListPath}` }, + sort_order: 0, + continue_on_error: false, + }, + { + id: randomUUID(), + name: 'Send daily summary', + type: 'send', + config: { + template: `Daily summary (${firstGetListPath}):\n{{data}}`, + }, + sort_order: 1, + continue_on_error: false, + }, + ]; + workflows.push({ + id: 'daily-summary', + name: 'Daily summary', + description: + `Every day at 9 AM UTC, fetch records from GET ${firstGetListPath} ` + + 'and send a summary. Useful for monitoring totals, recent activity, or inventory levels.', + trigger, + steps, + status: 'draft', + run_count: 0, + error_count: 0, + created_at: now, + }); + } + + // ── Workflow 3: status change alerts ───────────────────────────────────── + if (statusChangePath) { + const trigger: WorkflowTrigger = { + type: 'webhook', + event: 'record.updated', + field: 'status', + }; + const steps: WorkflowStep[] = [ + { + id: randomUUID(), + name: 'Alert on status change', + type: 'send', + config: { + template: + `Status changed on ${statusChangePath} (${statusChangeMethod}): ` + '{{data.status}}', + }, + sort_order: 0, + continue_on_error: false, + }, + ]; + workflows.push({ + id: 'status-change-alert', + name: 'Status change alerts', + description: + `Send an alert when a record's status field changes via ` + + `${statusChangeMethod} ${statusChangePath}. ` + + 'Useful for order status changes, ticket updates, or approval state transitions.', + trigger, + steps, + status: 'draft', + run_count: 0, + error_count: 0, + created_at: now, + }); + } + + return workflows; +} + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +/** Return true if the request body contains a status-like field. */ +function hasStatusField(op: OpenAPIV3.OperationObject, doc: OpenAPIV3.Document): boolean { + const body = op.requestBody as OpenAPIV3.RequestBodyObject | undefined; + if (!body?.content) return false; + + for (const mediaType of Object.values(body.content)) { + const schemaRef = mediaType.schema; + if (!schemaRef) continue; + + const schema = resolveSchema(schemaRef, doc); + if (!schema?.properties) continue; + + const keys = Object.keys(schema.properties).map((k) => k.toLowerCase()); + if (keys.some((k) => k === 'status' || k === 'state' || k === 'condition')) { + return true; + } + } + + return false; +} + +/** Dereference a $ref to a local schema component, or return the schema as-is. */ +function resolveSchema( + ref: OpenAPIV3.SchemaObject | OpenAPIV3.ReferenceObject, + doc: OpenAPIV3.Document, +): OpenAPIV3.SchemaObject | undefined { + if ('$ref' in ref) { + const name = ref.$ref.split('/').pop(); + if (!name) return undefined; + const resolved = doc.components?.schemas?.[name]; + if (!resolved || '$ref' in resolved) return undefined; + return resolved; + } + return ref; +} diff --git a/src/intelligence/activity-patterns.ts b/src/intelligence/activity-patterns.ts new file mode 100644 index 00000000..ba904339 --- /dev/null +++ b/src/intelligence/activity-patterns.ts @@ -0,0 +1,389 @@ +import type Database from 'better-sqlite3'; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +/** Categories of client activity patterns */ +export type PatternType = 'repeat-customer' | 'seasonal' | 'churn-risk' | 'growth-trend'; + +/** A detected activity pattern for a DocType */ +export interface ActivityPattern { + /** Category of the pattern */ + type: PatternType; + /** Entity identifier (e.g. customer name/ID) when the pattern is entity-scoped */ + entityId?: string; + /** Human-readable description suitable for inclusion in a business report */ + description: string; + /** Confidence score 0–1 (higher = stronger signal) */ + confidence: number; + /** Supporting metrics for the pattern */ + metadata: Record; +} + +// --------------------------------------------------------------------------- +// Internal helpers +// --------------------------------------------------------------------------- + +interface ColumnInfo { + name: string; + type: string; +} + +/** Return column names for a table via PRAGMA table_info. */ +function getTableColumns(db: Database.Database, table: string): ColumnInfo[] { + try { + return db.prepare(`PRAGMA table_info("${table.replace(/"/g, '""')}")`).all() as ColumnInfo[]; + } catch { + return []; + } +} + +/** + * Identify the most likely "customer / entity" column from available columns. + * We prefer columns whose names contain common CRM identifiers. + */ +function findEntityColumn(columns: ColumnInfo[]): string | null { + const names = columns.map((c) => c.name.toLowerCase()); + const candidates = [ + 'customer', + 'client', + 'customer_id', + 'client_id', + 'contact', + 'buyer', + 'vendor', + 'supplier', + 'partner', + 'user', + 'account', + ]; + for (const candidate of candidates) { + if (names.includes(candidate)) { + return columns[names.indexOf(candidate)]!.name; + } + } + return null; +} + +/** Check whether a column exists in the column list. */ +function hasColumn(columns: ColumnInfo[], name: string): boolean { + return columns.some((c) => c.name.toLowerCase() === name.toLowerCase()); +} + +// --------------------------------------------------------------------------- +// Pattern detectors +// --------------------------------------------------------------------------- + +/** + * Detect repeat customers — entities with more than one record. + * Returns patterns for the top repeat entities (up to 5). + */ +function detectRepeatCustomers( + db: Database.Database, + table: string, + entityCol: string, +): ActivityPattern[] { + interface RepeatRow { + entity: string | null; + cnt: number; + } + + let rows: RepeatRow[]; + try { + rows = db + .prepare( + `SELECT "${entityCol.replace(/"/g, '""')}" AS entity, + COUNT(*) AS cnt + FROM "${table.replace(/"/g, '""')}" + WHERE "${entityCol.replace(/"/g, '""')}" IS NOT NULL + AND "${entityCol.replace(/"/g, '""')}" != '' + GROUP BY "${entityCol.replace(/"/g, '""')}" + HAVING COUNT(*) > 1 + ORDER BY cnt DESC + LIMIT 5`, + ) + .all() as RepeatRow[]; + } catch { + return []; + } + + if (rows.length === 0) return []; + + return rows.map((r) => ({ + type: 'repeat-customer' as const, + entityId: r.entity ?? undefined, + description: `${r.entity ?? 'Unknown'} has placed ${r.cnt} orders — a repeat customer.`, + confidence: Math.min(0.5 + r.cnt * 0.05, 1), + metadata: { order_count: r.cnt, entity_column: entityCol }, + })); +} + +/** + * Detect seasonal patterns — days-of-week or months with above-average activity. + * Requires a `created_at` column. + */ +function detectSeasonalPatterns(db: Database.Database, table: string): ActivityPattern[] { + interface DowRow { + dow: number; + cnt: number; + } + + const t = `"${table.replace(/"/g, '""')}"`; + let dowRows: DowRow[]; + try { + dowRows = db + .prepare( + `SELECT strftime('%w', created_at) AS dow, COUNT(*) AS cnt + FROM ${t} + WHERE created_at IS NOT NULL + GROUP BY dow + ORDER BY cnt DESC`, + ) + .all() as DowRow[]; + } catch { + return []; + } + + if (dowRows.length === 0) return []; + + const total = dowRows.reduce((s, r) => s + r.cnt, 0); + const avg = total / dowRows.length; + const dayNames = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday']; + const patterns: ActivityPattern[] = []; + + for (const row of dowRows) { + if (row.cnt > avg * 1.5) { + const dayName = dayNames[row.dow] ?? `Day ${row.dow}`; + patterns.push({ + type: 'seasonal', + description: `${dayName} is a peak activity day with ${row.cnt} records (${Math.round((row.cnt / avg - 1) * 100)}% above average).`, + confidence: Math.min(0.4 + (row.cnt / avg - 1) * 0.3, 1), + metadata: { day_of_week: dayName, record_count: row.cnt, average: Math.round(avg) }, + }); + } + } + + return patterns.slice(0, 3); +} + +/** + * Detect churn risk — entities whose last activity is more than 2× their average interval ago. + * Requires `created_at` column and an entity column. + */ +function detectChurnRisk( + db: Database.Database, + table: string, + entityCol: string, +): ActivityPattern[] { + interface EntityTimestampRow { + entity: string | null; + last_at: string; + first_at: string; + cnt: number; + } + + const t = `"${table.replace(/"/g, '""')}"`; + const col = `"${entityCol.replace(/"/g, '""')}"`; + + let rows: EntityTimestampRow[]; + try { + rows = db + .prepare( + `SELECT ${col} AS entity, + MAX(created_at) AS last_at, + MIN(created_at) AS first_at, + COUNT(*) AS cnt + FROM ${t} + WHERE ${col} IS NOT NULL AND ${col} != '' AND created_at IS NOT NULL + GROUP BY ${col} + HAVING COUNT(*) >= 2`, + ) + .all() as EntityTimestampRow[]; + } catch { + return []; + } + + const now = Date.now(); + const patterns: ActivityPattern[] = []; + + for (const row of rows) { + const lastMs = new Date(row.last_at).getTime(); + const firstMs = new Date(row.first_at).getTime(); + if (isNaN(lastMs) || isNaN(firstMs)) continue; + + const spanMs = lastMs - firstMs; + const avgIntervalMs = spanMs / (row.cnt - 1); + if (avgIntervalMs <= 0) continue; + + const idleSince = now - lastMs; + if (idleSince > 2 * avgIntervalMs) { + const idleDays = Math.round(idleSince / 86_400_000); + const avgIntervalDays = Math.round(avgIntervalMs / 86_400_000); + patterns.push({ + type: 'churn-risk', + entityId: row.entity ?? undefined, + description: + `${row.entity ?? 'Unknown'} has been inactive for ${idleDays} days ` + + `(average interval: ${avgIntervalDays} days) — possible churn risk.`, + confidence: Math.min(0.4 + idleSince / avgIntervalMs / 10, 0.95), + metadata: { + idle_days: idleDays, + avg_interval_days: avgIntervalDays, + order_count: row.cnt, + last_activity: row.last_at, + }, + }); + } + } + + // Return top-5 highest confidence churn risks + return patterns.sort((a, b) => b.confidence - a.confidence).slice(0, 5); +} + +/** + * Detect growth trends — compare record counts in the last 30 days vs the prior 30 days. + * Requires `created_at` column. + */ +function detectGrowthTrend(db: Database.Database, table: string): ActivityPattern[] { + interface PeriodRow { + cnt: number; + } + + const t = `"${table.replace(/"/g, '""')}"`; + const now = new Date(); + const p1Start = new Date(now.getTime() - 30 * 86_400_000).toISOString().slice(0, 10); + const p2Start = new Date(now.getTime() - 60 * 86_400_000).toISOString().slice(0, 10); + const today = now.toISOString().slice(0, 10); + + let current: PeriodRow | undefined; + let previous: PeriodRow | undefined; + try { + current = db.prepare(`SELECT COUNT(*) AS cnt FROM ${t} WHERE created_at >= ?`).get(p1Start) as + | PeriodRow + | undefined; + previous = db + .prepare(`SELECT COUNT(*) AS cnt FROM ${t} WHERE created_at >= ? AND created_at < ?`) + .get(p2Start, p1Start) as PeriodRow | undefined; + } catch { + return []; + } + + const curr = current?.cnt ?? 0; + const prev = previous?.cnt ?? 0; + + if (prev === 0 && curr === 0) return []; + + const changeRatio = prev > 0 ? (curr - prev) / prev : curr > 0 ? 1 : 0; + const absChange = Math.abs(changeRatio); + + // Only report if change is >= 20% + if (absChange < 0.2) return []; + + const direction = changeRatio > 0 ? 'growth' : 'decline'; + const pct = Math.round(absChange * 100); + + return [ + { + type: 'growth-trend', + description: + `${direction === 'growth' ? 'Growth' : 'Decline'} detected: ` + + `${curr} records in the last 30 days vs ${prev} in the prior 30 days (${pct}% ${direction}).`, + confidence: Math.min(0.3 + absChange * 0.5, 0.95), + metadata: { + current_period_count: curr, + previous_period_count: prev, + change_percent: pct, + direction, + period_start: p1Start, + today, + }, + }, + ]; +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/** + * Detect client activity patterns for the given DocType. + * + * Analyzes the DocType's records for: + * - **Repeat customers** — entities with multiple records + * - **Seasonal patterns** — days-of-week or months with above-average activity + * - **Churn risk** — entities idle for more than 2× their average interaction interval + * - **Growth trends** — 30-day growth/decline vs prior 30-day period + * + * Returns an empty array if the DocType table cannot be found or has fewer than + * 3 records (not enough signal). + * + * @param db - SQLite database instance + * @param doctype - DocType name (e.g. "order", "customer") + */ +export function detectActivityPatterns(db: Database.Database, doctype: string): ActivityPattern[] { + // Resolve the table name from the doctypes registry + let tableName: string; + try { + const row = db.prepare('SELECT table_name FROM doctypes WHERE name = ?').get(doctype) as + | { table_name: string } + | undefined; + if (!row) return []; + tableName = row.table_name; + } catch { + return []; + } + + // Need at least 3 records for meaningful analysis + try { + const countRow = db + .prepare(`SELECT COUNT(*) AS c FROM "${tableName.replace(/"/g, '""')}"`) + .get() as { c: number } | undefined; + if (!countRow || countRow.c < 3) return []; + } catch { + return []; + } + + const columns = getTableColumns(db, tableName); + const entityCol = findEntityColumn(columns); + const hasCreatedAt = hasColumn(columns, 'created_at'); + + const patterns: ActivityPattern[] = []; + + if (entityCol) { + patterns.push(...detectRepeatCustomers(db, tableName, entityCol)); + if (hasCreatedAt) { + patterns.push(...detectChurnRisk(db, tableName, entityCol)); + } + } + + if (hasCreatedAt) { + patterns.push(...detectSeasonalPatterns(db, tableName)); + patterns.push(...detectGrowthTrend(db, tableName)); + } + + return patterns; +} + +/** + * Summarise activity patterns across multiple DocTypes into a plain-text block + * suitable for injection into an AI prompt. + * + * @param db - SQLite database instance + * @param doctypeNames - List of DocType names to analyse + */ +export function summariseActivityPatterns(db: Database.Database, doctypeNames: string[]): string { + const lines: string[] = []; + + for (const name of doctypeNames) { + const patterns = detectActivityPatterns(db, name); + if (patterns.length === 0) continue; + + lines.push(`**${name}**`); + for (const p of patterns) { + lines.push(` [${p.type}] ${p.description}`); + } + } + + return lines.length > 0 ? lines.join('\n') : 'No notable activity patterns detected.'; +} diff --git a/src/intelligence/audit-log.ts b/src/intelligence/audit-log.ts new file mode 100644 index 00000000..15946e67 --- /dev/null +++ b/src/intelligence/audit-log.ts @@ -0,0 +1,120 @@ +import type Database from 'better-sqlite3'; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +/** A single audit log entry from the dt_audit_log table. */ +export interface AuditEntry { + id: number; + doctype: string; + record_id: string; + /** e.g. 'transition', 'update', 'create', 'delete' */ + event: string; + /** Old value (state name, field value serialised as string, or null) */ + old_value: string | null; + /** New value (state name, field value serialised as string, or null) */ + new_value: string | null; + /** Who made the change (role name, user identifier, or null) */ + changed_by: string | null; + changed_at: string; +} + +interface AuditRow { + id: number; + doctype: string; + record_id: string; + event: string; + old_value: string | null; + new_value: string | null; + changed_by: string | null; + changed_at: string; +} + +// --------------------------------------------------------------------------- +// Schema +// --------------------------------------------------------------------------- + +/** + * Set of database instances that have already had the audit log table created. + * Using WeakSet ensures entries are garbage-collected when the DB is closed. + */ +const initializedDbs = new WeakSet(); + +/** + * Ensure the dt_audit_log table exists. + * Safe to call multiple times and across different database instances. + * Must be called OUTSIDE any active transaction. + */ +export function ensureAuditLogTable(db: Database.Database): void { + if (initializedDbs.has(db)) return; + db.exec(` + CREATE TABLE IF NOT EXISTS dt_audit_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + doctype TEXT NOT NULL, + record_id TEXT NOT NULL, + event TEXT NOT NULL, + old_value TEXT, + new_value TEXT, + changed_by TEXT, + changed_at TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_dt_audit_log_record + ON dt_audit_log(doctype, record_id); + `); + initializedDbs.add(db); +} + +// --------------------------------------------------------------------------- +// Write +// --------------------------------------------------------------------------- + +/** + * Insert a single audit log entry. + * Callers must ensure the dt_audit_log table exists by calling + * ensureAuditLogTable(db) before any active transaction. + */ +export function insertAuditEntry(db: Database.Database, entry: Omit): void { + db.prepare( + `INSERT INTO dt_audit_log + (doctype, record_id, event, old_value, new_value, changed_by, changed_at) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + ).run( + entry.doctype, + entry.record_id, + entry.event, + entry.old_value ?? null, + entry.new_value ?? null, + entry.changed_by ?? null, + entry.changed_at, + ); +} + +// --------------------------------------------------------------------------- +// Read +// --------------------------------------------------------------------------- + +/** + * Retrieve the full audit history for a specific record, ordered by + * changed_at ascending (oldest first). + * + * @param db - better-sqlite3 Database instance + * @param doctype - DocType name (e.g. "Invoice") + * @param recordId - The record's UUID + */ +export function getAuditLog( + db: Database.Database, + doctype: string, + recordId: string, +): AuditEntry[] { + // Safe to call here — getAuditLog is never called inside a transaction + ensureAuditLogTable(db); + const rows = db + .prepare( + `SELECT * FROM dt_audit_log + WHERE doctype = ? AND record_id = ? + ORDER BY changed_at ASC, id ASC`, + ) + .all(doctype, recordId) as AuditRow[]; + return rows; +} diff --git a/src/intelligence/branding.ts b/src/intelligence/branding.ts new file mode 100644 index 00000000..79b29b25 --- /dev/null +++ b/src/intelligence/branding.ts @@ -0,0 +1,55 @@ +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; + +/** Business branding configuration stored in .openbridge/context/branding.json */ +export interface Branding { + /** Company / business name */ + companyName: string; + /** Street address, city, state, zip */ + companyAddress?: string; + /** Business phone number */ + companyPhone?: string; + /** Business email address */ + companyEmail?: string; + /** Tax / VAT / GST registration number */ + taxId?: string; + /** Absolute or workspace-relative path to the company logo file */ + logoPath?: string; + /** Primary brand colour (CSS hex, e.g. "#1a73e8") */ + primaryColor?: string; + /** Secondary brand colour (CSS hex, e.g. "#f8f9fa") */ + secondaryColor?: string; +} + +const BRANDING_PATH = path.join('.openbridge', 'context', 'branding.json'); + +const DEFAULT_BRANDING: Branding = { + companyName: 'My Company', + primaryColor: '#1a73e8', + secondaryColor: '#f8f9fa', +}; + +/** + * Load branding config from `/.openbridge/context/branding.json`. + * Returns sensible defaults when the file is missing or cannot be parsed. + */ +export async function loadBranding(workspacePath: string): Promise { + const filePath = path.join(workspacePath, BRANDING_PATH); + try { + const raw = await fs.readFile(filePath, 'utf8'); + const parsed = JSON.parse(raw) as Partial; + return { ...DEFAULT_BRANDING, ...parsed }; + } catch { + return { ...DEFAULT_BRANDING }; + } +} + +/** + * Persist branding config to `/.openbridge/context/branding.json`. + * Creates intermediate directories if they do not exist. + */ +export async function saveBranding(workspacePath: string, branding: Branding): Promise { + const filePath = path.join(workspacePath, BRANDING_PATH); + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, JSON.stringify(branding, null, 2), 'utf8'); +} diff --git a/src/intelligence/doctype-api.ts b/src/intelligence/doctype-api.ts new file mode 100644 index 00000000..3d9a854e --- /dev/null +++ b/src/intelligence/doctype-api.ts @@ -0,0 +1,654 @@ +import type Database from 'better-sqlite3'; +import type { IncomingMessage, ServerResponse } from 'node:http'; +import { randomUUID } from 'node:crypto'; +import { z } from 'zod/v3'; +import type { DocType, DocTypeField } from '../types/doctype.js'; +import { getDocTypeByName, type FullDocType } from './doctype-store.js'; +import { createLogger } from '../core/logger.js'; +import { ensureAuditLogTable, insertAuditEntry } from './audit-log.js'; + +const logger = createLogger('doctype-api'); + +/** CORS headers for API responses */ +const CORS_HEADERS: Record = { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS', + 'Access-Control-Allow-Headers': 'Content-Type', +}; + +/** Default page size for list queries */ +const DEFAULT_PAGE_SIZE = 20; + +/** Maximum page size to prevent abuse */ +const MAX_PAGE_SIZE = 100; + +// --------------------------------------------------------------------------- +// Zod validator generation from DocType field schema +// --------------------------------------------------------------------------- + +/** Maps a FieldType to a Zod schema for input validation */ +function fieldTypeToZod(field: DocTypeField): z.ZodTypeAny { + let schema: z.ZodTypeAny; + + switch (field.field_type) { + case 'text': + case 'longtext': + case 'link': + case 'image': + case 'select': + case 'multiselect': + schema = z.string(); + break; + case 'email': + schema = z.string().email(); + break; + case 'phone': + schema = z.string(); + break; + case 'url': + schema = z.string().url(); + break; + case 'date': + case 'datetime': + schema = z.string(); + break; + case 'number': + case 'currency': + schema = z.number(); + break; + case 'checkbox': + schema = z.union([z.boolean(), z.literal(0), z.literal(1)]); + break; + case 'table': + // Table fields hold child data — skip in validation (handled separately) + schema = z.unknown(); + break; + default: + schema = z.unknown(); + } + + // Apply options constraint for select fields + if ( + (field.field_type === 'select' || field.field_type === 'multiselect') && + field.options && + field.options.length > 0 + ) { + schema = z.enum(field.options as [string, ...string[]]); + } + + // Make optional if not required + if (!field.required) { + schema = schema.optional(); + } + + return schema; +} + +/** + * Build a Zod object schema from DocType field definitions. + * Excludes GENERATED (formula) fields since they are computed by SQLite. + */ +export function buildZodValidator(fields: DocTypeField[]): z.ZodObject { + const shape: z.ZodRawShape = {}; + + for (const field of fields) { + // Skip GENERATED columns — these are computed by SQLite, not user-provided + if (field.formula != null) continue; + // Skip table-type fields — child table data is handled separately + if (field.field_type === 'table') continue; + + shape[field.name] = fieldTypeToZod(field); + } + + return z.object(shape).passthrough(); +} + +// --------------------------------------------------------------------------- +// Route handler +// --------------------------------------------------------------------------- + +/** + * Registered DocType route set — stored so we can match incoming requests + * against registered doctypes. + */ +interface DocTypeRouteConfig { + doctype: DocType; + fields: DocTypeField[]; + validator: z.ZodObject; + childTables: Array<{ fieldName: string; childDoctype: string }>; +} + +/** Registry of registered DocType routes, keyed by lowercase doctype name */ +const routeRegistry = new Map(); + +/** + * Register REST API routes for a DocType on the file-server's HTTP server. + * + * Routes: + * GET /api/dt/:doctype — list with pagination/filters + * GET /api/dt/:doctype/:id — get with child tables + * POST /api/dt/:doctype — create (runs hooks) + * PUT /api/dt/:doctype/:id — update + * DELETE /api/dt/:doctype/:id — soft-delete + * + * @param db — SQLite database instance + * @param doctype — full DocType definition (from getDocTypeByName) + */ +export function registerDocTypeRoutes(db: Database.Database, doctype: FullDocType): void { + const nonFormulaFields = doctype.fields.filter((f) => f.formula == null); + const validator = buildZodValidator(doctype.fields); + + const childTables: Array<{ fieldName: string; childDoctype: string }> = []; + for (const field of doctype.fields) { + if (field.field_type === 'table' && field.child_doctype) { + childTables.push({ fieldName: field.name, childDoctype: field.child_doctype }); + } + } + + routeRegistry.set(doctype.doctype.name.toLowerCase(), { + doctype: doctype.doctype, + fields: nonFormulaFields, + validator, + childTables, + }); + + logger.info({ doctype: doctype.doctype.name, routes: 5 }, 'Registered DocType REST API routes'); +} + +/** + * Unregister all DocType routes. Useful for cleanup in tests. + */ +export function clearDocTypeRoutes(): void { + routeRegistry.clear(); +} + +/** + * Handle an incoming HTTP request for DocType API routes. + * Returns true if the request was handled, false if it doesn't match any DocType route. + * + * Integrate this into the file-server's handleRequest method: + * if (await handleDocTypeRequest(db, req, res)) return; + */ +export async function handleDocTypeRequest( + db: Database.Database, + req: IncomingMessage, + res: ServerResponse, +): Promise { + const url = req.url ?? '/'; + const method = req.method ?? 'GET'; + + // Handle CORS preflight for /api/dt/ routes + if (method === 'OPTIONS' && url.startsWith('/api/dt/')) { + res.writeHead(204, CORS_HEADERS); + res.end(); + return true; + } + + // Match /api/dt/:doctype/:id + const itemMatch = url.match(/^\/api\/dt\/([^/?]+)\/([^/?]+)(?:\?.*)?$/); + if (itemMatch) { + const doctypeName = decodeURIComponent(itemMatch[1]!); + const recordId = decodeURIComponent(itemMatch[2]!); + const config = routeRegistry.get(doctypeName.toLowerCase()); + if (!config) return false; + + switch (method) { + case 'GET': + handleGetRecord(db, config, recordId, res); + return true; + case 'PUT': + await handleUpdateRecord(db, config, recordId, req, res); + return true; + case 'DELETE': + handleDeleteRecord(db, config, recordId, res); + return true; + default: + sendJson(res, 405, { error: 'Method not allowed' }); + return true; + } + } + + // Match /api/dt/:doctype + const listMatch = url.match(/^\/api\/dt\/([^/?]+)(?:\?(.*))?$/); + if (listMatch) { + const doctypeName = decodeURIComponent(listMatch[1]!); + const config = routeRegistry.get(doctypeName.toLowerCase()); + if (!config) return false; + + switch (method) { + case 'GET': + handleListRecords(db, config, url, res); + return true; + case 'POST': + await handleCreateRecord(db, config, req, res); + return true; + default: + sendJson(res, 405, { error: 'Method not allowed' }); + return true; + } + } + + return false; +} + +// --------------------------------------------------------------------------- +// Route handlers +// --------------------------------------------------------------------------- + +/** GET /api/dt/:doctype — list with pagination and filters */ +function handleListRecords( + db: Database.Database, + config: DocTypeRouteConfig, + rawUrl: string, + res: ServerResponse, +): void { + try { + const urlObj = new URL(rawUrl, 'http://localhost'); + const page = Math.max(1, parseInt(urlObj.searchParams.get('page') ?? '1', 10) || 1); + const pageSize = Math.min( + MAX_PAGE_SIZE, + Math.max( + 1, + parseInt(urlObj.searchParams.get('page_size') ?? String(DEFAULT_PAGE_SIZE), 10) || + DEFAULT_PAGE_SIZE, + ), + ); + const orderBy = urlObj.searchParams.get('order_by') ?? 'created_at'; + const orderDir = urlObj.searchParams.get('order_dir')?.toUpperCase() === 'ASC' ? 'ASC' : 'DESC'; + + // Build WHERE clause from query params that match field names + const fieldNames = new Set(config.fields.map((f) => f.name)); + const whereClauses: string[] = []; + const whereParams: unknown[] = []; + + for (const [key, value] of urlObj.searchParams.entries()) { + if (fieldNames.has(key)) { + whereClauses.push(`"${key.replace(/"/g, '""')}" = ?`); + whereParams.push(value); + } + } + + const whereSQL = whereClauses.length > 0 ? `WHERE ${whereClauses.join(' AND ')}` : ''; + + // Validate orderBy is a known column + const validOrderColumns = new Set([ + 'id', + 'created_at', + 'updated_at', + 'created_by', + ...fieldNames, + ]); + const safeOrderBy = validOrderColumns.has(orderBy) + ? `"${orderBy.replace(/"/g, '""')}"` + : '"created_at"'; + + const tableName = quoteIdentifier(config.doctype.table_name); + + // Count total + const countSQL = `SELECT COUNT(*) as total FROM ${tableName} ${whereSQL}`; + const countRow = db.prepare(countSQL).get(...whereParams) as { total: number } | undefined; + const total = countRow?.total ?? 0; + + // Fetch page + const offset = (page - 1) * pageSize; + const dataSQL = `SELECT * FROM ${tableName} ${whereSQL} ORDER BY ${safeOrderBy} ${orderDir} LIMIT ? OFFSET ?`; + const rows = db.prepare(dataSQL).all(...whereParams, pageSize, offset) as Record< + string, + unknown + >[]; + + sendJson(res, 200, { + data: rows, + meta: { + total, + page, + page_size: pageSize, + total_pages: Math.ceil(total / pageSize), + }, + }); + } catch (err) { + logger.error({ err, doctype: config.doctype.name }, 'Error listing records'); + sendJson(res, 500, { error: 'Internal server error' }); + } +} + +/** GET /api/dt/:doctype/:id — get single record with child tables */ +function handleGetRecord( + db: Database.Database, + config: DocTypeRouteConfig, + recordId: string, + res: ServerResponse, +): void { + try { + const tableName = quoteIdentifier(config.doctype.table_name); + const row = db.prepare(`SELECT * FROM ${tableName} WHERE id = ?`).get(recordId) as + | Record + | undefined; + + if (!row) { + sendJson(res, 404, { error: 'Record not found' }); + return; + } + + // Fetch child table records + const children: Record = {}; + for (const child of config.childTables) { + const childTableName = `dt_${config.doctype.name.toLowerCase()}__${child.childDoctype}`; + try { + const childRows = db + .prepare( + `SELECT * FROM ${quoteIdentifier(childTableName)} WHERE parent_id = ? ORDER BY idx`, + ) + .all(recordId) as Record[]; + children[child.fieldName] = childRows; + } catch { + // Child table may not exist yet — return empty array + children[child.fieldName] = []; + } + } + + sendJson(res, 200, { + data: { ...row, ...children }, + }); + } catch (err) { + logger.error({ err, doctype: config.doctype.name, recordId }, 'Error getting record'); + sendJson(res, 500, { error: 'Internal server error' }); + } +} + +/** POST /api/dt/:doctype — create a new record */ +async function handleCreateRecord( + db: Database.Database, + config: DocTypeRouteConfig, + req: IncomingMessage, + res: ServerResponse, +): Promise { + try { + const body = await readJsonBody(req); + if (body === null) { + sendJson(res, 400, { error: 'Invalid JSON body' }); + return; + } + + // Validate against auto-generated Zod schema + const result = config.validator.safeParse(body); + if (!result.success) { + sendJson(res, 422, { + error: 'Validation failed', + details: result.error.issues.map((i) => ({ + field: i.path.join('.'), + message: i.message, + })), + }); + return; + } + + const validated = result.data; + const id = randomUUID(); + const now = new Date().toISOString(); + + // Build INSERT statement from non-formula, non-table fields + const insertableFields = config.fields.filter( + (f) => f.formula == null && f.field_type !== 'table', + ); + + const columns = ['id', 'created_at', 'updated_at', 'created_by']; + const placeholders = ['?', '?', '?', '?']; + const createdBy = typeof body['created_by'] === 'string' ? body['created_by'] : 'api'; + const values: unknown[] = [id, now, now, createdBy]; + + for (const field of insertableFields) { + const value: unknown = validated[field.name]; + if (value !== undefined) { + columns.push(`"${field.name.replace(/"/g, '""')}"`); + placeholders.push('?'); + values.push(field.field_type === 'checkbox' ? (value ? 1 : 0) : value); + } + } + + const tableName = quoteIdentifier(config.doctype.table_name); + const sql = `INSERT INTO ${tableName} (${columns.join(', ')}) VALUES (${placeholders.join(', ')})`; + + db.prepare(sql).run(...values); + + // Return the created record + const created = db.prepare(`SELECT * FROM ${tableName} WHERE id = ?`).get(id) as + | Record + | undefined; + + // Run lifecycle hooks for 'after_insert' event + runHooks(db, config.doctype.name, 'after_insert', id); + + sendJson(res, 201, { data: created }); + } catch (err) { + logger.error({ err, doctype: config.doctype.name }, 'Error creating record'); + sendJson(res, 500, { error: 'Internal server error' }); + } +} + +/** PUT /api/dt/:doctype/:id — update a record */ +async function handleUpdateRecord( + db: Database.Database, + config: DocTypeRouteConfig, + recordId: string, + req: IncomingMessage, + res: ServerResponse, +): Promise { + try { + const tableName = quoteIdentifier(config.doctype.table_name); + + // Ensure audit log table exists before any potential writes + ensureAuditLogTable(db); + + // Check record exists and capture current values for audit + const existingRecord = db.prepare(`SELECT * FROM ${tableName} WHERE id = ?`).get(recordId) as + | Record + | undefined; + if (!existingRecord) { + sendJson(res, 404, { error: 'Record not found' }); + return; + } + + const body = await readJsonBody(req); + if (body === null) { + sendJson(res, 400, { error: 'Invalid JSON body' }); + return; + } + + // Validate with partial schema (all fields optional for update) + const partialValidator = config.validator.partial(); + const result = partialValidator.safeParse(body); + if (!result.success) { + sendJson(res, 422, { + error: 'Validation failed', + details: result.error.issues.map((i) => ({ + field: i.path.join('.'), + message: i.message, + })), + }); + return; + } + + const validated = result.data; + const now = new Date().toISOString(); + + const setClauses: string[] = ['updated_at = ?']; + const values: unknown[] = [now]; + + const insertableFields = config.fields.filter( + (f) => f.formula == null && f.field_type !== 'table', + ); + + const changedFields: Array<{ name: string; oldValue: string | null; newValue: string | null }> = + []; + for (const field of insertableFields) { + const value: unknown = validated[field.name]; + if (value !== undefined) { + const oldValue = existingRecord[field.name]; + const newValue = field.field_type === 'checkbox' ? (value ? 1 : 0) : value; + const oldStr = serializeValue(oldValue); + const newStr = serializeValue(newValue); + if (oldStr !== newStr) { + changedFields.push({ name: field.name, oldValue: oldStr, newValue: newStr }); + } + setClauses.push(`"${field.name.replace(/"/g, '""')}" = ?`); + values.push(newValue); + } + } + + values.push(recordId); + const sql = `UPDATE ${tableName} SET ${setClauses.join(', ')} WHERE id = ?`; + db.prepare(sql).run(...values); + + const updated = db.prepare(`SELECT * FROM ${tableName} WHERE id = ?`).get(recordId) as + | Record + | undefined; + + // Insert audit log entries for each changed field + const changedBy = typeof body['updated_by'] === 'string' ? body['updated_by'] : null; + for (const changed of changedFields) { + insertAuditEntry(db, { + doctype: config.doctype.name, + record_id: recordId, + event: 'update', + old_value: changed.oldValue, + new_value: changed.newValue, + changed_by: changedBy, + changed_at: now, + }); + } + + // Run lifecycle hooks for 'after_update' event + runHooks(db, config.doctype.name, 'after_update', recordId); + + sendJson(res, 200, { data: updated }); + } catch (err) { + logger.error({ err, doctype: config.doctype.name, recordId }, 'Error updating record'); + sendJson(res, 500, { error: 'Internal server error' }); + } +} + +/** DELETE /api/dt/:doctype/:id — soft-delete by setting _deleted = 1, or hard-delete if no _deleted column */ +function handleDeleteRecord( + db: Database.Database, + config: DocTypeRouteConfig, + recordId: string, + res: ServerResponse, +): void { + try { + const tableName = quoteIdentifier(config.doctype.table_name); + + // Check record exists + const existing = db.prepare(`SELECT id FROM ${tableName} WHERE id = ?`).get(recordId); + if (!existing) { + sendJson(res, 404, { error: 'Record not found' }); + return; + } + + // Attempt soft-delete: update updated_at and mark as deleted + // Since DocType tables may not have a _deleted column, we add a pragmatic approach: + // check if the table has a _deleted column, otherwise hard-delete + const tableInfo = db.prepare(`PRAGMA table_info(${tableName})`).all() as Array<{ + name: string; + }>; + const hasDeletedColumn = tableInfo.some((col) => col.name === '_deleted'); + + if (hasDeletedColumn) { + const now = new Date().toISOString(); + db.prepare(`UPDATE ${tableName} SET "_deleted" = 1, "updated_at" = ? WHERE id = ?`).run( + now, + recordId, + ); + } else { + db.prepare(`DELETE FROM ${tableName} WHERE id = ?`).run(recordId); + } + + // Run lifecycle hooks for 'after_delete' event + runHooks(db, config.doctype.name, 'after_delete', recordId); + + sendJson(res, 200, { data: { id: recordId, deleted: true } }); + } catch (err) { + logger.error({ err, doctype: config.doctype.name, recordId }, 'Error deleting record'); + sendJson(res, 500, { error: 'Internal server error' }); + } +} + +// --------------------------------------------------------------------------- +// Hooks (lightweight — logs hook events; full hook execution is a future task) +// --------------------------------------------------------------------------- + +/** Fire lifecycle hooks for a DocType event. Currently logs the event for future hook executor integration. */ +function runHooks( + db: Database.Database, + doctypeName: string, + event: string, + recordId: string, +): void { + try { + const fullDocType = getDocTypeByName(db, doctypeName); + if (!fullDocType) return; + + const hooks = fullDocType.hooks.filter((h) => h.event === event && h.enabled); + for (const hook of hooks) { + logger.info( + { doctype: doctypeName, event, recordId, actionType: hook.action_type }, + 'DocType lifecycle hook fired', + ); + } + } catch { + // Non-critical — don't fail the request if hook lookup fails + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** Send a JSON response with proper headers */ +function sendJson(res: ServerResponse, statusCode: number, data: unknown): void { + const body = JSON.stringify(data); + res.writeHead(statusCode, { + 'Content-Type': 'application/json; charset=utf-8', + 'Content-Length': Buffer.byteLength(body), + ...CORS_HEADERS, + }); + res.end(body); +} + +/** Read the request body as JSON */ +async function readJsonBody(req: IncomingMessage): Promise | null> { + return new Promise((resolve) => { + const chunks: Buffer[] = []; + req.on('data', (chunk: Buffer) => chunks.push(chunk)); + req.on('end', () => { + try { + const raw = Buffer.concat(chunks).toString('utf-8'); + const parsed: unknown = JSON.parse(raw); + if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) { + resolve(parsed as Record); + } else { + resolve(null); + } + } catch { + resolve(null); + } + }); + req.on('error', () => resolve(null)); + }); +} + +/** Wraps a SQLite identifier in double-quotes, escaping any embedded double-quotes. */ +function quoteIdentifier(name: string): string { + return `"${name.replace(/"/g, '""')}"`; +} + +/** + * Serialize a primitive field value to a string for audit log storage. + * Returns null when the value is null or undefined. + */ +function serializeValue(val: unknown): string | null { + if (val === null || val === undefined) return null; + if (typeof val === 'string') return val; + if (typeof val === 'number' || typeof val === 'boolean') return String(val); + return JSON.stringify(val); +} diff --git a/src/intelligence/doctype-exporter.ts b/src/intelligence/doctype-exporter.ts new file mode 100644 index 00000000..7aef8b3c --- /dev/null +++ b/src/intelligence/doctype-exporter.ts @@ -0,0 +1,233 @@ +/** + * DocType Exporter — Export DocType records to CSV or XLSX files. + * + * Queries the DocType's dynamic SQLite table with optional field filters, + * maps records to tabular form, and writes the output to + * `/{doctype}-{timestamp}.{ext}` using the SheetJS (xlsx) package. + * + * For XLSX exports, child table records are included as additional sheets + * (one sheet per child field whose `child_doctype` is set). + */ + +import { mkdirSync } from 'node:fs'; +import { join } from 'node:path'; +import type Database from 'better-sqlite3'; +import { getDocTypeByName } from './doctype-store.js'; +import { createLogger } from '../core/logger.js'; + +// Lazy-loaded xlsx module (optional dependency) + +interface _XLSXSheet { + [key: string]: unknown; +} + +interface _XLSXWorkbook { + SheetNames: string[]; + Sheets: Record; + Props?: Record; +} + +interface XLSXUtils { + book_new(): _XLSXWorkbook; + aoa_to_sheet(data: unknown[][]): _XLSXSheet; + book_append_sheet(wb: _XLSXWorkbook, ws: _XLSXSheet, name: string): void; + sheet_to_csv(ws: _XLSXSheet): string; +} + +interface XLSXModule { + utils: XLSXUtils; + readFile(path: string, opts?: Record): _XLSXWorkbook; + writeFile(wb: _XLSXWorkbook, path: string): void; +} + +let xlsxModule: XLSXModule | undefined; + +async function getXLSX(): Promise { + if (!xlsxModule) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + xlsxModule = (await import('xlsx')) as any as XLSXModule; + } + return xlsxModule; +} + +const logger = createLogger('doctype-exporter'); + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** Quote a SQLite identifier. */ +function quoteId(name: string): string { + return `"${name.replace(/"/g, '""')}"`; +} + +/** Slugify a name to a safe sheet name (max 31 chars, no special chars). */ +function toSheetName(name: string, maxLen = 31): string { + return name.replace(/[:\\/?*[\]]/g, '_').slice(0, maxLen); +} + +/** + * Build a WHERE clause + parameter array from a flat filters map. + * + * Only scalar equality filters are supported (no nested operators). + * Each key must match an actual column name to prevent injection; unknown + * keys are silently skipped. + */ +function buildWhereClause( + filters: Record, + allowedColumns: Set, +): { clause: string; params: unknown[] } { + const conditions: string[] = []; + const params: unknown[] = []; + + for (const [key, value] of Object.entries(filters)) { + if (!allowedColumns.has(key)) continue; // skip unknown columns + conditions.push(`${quoteId(key)} = ?`); + params.push(value); + } + + return { + clause: conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '', + params, + }; +} + +/** + * Query all rows from a table, returning headers and rows as a 2-D array. + * The first element of the result is the header row. + */ +function queryTable( + db: Database.Database, + tableName: string, + filters: Record, +): { headers: string[]; rows: (string | number | boolean | null)[][] } { + // Retrieve column names via PRAGMA so we can validate filter keys + const pragmaRows = db.prepare(`PRAGMA table_info(${quoteId(tableName)})`).all() as Array<{ + name: string; + }>; + + if (pragmaRows.length === 0) { + return { headers: [], rows: [] }; + } + + const columnNames = pragmaRows.map((r) => r.name); + const allowedColumns = new Set(columnNames); + + const { clause, params } = buildWhereClause(filters, allowedColumns); + const sql = `SELECT * FROM ${quoteId(tableName)} ${clause}`; + + const dataRows = db.prepare(sql).all(...params) as Record[]; + + const rows = dataRows.map((row) => + columnNames.map((col) => { + const v = row[col]; + if (v === null || v === undefined) return null; + if (typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean') return v; + return JSON.stringify(v); + }), + ); + + return { headers: columnNames, rows }; +} + +// --------------------------------------------------------------------------- +// Main export +// --------------------------------------------------------------------------- + +/** + * Export records from a DocType table to a CSV or XLSX file. + * + * @param db - Open SQLite database handle. + * @param outputDir - Absolute path to the output directory (created if absent). + * @param doctypeName - Name of the DocType to export. + * @param format - Output format: `'csv'` or `'xlsx'`. + * @param filters - Optional equality filters applied to the main table. + * @returns Absolute path to the written file. + */ +export async function exportToFile( + db: Database.Database, + outputDir: string, + doctypeName: string, + format: 'csv' | 'xlsx', + filters: Record = {}, +): Promise { + const XLSX = await getXLSX(); + // ── 1. Load DocType metadata ─────────────────────────────────────────────── + const fullDocType = getDocTypeByName(db, doctypeName); + if (!fullDocType) { + throw new Error(`DocType "${doctypeName}" not found`); + } + + const { doctype, fields } = fullDocType; + logger.info({ doctype: doctypeName, format }, 'Starting export'); + + // ── 2. Query main table ──────────────────────────────────────────────────── + const mainTable = queryTable(db, doctype.table_name, filters); + + if (mainTable.headers.length === 0) { + throw new Error(`Table "${doctype.table_name}" does not exist or has no columns`); + } + + // ── 3. Determine child tables ────────────────────────────────────────────── + const childFields = fields.filter( + (f) => f.field_type === 'table' && typeof f.child_doctype === 'string', + ); + + // ── 4. Build output path ─────────────────────────────────────────────────── + mkdirSync(outputDir, { recursive: true }); + + const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19); + const safeName = doctypeName.replace(/[^a-zA-Z0-9_-]/g, '_'); + const ext = format === 'csv' ? 'csv' : 'xlsx'; + const filePath = join(outputDir, `${safeName}-${timestamp}.${ext}`); + + // ── 5. Write file ────────────────────────────────────────────────────────── + if (format === 'csv') { + const wb = XLSX.utils.book_new(); + const ws = XLSX.utils.aoa_to_sheet([mainTable.headers, ...mainTable.rows]); + XLSX.utils.book_append_sheet(wb, ws, toSheetName(doctype.label_singular || doctypeName)); + const csv: string = XLSX.utils.sheet_to_csv(ws); + + // writeFile supports CSV via the bookType option; however, sheet_to_csv + + // writeFile('.csv') is simpler and consistent across SheetJS versions. + const { writeFileSync } = await import('node:fs'); + writeFileSync(filePath, csv, 'utf8'); + + logger.info({ filePath, rows: mainTable.rows.length }, 'CSV export complete'); + } else { + // XLSX: main sheet + one sheet per child table + const wb = XLSX.utils.book_new(); + + // Main sheet + const mainWs = XLSX.utils.aoa_to_sheet([mainTable.headers, ...mainTable.rows]); + XLSX.utils.book_append_sheet(wb, mainWs, toSheetName(doctype.label_singular || doctypeName)); + + // Child sheets — query `dt_{parent}__{child}` pattern + for (const field of childFields) { + const childDoctype = field.child_doctype as string; + // Naming follows the table-builder pattern: dt_{parent}__{child} + const childTableName = `dt_${doctype.name}__${childDoctype}`; + const childTable = queryTable(db, childTableName, {}); + + if (childTable.headers.length === 0) { + logger.warn({ childTableName }, 'Child table not found or empty — skipping sheet'); + continue; + } + + const sheetName = toSheetName(field.label || childDoctype); + const childWs = XLSX.utils.aoa_to_sheet([childTable.headers, ...childTable.rows]); + XLSX.utils.book_append_sheet(wb, childWs, sheetName); + + logger.debug({ sheetName, rows: childTable.rows.length }, 'Child sheet added'); + } + + XLSX.writeFile(wb, filePath); + + logger.info( + { filePath, rows: mainTable.rows.length, sheets: 1 + childFields.length }, + 'XLSX export complete', + ); + } + + return filePath; +} diff --git a/src/intelligence/doctype-importer.ts b/src/intelligence/doctype-importer.ts new file mode 100644 index 00000000..c6d033a2 --- /dev/null +++ b/src/intelligence/doctype-importer.ts @@ -0,0 +1,322 @@ +/** + * DocType Importer — Import data from CSV/Excel files into DocType tables. + * + * Detects the file type (CSV or Excel), calls the document-processor to + * extract table data, maps column headers to DocType field names using a + * normalisation-based heuristic, and inserts valid rows via the database. + * Skipped rows are reported with a human-readable reason. + */ + +import { randomUUID } from 'node:crypto'; +import type Database from 'better-sqlite3'; +import type { ExtractedTable } from '../types/intelligence.js'; +import { processCsv } from './processors/csv-processor.js'; +import { processExcel } from './processors/excel-processor.js'; +import { getDocTypeByName } from './doctype-store.js'; +import { createLogger } from '../core/logger.js'; + +const logger = createLogger('doctype-importer'); + +// --------------------------------------------------------------------------- +// Result type +// --------------------------------------------------------------------------- + +/** Result returned by importFromFile */ +export interface ImportResult { + /** Number of rows successfully imported */ + imported: number; + /** Human-readable error messages for skipped rows */ + errors: string[]; +} + +// --------------------------------------------------------------------------- +// Header → field mapping +// --------------------------------------------------------------------------- + +/** + * Normalise a string for fuzzy column-to-field matching. + * Lowercases, strips leading/trailing whitespace, and collapses spaces/hyphens/dots to underscores. + */ +function normalise(str: string): string { + return str + .trim() + .toLowerCase() + .replace(/[\s\-.]+/g, '_') + .replace(/[^a-z0-9_]/g, ''); +} + +/** + * Build a mapping from column index → DocType field name. + * + * Matching priority (highest first): + * 1. Exact field name match (case-insensitive) + * 2. Exact field label match (case-insensitive) + * 3. Normalised name / label match + * + * Returns a sparse array — unmapped column indices are left undefined. + */ +function buildColumnMap( + headers: string[], + fields: Array<{ name: string; label: string; field_type: string; formula?: string | null }>, +): Array { + // Only map non-formula, non-table fields + const mappableFields = fields.filter((f) => !f.formula && f.field_type !== 'table'); + + return headers.map((header) => { + const normHeader = normalise(header); + + // 1. Exact field-name match + const exact = mappableFields.find((f) => f.name.toLowerCase() === header.trim().toLowerCase()); + if (exact) return exact.name; + + // 2. Exact label match + const labelExact = mappableFields.find( + (f) => f.label.toLowerCase() === header.trim().toLowerCase(), + ); + if (labelExact) return labelExact.name; + + // 3. Normalised match on name or label + const fuzzy = mappableFields.find( + (f) => normalise(f.name) === normHeader || normalise(f.label) === normHeader, + ); + if (fuzzy) return fuzzy.name; + + return undefined; + }); +} + +// --------------------------------------------------------------------------- +// Value coercion +// --------------------------------------------------------------------------- + +/** Coerce a raw cell value into the appropriate SQLite-ready value for the given field type. */ +function coerceValue( + raw: unknown, + fieldType: string, +): { ok: true; value: unknown } | { ok: false; reason: string } { + // Treat empty string / null / undefined as NULL (omit from INSERT) + if (raw === null || raw === undefined || raw === '') { + return { ok: true, value: null }; + } + + if (typeof raw !== 'string' && typeof raw !== 'number' && typeof raw !== 'boolean') { + return { ok: false, reason: `Unexpected value type: ${typeof raw}` }; + } + + const str = String(raw).trim(); + + switch (fieldType) { + case 'number': + case 'currency': { + // Strip common currency symbols before parsing + const cleaned = str.replace(/[$£€,\s]/g, ''); + const num = Number(cleaned); + if (isNaN(num)) return { ok: false, reason: `Cannot parse "${str}" as number` }; + return { ok: true, value: num }; + } + case 'checkbox': { + const lower = str.toLowerCase(); + if (['1', 'true', 'yes', 'y', 'on'].includes(lower)) return { ok: true, value: 1 }; + if (['0', 'false', 'no', 'n', 'off', ''].includes(lower)) return { ok: true, value: 0 }; + return { ok: false, reason: `Cannot parse "${str}" as checkbox` }; + } + case 'date': { + // Accept ISO format; try to normalise common MM/DD/YYYY or DD/MM/YYYY + if (/^\d{4}-\d{2}-\d{2}$/.test(str)) return { ok: true, value: str }; + const d = new Date(str); + if (!isNaN(d.getTime())) return { ok: true, value: d.toISOString().slice(0, 10) }; + return { ok: false, reason: `Cannot parse "${str}" as date` }; + } + case 'datetime': { + const d = new Date(str); + if (!isNaN(d.getTime())) return { ok: true, value: d.toISOString() }; + return { ok: false, reason: `Cannot parse "${str}" as datetime` }; + } + default: + return { ok: true, value: str }; + } +} + +// --------------------------------------------------------------------------- +// Row insertion +// --------------------------------------------------------------------------- + +/** Quote a SQLite identifier */ +function quoteId(name: string): string { + return `"${name.replace(/"/g, '""')}"`; +} + +/** + * Insert a single data row into the DocType table. + * Returns null on success, or an error message on failure. + */ +function insertRow( + db: Database.Database, + tableName: string, + fieldMap: Array<{ fieldName: string; fieldType: string; colIndex: number }>, + row: unknown[], + rowIndex: number, +): string | null { + const columns: string[] = ['id', 'created_at', 'updated_at', 'created_by']; + const placeholders: string[] = ['?', '?', '?', '?']; + const values: unknown[] = [ + randomUUID(), + new Date().toISOString(), + new Date().toISOString(), + 'import', + ]; + + for (const { fieldName, fieldType, colIndex } of fieldMap) { + const raw = row[colIndex]; + const coerced = coerceValue(raw, fieldType); + if (!coerced.ok) { + return `Row ${rowIndex + 1}: field "${fieldName}" — ${coerced.reason}`; + } + if (coerced.value !== null) { + columns.push(quoteId(fieldName)); + placeholders.push('?'); + values.push(coerced.value); + } + } + + try { + const sql = `INSERT INTO ${quoteId(tableName)} (${columns.join(', ')}) VALUES (${placeholders.join(', ')})`; + db.prepare(sql).run(...values); + return null; + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + return `Row ${rowIndex + 1}: insert failed — ${msg}`; + } +} + +// --------------------------------------------------------------------------- +// File type detection +// --------------------------------------------------------------------------- + +function isCsvPath(filePath: string): boolean { + return filePath.toLowerCase().endsWith('.csv'); +} + +function isExcelPath(filePath: string): boolean { + const lower = filePath.toLowerCase(); + return lower.endsWith('.xlsx') || lower.endsWith('.xls'); +} + +// --------------------------------------------------------------------------- +// Main export +// --------------------------------------------------------------------------- + +/** + * Import data from a CSV or Excel file into the named DocType's SQLite table. + * + * Steps: + * 1. Detect file type and extract tables via the document processor. + * 2. Pick the first (or largest) table from the extracted output. + * 3. Map column headers to DocType field names via normalised heuristic. + * 4. Insert each row; collect errors for skipped rows. + * + * @param db - Open SQLite database handle. + * @param filePath - Absolute path to the file to import (CSV or XLSX/XLS). + * @param doctypeName - Name of the target DocType (must exist in doctypes table). + * @returns Promise resolving to { imported, errors }. + */ +export async function importFromFile( + db: Database.Database, + filePath: string, + doctypeName: string, +): Promise { + // ── 1. Load DocType metadata ─────────────────────────────────────────────── + const fullDocType = getDocTypeByName(db, doctypeName); + if (!fullDocType) { + return { imported: 0, errors: [`DocType "${doctypeName}" not found`] }; + } + + const { doctype, fields } = fullDocType; + logger.info({ doctype: doctypeName, filePath }, 'Starting import'); + + // ── 2. Extract tables from file ─────────────────────────────────────────── + let result; + try { + if (isCsvPath(filePath)) { + result = await processCsv(filePath); + } else if (isExcelPath(filePath)) { + result = await processExcel(filePath); + } else { + return { + imported: 0, + errors: [`Unsupported file type: ${filePath} (expected .csv, .xlsx, or .xls)`], + }; + } + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + return { imported: 0, errors: [`Failed to read file: ${msg}`] }; + } + + if (result.tables.length === 0) { + return { imported: 0, errors: ['No tables found in file'] }; + } + + // Pick the table with the most data rows; fall back to the first table + const table: ExtractedTable = result.tables.reduce( + (best, t) => (t.rows.length > best.rows.length ? t : best), + result.tables[0]!, + ); + + if (table.headers.length === 0) { + return { imported: 0, errors: ['Table has no headers'] }; + } + + if (table.rows.length === 0) { + return { imported: 0, errors: ['Table has no data rows'] }; + } + + // ── 3. Build column → field mapping ─────────────────────────────────────── + const columnMap = buildColumnMap(table.headers, fields); + + // Build a flat list of mapped columns for insertion + const fieldMap: Array<{ fieldName: string; fieldType: string; colIndex: number }> = []; + const unmappedHeaders: string[] = []; + + for (let i = 0; i < table.headers.length; i++) { + const fieldName = columnMap[i]; + if (fieldName) { + const field = fields.find((f) => f.name === fieldName)!; + fieldMap.push({ fieldName, fieldType: field.field_type, colIndex: i }); + } else { + unmappedHeaders.push(table.headers[i] ?? `col_${i}`); + } + } + + if (fieldMap.length === 0) { + return { + imported: 0, + errors: [ + `No columns could be mapped to DocType fields. ` + + `File headers: [${table.headers.join(', ')}]. ` + + `DocType fields: [${fields.map((f) => f.name).join(', ')}].`, + ], + }; + } + + if (unmappedHeaders.length > 0) { + logger.warn({ doctype: doctypeName, unmappedHeaders }, 'Some columns could not be mapped'); + } + + // ── 4. Insert rows ───────────────────────────────────────────────────────── + let imported = 0; + const errors: string[] = []; + + for (let rowIndex = 0; rowIndex < table.rows.length; rowIndex++) { + const row = table.rows[rowIndex]!; + const error = insertRow(db, doctype.table_name, fieldMap, row, rowIndex); + if (error) { + errors.push(error); + } else { + imported++; + } + } + + logger.info({ doctype: doctypeName, imported, skipped: errors.length }, 'Import complete'); + + return { imported, errors }; +} diff --git a/src/intelligence/doctype-store.ts b/src/intelligence/doctype-store.ts new file mode 100644 index 00000000..bcf540bf --- /dev/null +++ b/src/intelligence/doctype-store.ts @@ -0,0 +1,732 @@ +import type Database from 'better-sqlite3'; +import type { + DocType, + DocTypeField, + DocTypeState, + DocTypeTransition, + DocTypeHook, + DocTypeRelation, +} from '../types/doctype.js'; + +// --------------------------------------------------------------------------- +// Update type (avoids index-signature issues from Partial) +// --------------------------------------------------------------------------- + +export interface DocTypeUpdate { + id: string; + name?: string; + label_singular?: string; + label_plural?: string; + icon?: string | null; + table_name?: string; + source?: string; + template_id?: string | null; +} + +// --------------------------------------------------------------------------- +// Row types (SQLite representation — JSON columns stored as TEXT) +// --------------------------------------------------------------------------- + +interface DocTypeRow { + id: string; + name: string; + label_singular: string; + label_plural: string; + icon: string | null; + table_name: string; + source: string; + template_id: string | null; + created_at: string; + updated_at: string; +} + +interface DocTypeFieldRow { + id: string; + doctype_id: string; + name: string; + label: string; + field_type: string; + required: number; + default_value: string | null; + options: string | null; + formula: string | null; + depends_on: string | null; + searchable: number; + sort_order: number; + link_doctype: string | null; + child_doctype: string | null; +} + +interface DocTypeStateRow { + id: string; + doctype_id: string; + name: string; + label: string; + color: string; + is_initial: number; + is_terminal: number; + sort_order: number; +} + +interface DocTypeTransitionRow { + id: string; + doctype_id: string; + from_state: string; + to_state: string; + action_name: string; + action_label: string; + allowed_roles: string | null; + condition: string | null; +} + +interface DocTypeHookRow { + id: string; + doctype_id: string; + event: string; + action_type: string; + action_config: string; + sort_order: number; + enabled: number; +} + +interface DocTypeRelationRow { + id: string; + from_doctype: string; + to_doctype: string; + relation_type: string; + from_field: string; + to_field: string; + label: string | null; +} + +// --------------------------------------------------------------------------- +// Row → domain converters +// --------------------------------------------------------------------------- + +function rowToDocType(row: DocTypeRow): DocType { + return { + id: row.id, + name: row.name, + label_singular: row.label_singular, + label_plural: row.label_plural, + icon: row.icon ?? undefined, + table_name: row.table_name, + source: row.source as DocType['source'], + template_id: row.template_id ?? undefined, + created_at: row.created_at, + updated_at: row.updated_at, + }; +} + +function rowToField(row: DocTypeFieldRow): DocTypeField { + return { + id: row.id, + doctype_id: row.doctype_id, + name: row.name, + label: row.label, + field_type: row.field_type as DocTypeField['field_type'], + required: row.required === 1, + default_value: row.default_value ?? undefined, + options: row.options ? (JSON.parse(row.options) as string[]) : undefined, + formula: row.formula ?? undefined, + depends_on: row.depends_on ?? undefined, + searchable: row.searchable === 1, + sort_order: row.sort_order, + link_doctype: row.link_doctype ?? undefined, + child_doctype: row.child_doctype ?? undefined, + }; +} + +function rowToState(row: DocTypeStateRow): DocTypeState { + return { + id: row.id, + doctype_id: row.doctype_id, + name: row.name, + label: row.label, + color: row.color, + is_initial: row.is_initial === 1, + is_terminal: row.is_terminal === 1, + sort_order: row.sort_order, + }; +} + +function rowToTransition(row: DocTypeTransitionRow): DocTypeTransition { + return { + id: row.id, + doctype_id: row.doctype_id, + from_state: row.from_state, + to_state: row.to_state, + action_name: row.action_name, + action_label: row.action_label, + allowed_roles: row.allowed_roles ? (JSON.parse(row.allowed_roles) as string[]) : undefined, + condition: row.condition ?? undefined, + }; +} + +function rowToHook(row: DocTypeHookRow): DocTypeHook { + return { + id: row.id, + doctype_id: row.doctype_id, + event: row.event, + action_type: row.action_type as DocTypeHook['action_type'], + action_config: JSON.parse(row.action_config) as Record, + sort_order: row.sort_order, + enabled: row.enabled === 1, + }; +} + +function rowToRelation(row: DocTypeRelationRow): DocTypeRelation { + return { + id: row.id, + from_doctype: row.from_doctype, + to_doctype: row.to_doctype, + relation_type: row.relation_type as DocTypeRelation['relation_type'], + from_field: row.from_field, + to_field: row.to_field, + label: row.label ?? undefined, + }; +} + +// --------------------------------------------------------------------------- +// Schema creation +// --------------------------------------------------------------------------- + +/** + * Ensure all DocType metadata tables exist. + * Safe to call multiple times (uses CREATE TABLE IF NOT EXISTS). + */ +export function ensureDocTypeStoreSchema(db: Database.Database): void { + db.exec(` + CREATE TABLE IF NOT EXISTS doctypes ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + label_singular TEXT NOT NULL, + label_plural TEXT NOT NULL, + icon TEXT, + table_name TEXT NOT NULL UNIQUE, + source TEXT NOT NULL, + template_id TEXT, + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP + ); + + CREATE TABLE IF NOT EXISTS doctype_fields ( + id TEXT PRIMARY KEY, + doctype_id TEXT NOT NULL REFERENCES doctypes(id) ON DELETE CASCADE, + name TEXT NOT NULL, + label TEXT NOT NULL, + field_type TEXT NOT NULL, + required INTEGER DEFAULT 0, + default_value TEXT, + options TEXT, + formula TEXT, + depends_on TEXT, + searchable INTEGER DEFAULT 0, + sort_order INTEGER NOT NULL, + link_doctype TEXT, + child_doctype TEXT, + UNIQUE(doctype_id, name) + ); + + CREATE TABLE IF NOT EXISTS doctype_states ( + id TEXT PRIMARY KEY, + doctype_id TEXT NOT NULL REFERENCES doctypes(id) ON DELETE CASCADE, + name TEXT NOT NULL, + label TEXT NOT NULL, + color TEXT DEFAULT 'gray', + is_initial INTEGER DEFAULT 0, + is_terminal INTEGER DEFAULT 0, + sort_order INTEGER NOT NULL, + UNIQUE(doctype_id, name) + ); + + CREATE TABLE IF NOT EXISTS doctype_transitions ( + id TEXT PRIMARY KEY, + doctype_id TEXT NOT NULL REFERENCES doctypes(id) ON DELETE CASCADE, + from_state TEXT NOT NULL, + to_state TEXT NOT NULL, + action_name TEXT NOT NULL, + action_label TEXT NOT NULL, + allowed_roles TEXT, + condition TEXT, + UNIQUE(doctype_id, from_state, action_name) + ); + + CREATE TABLE IF NOT EXISTS doctype_hooks ( + id TEXT PRIMARY KEY, + doctype_id TEXT NOT NULL REFERENCES doctypes(id) ON DELETE CASCADE, + event TEXT NOT NULL, + action_type TEXT NOT NULL, + action_config TEXT NOT NULL, + sort_order INTEGER DEFAULT 0, + enabled INTEGER DEFAULT 1 + ); + + CREATE TABLE IF NOT EXISTS doctype_relations ( + id TEXT PRIMARY KEY, + from_doctype TEXT NOT NULL REFERENCES doctypes(id) ON DELETE CASCADE, + to_doctype TEXT NOT NULL REFERENCES doctypes(id) ON DELETE CASCADE, + relation_type TEXT NOT NULL, + from_field TEXT NOT NULL, + to_field TEXT DEFAULT 'id', + label TEXT + ); + + CREATE TABLE IF NOT EXISTS dt_series ( + prefix TEXT PRIMARY KEY, + current_value INTEGER DEFAULT 0 + ); + + CREATE INDEX IF NOT EXISTS idx_doctype_fields_doctype + ON doctype_fields(doctype_id); + CREATE INDEX IF NOT EXISTS idx_doctype_states_doctype + ON doctype_states(doctype_id); + CREATE INDEX IF NOT EXISTS idx_doctype_transitions_doctype + ON doctype_transitions(doctype_id); + `); +} + +// --------------------------------------------------------------------------- +// Full DocType with all child records +// --------------------------------------------------------------------------- + +export interface FullDocType { + doctype: DocType; + fields: DocTypeField[]; + states: DocTypeState[]; + transitions: DocTypeTransition[]; + hooks: DocTypeHook[]; + relations: DocTypeRelation[]; +} + +// --------------------------------------------------------------------------- +// CRUD — Create +// --------------------------------------------------------------------------- + +/** + * Create a DocType and its associated fields, states, transitions, hooks, + * and relations in a single transaction. + */ +export function createDocType( + db: Database.Database, + data: { + doctype: DocType; + fields?: DocTypeField[]; + states?: DocTypeState[]; + transitions?: DocTypeTransition[]; + hooks?: DocTypeHook[]; + relations?: DocTypeRelation[]; + }, +): string { + ensureDocTypeStoreSchema(db); + + const insertDocType = db.prepare(` + INSERT INTO doctypes + (id, name, label_singular, label_plural, icon, table_name, source, template_id, created_at, updated_at) + VALUES + (@id, @name, @label_singular, @label_plural, @icon, @table_name, @source, @template_id, @created_at, @updated_at) + `); + + const insertField = db.prepare(` + INSERT INTO doctype_fields + (id, doctype_id, name, label, field_type, required, default_value, options, formula, depends_on, searchable, sort_order, link_doctype, child_doctype) + VALUES + (@id, @doctype_id, @name, @label, @field_type, @required, @default_value, @options, @formula, @depends_on, @searchable, @sort_order, @link_doctype, @child_doctype) + `); + + const insertState = db.prepare(` + INSERT INTO doctype_states + (id, doctype_id, name, label, color, is_initial, is_terminal, sort_order) + VALUES + (@id, @doctype_id, @name, @label, @color, @is_initial, @is_terminal, @sort_order) + `); + + const insertTransition = db.prepare(` + INSERT INTO doctype_transitions + (id, doctype_id, from_state, to_state, action_name, action_label, allowed_roles, condition) + VALUES + (@id, @doctype_id, @from_state, @to_state, @action_name, @action_label, @allowed_roles, @condition) + `); + + const insertHook = db.prepare(` + INSERT INTO doctype_hooks + (id, doctype_id, event, action_type, action_config, sort_order, enabled) + VALUES + (@id, @doctype_id, @event, @action_type, @action_config, @sort_order, @enabled) + `); + + const insertRelation = db.prepare(` + INSERT INTO doctype_relations + (id, from_doctype, to_doctype, relation_type, from_field, to_field, label) + VALUES + (@id, @from_doctype, @to_doctype, @relation_type, @from_field, @to_field, @label) + `); + + const now = new Date().toISOString(); + const dt = data.doctype; + + db.transaction(() => { + insertDocType.run({ + id: dt.id, + name: dt.name, + label_singular: dt.label_singular, + label_plural: dt.label_plural, + icon: dt.icon ?? null, + table_name: dt.table_name, + source: dt.source, + template_id: dt.template_id ?? null, + created_at: dt.created_at ?? now, + updated_at: dt.updated_at ?? now, + }); + + for (const f of data.fields ?? []) { + insertField.run({ + id: f.id, + doctype_id: f.doctype_id, + name: f.name, + label: f.label, + field_type: f.field_type, + required: f.required ? 1 : 0, + default_value: f.default_value ?? null, + options: f.options ? JSON.stringify(f.options) : null, + formula: f.formula ?? null, + depends_on: f.depends_on ?? null, + searchable: f.searchable ? 1 : 0, + sort_order: f.sort_order, + link_doctype: f.link_doctype ?? null, + child_doctype: f.child_doctype ?? null, + }); + } + + for (const s of data.states ?? []) { + insertState.run({ + id: s.id, + doctype_id: s.doctype_id, + name: s.name, + label: s.label, + color: s.color ?? 'gray', + is_initial: s.is_initial ? 1 : 0, + is_terminal: s.is_terminal ? 1 : 0, + sort_order: s.sort_order, + }); + } + + for (const t of data.transitions ?? []) { + insertTransition.run({ + id: t.id, + doctype_id: t.doctype_id, + from_state: t.from_state, + to_state: t.to_state, + action_name: t.action_name, + action_label: t.action_label, + allowed_roles: t.allowed_roles ? JSON.stringify(t.allowed_roles) : null, + condition: t.condition ?? null, + }); + } + + for (const h of data.hooks ?? []) { + insertHook.run({ + id: h.id, + doctype_id: h.doctype_id, + event: h.event, + action_type: h.action_type, + action_config: JSON.stringify(h.action_config), + sort_order: h.sort_order ?? 0, + enabled: h.enabled !== false ? 1 : 0, + }); + } + + for (const r of data.relations ?? []) { + insertRelation.run({ + id: r.id, + from_doctype: r.from_doctype, + to_doctype: r.to_doctype, + relation_type: r.relation_type, + from_field: r.from_field, + to_field: r.to_field ?? 'id', + label: r.label ?? null, + }); + } + })(); + + return dt.id; +} + +// --------------------------------------------------------------------------- +// CRUD — Read +// --------------------------------------------------------------------------- + +/** + * Retrieve a DocType by ID with all associated child records. + */ +export function getDocType(db: Database.Database, doctypeId: string): FullDocType | null { + ensureDocTypeStoreSchema(db); + + const row = db.prepare('SELECT * FROM doctypes WHERE id = ?').get(doctypeId) as + | DocTypeRow + | undefined; + if (!row) return null; + + return loadFullDocType(db, row); +} + +/** + * Retrieve a DocType by its unique name with all associated child records. + */ +export function getDocTypeByName(db: Database.Database, name: string): FullDocType | null { + ensureDocTypeStoreSchema(db); + + const row = db.prepare('SELECT * FROM doctypes WHERE name = ?').get(name) as + | DocTypeRow + | undefined; + if (!row) return null; + + return loadFullDocType(db, row); +} + +/** + * List all DocTypes. Returns only the top-level metadata (no child records). + */ +export function listDocTypes(db: Database.Database): DocType[] { + ensureDocTypeStoreSchema(db); + + const rows = db.prepare('SELECT * FROM doctypes ORDER BY name').all() as DocTypeRow[]; + + return rows.map(rowToDocType); +} + +// --------------------------------------------------------------------------- +// CRUD — Update +// --------------------------------------------------------------------------- + +/** + * Update an existing DocType and optionally replace its child records. + * Only the provided child arrays are replaced; omitted arrays are left unchanged. + */ +export function updateDocType( + db: Database.Database, + data: { + doctype: DocTypeUpdate; + fields?: DocTypeField[]; + states?: DocTypeState[]; + transitions?: DocTypeTransition[]; + hooks?: DocTypeHook[]; + relations?: DocTypeRelation[]; + }, +): boolean { + ensureDocTypeStoreSchema(db); + + const existing = db.prepare('SELECT id FROM doctypes WHERE id = ?').get(data.doctype.id) as + | { id: string } + | undefined; + if (!existing) return false; + + const now = new Date().toISOString(); + + db.transaction(() => { + const dt = data.doctype; + const setClauses: string[] = ['updated_at = @updated_at']; + const params: Record = { id: dt.id, updated_at: now }; + + const updatableFields: Array<[keyof DocTypeUpdate, string]> = [ + ['name', 'name'], + ['label_singular', 'label_singular'], + ['label_plural', 'label_plural'], + ['icon', 'icon'], + ['table_name', 'table_name'], + ['source', 'source'], + ['template_id', 'template_id'], + ]; + + for (const [key, col] of updatableFields) { + if (dt[key] !== undefined) { + setClauses.push(`${col} = @${col}`); + params[col] = dt[key] ?? null; + } + } + + db.prepare(`UPDATE doctypes SET ${setClauses.join(', ')} WHERE id = @id`).run(params); + + // Replace child collections if provided + if (data.fields) { + db.prepare('DELETE FROM doctype_fields WHERE doctype_id = ?').run(dt.id); + const insertField = db.prepare(` + INSERT INTO doctype_fields + (id, doctype_id, name, label, field_type, required, default_value, options, formula, depends_on, searchable, sort_order, link_doctype, child_doctype) + VALUES + (@id, @doctype_id, @name, @label, @field_type, @required, @default_value, @options, @formula, @depends_on, @searchable, @sort_order, @link_doctype, @child_doctype) + `); + for (const f of data.fields) { + insertField.run({ + id: f.id, + doctype_id: f.doctype_id, + name: f.name, + label: f.label, + field_type: f.field_type, + required: f.required ? 1 : 0, + default_value: f.default_value ?? null, + options: f.options ? JSON.stringify(f.options) : null, + formula: f.formula ?? null, + depends_on: f.depends_on ?? null, + searchable: f.searchable ? 1 : 0, + sort_order: f.sort_order, + link_doctype: f.link_doctype ?? null, + child_doctype: f.child_doctype ?? null, + }); + } + } + + if (data.states) { + db.prepare('DELETE FROM doctype_states WHERE doctype_id = ?').run(dt.id); + const insertState = db.prepare(` + INSERT INTO doctype_states + (id, doctype_id, name, label, color, is_initial, is_terminal, sort_order) + VALUES + (@id, @doctype_id, @name, @label, @color, @is_initial, @is_terminal, @sort_order) + `); + for (const s of data.states) { + insertState.run({ + id: s.id, + doctype_id: s.doctype_id, + name: s.name, + label: s.label, + color: s.color ?? 'gray', + is_initial: s.is_initial ? 1 : 0, + is_terminal: s.is_terminal ? 1 : 0, + sort_order: s.sort_order, + }); + } + } + + if (data.transitions) { + db.prepare('DELETE FROM doctype_transitions WHERE doctype_id = ?').run(dt.id); + const insertTransition = db.prepare(` + INSERT INTO doctype_transitions + (id, doctype_id, from_state, to_state, action_name, action_label, allowed_roles, condition) + VALUES + (@id, @doctype_id, @from_state, @to_state, @action_name, @action_label, @allowed_roles, @condition) + `); + for (const t of data.transitions) { + insertTransition.run({ + id: t.id, + doctype_id: t.doctype_id, + from_state: t.from_state, + to_state: t.to_state, + action_name: t.action_name, + action_label: t.action_label, + allowed_roles: t.allowed_roles ? JSON.stringify(t.allowed_roles) : null, + condition: t.condition ?? null, + }); + } + } + + if (data.hooks) { + db.prepare('DELETE FROM doctype_hooks WHERE doctype_id = ?').run(dt.id); + const insertHook = db.prepare(` + INSERT INTO doctype_hooks + (id, doctype_id, event, action_type, action_config, sort_order, enabled) + VALUES + (@id, @doctype_id, @event, @action_type, @action_config, @sort_order, @enabled) + `); + for (const h of data.hooks) { + insertHook.run({ + id: h.id, + doctype_id: h.doctype_id, + event: h.event, + action_type: h.action_type, + action_config: JSON.stringify(h.action_config), + sort_order: h.sort_order ?? 0, + enabled: h.enabled !== false ? 1 : 0, + }); + } + } + + if (data.relations) { + db.prepare('DELETE FROM doctype_relations WHERE from_doctype = ?').run(dt.id); + const insertRelation = db.prepare(` + INSERT INTO doctype_relations + (id, from_doctype, to_doctype, relation_type, from_field, to_field, label) + VALUES + (@id, @from_doctype, @to_doctype, @relation_type, @from_field, @to_field, @label) + `); + for (const r of data.relations) { + insertRelation.run({ + id: r.id, + from_doctype: r.from_doctype, + to_doctype: r.to_doctype, + relation_type: r.relation_type, + from_field: r.from_field, + to_field: r.to_field ?? 'id', + label: r.label ?? null, + }); + } + } + })(); + + return true; +} + +// --------------------------------------------------------------------------- +// CRUD — Delete +// --------------------------------------------------------------------------- + +/** + * Delete a DocType and all associated child records (cascades via FK). + * Returns true if the DocType existed and was deleted. + */ +export function deleteDocType(db: Database.Database, doctypeId: string): boolean { + ensureDocTypeStoreSchema(db); + + // Ensure FK cascades are honoured in this connection + db.pragma('foreign_keys = ON'); + + const result = db.prepare('DELETE FROM doctypes WHERE id = ?').run(doctypeId); + return result.changes > 0; +} + +// --------------------------------------------------------------------------- +// Internal helpers +// --------------------------------------------------------------------------- + +function loadFullDocType(db: Database.Database, row: DocTypeRow): FullDocType { + const doctypeId = row.id; + + const fields = ( + db + .prepare('SELECT * FROM doctype_fields WHERE doctype_id = ? ORDER BY sort_order') + .all(doctypeId) as DocTypeFieldRow[] + ).map(rowToField); + + const states = ( + db + .prepare('SELECT * FROM doctype_states WHERE doctype_id = ? ORDER BY sort_order') + .all(doctypeId) as DocTypeStateRow[] + ).map(rowToState); + + const transitions = ( + db + .prepare('SELECT * FROM doctype_transitions WHERE doctype_id = ?') + .all(doctypeId) as DocTypeTransitionRow[] + ).map(rowToTransition); + + const hooks = ( + db + .prepare('SELECT * FROM doctype_hooks WHERE doctype_id = ? ORDER BY sort_order') + .all(doctypeId) as DocTypeHookRow[] + ).map(rowToHook); + + const relations = ( + db + .prepare('SELECT * FROM doctype_relations WHERE from_doctype = ?') + .all(doctypeId) as DocTypeRelationRow[] + ).map(rowToRelation); + + return { + doctype: rowToDocType(row), + fields, + states, + transitions, + hooks, + relations, + }; +} diff --git a/src/intelligence/document-processor.ts b/src/intelligence/document-processor.ts new file mode 100644 index 00000000..c2734ee4 --- /dev/null +++ b/src/intelligence/document-processor.ts @@ -0,0 +1,149 @@ +/** + * Document Processor — Unified entry point for document processing pipeline + * + * Routes incoming files (PDF, XLSX, DOCX, images, etc.) to appropriate processors + * based on MIME type detection using file-type (magic bytes). + * Falls back to extension-based detection for plain-text formats (CSV, JSON). + */ + +import { readFile } from 'fs/promises'; +import { extname, basename } from 'path'; +import { randomUUID } from 'crypto'; +import { fileTypeFromBuffer } from 'file-type'; + +import type { ProcessedDocument, ProcessorResult } from '../types/intelligence.js'; +import { processPdf } from './processors/pdf-processor.js'; +import { processExcel } from './processors/excel-processor.js'; +import { processCsv } from './processors/csv-processor.js'; +import { processWord } from './processors/word-processor.js'; +import { processImage } from './processors/image-processor.js'; +import { processEmail } from './processors/email-processor.js'; +import { processStructured } from './processors/structured-processor.js'; + +// ── MIME type constants ─────────────────────────────────────────── + +const MIME_PDF = 'application/pdf'; +const MIME_XLSX = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; +const MIME_XLS = 'application/vnd.ms-excel'; +const MIME_DOCX = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; +const MIME_DOC = 'application/msword'; +const MIME_CSV = 'text/csv'; +const MIME_JSON = 'application/json'; +const MIME_XML = 'application/xml'; +const MIME_XML_TEXT = 'text/xml'; +const MIME_EML = 'message/rfc822'; + +// ── Extension → MIME fallback map ──────────────────────────────── + +const EXTENSION_MIME_MAP: Record = { + '.csv': MIME_CSV, + '.json': MIME_JSON, + '.xml': MIME_XML, + '.eml': MIME_EML, + '.pdf': MIME_PDF, + '.xlsx': MIME_XLSX, + '.xls': MIME_XLS, + '.docx': MIME_DOCX, + '.doc': MIME_DOC, +}; + +// ── MIME detection ──────────────────────────────────────────────── + +/** + * Detect the MIME type of a file. + * + * Uses `file-type` (magic bytes) as the primary mechanism. + * Falls back to extension-based detection for plain-text formats + * that have no magic bytes (CSV, JSON, XML, EML). + */ +async function detectMimeType(filePath: string, buffer: Buffer): Promise { + const detected = await fileTypeFromBuffer(buffer); + if (detected) { + return detected.mime; + } + + // Plain-text formats (no magic bytes) — use extension fallback + const ext = extname(filePath).toLowerCase(); + const fallback = EXTENSION_MIME_MAP[ext]; + if (fallback) { + return fallback; + } + + return 'application/octet-stream'; +} + +// ── Processor routing ───────────────────────────────────────────── + +/** + * Route a file to the appropriate format-specific processor. + * + * Individual processors are implemented in ./processors/ and will be + * fully wired in as they are completed (OB-1336 through OB-1343). + */ +async function routeToProcessor(filePath: string, mimeType: string): Promise { + if (mimeType === MIME_PDF) { + return processPdf(filePath); + } + + if (mimeType === MIME_XLSX || mimeType === MIME_XLS) { + return processExcel(filePath); + } + + if (mimeType === MIME_CSV) { + return processCsv(filePath); + } + + if (mimeType === MIME_DOCX || mimeType === MIME_DOC) { + return processWord(filePath); + } + + if (mimeType.startsWith('image/')) { + return processImage(filePath); + } + + if (mimeType === MIME_EML) { + return processEmail(filePath); + } + + if (mimeType === MIME_JSON || mimeType === MIME_XML || mimeType === MIME_XML_TEXT) { + return processStructured(filePath, mimeType); + } + + // Unknown format — return empty result + return { + rawText: '', + tables: [], + images: [], + metadata: { mimeType, unrecognized: true }, + }; +} + +// ── Public API ──────────────────────────────────────────────────── + +/** + * Process a document file and extract structured content. + * + * @param filePath - Absolute path to the file to process + * @returns Fully structured `ProcessedDocument` with detected MIME, raw text, tables, and metadata + */ +export async function processDocument(filePath: string): Promise { + const buffer = await readFile(filePath); + const mimeType = await detectMimeType(filePath, buffer); + + const result = await routeToProcessor(filePath, mimeType); + + return { + id: randomUUID(), + filename: basename(filePath), + mimeType, + filePath, + docType: 'unknown', + rawText: result.rawText, + tables: result.tables, + images: result.images as [], + entities: [], + relations: [], + metadata: result.metadata, + processedAt: new Date().toISOString(), + }; +} diff --git a/src/intelligence/document-store.ts b/src/intelligence/document-store.ts new file mode 100644 index 00000000..3e0f3bd4 --- /dev/null +++ b/src/intelligence/document-store.ts @@ -0,0 +1,188 @@ +import type Database from 'better-sqlite3'; +import type { ProcessedDocument } from '../types/intelligence.js'; + +// --------------------------------------------------------------------------- +// FTS5 query sanitization (same approach as chunk-store.ts) +// --------------------------------------------------------------------------- + +function sanitizeFts5Query(raw: string): string { + const cleaned = raw.replace(/["*(){}[\]:^~?@#$%&\\|<>=!+,;]/g, ' '); + const tokens = cleaned.split(/\s+/).filter((t) => t.length > 0); + if (tokens.length === 0) return ''; + return tokens.map((t) => `"${t}"`).join(' '); +} + +// --------------------------------------------------------------------------- +// Row types +// --------------------------------------------------------------------------- + +interface ProcessedDocumentRow { + id: string; + filename: string; + mime_type: string; + file_path: string; + doc_type: string; + raw_text: string; + entities: string; + relations: string; + tables: string; + metadata: string; + processed_at: string; + source: string | null; +} + +function rowToDocument(row: ProcessedDocumentRow): ProcessedDocument { + return { + id: row.id, + filename: row.filename, + mimeType: row.mime_type, + filePath: row.file_path, + docType: row.doc_type as ProcessedDocument['docType'], + rawText: row.raw_text, + entities: JSON.parse(row.entities) as ProcessedDocument['entities'], + relations: JSON.parse(row.relations) as ProcessedDocument['relations'], + tables: JSON.parse(row.tables) as ProcessedDocument['tables'], + images: [], + metadata: JSON.parse(row.metadata) as ProcessedDocument['metadata'], + processedAt: row.processed_at, + source: row.source ?? undefined, + }; +} + +// --------------------------------------------------------------------------- +// Schema creation +// --------------------------------------------------------------------------- + +/** + * Ensure the `processed_documents` and `processed_documents_fts` tables exist. + * Safe to call multiple times (uses CREATE TABLE IF NOT EXISTS). + */ +export function ensureDocumentStoreSchema(db: Database.Database): void { + db.exec(` + CREATE TABLE IF NOT EXISTS processed_documents ( + id TEXT PRIMARY KEY, + filename TEXT NOT NULL, + mime_type TEXT NOT NULL, + file_path TEXT NOT NULL, + doc_type TEXT NOT NULL DEFAULT 'unknown', + raw_text TEXT NOT NULL DEFAULT '', + entities TEXT NOT NULL DEFAULT '[]', + relations TEXT NOT NULL DEFAULT '[]', + tables TEXT NOT NULL DEFAULT '[]', + metadata TEXT NOT NULL DEFAULT '{}', + processed_at TEXT NOT NULL, + source TEXT + ); + + CREATE VIRTUAL TABLE IF NOT EXISTS processed_documents_fts + USING fts5(raw_text, filename, content='processed_documents', content_rowid='rowid'); + `); +} + +// --------------------------------------------------------------------------- +// CRUD +// --------------------------------------------------------------------------- + +/** + * Persist a fully processed document to SQLite and index it in FTS5. + * If a document with the same `id` already exists it is replaced. + * + * @param db - Open better-sqlite3 database instance + * @param doc - The processed document to store + * @returns The document ID + */ +export function storeDocument(db: Database.Database, doc: ProcessedDocument): string { + ensureDocumentStoreSchema(db); + + const upsert = db.prepare(` + INSERT OR REPLACE INTO processed_documents + (id, filename, mime_type, file_path, doc_type, raw_text, entities, relations, tables, metadata, processed_at, source) + VALUES + (@id, @filename, @mime_type, @file_path, @doc_type, @raw_text, @entities, @relations, @tables, @metadata, @processed_at, @source) + `); + + const rebuildFts = db.prepare(` + INSERT OR REPLACE INTO processed_documents_fts (rowid, raw_text, filename) + SELECT rowid, raw_text, filename FROM processed_documents WHERE id = ? + `); + + db.transaction(() => { + upsert.run({ + id: doc.id, + filename: doc.filename, + mime_type: doc.mimeType, + file_path: doc.filePath, + doc_type: doc.docType ?? 'unknown', + raw_text: doc.rawText, + entities: JSON.stringify(doc.entities ?? []), + relations: JSON.stringify(doc.relations ?? []), + tables: JSON.stringify(doc.tables ?? []), + metadata: JSON.stringify(doc.metadata ?? {}), + processed_at: doc.processedAt, + source: doc.source ?? null, + }); + + rebuildFts.run(doc.id); + })(); + + return doc.id; +} + +/** + * Retrieve a stored document by its ID. + * + * @param db - Open better-sqlite3 database instance + * @param documentId - UUID of the document + * @returns The document record, or `null` if not found + */ +export function getDocument(db: Database.Database, documentId: string): ProcessedDocument | null { + ensureDocumentStoreSchema(db); + + const row = db.prepare(`SELECT * FROM processed_documents WHERE id = ?`).get(documentId) as + | ProcessedDocumentRow + | undefined; + + return row ? rowToDocument(row) : null; +} + +/** + * Full-text search over processed documents using the FTS5 index. + * Searches both `raw_text` and `filename` columns. + * + * @param db - Open better-sqlite3 database instance + * @param query - Search query string + * @param limit - Maximum number of results (default 10) + * @returns Array of matching documents ordered by relevance + */ +export function searchDocuments( + db: Database.Database, + query: string, + limit = 10, +): ProcessedDocument[] { + ensureDocumentStoreSchema(db); + + if (!query.trim()) return []; + + const sanitized = sanitizeFts5Query(query); + if (!sanitized) return []; + + try { + const rows = db + .prepare( + `SELECT d.* + FROM processed_documents d + JOIN ( + SELECT rowid + FROM processed_documents_fts + WHERE processed_documents_fts MATCH ? + ORDER BY rank + LIMIT ? + ) AS ranked ON d.rowid = ranked.rowid`, + ) + .all(sanitized, limit) as ProcessedDocumentRow[]; + + return rows.map(rowToDocument); + } catch { + return []; + } +} diff --git a/src/intelligence/entity-extractor.ts b/src/intelligence/entity-extractor.ts new file mode 100644 index 00000000..0b05e489 --- /dev/null +++ b/src/intelligence/entity-extractor.ts @@ -0,0 +1,283 @@ +/** + * Entity Extractor — AI-powered entity extraction from document content + * + * Spawns a read-only worker via AgentRunner to analyze document text and extract: + * - Document type classification (invoice, receipt, contract, etc.) + * - Key entities (people, companies, products, amounts, dates) + * - Relationships between entities + * + * Uses result-parser.ts for robust JSON extraction from AI output. + */ + +import { randomUUID } from 'crypto'; +import { createLogger } from '../core/logger.js'; +import type { + ProcessorResult, + ExtractedEntity, + EntityRelation, + DocumentType, +} from '../types/intelligence.js'; +import { DocumentTypeSchema } from '../types/intelligence.js'; +import { parseAIResult } from '../master/result-parser.js'; + +const logger = createLogger('entity-extractor'); + +/** Maximum raw text length sent to the AI worker (chars) */ +const MAX_TEXT_LENGTH = 16_000; + +/** Maximum number of table rows included in the prompt */ +const MAX_TABLE_ROWS = 50; + +/** Minimal AgentRunner typings to avoid circular import */ +interface AgentRunnerResult { + stdout: string; + exitCode: number; +} + +interface AgentRunnerLike { + spawn(opts: { + prompt: string; + workspacePath: string; + model?: string; + allowedTools?: string[]; + maxTurns?: number; + timeout?: number; + retries?: number; + }): Promise; +} + +/** Shape of the JSON we ask the AI to produce */ +interface AIExtractionResponse { + documentType?: string; + entities?: Array<{ + type: string; + name: string; + attributes?: Record; + }>; + relations?: Array<{ + fromName: string; + toName: string; + relation: string; + attributes?: Record; + }>; +} + +/** Result returned by extractEntities */ +export interface EntityExtractionResult { + docType: DocumentType; + entities: ExtractedEntity[]; + relations: EntityRelation[]; +} + +/** + * Truncate text to a maximum character length, appending a note if truncated. + */ +function truncateText(text: string, maxLen: number): string { + if (text.length <= maxLen) return text; + return text.slice(0, maxLen) + '\n\n[...text truncated...]'; +} + +/** + * Format tables into a readable text representation for the AI prompt. + */ +function formatTables(tables: ProcessorResult['tables']): string { + if (tables.length === 0) return ''; + + const parts: string[] = ['\n## Tables\n']; + for (const table of tables) { + if (table.sheetName) { + parts.push(`### ${table.sheetName}\n`); + } + if (table.headers.length > 0) { + parts.push(`| ${table.headers.join(' | ')} |`); + parts.push(`| ${table.headers.map(() => '---').join(' | ')} |`); + } + const rows = table.rows.slice(0, MAX_TABLE_ROWS); + for (const row of rows) { + parts.push( + `| ${row.map((cell) => (cell == null ? '' : typeof cell === 'object' ? JSON.stringify(cell) : String(cell as string | number | boolean))).join(' | ')} |`, + ); + } + if (table.rows.length > MAX_TABLE_ROWS) { + parts.push(`\n_...${table.rows.length - MAX_TABLE_ROWS} more rows omitted..._\n`); + } + parts.push(''); + } + return parts.join('\n'); +} + +/** + * Build the extraction prompt for the AI worker. + */ +function buildExtractionPrompt(processed: ProcessorResult, context?: string): string { + const sections: string[] = [ + 'Analyze the following document content and extract structured information.', + 'Return ONLY a JSON object (no markdown fences, no explanation) with this exact structure:', + '', + '{', + ' "documentType": "invoice" | "receipt" | "contract" | "catalog" | "report" | "spreadsheet" | "email" | "image" | "unknown",', + ' "entities": [', + ' { "type": "person|company|product|amount|date|address|phone|email|reference", "name": "", "attributes": { ... } }', + ' ],', + ' "relations": [', + ' { "fromName": "", "toName": "", "relation": "issued_by|belongs_to|paid_to|references|contains|sent_by|received_by" }', + ' ]', + '}', + '', + 'Rules:', + '- For amounts, set "name" to the formatted value (e.g. "$1,234.56") and add numeric "value" and "currency" in attributes.', + '- For dates, set "name" to the date string and add "iso" in attributes (e.g. "2024-01-15").', + '- Include ALL entities you can identify — people, companies, products, monetary amounts, dates, addresses, reference numbers.', + '- Relations should link entities by their "name" field.', + '- If you cannot determine the document type, use "unknown".', + '', + ]; + + if (context) { + sections.push(`## Context\n${context}\n`); + } + + const rawText = truncateText(processed.rawText, MAX_TEXT_LENGTH); + if (rawText.length > 0) { + sections.push(`## Document Text\n${rawText}\n`); + } + + const tableText = formatTables(processed.tables); + if (tableText.length > 0) { + sections.push(tableText); + } + + if (Object.keys(processed.metadata).length > 0) { + sections.push(`## Metadata\n${JSON.stringify(processed.metadata, null, 2)}\n`); + } + + return sections.join('\n'); +} + +/** + * Parse the AI response into structured entities and relations. + */ +function parseExtractionResponse(stdout: string): EntityExtractionResult { + const parsed = parseAIResult(stdout, 'entity-extraction'); + + if (!parsed.success) { + logger.warn({ error: parsed.error }, 'Failed to parse AI extraction response'); + return { docType: 'unknown', entities: [], relations: [] }; + } + + const data = parsed.data; + + // Validate document type + const docTypeResult = DocumentTypeSchema.safeParse(data.documentType); + const docType: DocumentType = docTypeResult.success ? docTypeResult.data : 'unknown'; + + // Build entity map (name → id) for relation linking + const entityMap = new Map(); + const entities: ExtractedEntity[] = (data.entities ?? []).map((e) => { + const id = randomUUID(); + entityMap.set(e.name, id); + return { + id, + type: e.type, + name: e.name, + attributes: e.attributes, + }; + }); + + // Build relations using entity name → id mapping + const relations: EntityRelation[] = []; + for (const r of data.relations ?? []) { + const fromId = entityMap.get(r.fromName); + const toId = entityMap.get(r.toName); + if (!fromId || !toId) { + logger.debug( + { fromName: r.fromName, toName: r.toName }, + 'Skipping relation with unknown entity reference', + ); + continue; + } + const rel: EntityRelation = { + fromId, + toId, + relation: r.relation, + }; + if (r.attributes) { + rel.attributes = r.attributes; + } + relations.push(rel); + } + + return { docType, entities, relations }; +} + +/** + * Extract structured entities from processed document content. + * + * Spawns a read-only worker via AgentRunner with a prompt containing + * the raw text and tables. Parses the worker's JSON output via result-parser. + * + * @param processed - ProcessorResult from a format-specific processor + * @param context - Optional context hint (e.g. "this was sent via WhatsApp") + * @returns Extracted entities, relations, and document type classification + */ +export async function extractEntities( + processed: ProcessorResult, + context?: string, +): Promise { + // Skip extraction if there's no meaningful content + if (processed.rawText.trim().length === 0 && processed.tables.length === 0) { + logger.debug('No text or tables to extract entities from'); + return { docType: 'unknown', entities: [], relations: [] }; + } + + const prompt = buildExtractionPrompt(processed, context); + + // Dynamically import AgentRunner to avoid circular dependencies + let AgentRunner: new () => AgentRunnerLike; + try { + const mod = (await import('../core/agent-runner.js')) as { + AgentRunner: new () => AgentRunnerLike; + }; + AgentRunner = mod.AgentRunner; + } catch { + logger.warn('AgentRunner not available, returning empty extraction'); + return { docType: 'unknown', entities: [], relations: [] }; + } + + const runner = new AgentRunner(); + try { + const result = await runner.spawn({ + prompt, + workspacePath: '.', + allowedTools: ['Read', 'Glob', 'Grep'], + maxTurns: 3, + // Sonnet-class models need 90-130s for image analysis (OB-F206) + timeout: 180_000, + retries: 0, + }); + + if (result.exitCode !== 0 || result.stdout.trim().length === 0) { + logger.warn( + { exitCode: result.exitCode, outputLen: result.stdout.length }, + 'Entity extraction worker returned no useful output', + ); + return { docType: 'unknown', entities: [], relations: [] }; + } + + const extraction = parseExtractionResponse(result.stdout); + + logger.info( + { + docType: extraction.docType, + entityCount: extraction.entities.length, + relationCount: extraction.relations.length, + }, + 'Entity extraction complete', + ); + + return extraction; + } catch (err) { + logger.error({ err }, 'Entity extraction worker failed'); + return { docType: 'unknown', entities: [], relations: [] }; + } +} diff --git a/src/intelligence/form-generator.ts b/src/intelligence/form-generator.ts new file mode 100644 index 00000000..7aa77d7a --- /dev/null +++ b/src/intelligence/form-generator.ts @@ -0,0 +1,430 @@ +import type { DocType, DocTypeField, DocTypeState, DocTypeTransition } from '../types/doctype.js'; + +/** + * Generate a self-contained HTML form page from DocType metadata. + * + * Maps each field type to the appropriate HTML input element and includes + * state machine action buttons when states/transitions are provided. + */ +export function generateForm( + doctype: DocType, + fields: DocTypeField[], + options?: { + record?: Record; + states?: DocTypeState[]; + transitions?: DocTypeTransition[]; + currentState?: string; + apiBase?: string; + }, +): string { + const record = options?.record; + const states = options?.states ?? []; + const transitions = options?.transitions ?? []; + const currentState = options?.currentState ?? (record?.['_state'] as string | undefined); + const apiBase = options?.apiBase ?? '/api/dt'; + const isEdit = record != null && record['id'] != null; + const recordId = isEdit ? String(record['id']) : ''; + const title = isEdit ? `Edit ${doctype.label_singular}` : `New ${doctype.label_singular}`; + + // Find available transitions from current state + const availableTransitions = currentState + ? transitions.filter((t) => t.from_state === currentState) + : []; + + // Find the state object for badge display + const stateObj = currentState ? states.find((s) => s.name === currentState) : undefined; + + const fieldHtml = fields + .filter((f) => f.formula == null) // Skip GENERATED columns + .sort((a, b) => a.sort_order - b.sort_order) + .map((f) => renderField(f, record)) + .join('\n'); + + return ` + + + + + ${escapeHtml(title)} + + + +
+
+
+ ← ${escapeHtml(doctype.label_plural)} +

${escapeHtml(title)}

+
+
+ ${stateObj ? `${escapeHtml(stateObj.label)}` : ''} +
+
+ + ${ + availableTransitions.length > 0 + ? `
${availableTransitions + .map( + (t) => + ``, + ) + .join('\n')}
` + : '' + } + +
+ ${fieldHtml} +
+ + Cancel +
+
+
+ + + +`; +} + +// --------------------------------------------------------------------------- +// Field renderers +// --------------------------------------------------------------------------- + +function toStr(val: unknown): string { + if (val == null) return ''; + if (typeof val === 'string') return val; + if (typeof val === 'number' || typeof val === 'boolean') return String(val); + return JSON.stringify(val); +} + +function renderField(field: DocTypeField, record?: Record): string { + const value = record?.[field.name]; + const val = toStr(value || field.default_value); + const required = field.required ? 'required' : ''; + const requiredMark = field.required ? '*' : ''; + const id = `field-${field.name}`; + + switch (field.field_type) { + case 'text': + case 'phone': + return fieldWrapper( + id, + field.label, + requiredMark, + ``, + ); + + case 'email': + return fieldWrapper( + id, + field.label, + requiredMark, + ``, + ); + + case 'url': + return fieldWrapper( + id, + field.label, + requiredMark, + ``, + ); + + case 'number': + return fieldWrapper( + id, + field.label, + requiredMark, + ``, + ); + + case 'currency': + return fieldWrapper( + id, + field.label, + requiredMark, + ``, + ); + + case 'date': + return fieldWrapper( + id, + field.label, + requiredMark, + ``, + ); + + case 'datetime': + return fieldWrapper( + id, + field.label, + requiredMark, + ``, + ); + + case 'longtext': + return fieldWrapper( + id, + field.label, + requiredMark, + ``, + ); + + case 'checkbox': + return fieldWrapper( + id, + field.label, + requiredMark, + ``, + 'checkbox-field', + ); + + case 'select': + return fieldWrapper( + id, + field.label, + requiredMark, + renderSelect(id, field.name, field.options ?? [], val, required), + ); + + case 'multiselect': + return fieldWrapper( + id, + field.label, + requiredMark, + renderSelect(id, field.name, field.options ?? [], val, required, true), + ); + + case 'link': { + const linkVal = toStr(value); + return fieldWrapper( + id, + field.label, + requiredMark, + `` + + (field.link_doctype + ? `Links to ${escapeHtml(field.link_doctype)}` + : ''), + ); + } + + case 'image': { + const imgVal = toStr(value); + return fieldWrapper( + id, + field.label, + requiredMark, + `` + + (imgVal ? `` : ''), + ); + } + + case 'table': + return renderTableField(field, record); + + default: { + const defVal = toStr(value); + return fieldWrapper( + id, + field.label, + requiredMark, + ``, + ); + } + } +} + +function fieldWrapper( + id: string, + label: string, + requiredMark: string, + inputHtml: string, + extraClass?: string, +): string { + return `
+ + ${inputHtml} +
`; +} + +function renderSelect( + id: string, + name: string, + options: string[], + selectedValue: string, + required: string, + multiple?: boolean, +): string { + const optionHtml = options + .map( + (opt) => + ``, + ) + .join('\n'); + return ``; +} + +function renderTableField(field: DocTypeField, record?: Record): string { + const rows = (record?.[field.name] as Record[] | undefined) ?? []; + const childDoctype = field.child_doctype ?? field.name; + + // Render existing rows as a simple inline table + let tableBody = ''; + if (rows.length > 0) { + const columns = Object.keys(rows[0]!).filter( + (k) => k !== 'id' && k !== 'parent_id' && k !== 'idx', + ); + const headerRow = columns.map((c) => `${escapeHtml(c)}`).join(''); + const bodyRows = rows + .map( + (row, idx) => + `${columns.map((c) => ``).join('')}`, + ) + .join('\n'); + + tableBody = ` + ${headerRow} + ${bodyRows} +
`; + } + + return `
+ + Child DocType: ${escapeHtml(childDoctype)} + ${tableBody || '

No rows yet.

'} +
`; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function escapeHtml(str: string): string { + return str + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +// --------------------------------------------------------------------------- +// Embedded CSS +// --------------------------------------------------------------------------- + +const CSS = ` + * { box-sizing: border-box; margin: 0; padding: 0; } + body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; background: #f5f5f5; color: #333; line-height: 1.5; } + .container { max-width: 720px; margin: 2rem auto; background: #fff; border-radius: 8px; box-shadow: 0 1px 3px rgba(0,0,0,0.12); padding: 2rem; } + .form-header { display: flex; justify-content: space-between; align-items: flex-start; margin-bottom: 1.5rem; } + .header-left { display: flex; flex-direction: column; gap: 0.25rem; } + .back-link { color: #666; text-decoration: none; font-size: 0.875rem; } + .back-link:hover { color: #333; } + h1 { font-size: 1.5rem; font-weight: 600; } + .state-badge { display: inline-block; padding: 0.25rem 0.75rem; border-radius: 999px; color: #fff; font-size: 0.8rem; font-weight: 500; } + .action-bar { display: flex; gap: 0.5rem; margin-bottom: 1.5rem; padding: 0.75rem; background: #f9f9f9; border-radius: 6px; border: 1px solid #e0e0e0; } + .doctype-form { display: flex; flex-direction: column; gap: 1.25rem; } + .form-group { display: flex; flex-direction: column; gap: 0.375rem; } + .form-group label { font-weight: 500; font-size: 0.875rem; color: #555; } + .required { color: #e53e3e; margin-left: 2px; } + input[type="text"], input[type="email"], input[type="url"], input[type="number"], + input[type="date"], input[type="datetime-local"], select, textarea { + padding: 0.5rem 0.75rem; border: 1px solid #d1d5db; border-radius: 6px; font-size: 0.9375rem; + transition: border-color 0.15s; width: 100%; + } + input:focus, select:focus, textarea:focus { outline: none; border-color: #3b82f6; box-shadow: 0 0 0 3px rgba(59,130,246,0.1); } + .checkbox-field { flex-direction: row; align-items: center; gap: 0.5rem; } + .checkbox-field input[type="checkbox"] { width: 1.125rem; height: 1.125rem; } + .help-text { color: #888; font-size: 0.8rem; } + .image-preview { max-width: 200px; max-height: 120px; margin-top: 0.5rem; border-radius: 4px; } + .child-table { width: 100%; border-collapse: collapse; margin-top: 0.5rem; } + .child-table th { background: #f3f4f6; padding: 0.375rem 0.5rem; text-align: left; font-size: 0.8rem; font-weight: 500; border-bottom: 1px solid #e5e7eb; } + .child-table td { padding: 0.25rem 0.5rem; border-bottom: 1px solid #f0f0f0; } + .child-table input { padding: 0.25rem 0.5rem; font-size: 0.85rem; border: 1px solid #e5e7eb; border-radius: 4px; } + .btn-remove { background: none; border: none; color: #e53e3e; cursor: pointer; font-size: 1.1rem; } + .empty-table { color: #999; font-size: 0.875rem; font-style: italic; } + .table-field { border: 1px solid #e5e7eb; border-radius: 6px; padding: 1rem; } + .form-actions { display: flex; gap: 0.75rem; margin-top: 0.5rem; padding-top: 1rem; border-top: 1px solid #e5e7eb; } + .btn { display: inline-block; padding: 0.5rem 1.25rem; border-radius: 6px; font-size: 0.9375rem; font-weight: 500; cursor: pointer; text-decoration: none; text-align: center; border: none; transition: background 0.15s; } + .btn-primary { background: #3b82f6; color: #fff; } + .btn-primary:hover { background: #2563eb; } + .btn-secondary { background: #e5e7eb; color: #374151; } + .btn-secondary:hover { background: #d1d5db; } + .btn-action { background: #10b981; color: #fff; } + .btn-action:hover { background: #059669; } + @media (max-width: 640px) { .container { margin: 1rem; padding: 1rem; } } +`; diff --git a/src/intelligence/hook-executor.ts b/src/intelligence/hook-executor.ts new file mode 100644 index 00000000..40815921 --- /dev/null +++ b/src/intelligence/hook-executor.ts @@ -0,0 +1,1224 @@ +import { mkdir, readFile, stat } from 'node:fs/promises'; +import { extname, join } from 'node:path'; +import { randomUUID } from 'node:crypto'; +import type Database from 'better-sqlite3'; +import type { DocType, DocTypeHook, HookActionType } from '../types/doctype.js'; +import { createLogger } from '../core/logger.js'; +import { generateNextNumber } from './naming-series.js'; +import { generatePdf } from './pdf-generator.js'; +import { loadBranding } from './branding.js'; +import { buildInvoiceDefinition } from './templates/invoice-template.js'; +import { buildQuoteDefinition } from './templates/quote-template.js'; +import { buildReceiptDefinition } from './templates/receipt-template.js'; +import { buildReportDefinition } from './templates/report-template.js'; +import type { InvoiceData, InvoiceItem } from './templates/invoice-template.js'; +import type { QuoteData, QuoteItem } from './templates/quote-template.js'; +import type { ReceiptData, ReceiptItem } from './templates/receipt-template.js'; +import type { ReportSection } from './templates/report-template.js'; + +const logger = createLogger('hook-executor'); + +// --------------------------------------------------------------------------- +// Notification sender registry +// --------------------------------------------------------------------------- + +/** + * Sender function for a messaging channel (WhatsApp, etc.). + * `to` is the channel-specific recipient identifier (phone number, chat ID, etc.). + */ +export type ChannelSenderFn = ( + to: string, + message: string, + attachments?: AttachmentPayload[], +) => Promise; + +/** Email sender function — sends a formatted email. */ +export type EmailSenderFn = ( + to: string, + subject: string, + body: string, + attachments?: AttachmentPayload[], +) => Promise; + +/** A resolved attachment ready to send. */ +export interface AttachmentPayload { + filename: string; + content: Buffer; + contentType: string; +} + +/** + * Registry of notification sender functions. + * Wire up by calling `registerNotificationSenders()` from bridge.ts. + */ +const notificationSenders: { + whatsapp?: ChannelSenderFn; + telegram?: ChannelSenderFn; + email?: EmailSenderFn; +} = {}; + +/** + * Register sender functions so the send_notification hook can deliver messages. + * Call this during bridge startup before any hooks execute. + */ +export function registerNotificationSenders(senders: { + whatsapp?: ChannelSenderFn; + telegram?: ChannelSenderFn; + email?: EmailSenderFn; +}): void { + if (senders.whatsapp) notificationSenders.whatsapp = senders.whatsapp; + if (senders.telegram) notificationSenders.telegram = senders.telegram; + if (senders.email) notificationSenders.email = senders.email; + logger.debug({ channels: Object.keys(senders) }, 'Notification senders registered'); +} + +// --------------------------------------------------------------------------- +// Worker spawner registry (used by spawn_worker) +// --------------------------------------------------------------------------- + +/** + * Spawner function that runs an AI worker and returns its stdout. + * + * @param prompt - The formatted prompt to send to the worker + * @param workspacePath - Working directory for the worker process + * @param skillPack - Optional skill pack / tool profile name (e.g. 'read-only', 'code-edit') + * @returns The worker's stdout output + */ +export type WorkerSpawnerFn = ( + prompt: string, + workspacePath: string, + skillPack?: string, +) => Promise; + +let workerSpawner: WorkerSpawnerFn | undefined; + +/** + * Register the worker spawner so the spawn_worker hook can launch AI workers. + * Call this during bridge startup (same place as registerNotificationSenders). + * Pass `undefined` to deregister (useful in tests). + */ +export function registerWorkerSpawner(spawner: WorkerSpawnerFn | undefined): void { + workerSpawner = spawner; + if (spawner) { + logger.debug('Worker spawner registered for spawn_worker hook'); + } else { + logger.debug('Worker spawner deregistered'); + } +} + +// --------------------------------------------------------------------------- +// Stripe adapter registry (used by create_payment_link) +// --------------------------------------------------------------------------- + +/** + * Stripe adapter interface for the create_payment_link hook. + * Phase 119 (Integration Hub) wires in the real Stripe adapter. + * Until then, the registry remains empty and the hook logs a warning and skips. + */ +export interface StripeAdapter { + /** + * Create a Stripe payment link. + * @param amount - Amount in the currency's smallest unit (e.g. cents for USD) + * @param description - Human-readable description shown on the payment page + * @returns The payment link URL + */ + createPaymentLink(amount: number, description: string): Promise; +} + +let stripeAdapter: StripeAdapter | undefined; + +/** + * Register the Stripe adapter so the create_payment_link hook can call it. + * Call this during bridge startup after Phase 119's Integration Hub is initialised. + * Pass `undefined` to deregister (useful in tests). + */ +export function registerStripeAdapter(adapter: StripeAdapter | undefined): void { + stripeAdapter = adapter; + if (adapter) { + logger.debug('Stripe adapter registered for create_payment_link hook'); + } else { + logger.debug('Stripe adapter deregistered'); + } +} + +// --------------------------------------------------------------------------- +// Workspace path registry (used by generate_pdf) +// --------------------------------------------------------------------------- + +let registeredWorkspacePath: string | undefined; + +/** + * Register the workspace path so the generate_pdf hook knows where to save files. + * Call this during bridge startup (same place as registerNotificationSenders). + */ +export function registerWorkspacePath(workspacePath: string): void { + registeredWorkspacePath = workspacePath; + logger.debug({ workspacePath }, 'Workspace path registered for hook-executor'); +} + +// --------------------------------------------------------------------------- +// Hook row type for SQLite reads +// --------------------------------------------------------------------------- + +interface HookRow { + id: string; + doctype_id: string; + event: string; + action_type: string; + action_config: string; + sort_order: number; + enabled: number; +} + +function rowToHook(row: HookRow): DocTypeHook { + return { + id: row.id, + doctype_id: row.doctype_id, + event: row.event, + action_type: row.action_type as HookActionType, + action_config: JSON.parse(row.action_config) as Record, + sort_order: row.sort_order, + enabled: row.enabled === 1, + }; +} + +// --------------------------------------------------------------------------- +// Hook handler dispatch map +// --------------------------------------------------------------------------- + +/** + * Handler function signature for a single hook type. + * Returns void — side effects are applied directly to the record object. + */ +type HookHandler = ( + hook: DocTypeHook, + record: Record, + db: Database.Database, +) => Promise | void; + +/** + * Dispatch table: action_type → handler. + * + * Each handler is implemented in a dedicated task (OB-1377 through OB-1381). + * Unknown types are handled by the fallback in executeHooks(). + */ +const HOOK_HANDLERS: Partial> = { + // OB-1377: generate_number — fills a naming-series field on create + generate_number: handleGenerateNumber, + + // OB-1378: update_field — evaluates an expression and sets a field + update_field: handleUpdateField, + + // OB-1379: send_notification — sends a formatted message via a channel + send_notification: handleSendNotification, + + // OB-1380: generate_pdf — renders a PDF and stores the file path + generate_pdf: handleGeneratePdf, + + // OB-1381: create_payment_link — calls Stripe to generate a payment URL + create_payment_link: handleCreatePaymentLink, + + // OB-1382: spawn_worker — launches an AI worker with an injected record prompt + spawn_worker: handleSpawnWorker, + + // OB-future: remaining action types + run_workflow: handleNotImplemented('run_workflow'), + call_integration: handleNotImplemented('call_integration'), +}; + +/** + * Handler for `generate_number` hook action type. + * On create event (before timing): parses action_config.pattern, + * calls generateNextNumber() to get the next number in the sequence, + * and sets the result on the field specified in action_config.field. + */ +function handleGenerateNumber( + hook: DocTypeHook, + record: Record, + db: Database.Database, +): void { + const config = hook.action_config; + + // Extract pattern and field from config + const pattern = config['pattern'] as string | undefined; + const field = config['field'] as string | undefined; + + if (!pattern) { + logger.warn({ hookId: hook.id }, 'generate_number hook missing "pattern" in action_config'); + return; + } + + if (!field) { + logger.warn({ hookId: hook.id }, 'generate_number hook missing "field" in action_config'); + return; + } + + // Generate the next number + const nextNumber = generateNextNumber(db, pattern); + + // Set the field value on the record + record[field] = nextNumber; + + logger.debug( + { hookId: hook.id, field, pattern, generatedNumber: nextNumber }, + 'generate_number hook executed successfully', + ); +} + +/** + * Handler for `update_field` hook action type. + * Evaluates a value expression and sets the result on a field in the record. + * + * Supported expression syntax: + * - `now()` → current ISO timestamp string + * - `{field_name}` → reference a field from the record + * - Literal values (strings, numbers, booleans) + * + * Examples: + * - `{ "field": "sent_at", "value": "now()" }` + * - `{ "field": "approved_by", "value": "{created_by}" }` + * - `{ "field": "is_active", "value": "true" }` + */ +function handleUpdateField( + hook: DocTypeHook, + record: Record, + _db: Database.Database, +): void { + const config = hook.action_config; + + // Extract field and value from config + const field = config['field'] as string | undefined; + const valueExpr = config['value'] as string | undefined; + + if (!field) { + logger.warn({ hookId: hook.id }, 'update_field hook missing "field" in action_config'); + return; + } + + if (valueExpr === undefined) { + logger.warn({ hookId: hook.id }, 'update_field hook missing "value" in action_config'); + return; + } + + // Evaluate the value expression + const evaluatedValue = evaluateValueExpression(valueExpr, record); + + // Set the field value on the record + record[field] = evaluatedValue; + + logger.debug( + { hookId: hook.id, field, expression: valueExpr, evaluatedValue }, + 'update_field hook executed successfully', + ); +} + +/** + * Evaluate a value expression against a record. + * + * Supported syntax: + * - `now()` → current ISO timestamp + * - `{field_name}` → field reference from the record + * - Literal values: strings, numbers, booleans (case-insensitive `true`/`false`/`null`) + * + * @param expression - The expression string to evaluate + * @param record - The record data to use for field references + * @returns The evaluated value + */ +function evaluateValueExpression(expression: string, record: Record): unknown { + const trimmed = expression.trim(); + + // Handle `now()` function + if (trimmed === 'now()') { + return new Date().toISOString(); + } + + // Handle field references `{field_name}` + if (trimmed.startsWith('{') && trimmed.endsWith('}')) { + const fieldName = trimmed.slice(1, -1); + return record[fieldName] ?? null; + } + + // Try to parse as a boolean literal + if (trimmed.toLowerCase() === 'true') { + return true; + } + if (trimmed.toLowerCase() === 'false') { + return false; + } + + // Try to parse as null + if (trimmed.toLowerCase() === 'null') { + return null; + } + + // Try to parse as a number + const asNumber = Number(trimmed); + if (!Number.isNaN(asNumber)) { + return asNumber; + } + + // String literal: remove surrounding quotes if present + if ( + (trimmed.startsWith('"') && trimmed.endsWith('"')) || + (trimmed.startsWith("'") && trimmed.endsWith("'")) + ) { + return trimmed.slice(1, -1); + } + + // Default: return as-is (bare string) + return trimmed; +} + +/** + * Format a Mustache-style template string with field values from a record. + * + * Replaces every `{{field}}` occurrence with the corresponding value from + * the record. Unknown field references are left as empty strings. + * + * @param template - Template string, e.g. `"Hello {{name}}, your total is {{total}}"` + * @param record - Record data to substitute into the template + * @returns Formatted string + */ +function formatTemplate(template: string, record: Record): string { + return template.replace(/\{\{(\w+)\}\}/g, (_match, field: string) => { + const value = record[field]; + if (value === undefined || value === null) { + return ''; + } + if (typeof value === 'object') { + return JSON.stringify(value); + } + return String(value as string | number | boolean); + }); +} + +/** + * Resolve a list of file paths to in-memory AttachmentPayload objects. + * Files that cannot be read are skipped with a warning. + */ +async function resolveAttachments(paths: string[]): Promise { + const attachments: AttachmentPayload[] = []; + + for (const filePath of paths) { + try { + const content = await readFile(filePath); + const ext = extname(filePath).toLowerCase(); + + // Derive a reasonable MIME type from extension + const mimeTypes: Record = { + '.pdf': 'application/pdf', + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.webp': 'image/webp', + '.txt': 'text/plain', + '.csv': 'text/csv', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + }; + + attachments.push({ + filename: filePath.split('/').pop() ?? filePath, + content, + contentType: mimeTypes[ext] ?? 'application/octet-stream', + }); + } catch (err) { + logger.warn({ filePath, err }, 'send_notification: could not read attachment — skipping'); + } + } + + return attachments; +} + +/** + * Handler for `send_notification` hook action type. + * + * Formats `action_config.template` using Mustache-style `{{field}}` substitution + * from the record, then delivers the message via the specified channel. + * + * Supported channels: + * - `whatsapp` — sends via registered WhatsApp connector sender + * - `telegram` — sends via registered Telegram connector sender + * - `email` — sends via registered email-sender (requires `subject` config) + * - `webhook` — sends JSON POST to `action_config.url` via HTTP fetch + * + * action_config fields: + * - `channel` (required) — delivery channel: `whatsapp` | `telegram` | `email` | `webhook` + * - `to` (required for whatsapp/telegram/email) — recipient identifier + * - `template` (required) — message body template with `{{field}}` placeholders + * - `subject` (optional, used by email channel) — email subject line + * - `url` (required for webhook) — URL to POST to + * - `attachments` (optional) — array of file paths to attach + */ +async function handleSendNotification( + hook: DocTypeHook, + record: Record, + _db: Database.Database, +): Promise { + const config = hook.action_config; + + const channel = config['channel'] as string | undefined; + const template = config['template'] as string | undefined; + const to = config['to'] as string | undefined; + const subject = config['subject'] as string | undefined; + const webhookUrl = config['url'] as string | undefined; + const attachmentPaths = Array.isArray(config['attachments']) + ? (config['attachments'] as string[]) + : []; + + if (!channel) { + logger.warn({ hookId: hook.id }, 'send_notification hook missing "channel" in action_config'); + return; + } + + if (!template) { + logger.warn({ hookId: hook.id }, 'send_notification hook missing "template" in action_config'); + return; + } + + // Format the message template with record field values + const message = formatTemplate(template, record); + + // Resolve attachments + const attachments = attachmentPaths.length > 0 ? await resolveAttachments(attachmentPaths) : []; + + logger.debug( + { hookId: hook.id, channel, to, attachmentCount: attachments.length }, + 'send_notification: delivering message', + ); + + if (channel === 'webhook') { + // Webhook delivery — HTTP POST with JSON body + if (!webhookUrl) { + logger.warn( + { hookId: hook.id }, + 'send_notification webhook channel missing "url" in action_config', + ); + return; + } + + const payload = { + message, + record, + attachments: attachmentPaths, + hook_id: hook.id, + }; + + const response = await fetch(webhookUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }); + + if (!response.ok) { + throw new Error( + `Webhook delivery failed: HTTP ${response.status.toString()} ${response.statusText}`, + ); + } + + logger.info({ hookId: hook.id, url: webhookUrl }, 'send_notification: webhook delivered'); + return; + } + + if (channel === 'email') { + if (!to) { + logger.warn( + { hookId: hook.id }, + 'send_notification email channel missing "to" in action_config', + ); + return; + } + + const emailSender = notificationSenders.email; + if (!emailSender) { + logger.warn({ hookId: hook.id }, 'send_notification: no email sender registered — skipping'); + return; + } + + await emailSender(to, subject ?? 'Notification', message, attachments); + logger.info({ hookId: hook.id, to }, 'send_notification: email delivered'); + return; + } + + if (channel === 'whatsapp') { + if (!to) { + logger.warn( + { hookId: hook.id }, + 'send_notification whatsapp channel missing "to" in action_config', + ); + return; + } + + const whatsappSender = notificationSenders.whatsapp; + if (!whatsappSender) { + logger.warn( + { hookId: hook.id }, + 'send_notification: no whatsapp sender registered — skipping', + ); + return; + } + + await whatsappSender(to, message, attachments); + logger.info({ hookId: hook.id, to }, 'send_notification: whatsapp message delivered'); + return; + } + + if (channel === 'telegram') { + if (!to) { + logger.warn( + { hookId: hook.id }, + 'send_notification telegram channel missing "to" in action_config', + ); + return; + } + + const telegramSender = notificationSenders.telegram; + if (!telegramSender) { + logger.warn( + { hookId: hook.id }, + 'send_notification: no telegram sender registered — skipping', + ); + return; + } + + await telegramSender(to, message, attachments); + logger.info({ hookId: hook.id, to }, 'send_notification: telegram message delivered'); + return; + } + + logger.warn( + { hookId: hook.id, channel }, + 'send_notification: unknown channel — no delivery performed', + ); +} + +// --------------------------------------------------------------------------- +// Record field extraction helpers +// --------------------------------------------------------------------------- + +/** Safely read a string value from a record, checking camelCase then snake_case. */ +function strField( + record: Record, + camelKey: string, + snakeKey?: string, +): string | undefined { + const v = record[camelKey] ?? (snakeKey ? record[snakeKey] : undefined); + if (v === undefined || v === null) return undefined; + return String(v as string | number | boolean); +} + +/** Safely read a numeric value from a record, checking camelCase then snake_case. */ +function numField( + record: Record, + camelKey: string, + snakeKey?: string, +): number | undefined { + const v = record[camelKey] ?? (snakeKey ? record[snakeKey] : undefined); + if (v === undefined || v === null) return undefined; + const n = Number(v); + return Number.isNaN(n) ? undefined : n; +} + +/** Map a flat record to InvoiceData, supporting both camelCase and snake_case field names. */ +function recordToInvoiceData(record: Record): InvoiceData { + return { + invoiceNumber: strField(record, 'invoiceNumber', 'invoice_number') ?? '', + date: strField(record, 'date') ?? new Date().toISOString().split('T')[0]!, + dueDate: strField(record, 'dueDate', 'due_date'), + customerName: strField(record, 'customerName', 'customer_name') ?? '', + customerEmail: strField(record, 'customerEmail', 'customer_email'), + customerAddress: strField(record, 'customerAddress', 'customer_address'), + customerPhone: strField(record, 'customerPhone', 'customer_phone'), + notes: strField(record, 'notes'), + terms: strField(record, 'terms'), + paymentLink: strField(record, 'paymentLink', 'payment_link'), + taxRate: numField(record, 'taxRate', 'tax_rate'), + currency: strField(record, 'currency'), + }; +} + +/** Extract InvoiceItem[] from `record.items` or `record.line_items`. */ +function recordToInvoiceItems(record: Record): InvoiceItem[] { + const raw = record['items'] ?? record['lineItems'] ?? record['line_items']; + if (!Array.isArray(raw)) return []; + return (raw as unknown[]).map((item) => { + const r = (item ?? {}) as Record; + return { + description: strField(r, 'description') ?? '', + quantity: numField(r, 'quantity') ?? 1, + unitPrice: numField(r, 'unitPrice', 'unit_price') ?? 0, + total: numField(r, 'total'), + }; + }); +} + +/** Map a flat record to QuoteData. */ +function recordToQuoteData(record: Record): QuoteData { + return { + quoteNumber: strField(record, 'quoteNumber', 'quote_number') ?? '', + date: strField(record, 'date') ?? new Date().toISOString().split('T')[0]!, + validUntil: strField(record, 'validUntil', 'valid_until'), + customerName: strField(record, 'customerName', 'customer_name') ?? '', + customerEmail: strField(record, 'customerEmail', 'customer_email'), + customerAddress: strField(record, 'customerAddress', 'customer_address'), + customerPhone: strField(record, 'customerPhone', 'customer_phone'), + notes: strField(record, 'notes'), + terms: strField(record, 'terms'), + taxRate: numField(record, 'taxRate', 'tax_rate'), + currency: strField(record, 'currency'), + }; +} + +/** Extract QuoteItem[] from `record.items` or `record.line_items`. */ +function recordToQuoteItems(record: Record): QuoteItem[] { + const raw = record['items'] ?? record['lineItems'] ?? record['line_items']; + if (!Array.isArray(raw)) return []; + return (raw as unknown[]).map((item) => { + const r = (item ?? {}) as Record; + return { + description: strField(r, 'description') ?? '', + quantity: numField(r, 'quantity') ?? 1, + unitPrice: numField(r, 'unitPrice', 'unit_price') ?? 0, + total: numField(r, 'total'), + }; + }); +} + +/** Map a flat record to ReceiptData. */ +function recordToReceiptData(record: Record): ReceiptData { + return { + receiptNumber: strField(record, 'receiptNumber', 'receipt_number'), + date: strField(record, 'date') ?? new Date().toISOString().split('T')[0]!, + time: strField(record, 'time'), + customerName: strField(record, 'customerName', 'customer_name'), + paymentMethod: strField(record, 'paymentMethod', 'payment_method'), + notes: strField(record, 'notes'), + currency: strField(record, 'currency'), + }; +} + +/** Extract ReceiptItem[] from `record.items`. */ +function recordToReceiptItems(record: Record): ReceiptItem[] { + const raw = record['items'] ?? record['lineItems'] ?? record['line_items']; + if (!Array.isArray(raw)) return []; + return (raw as unknown[]).map((item) => { + const r = (item ?? {}) as Record; + return { + description: strField(r, 'description') ?? '', + quantity: numField(r, 'quantity'), + amount: numField(r, 'amount') ?? numField(r, 'total') ?? 0, + }; + }); +} + +/** Extract ReportSection[] from `record.sections`. */ +function recordToReportSections(record: Record): ReportSection[] { + const raw = record['sections']; + if (!Array.isArray(raw)) return []; + return raw as ReportSection[]; +} + +// --------------------------------------------------------------------------- +// Minimal Puppeteer page type for PDF generation (fallback for custom templates) +// --------------------------------------------------------------------------- + +interface PuppeteerPdfPage { + setViewport(opts: { width: number; height: number }): Promise; + setContent(html: string, opts: { waitUntil: string }): Promise; + pdf(opts: { path: string; format: string; printBackground: boolean }): Promise; +} + +interface PuppeteerPdfBrowser { + newPage(): Promise; + close(): Promise; +} + +interface PuppeteerModule { + launch(opts: { headless: boolean; args: string[] }): Promise; +} + +/** + * Build a simple HTML document for a record, using an optional HTML template. + * + * If `.openbridge/templates/{templateName}.html` exists in the workspace, it is + * loaded and Mustache-style `{{field}}` placeholders are substituted with record + * values. Otherwise, a generic HTML table listing all record fields is generated. + */ +async function buildPdfHtml( + workspacePath: string, + templateName: string, + record: Record, +): Promise { + const templatePath = join(workspacePath, '.openbridge', 'templates', `${templateName}.html`); + + try { + const raw = await readFile(templatePath, 'utf-8'); + return formatTemplate(raw, record); + } catch { + // Template file not found — generate a generic HTML table + const rows = Object.entries(record) + .map(([k, v]) => { + const display = + v === null || v === undefined + ? '' + : typeof v === 'object' + ? JSON.stringify(v) + : String(v as string | number | boolean); + return `${escapeHtml(k)}${escapeHtml(display)}`; + }) + .join('\n'); + + return ` + + + + ${escapeHtml(templateName)} + + + +

${escapeHtml(templateName)}

+ + ${rows} +
+ +`; + } +} + +/** Minimal HTML entity escaping for text content. */ +function escapeHtml(str: string): string { + return str + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} + +/** Built-in pdfmake template names — these bypass Puppeteer entirely. */ +const PDFMAKE_TEMPLATES = new Set(['invoice', 'quote', 'receipt', 'report']); + +/** + * Handler for `generate_pdf` hook action type. + * + * Generates a PDF using: + * - pdfmake templates (invoice, quote, receipt, report) — no Chromium needed + * - Puppeteer HTML→PDF fallback for custom `.openbridge/templates/{name}.html` files + * + * Saves the file to `.openbridge/generated/` and updates `output_field` on the record. + * + * action_config fields: + * - `template` (required) — "invoice" | "quote" | "receipt" | "report", + * or a custom HTML template name + * - `output_field` (required) — record field to set with the generated PDF path + */ +async function handleGeneratePdf( + hook: DocTypeHook, + record: Record, + _db: Database.Database, +): Promise { + const config = hook.action_config; + const templateName = config['template'] as string | undefined; + const outputField = config['output_field'] as string | undefined; + + if (!templateName) { + logger.warn({ hookId: hook.id }, 'generate_pdf hook missing "template" in action_config'); + return; + } + + if (!outputField) { + logger.warn({ hookId: hook.id }, 'generate_pdf hook missing "output_field" in action_config'); + return; + } + + const workspacePath = registeredWorkspacePath; + if (!workspacePath) { + logger.warn( + { hookId: hook.id }, + 'generate_pdf hook: no workspace path registered — call registerWorkspacePath() on startup', + ); + return; + } + + let outputPath: string; + + if (PDFMAKE_TEMPLATES.has(templateName)) { + // ── pdfmake route for known business document templates ───────────────── + const branding = await loadBranding(workspacePath); + + let definition; + if (templateName === 'invoice') { + definition = buildInvoiceDefinition( + recordToInvoiceData(record), + recordToInvoiceItems(record), + branding, + ); + } else if (templateName === 'quote') { + definition = buildQuoteDefinition( + recordToQuoteData(record), + recordToQuoteItems(record), + branding, + ); + } else if (templateName === 'receipt') { + definition = buildReceiptDefinition( + recordToReceiptData(record), + recordToReceiptItems(record), + branding, + ); + } else { + // templateName === 'report' + const title = strField(record, 'title') ?? 'Report'; + definition = buildReportDefinition(title, recordToReportSections(record), branding); + } + + outputPath = await generatePdf(definition, workspacePath); + + logger.info( + { hookId: hook.id, outputPath, template: templateName }, + 'generate_pdf hook: PDF written via pdfmake', + ); + } else { + // ── Puppeteer fallback for custom HTML templates ───────────────────────── + const outputDir = join(workspacePath, '.openbridge', 'generated'); + await mkdir(outputDir, { recursive: true }); + + const html = await buildPdfHtml(workspacePath, templateName, record); + const outputFilename = `${templateName}-${randomUUID()}.pdf`; + outputPath = join(outputDir, outputFilename); + + let puppeteer: PuppeteerModule; + try { + const mod = (await import('puppeteer')) as { default?: PuppeteerModule } & PuppeteerModule; + puppeteer = mod.default ?? mod; + } catch { + throw new Error( + 'Puppeteer is not installed. Run `npm install puppeteer` to enable PDF generation.', + ); + } + + const browser = await puppeteer.launch({ + headless: true, + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage', + '--disable-gpu', + ], + }); + + try { + const page = await browser.newPage(); + await page.setViewport({ width: 1240, height: 1754 }); // A4-ish at 150 dpi + await page.setContent(html, { waitUntil: 'load' }); + await page.pdf({ path: outputPath, format: 'A4', printBackground: true }); + } finally { + await browser.close(); + } + + const fileStat = await stat(outputPath); + logger.info( + { hookId: hook.id, outputPath, sizeBytes: fileStat.size }, + 'generate_pdf hook: PDF written via Puppeteer', + ); + } + + // Update the record field with the generated file path + record[outputField] = outputPath; + + logger.debug( + { hookId: hook.id, outputField, outputPath }, + 'generate_pdf hook executed successfully', + ); +} + +/** + * Handler for `create_payment_link` hook action type. + * + * On transition events (after timing): reads amount and description from the + * record using the configured field names, calls the registered Stripe adapter's + * `createPaymentLink()`, and stores the returned URL in `action_config.output_field`. + * + * If the Stripe adapter has not been registered (Phase 119 not connected), the + * hook logs a warning and skips without throwing. + * + * action_config fields: + * - `amount_field` (required) — record field containing the payment amount + * (numeric, currency's smallest unit, e.g. cents) + * - `description_field` (required) — record field containing the payment description + * - `output_field` (required) — record field to set with the generated payment link URL + */ +async function handleCreatePaymentLink( + hook: DocTypeHook, + record: Record, + _db: Database.Database, +): Promise { + const config = hook.action_config; + const amountField = config['amount_field'] as string | undefined; + const descriptionField = config['description_field'] as string | undefined; + const outputField = config['output_field'] as string | undefined; + + if (!amountField) { + logger.warn( + { hookId: hook.id }, + 'create_payment_link hook missing "amount_field" in action_config', + ); + return; + } + + if (!descriptionField) { + logger.warn( + { hookId: hook.id }, + 'create_payment_link hook missing "description_field" in action_config', + ); + return; + } + + if (!outputField) { + logger.warn( + { hookId: hook.id }, + 'create_payment_link hook missing "output_field" in action_config', + ); + return; + } + + if (!stripeAdapter) { + logger.warn( + { hookId: hook.id }, + 'create_payment_link hook: Stripe integration not connected — skipping (register via registerStripeAdapter())', + ); + return; + } + + const amount = record[amountField]; + const description = record[descriptionField]; + + if (amount === undefined || amount === null) { + logger.warn( + { hookId: hook.id, amountField }, + 'create_payment_link hook: amount field is missing from record — skipping', + ); + return; + } + + const numericAmount = Number(amount); + if (Number.isNaN(numericAmount)) { + logger.warn( + { hookId: hook.id, amountField, amount }, + 'create_payment_link hook: amount field value is not numeric — skipping', + ); + return; + } + + let descriptionStr: string; + if (description === undefined || description === null) { + descriptionStr = ''; + } else if (typeof description === 'object') { + descriptionStr = JSON.stringify(description); + } else { + descriptionStr = String(description as string | number | boolean); + } + + logger.debug( + { hookId: hook.id, amountField, descriptionField, amount: numericAmount }, + 'create_payment_link: calling Stripe adapter', + ); + + const paymentUrl = await stripeAdapter.createPaymentLink(numericAmount, descriptionStr); + + record[outputField] = paymentUrl; + + logger.info( + { hookId: hook.id, outputField, paymentUrl }, + 'create_payment_link hook executed successfully', + ); +} + +/** + * Handler for `spawn_worker` hook action type. + * + * On any event (after timing): formats `action_config.prompt` using Mustache-style + * `{{field}}` substitution from the record, then spawns an AI worker via the + * registered WorkerSpawnerFn. Captures the worker's stdout and optionally stores it + * in the record field specified by `action_config.output_field`. + * + * If no worker spawner has been registered (registerWorkerSpawner() not called), + * the hook logs a warning and skips without throwing. + * + * action_config fields: + * - `prompt` (required) — worker prompt template with `{{field}}` placeholders + * - `skill_pack` (optional) — tool profile / skill pack name passed to the spawner + * (e.g. 'read-only', 'code-edit', 'full-access') + * - `output_field` (optional) — record field to set with the worker's stdout output + */ +async function handleSpawnWorker( + hook: DocTypeHook, + record: Record, + _db: Database.Database, +): Promise { + const config = hook.action_config; + + const promptTemplate = config['prompt'] as string | undefined; + const skillPack = config['skill_pack'] as string | undefined; + const outputField = config['output_field'] as string | undefined; + + if (!promptTemplate) { + logger.warn({ hookId: hook.id }, 'spawn_worker hook missing "prompt" in action_config'); + return; + } + + if (!workerSpawner) { + logger.warn( + { hookId: hook.id }, + 'spawn_worker hook: no worker spawner registered — skipping (register via registerWorkerSpawner())', + ); + return; + } + + const workspacePath = registeredWorkspacePath; + if (!workspacePath) { + logger.warn( + { hookId: hook.id }, + 'spawn_worker hook: no workspace path registered — call registerWorkspacePath() on startup', + ); + return; + } + + // Format the prompt template with record field values + const formattedPrompt = formatTemplate(promptTemplate, record); + + logger.debug( + { hookId: hook.id, skillPack, hasOutputField: Boolean(outputField) }, + 'spawn_worker: launching AI worker', + ); + + const workerOutput = await workerSpawner(formattedPrompt, workspacePath, skillPack); + + if (outputField) { + record[outputField] = workerOutput; + logger.debug( + { hookId: hook.id, outputField }, + 'spawn_worker: worker output stored in record field', + ); + } + + logger.info({ hookId: hook.id, skillPack }, 'spawn_worker hook executed successfully'); +} + +/** + * Returns a stub handler that logs a "not yet implemented" warning and returns + * without throwing, so it does not block other hooks in the sequence. + */ +function handleNotImplemented(actionType: string): HookHandler { + return (_hook, _record, _db) => { + logger.warn({ actionType }, 'Hook action type not yet implemented — skipping'); + }; +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/** + * Execute lifecycle hooks for a given (event, timing) combination. + * + * Loads hooks from the `doctype_hooks` metadata table filtered by: + * - `doctype_id` matches the given DocType + * - `event` matches `${timing}_${event}` (e.g. `before_create`, `after_transition`) + * - `enabled = 1` + * - Ordered by `sort_order` ASC + * + * Hooks are executed **sequentially** in sort_order. Errors are caught per hook + * so that one failing hook does not prevent subsequent hooks from running. + * + * @param db - better-sqlite3 Database instance (must have doctype_hooks table) + * @param doctype - DocType definition (used for its `id`) + * @param event - Event name without timing prefix (e.g. `create`, `update`, `transition`) + * @param record - The current data record (may be mutated by before-hooks like generate_number) + * @param timing - Whether these are `before` or `after` hooks + */ +export async function executeHooks( + db: Database.Database, + doctype: DocType, + event: string, + record: Record, + timing: 'before' | 'after', +): Promise { + const fullEvent = `${timing}_${event}`; + + // Load hooks from the metadata table + const rows = db + .prepare( + `SELECT * FROM doctype_hooks + WHERE doctype_id = ? AND event = ? AND enabled = 1 + ORDER BY sort_order ASC`, + ) + .all(doctype.id, fullEvent) as HookRow[]; + + if (rows.length === 0) { + logger.debug({ doctypeId: doctype.id, event: fullEvent }, 'No hooks to execute'); + return; + } + + const hooks = rows.map(rowToHook); + + logger.debug( + { doctypeId: doctype.id, event: fullEvent, hookCount: hooks.length }, + 'Executing lifecycle hooks', + ); + + for (const hook of hooks) { + logger.debug( + { hookId: hook.id, actionType: hook.action_type, sortOrder: hook.sort_order }, + 'Executing hook', + ); + + const handler = HOOK_HANDLERS[hook.action_type]; + + if (!handler) { + logger.warn( + { hookId: hook.id, actionType: hook.action_type }, + 'Unknown hook action_type — skipping', + ); + continue; + } + + try { + await handler(hook, record, db); + logger.debug({ hookId: hook.id, actionType: hook.action_type }, 'Hook executed successfully'); + } catch (err) { + // Error isolation: log and continue — one failed hook must not block others + logger.error( + { err, hookId: hook.id, actionType: hook.action_type, doctypeId: doctype.id }, + 'Hook execution failed — continuing with remaining hooks', + ); + } + } + + logger.debug( + { doctypeId: doctype.id, event: fullEvent, hookCount: hooks.length }, + 'All hooks processed', + ); +} + +// --------------------------------------------------------------------------- +// Handler registration (used by sub-tasks OB-1377 through OB-1382) +// --------------------------------------------------------------------------- + +/** + * Register a handler for a specific hook action type. + * Subsequent tasks (OB-1377+) call this to register their implementations, + * overriding the default stub handlers. + */ +export function registerHookHandler(actionType: HookActionType, handler: HookHandler): void { + HOOK_HANDLERS[actionType] = handler; +} + +// Re-export the handler type for sub-task use +export type { HookHandler }; diff --git a/src/intelligence/index.ts b/src/intelligence/index.ts new file mode 100644 index 00000000..d7d6c8bc --- /dev/null +++ b/src/intelligence/index.ts @@ -0,0 +1,33 @@ +/** + * Intelligence Module — Document processing and entity extraction + * + * Provides the document intelligence layer for OpenBridge: + * - Process business files (PDF, Excel, DOCX, images) + * - Extract structured entities (people, companies, amounts, dates) + * - Store processed documents with FTS5 search + * - Support AI-powered document understanding + * + * Part of the Document Intelligence Layer (OB-F184) + */ + +export * from './document-processor.js'; +export * from './entity-extractor.js'; +export * from './document-store.js'; +export * from './processors/index.js'; +export * from './doctype-store.js'; +export * from './naming-series.js'; +export * from './doctype-api.js'; +export * from './form-generator.js'; +export * from './list-generator.js'; +export * from './relation-manager.js'; +export * from './doctype-importer.js'; +export * from './doctype-exporter.js'; +export * from './knowledge-graph.js'; +export * from './state-machine.js'; +export * from './hook-executor.js'; +export * from './audit-log.js'; +export * from './pdf-generator.js'; +export * from './skill-creator.js'; +export * from './query-cache.js'; +export * from './user-preferences.js'; +export * from './activity-patterns.js'; diff --git a/src/intelligence/industry-detector.ts b/src/intelligence/industry-detector.ts new file mode 100644 index 00000000..0a9f155f --- /dev/null +++ b/src/intelligence/industry-detector.ts @@ -0,0 +1,182 @@ +/** + * Industry Detector — AI-powered business type classification + * + * Spawns a read-only worker via AgentRunner to analyze workspace context and + * user messages, then classifies the business type and returns the best-match + * industry template ID. + * + * Supported template IDs: + * - restaurant + * - retail + * - services + * - car-rental + * - construction + * - marketplace-seller + */ + +import { createLogger } from '../core/logger.js'; +import { parseAIResult } from '../master/result-parser.js'; + +const logger = createLogger('industry-detector'); + +/** Supported industry template IDs */ +export const INDUSTRY_TEMPLATE_IDS = [ + 'restaurant', + 'retail', + 'services', + 'car-rental', + 'construction', + 'marketplace-seller', +] as const; + +export type IndustryTemplateId = (typeof INDUSTRY_TEMPLATE_IDS)[number]; + +/** Fallback template ID when classification fails */ +const FALLBACK_TEMPLATE_ID: IndustryTemplateId = 'services'; + +/** Minimal AgentRunner typings to avoid circular import */ +interface AgentRunnerResult { + stdout: string; + exitCode: number; +} + +interface AgentRunnerLike { + spawn(opts: { + prompt: string; + workspacePath: string; + model?: string; + allowedTools?: string[]; + maxTurns?: number; + timeout?: number; + retries?: number; + }): Promise; +} + +/** Shape of the JSON we ask the AI to produce */ +interface AIClassificationResponse { + templateId?: string; + confidence?: string; + reasoning?: string; +} + +/** + * Build the classification prompt for the AI worker. + */ +function buildClassificationPrompt(workspaceContext: string, userMessages: string[]): string { + const sections: string[] = [ + 'Analyze the following workspace description and user messages to determine the best-match business type.', + 'Return ONLY a JSON object (no markdown fences, no explanation) with this exact structure:', + '', + '{', + ' "templateId": "restaurant" | "retail" | "services" | "car-rental" | "construction" | "marketplace-seller",', + ' "confidence": "high" | "medium" | "low",', + ' "reasoning": ""', + '}', + '', + 'Template descriptions:', + '- restaurant: Food & beverage businesses — cafes, restaurants, food trucks, bakeries, catering', + '- retail: Physical or online product sales — shops, stores, e-commerce, boutiques', + '- services: Professional services — consulting, accounting, legal, cleaning, maintenance, freelancing', + '- car-rental: Vehicle rental businesses — car rental, fleet management, vehicle leasing', + '- construction: Construction & contracting — builders, contractors, renovation, real estate development', + '- marketplace-seller: Multi-vendor or marketplace operations — platforms that connect buyers/sellers', + '', + 'Rules:', + '- Choose the single best-matching templateId from the list above.', + '- If unsure, pick "services" as the default.', + '- Do not invent new template IDs.', + '', + ]; + + if (workspaceContext.trim().length > 0) { + sections.push(`## Workspace Context\n${workspaceContext.trim()}\n`); + } + + if (userMessages.length > 0) { + sections.push('## Recent User Messages'); + for (const msg of userMessages) { + sections.push(`- ${msg}`); + } + sections.push(''); + } + + return sections.join('\n'); +} + +/** + * Parse the AI response and return a validated template ID. + */ +function parseClassificationResponse(stdout: string): IndustryTemplateId { + const parsed = parseAIResult(stdout, 'industry-detection'); + + if (!parsed.success) { + logger.warn({ error: parsed.error }, 'Failed to parse AI classification response'); + return FALLBACK_TEMPLATE_ID; + } + + const templateId = parsed.data.templateId?.trim().toLowerCase(); + if (templateId && (INDUSTRY_TEMPLATE_IDS as readonly string[]).includes(templateId)) { + return templateId as IndustryTemplateId; + } + + logger.warn({ templateId }, 'Unrecognised template ID from AI — falling back to default'); + return FALLBACK_TEMPLATE_ID; +} + +/** + * Detect the industry type for a workspace. + * + * Spawns a read-only worker via AgentRunner with a prompt containing the + * workspace description and recent user messages. Parses the worker's JSON + * output to return the best-match industry template ID. + * + * @param workspaceContext - Workspace description (e.g. from workspace-map.json or memory.md) + * @param userMessages - Recent user messages that may hint at the business type + * @returns The best-match template ID (e.g. "restaurant", "retail") + */ +export async function detectIndustry( + workspaceContext: string, + userMessages: string[], +): Promise { + const prompt = buildClassificationPrompt(workspaceContext, userMessages); + + // Dynamically import AgentRunner to avoid circular dependencies + let AgentRunner: new () => AgentRunnerLike; + try { + const mod = (await import('../core/agent-runner.js')) as { + AgentRunner: new () => AgentRunnerLike; + }; + AgentRunner = mod.AgentRunner; + } catch { + logger.warn('AgentRunner not available, returning fallback industry template'); + return FALLBACK_TEMPLATE_ID; + } + + const runner = new AgentRunner(); + try { + const result = await runner.spawn({ + prompt, + workspacePath: '.', + allowedTools: ['Read', 'Glob', 'Grep'], + maxTurns: 3, + timeout: 60_000, + retries: 1, + }); + + if (result.exitCode !== 0 || result.stdout.trim().length === 0) { + logger.warn( + { exitCode: result.exitCode, outputLen: result.stdout.length }, + 'Industry detection worker returned no useful output — falling back to default', + ); + return FALLBACK_TEMPLATE_ID; + } + + const templateId = parseClassificationResponse(result.stdout); + + logger.info({ templateId }, 'Industry detection complete'); + return templateId; + } catch (err) { + logger.error({ err }, 'Industry detection worker failed — falling back to default'); + return FALLBACK_TEMPLATE_ID; + } +} diff --git a/src/intelligence/industry-templates/car-rental/manifest.json b/src/intelligence/industry-templates/car-rental/manifest.json new file mode 100644 index 00000000..5758c1fc --- /dev/null +++ b/src/intelligence/industry-templates/car-rental/manifest.json @@ -0,0 +1,1003 @@ +{ + "id": "car-rental", + "name": "Car Rental", + "description": "Car rental, fleet management, vehicle leasing — fleet tracking, bookings, maintenance scheduling, rental contracts, and insurance management.", + "skillPack": "skill-pack.md", + "sampleQueries": [ + "Add a new vehicle: 2024 Toyota Camry, plate ABC-1234, silver, 15000 km", + "Show me all available vehicles for this weekend", + "Create a booking for John Smith, picking up tomorrow, returning in 3 days", + "Which vehicles have maintenance due this month?", + "Log maintenance for vehicle ABC-1234: oil change, $85, 20000 km", + "Show me all active rental contracts", + "Which vehicles have insurance expiring in the next 30 days?", + "What's our fleet utilization rate this week?", + "Record vehicle return for booking #1042, 350 km driven, no damage", + "Generate a damage assessment report for vehicle XYZ-5678" + ], + "doctypes": [ + { + "doctype": { + "name": "vehicle", + "label_singular": "Vehicle", + "label_plural": "Vehicles", + "icon": "car", + "table_name": "dt_vehicles", + "source": "template" + }, + "fields": [ + { + "name": "make", + "label": "Make", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 0 + }, + { + "name": "model", + "label": "Model", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 1 + }, + { + "name": "year", + "label": "Year", + "field_type": "number", + "required": true, + "sort_order": 2 + }, + { + "name": "license_plate", + "label": "License Plate", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 3 + }, + { + "name": "vin", + "label": "VIN", + "field_type": "text", + "sort_order": 4 + }, + { + "name": "color", + "label": "Color", + "field_type": "text", + "sort_order": 5 + }, + { + "name": "category", + "label": "Category", + "field_type": "select", + "required": true, + "sort_order": 6, + "options": ["economy", "compact", "midsize", "fullsize", "suv", "luxury", "van", "truck"] + }, + { + "name": "transmission", + "label": "Transmission", + "field_type": "select", + "sort_order": 7, + "options": ["automatic", "manual"] + }, + { + "name": "fuel_type", + "label": "Fuel Type", + "field_type": "select", + "sort_order": 8, + "options": ["gasoline", "diesel", "hybrid", "electric"] + }, + { + "name": "current_mileage", + "label": "Current Mileage (km)", + "field_type": "number", + "required": true, + "sort_order": 9 + }, + { + "name": "daily_rate", + "label": "Daily Rate", + "field_type": "currency", + "required": true, + "sort_order": 10 + }, + { + "name": "insurance_expiry", + "label": "Insurance Expiry Date", + "field_type": "date", + "sort_order": 11 + }, + { + "name": "insurance_policy", + "label": "Insurance Policy Number", + "field_type": "text", + "sort_order": 12 + }, + { + "name": "next_service_km", + "label": "Next Service At (km)", + "field_type": "number", + "sort_order": 13 + }, + { + "name": "image_url", + "label": "Photo", + "field_type": "image", + "sort_order": 14 + }, + { + "name": "notes", + "label": "Notes", + "field_type": "longtext", + "sort_order": 15 + } + ], + "states": [ + { + "name": "available", + "label": "Available", + "color": "green", + "is_initial": true, + "sort_order": 0 + }, + { "name": "rented", "label": "Rented", "color": "blue", "sort_order": 1 }, + { "name": "maintenance", "label": "In Maintenance", "color": "orange", "sort_order": 2 }, + { "name": "reserved", "label": "Reserved", "color": "purple", "sort_order": 3 }, + { + "name": "retired", + "label": "Retired", + "color": "gray", + "is_terminal": true, + "sort_order": 4 + } + ], + "transitions": [ + { + "from_state": "available", + "to_state": "rented", + "action_name": "rent_out", + "action_label": "Rent Out" + }, + { + "from_state": "available", + "to_state": "reserved", + "action_name": "reserve", + "action_label": "Reserve" + }, + { + "from_state": "available", + "to_state": "maintenance", + "action_name": "send_to_maintenance", + "action_label": "Send to Maintenance" + }, + { + "from_state": "rented", + "to_state": "available", + "action_name": "return_vehicle", + "action_label": "Return Vehicle" + }, + { + "from_state": "rented", + "to_state": "maintenance", + "action_name": "return_for_maintenance", + "action_label": "Return for Maintenance" + }, + { + "from_state": "reserved", + "to_state": "rented", + "action_name": "pick_up", + "action_label": "Pick Up" + }, + { + "from_state": "reserved", + "to_state": "available", + "action_name": "cancel_reservation", + "action_label": "Cancel Reservation" + }, + { + "from_state": "maintenance", + "to_state": "available", + "action_name": "complete_maintenance", + "action_label": "Complete Maintenance" + }, + { + "from_state": "available", + "to_state": "retired", + "action_name": "retire", + "action_label": "Retire Vehicle" + }, + { + "from_state": "maintenance", + "to_state": "retired", + "action_name": "retire", + "action_label": "Retire Vehicle" + } + ], + "hooks": [ + { + "event": "after_insert", + "action_type": "send_notification", + "action_config": { + "message": "New vehicle added: {{year}} {{make}} {{model}} ({{license_plate}}) — ${{daily_rate}}/day" + } + } + ], + "relations": [] + }, + { + "doctype": { + "name": "booking", + "label_singular": "Booking", + "label_plural": "Bookings", + "icon": "calendar", + "table_name": "dt_bookings", + "source": "template" + }, + "fields": [ + { + "name": "customer_name", + "label": "Customer Name", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 0 + }, + { + "name": "customer_phone", + "label": "Customer Phone", + "field_type": "phone", + "sort_order": 1 + }, + { + "name": "customer_email", + "label": "Customer Email", + "field_type": "email", + "sort_order": 2 + }, + { + "name": "customer_license", + "label": "Driver License Number", + "field_type": "text", + "required": true, + "sort_order": 3 + }, + { + "name": "vehicle_id", + "label": "Vehicle", + "field_type": "link", + "required": true, + "sort_order": 4, + "link_doctype": "vehicle" + }, + { + "name": "pickup_date", + "label": "Pickup Date", + "field_type": "date", + "required": true, + "sort_order": 5 + }, + { + "name": "return_date", + "label": "Return Date", + "field_type": "date", + "required": true, + "sort_order": 6 + }, + { + "name": "pickup_location", + "label": "Pickup Location", + "field_type": "text", + "sort_order": 7 + }, + { + "name": "return_location", + "label": "Return Location", + "field_type": "text", + "sort_order": 8 + }, + { + "name": "daily_rate", + "label": "Daily Rate", + "field_type": "currency", + "required": true, + "sort_order": 9 + }, + { + "name": "total_days", + "label": "Total Days", + "field_type": "number", + "sort_order": 10, + "formula": "return_date - pickup_date", + "depends_on": "pickup_date,return_date" + }, + { + "name": "total_amount", + "label": "Total Amount", + "field_type": "currency", + "sort_order": 11, + "formula": "daily_rate * total_days", + "depends_on": "daily_rate,total_days" + }, + { + "name": "mileage_out", + "label": "Mileage at Pickup", + "field_type": "number", + "sort_order": 12 + }, + { + "name": "mileage_in", + "label": "Mileage at Return", + "field_type": "number", + "sort_order": 13 + }, + { + "name": "fuel_level_out", + "label": "Fuel Level at Pickup", + "field_type": "select", + "sort_order": 14, + "options": ["full", "3/4", "1/2", "1/4", "empty"] + }, + { + "name": "fuel_level_in", + "label": "Fuel Level at Return", + "field_type": "select", + "sort_order": 15, + "options": ["full", "3/4", "1/2", "1/4", "empty"] + }, + { + "name": "damage_notes", + "label": "Damage Notes", + "field_type": "longtext", + "sort_order": 16 + }, + { + "name": "extras", + "label": "Extras", + "field_type": "multiselect", + "sort_order": 17, + "options": [ + "gps", + "child-seat", + "additional-driver", + "insurance-upgrade", + "roadside-assistance" + ] + }, + { + "name": "notes", + "label": "Notes", + "field_type": "longtext", + "sort_order": 18 + } + ], + "states": [ + { + "name": "pending", + "label": "Pending", + "color": "orange", + "is_initial": true, + "sort_order": 0 + }, + { "name": "confirmed", "label": "Confirmed", "color": "blue", "sort_order": 1 }, + { "name": "active", "label": "Active (Picked Up)", "color": "green", "sort_order": 2 }, + { "name": "completed", "label": "Completed", "color": "gray", "sort_order": 3 }, + { + "name": "cancelled", + "label": "Cancelled", + "color": "red", + "is_terminal": true, + "sort_order": 4 + } + ], + "transitions": [ + { + "from_state": "pending", + "to_state": "confirmed", + "action_name": "confirm", + "action_label": "Confirm Booking" + }, + { + "from_state": "pending", + "to_state": "cancelled", + "action_name": "cancel", + "action_label": "Cancel Booking" + }, + { + "from_state": "confirmed", + "to_state": "active", + "action_name": "pick_up", + "action_label": "Pick Up Vehicle" + }, + { + "from_state": "confirmed", + "to_state": "cancelled", + "action_name": "cancel", + "action_label": "Cancel Booking" + }, + { + "from_state": "active", + "to_state": "completed", + "action_name": "return_vehicle", + "action_label": "Return Vehicle" + } + ], + "hooks": [ + { + "event": "on_state_change", + "action_type": "run_workflow", + "action_config": { + "condition": "new_state == 'confirmed'", + "workflow": "booking-confirmation" + } + }, + { + "event": "on_state_change", + "action_type": "send_notification", + "action_config": { + "condition": "new_state == 'active'", + "message": "Vehicle {{vehicle_id}} picked up by {{customer_name}}. Mileage: {{mileage_out}} km." + } + } + ], + "relations": [ + { + "from_doctype": "booking", + "to_doctype": "vehicle", + "relation_type": "belongs_to", + "from_field": "vehicle_id", + "to_field": "id", + "label": "Rented vehicle" + } + ] + }, + { + "doctype": { + "name": "maintenance-log", + "label_singular": "Maintenance Log", + "label_plural": "Maintenance Logs", + "icon": "wrench", + "table_name": "dt_maintenance_logs", + "source": "template" + }, + "fields": [ + { + "name": "vehicle_id", + "label": "Vehicle", + "field_type": "link", + "required": true, + "sort_order": 0, + "link_doctype": "vehicle" + }, + { + "name": "date", + "label": "Service Date", + "field_type": "date", + "required": true, + "searchable": true, + "sort_order": 1 + }, + { + "name": "service_type", + "label": "Service Type", + "field_type": "select", + "required": true, + "sort_order": 2, + "options": [ + "oil-change", + "tire-rotation", + "brake-service", + "battery", + "ac-service", + "full-inspection", + "body-repair", + "engine-repair", + "transmission", + "electrical", + "other" + ] + }, + { + "name": "description", + "label": "Description", + "field_type": "longtext", + "required": true, + "searchable": true, + "sort_order": 3 + }, + { + "name": "mileage_at_service", + "label": "Mileage at Service (km)", + "field_type": "number", + "sort_order": 4 + }, + { + "name": "cost", + "label": "Cost", + "field_type": "currency", + "required": true, + "sort_order": 5 + }, + { + "name": "vendor", + "label": "Service Vendor", + "field_type": "text", + "sort_order": 6 + }, + { + "name": "next_service_km", + "label": "Next Service At (km)", + "field_type": "number", + "sort_order": 7 + }, + { + "name": "next_service_date", + "label": "Next Service Date", + "field_type": "date", + "sort_order": 8 + }, + { + "name": "parts_replaced", + "label": "Parts Replaced", + "field_type": "longtext", + "sort_order": 9 + }, + { + "name": "receipt_image", + "label": "Receipt / Invoice", + "field_type": "image", + "sort_order": 10 + }, + { + "name": "notes", + "label": "Notes", + "field_type": "longtext", + "sort_order": 11 + } + ], + "states": [ + { + "name": "scheduled", + "label": "Scheduled", + "color": "blue", + "is_initial": true, + "sort_order": 0 + }, + { "name": "in_progress", "label": "In Progress", "color": "orange", "sort_order": 1 }, + { + "name": "completed", + "label": "Completed", + "color": "green", + "is_terminal": true, + "sort_order": 2 + }, + { + "name": "cancelled", + "label": "Cancelled", + "color": "gray", + "is_terminal": true, + "sort_order": 3 + } + ], + "transitions": [ + { + "from_state": "scheduled", + "to_state": "in_progress", + "action_name": "start_service", + "action_label": "Start Service" + }, + { + "from_state": "scheduled", + "to_state": "cancelled", + "action_name": "cancel", + "action_label": "Cancel" + }, + { + "from_state": "in_progress", + "to_state": "completed", + "action_name": "complete", + "action_label": "Complete Service" + } + ], + "hooks": [ + { + "event": "on_state_change", + "action_type": "send_notification", + "action_config": { + "condition": "new_state == 'completed'", + "message": "Maintenance completed for vehicle {{vehicle_id}}: {{service_type}} — ${{cost}}" + } + } + ], + "relations": [ + { + "from_doctype": "maintenance-log", + "to_doctype": "vehicle", + "relation_type": "belongs_to", + "from_field": "vehicle_id", + "to_field": "id", + "label": "Serviced vehicle" + } + ] + }, + { + "doctype": { + "name": "rental-contract", + "label_singular": "Rental Contract", + "label_plural": "Rental Contracts", + "icon": "file-text", + "table_name": "dt_rental_contracts", + "source": "template" + }, + "fields": [ + { + "name": "contract_number", + "label": "Contract Number", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 0 + }, + { + "name": "booking_id", + "label": "Booking", + "field_type": "link", + "required": true, + "sort_order": 1, + "link_doctype": "booking" + }, + { + "name": "customer_name", + "label": "Customer Name", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 2 + }, + { + "name": "customer_license", + "label": "Driver License Number", + "field_type": "text", + "required": true, + "sort_order": 3 + }, + { + "name": "vehicle_id", + "label": "Vehicle", + "field_type": "link", + "required": true, + "sort_order": 4, + "link_doctype": "vehicle" + }, + { + "name": "start_date", + "label": "Start Date", + "field_type": "date", + "required": true, + "sort_order": 5 + }, + { + "name": "end_date", + "label": "End Date", + "field_type": "date", + "required": true, + "sort_order": 6 + }, + { + "name": "daily_rate", + "label": "Daily Rate", + "field_type": "currency", + "required": true, + "sort_order": 7 + }, + { + "name": "deposit_amount", + "label": "Security Deposit", + "field_type": "currency", + "sort_order": 8 + }, + { + "name": "insurance_type", + "label": "Insurance Type", + "field_type": "select", + "required": true, + "sort_order": 9, + "options": ["basic", "standard", "premium", "full-coverage"] + }, + { + "name": "mileage_limit", + "label": "Daily Mileage Limit (km)", + "field_type": "number", + "sort_order": 10 + }, + { + "name": "excess_mileage_rate", + "label": "Excess Mileage Rate (per km)", + "field_type": "currency", + "sort_order": 11 + }, + { + "name": "total_amount", + "label": "Total Contract Amount", + "field_type": "currency", + "required": true, + "sort_order": 12 + }, + { + "name": "payment_status", + "label": "Payment Status", + "field_type": "select", + "sort_order": 13, + "options": ["unpaid", "partial", "paid", "refunded"] + }, + { + "name": "terms_accepted", + "label": "Terms Accepted", + "field_type": "checkbox", + "required": true, + "sort_order": 14 + }, + { + "name": "notes", + "label": "Notes", + "field_type": "longtext", + "sort_order": 15 + } + ], + "states": [ + { + "name": "draft", + "label": "Draft", + "color": "gray", + "is_initial": true, + "sort_order": 0 + }, + { "name": "active", "label": "Active", "color": "green", "sort_order": 1 }, + { "name": "completed", "label": "Completed", "color": "blue", "sort_order": 2 }, + { "name": "disputed", "label": "Disputed", "color": "red", "sort_order": 3 }, + { + "name": "closed", + "label": "Closed", + "color": "gray", + "is_terminal": true, + "sort_order": 4 + } + ], + "transitions": [ + { + "from_state": "draft", + "to_state": "active", + "action_name": "activate", + "action_label": "Activate Contract" + }, + { + "from_state": "active", + "to_state": "completed", + "action_name": "complete", + "action_label": "Complete Contract" + }, + { + "from_state": "active", + "to_state": "disputed", + "action_name": "dispute", + "action_label": "Flag Dispute" + }, + { + "from_state": "completed", + "to_state": "disputed", + "action_name": "dispute", + "action_label": "Flag Dispute" + }, + { + "from_state": "completed", + "to_state": "closed", + "action_name": "close", + "action_label": "Close Contract" + }, + { + "from_state": "disputed", + "to_state": "closed", + "action_name": "resolve", + "action_label": "Resolve & Close" + } + ], + "hooks": [ + { + "event": "after_insert", + "action_type": "send_notification", + "action_config": { + "message": "New rental contract {{contract_number}} created for {{customer_name}} — Vehicle {{vehicle_id}}, ${{total_amount}}" + } + } + ], + "relations": [ + { + "from_doctype": "rental-contract", + "to_doctype": "booking", + "relation_type": "belongs_to", + "from_field": "booking_id", + "to_field": "id", + "label": "For booking" + }, + { + "from_doctype": "rental-contract", + "to_doctype": "vehicle", + "relation_type": "belongs_to", + "from_field": "vehicle_id", + "to_field": "id", + "label": "Rented vehicle" + } + ] + } + ], + "workflows": [ + { + "name": "Maintenance Due Alert", + "description": "Checks vehicles daily and sends an alert for any vehicles approaching their next service mileage or with overdue maintenance dates.", + "trigger": { + "type": "schedule", + "cron": "0 7 * * *" + }, + "steps": [ + { + "id": "query-due-mileage", + "name": "Find vehicles near service mileage", + "type": "query", + "config": { + "doctype": "vehicle", + "filter": "current_mileage >= (next_service_km - 500) AND status != 'retired'", + "fields": ["make", "model", "license_plate", "current_mileage", "next_service_km"] + }, + "sort_order": 0 + }, + { + "id": "check-has-vehicles", + "name": "Check if any vehicles need service", + "type": "condition", + "config": { + "expression": "results.length > 0", + "on_false": "skip_remaining" + }, + "sort_order": 1 + }, + { + "id": "format-alert", + "name": "Format maintenance alert", + "type": "transform", + "config": { + "template": "Maintenance Due Alert:\n{{#each results}}\n- {{year}} {{make}} {{model}} ({{license_plate}}): {{current_mileage}} km / next service at {{next_service_km}} km\n{{/each}}" + }, + "sort_order": 2 + }, + { + "id": "send-alert", + "name": "Send maintenance notification", + "type": "send", + "config": { + "channel": "default", + "message": "{{formatted_output}}" + }, + "sort_order": 3 + } + ], + "status": "active" + }, + { + "name": "Booking Confirmation", + "description": "Sends a confirmation message when a booking is confirmed, including vehicle details, dates, and pricing.", + "trigger": { + "type": "event", + "event": "booking.state_change", + "condition": "new_state == 'confirmed'" + }, + "steps": [ + { + "id": "get-booking", + "name": "Get booking details", + "type": "query", + "config": { + "doctype": "booking", + "filter": "id = '{{trigger.record_id}}'", + "fields": [ + "customer_name", + "customer_email", + "vehicle_id", + "pickup_date", + "return_date", + "daily_rate", + "total_amount", + "pickup_location", + "extras" + ] + }, + "sort_order": 0 + }, + { + "id": "get-vehicle", + "name": "Get vehicle details", + "type": "query", + "config": { + "doctype": "vehicle", + "filter": "id = '{{booking.vehicle_id}}'", + "fields": ["make", "model", "year", "license_plate", "color", "category"] + }, + "sort_order": 1 + }, + { + "id": "format-confirmation", + "name": "Format confirmation message", + "type": "transform", + "config": { + "template": "Booking Confirmed!\n\nCustomer: {{booking.customer_name}}\nVehicle: {{vehicle.year}} {{vehicle.make}} {{vehicle.model}} ({{vehicle.color}})\nPlate: {{vehicle.license_plate}}\nPickup: {{booking.pickup_date}} at {{booking.pickup_location}}\nReturn: {{booking.return_date}}\nRate: ${{booking.daily_rate}}/day\nTotal: ${{booking.total_amount}}" + }, + "sort_order": 2 + }, + { + "id": "send-confirmation", + "name": "Send booking confirmation", + "type": "send", + "config": { + "channel": "default", + "message": "{{formatted_output}}" + }, + "sort_order": 3 + } + ], + "status": "active" + }, + { + "name": "Insurance Expiry Alert", + "description": "Checks weekly for vehicles with insurance expiring within the next 30 days and sends an alert to schedule renewals.", + "trigger": { + "type": "schedule", + "cron": "0 9 * * 1" + }, + "steps": [ + { + "id": "query-expiring", + "name": "Find vehicles with expiring insurance", + "type": "query", + "config": { + "doctype": "vehicle", + "filter": "insurance_expiry <= date('now', '+30 days') AND insurance_expiry >= date('now') AND status != 'retired'", + "fields": ["make", "model", "license_plate", "insurance_expiry", "insurance_policy"] + }, + "sort_order": 0 + }, + { + "id": "check-has-expiring", + "name": "Check if any insurance is expiring", + "type": "condition", + "config": { + "expression": "results.length > 0", + "on_false": "skip_remaining" + }, + "sort_order": 1 + }, + { + "id": "format-alert", + "name": "Format insurance expiry alert", + "type": "transform", + "config": { + "template": "Insurance Expiry Alert — Renewals needed:\n{{#each results}}\n- {{year}} {{make}} {{model}} ({{license_plate}}): Policy {{insurance_policy}} expires {{insurance_expiry}}\n{{/each}}" + }, + "sort_order": 2 + }, + { + "id": "send-alert", + "name": "Send insurance expiry notification", + "type": "send", + "config": { + "channel": "default", + "message": "{{formatted_output}}" + }, + "sort_order": 3 + } + ], + "status": "active" + } + ] +} diff --git a/src/intelligence/industry-templates/car-rental/skill-pack.md b/src/intelligence/industry-templates/car-rental/skill-pack.md new file mode 100644 index 00000000..1e6a41d7 --- /dev/null +++ b/src/intelligence/industry-templates/car-rental/skill-pack.md @@ -0,0 +1,67 @@ +# Car Rental Operations Skill Pack + +You are managing a car rental, fleet management, or vehicle leasing business. Use the following guidance when handling user requests. + +## Data Types Available + +- **Vehicle**: Track fleet vehicles with make, model, year, license plate, mileage, daily rate, insurance, and maintenance schedule +- **Booking**: Manage reservations with customer details, pickup/return dates, mileage tracking, fuel levels, and extras +- **Maintenance Log**: Record service history with type, cost, vendor, mileage, and next service scheduling +- **Rental Contract**: Formal rental agreements with insurance type, mileage limits, deposit, payment status, and terms + +## Common Operations + +### Fleet Management + +- Add, update, or retire vehicles from the fleet +- Track vehicle availability, status (available, rented, maintenance, reserved) +- Monitor mileage across the fleet +- Categorize vehicles (economy, compact, midsize, fullsize, SUV, luxury, van, truck) +- Track fuel type and transmission for matching customer preferences + +### Booking & Reservations + +- Create, confirm, and cancel bookings +- Record pickup and return details (mileage, fuel level, damage) +- Calculate rental totals: `daily_rate * number_of_days` +- Manage extras (GPS, child seat, additional driver, insurance upgrade, roadside assistance) +- Track booking lifecycle: pending → confirmed → active → completed + +### Maintenance & Servicing + +- Schedule and track maintenance (oil change, tires, brakes, battery, inspection, body repair) +- Set next service thresholds by mileage or date +- Track maintenance costs per vehicle +- Flag vehicles approaching service mileage (within 500 km of next service) +- Record vendor and parts information + +### Contracts & Insurance + +- Generate rental contracts linked to bookings +- Track insurance types (basic, standard, premium, full-coverage) +- Set mileage limits and excess mileage rates +- Manage security deposits and payment status +- Handle disputes and resolution + +### Reporting + +- Fleet utilization rate: `rented vehicles / total available vehicles` +- Revenue per vehicle per month +- Maintenance cost trends by vehicle and service type +- Insurance expiry calendar +- Booking pipeline (pending → confirmed → active) + +## Key Metrics + +- **Fleet utilization target**: 65-80% (vehicles rented vs total fleet) +- **Maintenance cost ratio**: maintenance costs should stay under 15% of vehicle revenue +- **Average rental duration**: track to optimize pricing tiers +- Vehicles with high maintenance costs relative to revenue should be flagged for retirement + +## Tips + +- When a customer "picks up" a vehicle, record mileage and fuel level at pickup +- When a vehicle is "returned", record mileage and fuel level at return, note any damage +- "Utilization" means percentage of fleet currently rented out +- Always check insurance expiry before renting out a vehicle +- Flag vehicles approaching service mileage before they go out on rental diff --git a/src/intelligence/industry-templates/marketplace-seller/api-spec.json b/src/intelligence/industry-templates/marketplace-seller/api-spec.json new file mode 100644 index 00000000..40e6bfb4 --- /dev/null +++ b/src/intelligence/industry-templates/marketplace-seller/api-spec.json @@ -0,0 +1,290 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "Marketplace Seller API", + "description": "Sample OpenAPI spec for a marketplace seller API. Replace the server URL with your marketplace's actual API endpoint during onboarding.", + "version": "1.0.0" + }, + "servers": [ + { + "url": "https://api.your-marketplace.com/v1", + "description": "Marketplace API base URL — replace with your marketplace's actual endpoint" + } + ], + "security": [{ "ApiKeyAuth": [] }], + "components": { + "securitySchemes": { + "ApiKeyAuth": { + "type": "apiKey", + "in": "header", + "name": "X-API-Key" + } + }, + "schemas": { + "Order": { + "type": "object", + "properties": { + "id": { "type": "string" }, + "created_at": { "type": "string", "format": "date-time" }, + "status": { + "type": "string", + "enum": ["pending", "processing", "shipped", "delivered", "cancelled", "refunded"] + }, + "buyer_name": { "type": "string" }, + "buyer_email": { "type": "string" }, + "shipping_address": { "type": "string" }, + "items": { + "type": "array", + "items": { "$ref": "#/components/schemas/OrderItem" } + }, + "subtotal": { "type": "number" }, + "shipping_cost": { "type": "number" }, + "total": { "type": "number" }, + "currency": { "type": "string" }, + "tracking_number": { "type": "string" } + } + }, + "OrderItem": { + "type": "object", + "properties": { + "listing_id": { "type": "string" }, + "sku": { "type": "string" }, + "title": { "type": "string" }, + "quantity": { "type": "integer" }, + "unit_price": { "type": "number" }, + "subtotal": { "type": "number" } + } + }, + "Listing": { + "type": "object", + "properties": { + "id": { "type": "string" }, + "sku": { "type": "string" }, + "title": { "type": "string" }, + "description": { "type": "string" }, + "price": { "type": "number" }, + "currency": { "type": "string" }, + "stock_quantity": { "type": "integer" }, + "status": { + "type": "string", + "enum": ["active", "inactive", "out_of_stock", "suspended"] + }, + "category": { "type": "string" }, + "images": { "type": "array", "items": { "type": "string" } }, + "created_at": { "type": "string", "format": "date-time" }, + "updated_at": { "type": "string", "format": "date-time" } + } + }, + "SalesReport": { + "type": "object", + "properties": { + "period_start": { "type": "string", "format": "date" }, + "period_end": { "type": "string", "format": "date" }, + "total_orders": { "type": "integer" }, + "total_revenue": { "type": "number" }, + "total_units_sold": { "type": "integer" }, + "currency": { "type": "string" }, + "top_listings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "listing_id": { "type": "string" }, + "title": { "type": "string" }, + "units_sold": { "type": "integer" }, + "revenue": { "type": "number" } + } + } + } + } + } + } + }, + "paths": { + "/orders": { + "get": { + "summary": "List orders", + "operationId": "listOrders", + "parameters": [ + { "name": "status", "in": "query", "schema": { "type": "string" } }, + { + "name": "created_after", + "in": "query", + "schema": { "type": "string", "format": "date-time" } + }, + { + "name": "created_before", + "in": "query", + "schema": { "type": "string", "format": "date-time" } + }, + { "name": "page", "in": "query", "schema": { "type": "integer", "default": 1 } }, + { "name": "per_page", "in": "query", "schema": { "type": "integer", "default": 50 } } + ], + "responses": { + "200": { + "description": "List of orders", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "orders": { + "type": "array", + "items": { "$ref": "#/components/schemas/Order" } + }, + "total": { "type": "integer" }, + "page": { "type": "integer" }, + "per_page": { "type": "integer" } + } + } + } + } + } + } + } + }, + "/orders/{orderId}": { + "get": { + "summary": "Get order details", + "operationId": "getOrder", + "parameters": [ + { "name": "orderId", "in": "path", "required": true, "schema": { "type": "string" } } + ], + "responses": { + "200": { + "description": "Order details", + "content": { + "application/json": { "schema": { "$ref": "#/components/schemas/Order" } } + } + } + } + } + }, + "/orders/{orderId}/ship": { + "post": { + "summary": "Mark order as shipped", + "operationId": "shipOrder", + "parameters": [ + { "name": "orderId", "in": "path", "required": true, "schema": { "type": "string" } } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": ["tracking_number", "carrier"], + "properties": { + "tracking_number": { "type": "string" }, + "carrier": { "type": "string" } + } + } + } + } + }, + "responses": { + "200": { "description": "Order marked as shipped" } + } + } + }, + "/listings": { + "get": { + "summary": "List product listings", + "operationId": "listListings", + "parameters": [ + { "name": "status", "in": "query", "schema": { "type": "string" } }, + { "name": "page", "in": "query", "schema": { "type": "integer", "default": 1 } }, + { "name": "per_page", "in": "query", "schema": { "type": "integer", "default": 50 } } + ], + "responses": { + "200": { + "description": "List of product listings", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "listings": { + "type": "array", + "items": { "$ref": "#/components/schemas/Listing" } + }, + "total": { "type": "integer" } + } + } + } + } + } + } + } + }, + "/listings/{listingId}": { + "get": { + "summary": "Get listing details", + "operationId": "getListing", + "parameters": [ + { "name": "listingId", "in": "path", "required": true, "schema": { "type": "string" } } + ], + "responses": { + "200": { + "description": "Listing details", + "content": { + "application/json": { "schema": { "$ref": "#/components/schemas/Listing" } } + } + } + } + }, + "patch": { + "summary": "Update listing (price, stock, status)", + "operationId": "updateListing", + "parameters": [ + { "name": "listingId", "in": "path", "required": true, "schema": { "type": "string" } } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "price": { "type": "number" }, + "stock_quantity": { "type": "integer" }, + "status": { "type": "string" } + } + } + } + } + }, + "responses": { + "200": { "description": "Listing updated" } + } + } + }, + "/reports/sales": { + "get": { + "summary": "Get sales report for a period", + "operationId": "getSalesReport", + "parameters": [ + { + "name": "start_date", + "in": "query", + "required": true, + "schema": { "type": "string", "format": "date" } + }, + { + "name": "end_date", + "in": "query", + "required": true, + "schema": { "type": "string", "format": "date" } + } + ], + "responses": { + "200": { + "description": "Sales report", + "content": { + "application/json": { "schema": { "$ref": "#/components/schemas/SalesReport" } } + } + } + } + } + } + } +} diff --git a/src/intelligence/industry-templates/marketplace-seller/integrations.json b/src/intelligence/industry-templates/marketplace-seller/integrations.json new file mode 100644 index 00000000..7196f836 --- /dev/null +++ b/src/intelligence/industry-templates/marketplace-seller/integrations.json @@ -0,0 +1,4 @@ +{ + "required": ["api"], + "suggested_spec": "api-spec.json" +} diff --git a/src/intelligence/industry-templates/marketplace-seller/manifest.json b/src/intelligence/industry-templates/marketplace-seller/manifest.json new file mode 100644 index 00000000..b432d6ee --- /dev/null +++ b/src/intelligence/industry-templates/marketplace-seller/manifest.json @@ -0,0 +1,616 @@ +{ + "id": "marketplace-seller", + "name": "Marketplace Seller", + "description": "Online marketplace seller — Etsy, Amazon, eBay, Shopify, or any platform with an API. Track product listings, supplier orders, and sales performance. Connect to your marketplace API for live order sync.", + "skillPack": "skill-pack.md", + "integrations": "integrations.json", + "sampleQueries": [ + "Show me all active product listings", + "Add a new product listing: Handmade Ceramic Mug, SKU MUG-001, price $28, initial stock 50", + "Create a supplier order: 100 units of raw clay from Potter Supply Co, $0.80/unit", + "Which products are running low on stock?", + "How many orders have I received this week?", + "What's my total revenue this month?", + "Mark supplier order SUP-012 as received", + "Show me my top 5 products by units sold this month", + "Update the price of listing MUG-001 to $32", + "Which supplier orders are pending delivery?" + ], + "doctypes": [ + { + "doctype": { + "name": "product-listing", + "label_singular": "Product Listing", + "label_plural": "Product Listings", + "icon": "tag", + "table_name": "dt_product_listings", + "source": "template" + }, + "fields": [ + { + "name": "name", + "label": "Product Name", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 0 + }, + { + "name": "sku", + "label": "SKU", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 1 + }, + { + "name": "marketplace_listing_id", + "label": "Marketplace Listing ID", + "field_type": "text", + "searchable": true, + "sort_order": 2 + }, + { + "name": "description", + "label": "Description", + "field_type": "longtext", + "sort_order": 3 + }, + { + "name": "category", + "label": "Category", + "field_type": "text", + "searchable": true, + "sort_order": 4 + }, + { + "name": "price", + "label": "Selling Price", + "field_type": "currency", + "required": true, + "sort_order": 5 + }, + { + "name": "cost_price", + "label": "Cost Price", + "field_type": "currency", + "sort_order": 6 + }, + { + "name": "margin", + "label": "Margin (%)", + "field_type": "number", + "sort_order": 7, + "formula": "((price - cost_price) / price) * 100", + "depends_on": "price,cost_price" + }, + { + "name": "stock_quantity", + "label": "Stock Quantity", + "field_type": "number", + "required": true, + "sort_order": 8 + }, + { + "name": "reorder_threshold", + "label": "Reorder Threshold", + "field_type": "number", + "sort_order": 9 + }, + { + "name": "reorder_quantity", + "label": "Reorder Quantity", + "field_type": "number", + "sort_order": 10 + }, + { + "name": "supplier_id", + "label": "Primary Supplier", + "field_type": "text", + "searchable": true, + "sort_order": 11 + }, + { + "name": "units_sold_total", + "label": "Total Units Sold", + "field_type": "number", + "sort_order": 12 + }, + { + "name": "total_revenue", + "label": "Total Revenue", + "field_type": "currency", + "sort_order": 13 + }, + { "name": "notes", "label": "Notes", "field_type": "longtext", "sort_order": 14 } + ], + "states": [ + { + "name": "draft", + "label": "Draft", + "color": "gray", + "is_initial": true, + "sort_order": 0 + }, + { "name": "active", "label": "Active", "color": "green", "sort_order": 1 }, + { "name": "low_stock", "label": "Low Stock", "color": "orange", "sort_order": 2 }, + { + "name": "out_of_stock", + "label": "Out of Stock", + "color": "red", + "sort_order": 3 + }, + { + "name": "inactive", + "label": "Inactive", + "color": "gray", + "is_terminal": true, + "sort_order": 4 + } + ], + "transitions": [ + { + "from_state": "draft", + "to_state": "active", + "action_name": "publish", + "action_label": "Publish Listing" + }, + { + "from_state": "active", + "to_state": "low_stock", + "action_name": "flag_low_stock", + "action_label": "Flag Low Stock" + }, + { + "from_state": "low_stock", + "to_state": "active", + "action_name": "restock", + "action_label": "Mark Restocked" + }, + { + "from_state": "low_stock", + "to_state": "out_of_stock", + "action_name": "deplete", + "action_label": "Mark Out of Stock" + }, + { + "from_state": "out_of_stock", + "to_state": "active", + "action_name": "restock", + "action_label": "Mark Restocked" + }, + { + "from_state": "active", + "to_state": "inactive", + "action_name": "deactivate", + "action_label": "Deactivate" + }, + { + "from_state": "inactive", + "to_state": "active", + "action_name": "reactivate", + "action_label": "Reactivate" + } + ], + "hooks": [ + { + "event": "after_update", + "action_type": "run_workflow", + "action_config": { + "condition": "stock_quantity <= reorder_threshold AND reorder_threshold > 0", + "workflow": "low-stock-reorder" + } + } + ], + "relations": [] + }, + { + "doctype": { + "name": "supplier-order", + "label_singular": "Supplier Order", + "label_plural": "Supplier Orders", + "icon": "truck", + "table_name": "dt_supplier_orders", + "source": "template" + }, + "fields": [ + { + "name": "order_number", + "label": "Order Number", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 0 + }, + { + "name": "supplier_name", + "label": "Supplier Name", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 1 + }, + { + "name": "supplier_contact", + "label": "Supplier Contact", + "field_type": "text", + "sort_order": 2 + }, + { + "name": "supplier_email", + "label": "Supplier Email", + "field_type": "email", + "sort_order": 3 + }, + { + "name": "product_listing_id", + "label": "Product", + "field_type": "link", + "sort_order": 4, + "link_doctype": "product-listing" + }, + { + "name": "product_description", + "label": "Product / Item Description", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 5 + }, + { + "name": "quantity_ordered", + "label": "Quantity Ordered", + "field_type": "number", + "required": true, + "sort_order": 6 + }, + { + "name": "quantity_received", + "label": "Quantity Received", + "field_type": "number", + "sort_order": 7 + }, + { + "name": "unit_cost", + "label": "Unit Cost", + "field_type": "currency", + "required": true, + "sort_order": 8 + }, + { + "name": "total_cost", + "label": "Total Cost", + "field_type": "currency", + "sort_order": 9, + "formula": "quantity_ordered * unit_cost", + "depends_on": "quantity_ordered,unit_cost" + }, + { + "name": "order_date", + "label": "Order Date", + "field_type": "date", + "required": true, + "sort_order": 10 + }, + { + "name": "expected_delivery_date", + "label": "Expected Delivery", + "field_type": "date", + "sort_order": 11 + }, + { + "name": "received_date", + "label": "Date Received", + "field_type": "date", + "sort_order": 12 + }, + { + "name": "tracking_number", + "label": "Tracking Number", + "field_type": "text", + "sort_order": 13 + }, + { "name": "notes", "label": "Notes", "field_type": "longtext", "sort_order": 14 } + ], + "states": [ + { + "name": "draft", + "label": "Draft", + "color": "gray", + "is_initial": true, + "sort_order": 0 + }, + { "name": "ordered", "label": "Ordered", "color": "blue", "sort_order": 1 }, + { "name": "in_transit", "label": "In Transit", "color": "orange", "sort_order": 2 }, + { + "name": "received", + "label": "Received", + "color": "green", + "is_terminal": true, + "sort_order": 3 + }, + { + "name": "cancelled", + "label": "Cancelled", + "color": "red", + "is_terminal": true, + "sort_order": 4 + } + ], + "transitions": [ + { + "from_state": "draft", + "to_state": "ordered", + "action_name": "place_order", + "action_label": "Place Order" + }, + { + "from_state": "ordered", + "to_state": "in_transit", + "action_name": "mark_shipped", + "action_label": "Mark Shipped" + }, + { + "from_state": "in_transit", + "to_state": "received", + "action_name": "receive", + "action_label": "Mark Received" + }, + { + "from_state": "ordered", + "to_state": "received", + "action_name": "receive", + "action_label": "Mark Received" + }, + { + "from_state": "ordered", + "to_state": "cancelled", + "action_name": "cancel", + "action_label": "Cancel Order" + }, + { + "from_state": "draft", + "to_state": "cancelled", + "action_name": "cancel", + "action_label": "Cancel" + } + ], + "hooks": [ + { + "event": "after_insert", + "action_type": "send_notification", + "action_config": { + "message": "New supplier order {{order_number}}: {{quantity_ordered}} units of {{product_description}} from {{supplier_name}} — expected {{expected_delivery_date}}" + } + }, + { + "event": "on_state_change", + "action_type": "send_notification", + "action_config": { + "condition": "new_state == 'received'", + "message": "Supplier order {{order_number}} received: {{quantity_received}} units of {{product_description}} from {{supplier_name}}" + } + } + ], + "relations": [ + { + "from_doctype": "supplier-order", + "to_doctype": "product-listing", + "relation_type": "belongs_to", + "from_field": "product_listing_id", + "to_field": "id", + "label": "Replenishes product" + } + ] + } + ], + "workflows": [ + { + "name": "Low Stock Reorder", + "description": "Triggered when a product's stock quantity drops to or below its reorder threshold. Checks if there's already a pending supplier order for the product before creating a new one.", + "trigger": { + "type": "event", + "on": "product-listing.after_update" + }, + "steps": [ + { + "id": "get-product-details", + "name": "Get product details", + "type": "query", + "config": { + "doctype": "product-listing", + "filter": "id = {{trigger.record_id}}", + "fields": [ + "name", + "sku", + "stock_quantity", + "reorder_threshold", + "reorder_quantity", + "supplier_id", + "cost_price" + ] + }, + "sort_order": 0 + }, + { + "id": "check-threshold", + "name": "Check if reorder is needed", + "type": "condition", + "config": { + "expression": "product.stock_quantity <= product.reorder_threshold && product.reorder_threshold > 0", + "on_false": "skip_remaining" + }, + "sort_order": 1 + }, + { + "id": "check-pending-orders", + "name": "Check for existing pending supplier orders", + "type": "query", + "config": { + "doctype": "supplier-order", + "filter": "product_listing_id = {{trigger.record_id}} AND status IN ('draft', 'ordered', 'in_transit')", + "fields": ["order_number", "status", "quantity_ordered"] + }, + "sort_order": 2 + }, + { + "id": "skip-if-pending", + "name": "Skip if there's already a pending order", + "type": "condition", + "config": { + "expression": "pending_orders.length === 0", + "on_false": "skip_remaining" + }, + "sort_order": 3 + }, + { + "id": "send-reorder-alert", + "name": "Send low stock reorder alert", + "type": "send", + "config": { + "channel": "default", + "message": "Low Stock Alert — {{product.name}} ({{product.sku}})\nCurrent stock: {{product.stock_quantity}} units (threshold: {{product.reorder_threshold}})\nSuggested reorder: {{product.reorder_quantity}} units from {{product.supplier_id}}\nShall I create a supplier order?" + }, + "sort_order": 4 + } + ], + "status": "active" + }, + { + "name": "New Order Notification", + "description": "Polls the marketplace API for new orders every 15 minutes and sends a notification for each new order received. Syncs order data to the local supplier-order log for fulfilment tracking.", + "trigger": { + "type": "schedule", + "cron": "*/15 * * * *" + }, + "steps": [ + { + "id": "fetch-new-orders", + "name": "Fetch new orders from marketplace API", + "type": "api", + "config": { + "integration": "api", + "operation": "listOrders", + "params": { + "status": "pending", + "created_after": "{{last_run_at}}" + } + }, + "sort_order": 0 + }, + { + "id": "check-has-orders", + "name": "Check if any new orders arrived", + "type": "condition", + "config": { + "expression": "api_result.orders.length > 0", + "on_false": "skip_remaining" + }, + "sort_order": 1 + }, + { + "id": "format-order-summary", + "name": "Format new order summary", + "type": "transform", + "config": { + "template": "New Orders ({{api_result.orders.length}}):\n{{#each api_result.orders}}\n- Order {{id}}: {{#each items}}{{quantity}}× {{title}}{{/each}} — ${{total}} from {{buyer_name}}\n{{/each}}" + }, + "sort_order": 2 + }, + { + "id": "send-order-notification", + "name": "Send new order notification", + "type": "send", + "config": { + "channel": "default", + "message": "New Marketplace Orders:\n{{formatted_output}}" + }, + "sort_order": 3 + } + ], + "status": "active" + }, + { + "name": "Weekly Sales Report", + "description": "Every Monday at 8am, fetches last week's sales data from the marketplace API and combines it with local product listing data to produce a ranked performance report.", + "trigger": { + "type": "schedule", + "cron": "0 8 * * 1" + }, + "steps": [ + { + "id": "fetch-weekly-sales", + "name": "Fetch last week's sales from marketplace API", + "type": "api", + "config": { + "integration": "api", + "operation": "getSalesReport", + "params": { + "start_date": "{{date('now', '-7 days', 'start of day')}}", + "end_date": "{{date('now', '-1 day', 'end of day')}}" + } + }, + "sort_order": 0 + }, + { + "id": "get-product-listings", + "name": "Get all active product listings", + "type": "query", + "config": { + "doctype": "product-listing", + "filter": "status IN ('active', 'low_stock')", + "fields": [ + "name", + "sku", + "marketplace_listing_id", + "price", + "cost_price", + "stock_quantity", + "reorder_threshold" + ] + }, + "sort_order": 1 + }, + { + "id": "get-pending-supplier-orders", + "name": "Get pending supplier orders", + "type": "query", + "config": { + "doctype": "supplier-order", + "filter": "status IN ('ordered', 'in_transit')", + "fields": [ + "order_number", + "product_listing_id", + "quantity_ordered", + "expected_delivery_date", + "status" + ] + }, + "sort_order": 2 + }, + { + "id": "generate-weekly-report", + "name": "AI generates weekly sales report", + "type": "ai", + "config": { + "prompt": "Generate a weekly sales report from the following data:\n\nMarketplace Sales (last 7 days): {{api_sales_report}}\n\nActive Product Listings: {{product_listings}}\n\nPending Supplier Orders: {{pending_supplier_orders}}\n\nInclude: (1) Total orders and revenue for the week, (2) Top 5 products by units sold, (3) Top 5 products by revenue, (4) Products with stock below reorder threshold (flag urgently if stock < 5), (5) Pending restocks and expected delivery dates, (6) Estimated days of stock remaining for top sellers (stock_quantity / avg daily sales). Keep it concise — bullet points preferred.", + "tool_profile": "read-only", + "max_turns": 2 + }, + "sort_order": 3 + }, + { + "id": "send-weekly-report", + "name": "Send weekly sales report", + "type": "send", + "config": { + "channel": "default", + "message": "Weekly Sales Report:\n{{ai_output}}" + }, + "sort_order": 4 + } + ], + "status": "active" + } + ] +} diff --git a/src/intelligence/industry-templates/marketplace-seller/skill-pack.md b/src/intelligence/industry-templates/marketplace-seller/skill-pack.md new file mode 100644 index 00000000..f631c713 --- /dev/null +++ b/src/intelligence/industry-templates/marketplace-seller/skill-pack.md @@ -0,0 +1,90 @@ +# Marketplace Seller Operations Skill Pack + +You are managing an online marketplace seller business — selling on platforms like Etsy, Amazon, eBay, Shopify, or any marketplace with an API. Use the following guidance when handling user requests. + +## Data Types Available + +- **Product Listing**: Track products with SKU, price, cost, stock level, reorder threshold, and marketplace listing ID +- **Supplier Order**: Manage purchase orders to suppliers — quantities, unit cost, delivery tracking, and fulfilment status + +## Marketplace API Integration + +This template uses a universal API adapter connected to your marketplace. The API provides: + +- **listOrders** — fetch recent orders (filter by status, date range) +- **getOrder** — get full order details including line items and buyer info +- **shipOrder** — mark an order as shipped with tracking number and carrier +- **listListings** — fetch your live product listings from the marketplace +- **getListing** — get listing details including current stock on the platform +- **updateListing** — update price, stock quantity, or listing status +- **getSalesReport** — get aggregated sales data for a date range + +When the user asks about orders, always use the marketplace API for live data. Use local DocTypes to track supplier orders and product details not available via the API. + +## Common Operations + +### Product Listing Management + +- Add, update, or deactivate product listings +- Track selling price, cost price, and calculated margin: `((price - cost_price) / price) × 100` +- Monitor stock levels and set reorder thresholds per product +- Link products to their primary supplier for quick reorder +- Sync marketplace listing IDs to connect local records with live marketplace data + +### Supplier Order Management + +- Create purchase orders for restocking — link to the relevant product listing +- Track supplier orders through: draft → ordered → in_transit → received +- Log quantities ordered vs received (useful for partial deliveries) +- Calculate total cost: `quantity_ordered × unit_cost` +- Record tracking numbers and expected delivery dates + +### Stock Management + +- Monitor stock levels across all active listings +- Query products below their reorder threshold: `stock_quantity <= reorder_threshold` +- Check for pending supplier orders before raising a new reorder alert (avoid duplicate orders) +- Estimate days of stock remaining: `stock_quantity / average_daily_sales` +- After receiving supplier order: update `stock_quantity` on the product listing and record `quantity_received` on the order + +### Order Fulfilment + +- Fetch new orders from the marketplace API (status: `pending`) +- For each order, check available stock before committing to fulfil +- Mark orders as shipped via the API with tracking number and carrier +- Flag orders with items that are out of stock for manual review + +### Sales Reporting + +- Weekly summary: total orders, revenue, top products by units and revenue +- Inventory health: stock levels vs reorder thresholds, pending restocks +- Margin analysis: compare selling price vs cost price per product +- Stock days remaining: identify which top sellers need reordering soonest + +## Key Metrics + +- **Gross margin**: `((price - cost_price) / price) × 100` per product (target: ≥ 40% for physical goods) +- **Stock days remaining**: `stock_quantity / avg_daily_sales` — reorder when < 14 days +- **Sell-through rate**: `units_sold / (units_sold + stock_quantity)` — flag slow movers < 20% +- **Reorder rate**: how often each product triggers a restock (high rate = best seller) +- **Supplier lead time**: days between `order_date` and `received_date` — use to set accurate reorder points + +## Tips + +- Always check for pending supplier orders before creating a new one — avoid double-ordering +- When stock hits the reorder threshold, suggest an order for `reorder_quantity` units (pre-filled from the product listing) +- For fast-moving products, estimate days of stock remaining and reorder before hitting the threshold +- Use `marketplace_listing_id` to call `getListing` on the API and get the live stock figure — compare to local `stock_quantity` to detect discrepancies +- When user asks "how many orders today/this week?", call `listOrders` with `created_after` filter for live data +- After marking an order shipped via the API, note the tracking number in the supplier order if relevant +- Slow movers (low sell-through) may need a price reduction — suggest checking competitor pricing before updating +- For seasonal products, track year-over-year patterns using sales report data + +## Onboarding Checklist + +1. Connect your marketplace API: provide your marketplace's base URL and API key (see `api-spec.json` for the expected endpoint format) +2. Import your existing product listings (or add them manually via Product Listing) +3. Set reorder thresholds and reorder quantities for each product +4. Link each product to its primary supplier (name + contact email) +5. The New Order Notification workflow will start polling every 15 minutes once connected +6. The Weekly Sales Report runs every Monday at 8am automatically diff --git a/src/intelligence/industry-templates/restaurant/manifest.json b/src/intelligence/industry-templates/restaurant/manifest.json new file mode 100644 index 00000000..88fffbc2 --- /dev/null +++ b/src/intelligence/industry-templates/restaurant/manifest.json @@ -0,0 +1,795 @@ +{ + "id": "restaurant", + "name": "Restaurant", + "description": "Restaurant, cafe, food truck, bakery, or catering business — menu management, inventory tracking, daily sales, supplier management, and food cost analysis.", + "skillPack": "skill-pack.md", + "sampleQueries": [ + "Add a new menu item: Grilled Salmon, main course, $24.99, food cost $8.50", + "What's my food cost percentage this week?", + "Show me low-stock inventory items", + "Record today's sales: $3,200 revenue, 85 covers", + "Add expense: $450 food delivery from Fresh Foods Inc", + "86 the lobster bisque", + "Generate a prep list for tomorrow", + "Which menu items have a food cost above 35%?", + "Show me this week's expenses by category", + "Add a new supplier: Ocean Fresh Seafood, contact@oceanfresh.com, net-30 terms" + ], + "doctypes": [ + { + "doctype": { + "name": "menu-item", + "label_singular": "Menu Item", + "label_plural": "Menu Items", + "icon": "utensils", + "table_name": "dt_menu_items", + "source": "template" + }, + "fields": [ + { + "name": "name", + "label": "Item Name", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 0 + }, + { + "name": "category", + "label": "Category", + "field_type": "select", + "required": true, + "sort_order": 1, + "options": ["appetizer", "main", "dessert", "beverage", "side"] + }, + { + "name": "description", + "label": "Description", + "field_type": "longtext", + "sort_order": 2 + }, + { + "name": "price", + "label": "Menu Price", + "field_type": "currency", + "required": true, + "sort_order": 3 + }, + { "name": "cost", "label": "Food Cost", "field_type": "currency", "sort_order": 4 }, + { + "name": "food_cost_pct", + "label": "Food Cost %", + "field_type": "number", + "sort_order": 5, + "formula": "(cost / price) * 100", + "depends_on": "cost,price" + }, + { + "name": "allergens", + "label": "Allergens", + "field_type": "multiselect", + "sort_order": 6, + "options": ["gluten", "dairy", "nuts", "shellfish", "eggs", "soy", "fish"] + }, + { "name": "image_url", "label": "Photo", "field_type": "image", "sort_order": 7 }, + { + "name": "is_vegetarian", + "label": "Vegetarian", + "field_type": "checkbox", + "sort_order": 8 + }, + { "name": "is_vegan", "label": "Vegan", "field_type": "checkbox", "sort_order": 9 } + ], + "states": [ + { + "name": "available", + "label": "Available", + "color": "green", + "is_initial": true, + "sort_order": 0 + }, + { "name": "unavailable", "label": "Unavailable (86'd)", "color": "red", "sort_order": 1 }, + { "name": "seasonal", "label": "Seasonal", "color": "orange", "sort_order": 2 }, + { + "name": "discontinued", + "label": "Discontinued", + "color": "gray", + "is_terminal": true, + "sort_order": 3 + } + ], + "transitions": [ + { + "from_state": "available", + "to_state": "unavailable", + "action_name": "mark_unavailable", + "action_label": "86 Item" + }, + { + "from_state": "unavailable", + "to_state": "available", + "action_name": "mark_available", + "action_label": "Bring Back" + }, + { + "from_state": "available", + "to_state": "seasonal", + "action_name": "mark_seasonal", + "action_label": "Mark Seasonal" + }, + { + "from_state": "seasonal", + "to_state": "available", + "action_name": "activate", + "action_label": "Activate" + }, + { + "from_state": "available", + "to_state": "discontinued", + "action_name": "discontinue", + "action_label": "Discontinue" + }, + { + "from_state": "unavailable", + "to_state": "discontinued", + "action_name": "discontinue", + "action_label": "Discontinue" + } + ], + "hooks": [ + { + "event": "after_insert", + "action_type": "send_notification", + "action_config": { + "message": "New menu item added: {{name}} ({{category}}) — ${{price}}" + } + } + ], + "relations": [] + }, + { + "doctype": { + "name": "supplier", + "label_singular": "Supplier", + "label_plural": "Suppliers", + "icon": "truck", + "table_name": "dt_suppliers", + "source": "template" + }, + "fields": [ + { + "name": "name", + "label": "Company Name", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 0 + }, + { + "name": "contact_name", + "label": "Contact Person", + "field_type": "text", + "sort_order": 1 + }, + { "name": "email", "label": "Email", "field_type": "email", "sort_order": 2 }, + { "name": "phone", "label": "Phone", "field_type": "phone", "sort_order": 3 }, + { + "name": "categories", + "label": "Product Categories", + "field_type": "multiselect", + "sort_order": 4, + "options": [ + "produce", + "meat", + "seafood", + "dairy", + "dry-goods", + "beverages", + "equipment", + "cleaning" + ] + }, + { + "name": "payment_terms", + "label": "Payment Terms", + "field_type": "select", + "sort_order": 5, + "options": ["cod", "net-15", "net-30", "net-60"] + }, + { + "name": "delivery_days", + "label": "Delivery Days", + "field_type": "multiselect", + "sort_order": 6, + "options": ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"] + }, + { + "name": "min_order", + "label": "Minimum Order", + "field_type": "currency", + "sort_order": 7 + }, + { "name": "notes", "label": "Notes", "field_type": "longtext", "sort_order": 8 } + ], + "states": [ + { + "name": "active", + "label": "Active", + "color": "green", + "is_initial": true, + "sort_order": 0 + }, + { "name": "on_hold", "label": "On Hold", "color": "orange", "sort_order": 1 }, + { + "name": "inactive", + "label": "Inactive", + "color": "gray", + "is_terminal": true, + "sort_order": 2 + } + ], + "transitions": [ + { + "from_state": "active", + "to_state": "on_hold", + "action_name": "pause", + "action_label": "Put On Hold" + }, + { + "from_state": "on_hold", + "to_state": "active", + "action_name": "reactivate", + "action_label": "Reactivate" + }, + { + "from_state": "active", + "to_state": "inactive", + "action_name": "deactivate", + "action_label": "Deactivate" + }, + { + "from_state": "on_hold", + "to_state": "inactive", + "action_name": "deactivate", + "action_label": "Deactivate" + } + ], + "hooks": [], + "relations": [] + }, + { + "doctype": { + "name": "inventory-item", + "label_singular": "Inventory Item", + "label_plural": "Inventory Items", + "icon": "box", + "table_name": "dt_inventory_items", + "source": "template" + }, + "fields": [ + { + "name": "name", + "label": "Item Name", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 0 + }, + { + "name": "category", + "label": "Category", + "field_type": "select", + "required": true, + "sort_order": 1, + "options": [ + "produce", + "meat", + "seafood", + "dairy", + "dry-goods", + "beverages", + "cleaning", + "packaging" + ] + }, + { + "name": "unit", + "label": "Unit", + "field_type": "select", + "required": true, + "sort_order": 2, + "options": ["kg", "lb", "liter", "gallon", "each", "case", "bag", "box"] + }, + { + "name": "current_stock", + "label": "Current Stock", + "field_type": "number", + "required": true, + "sort_order": 3 + }, + { + "name": "reorder_point", + "label": "Reorder Point", + "field_type": "number", + "sort_order": 4 + }, + { + "name": "reorder_qty", + "label": "Reorder Quantity", + "field_type": "number", + "sort_order": 5 + }, + { "name": "unit_cost", "label": "Unit Cost", "field_type": "currency", "sort_order": 6 }, + { + "name": "supplier_id", + "label": "Supplier", + "field_type": "link", + "sort_order": 7, + "link_doctype": "supplier" + }, + { + "name": "storage_location", + "label": "Storage Location", + "field_type": "select", + "sort_order": 8, + "options": ["walk-in-cooler", "freezer", "dry-storage", "bar", "prep-station"] + }, + { "name": "expiry_date", "label": "Expiry Date", "field_type": "date", "sort_order": 9 }, + { "name": "last_ordered", "label": "Last Ordered", "field_type": "date", "sort_order": 10 } + ], + "states": [ + { + "name": "in_stock", + "label": "In Stock", + "color": "green", + "is_initial": true, + "sort_order": 0 + }, + { "name": "low_stock", "label": "Low Stock", "color": "orange", "sort_order": 1 }, + { "name": "out_of_stock", "label": "Out of Stock", "color": "red", "sort_order": 2 }, + { + "name": "discontinued", + "label": "Discontinued", + "color": "gray", + "is_terminal": true, + "sort_order": 3 + } + ], + "transitions": [ + { + "from_state": "in_stock", + "to_state": "low_stock", + "action_name": "flag_low", + "action_label": "Flag Low Stock" + }, + { + "from_state": "low_stock", + "to_state": "in_stock", + "action_name": "restock", + "action_label": "Restock" + }, + { + "from_state": "low_stock", + "to_state": "out_of_stock", + "action_name": "deplete", + "action_label": "Mark Out of Stock" + }, + { + "from_state": "out_of_stock", + "to_state": "in_stock", + "action_name": "restock", + "action_label": "Restock" + }, + { + "from_state": "in_stock", + "to_state": "discontinued", + "action_name": "discontinue", + "action_label": "Discontinue" + } + ], + "hooks": [ + { + "event": "after_update", + "action_type": "run_workflow", + "action_config": { + "workflow": "low-stock-alert", + "condition": "current_stock <= reorder_point" + } + } + ], + "relations": [ + { + "from_doctype": "inventory-item", + "to_doctype": "supplier", + "relation_type": "belongs_to", + "from_field": "supplier_id", + "to_field": "id", + "label": "Supplied by" + } + ] + }, + { + "doctype": { + "name": "daily-sales", + "label_singular": "Daily Sales Record", + "label_plural": "Daily Sales Records", + "icon": "chart-bar", + "table_name": "dt_daily_sales", + "source": "template" + }, + "fields": [ + { + "name": "date", + "label": "Date", + "field_type": "date", + "required": true, + "searchable": true, + "sort_order": 0 + }, + { + "name": "total_revenue", + "label": "Total Revenue", + "field_type": "currency", + "required": true, + "sort_order": 1 + }, + { + "name": "covers", + "label": "Covers (Customers)", + "field_type": "number", + "sort_order": 2 + }, + { + "name": "avg_per_cover", + "label": "Avg Revenue / Cover", + "field_type": "currency", + "sort_order": 3, + "formula": "total_revenue / covers", + "depends_on": "total_revenue,covers" + }, + { + "name": "food_revenue", + "label": "Food Revenue", + "field_type": "currency", + "sort_order": 4 + }, + { + "name": "beverage_revenue", + "label": "Beverage Revenue", + "field_type": "currency", + "sort_order": 5 + }, + { + "name": "top_items", + "label": "Top Selling Items", + "field_type": "longtext", + "sort_order": 6 + }, + { + "name": "weather", + "label": "Weather", + "field_type": "select", + "sort_order": 7, + "options": ["sunny", "cloudy", "rainy", "snowy", "hot", "cold"] + }, + { + "name": "special_event", + "label": "Special Event", + "field_type": "text", + "sort_order": 8 + }, + { "name": "notes", "label": "Notes", "field_type": "longtext", "sort_order": 9 } + ], + "states": [ + { "name": "draft", "label": "Draft", "color": "gray", "is_initial": true, "sort_order": 0 }, + { + "name": "confirmed", + "label": "Confirmed", + "color": "green", + "is_terminal": true, + "sort_order": 1 + } + ], + "transitions": [ + { + "from_state": "draft", + "to_state": "confirmed", + "action_name": "confirm", + "action_label": "Confirm Sales" + } + ], + "hooks": [ + { + "event": "after_insert", + "action_type": "send_notification", + "action_config": { + "message": "Daily sales recorded: ${{total_revenue}} revenue, {{covers}} covers on {{date}}" + } + } + ], + "relations": [] + }, + { + "doctype": { + "name": "expense", + "label_singular": "Expense", + "label_plural": "Expenses", + "icon": "receipt", + "table_name": "dt_expenses", + "source": "template" + }, + "fields": [ + { + "name": "date", + "label": "Date", + "field_type": "date", + "required": true, + "searchable": true, + "sort_order": 0 + }, + { + "name": "category", + "label": "Category", + "field_type": "select", + "required": true, + "sort_order": 1, + "options": ["food", "labor", "rent", "utilities", "equipment", "marketing", "other"] + }, + { + "name": "description", + "label": "Description", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 2 + }, + { + "name": "amount", + "label": "Amount", + "field_type": "currency", + "required": true, + "sort_order": 3 + }, + { + "name": "supplier_id", + "label": "Supplier", + "field_type": "link", + "sort_order": 4, + "link_doctype": "supplier" + }, + { + "name": "receipt_image", + "label": "Receipt Photo", + "field_type": "image", + "sort_order": 5 + }, + { "name": "notes", "label": "Notes", "field_type": "longtext", "sort_order": 6 } + ], + "states": [ + { + "name": "pending", + "label": "Pending", + "color": "orange", + "is_initial": true, + "sort_order": 0 + }, + { "name": "paid", "label": "Paid", "color": "green", "sort_order": 1 }, + { "name": "overdue", "label": "Overdue", "color": "red", "sort_order": 2 }, + { + "name": "cancelled", + "label": "Cancelled", + "color": "gray", + "is_terminal": true, + "sort_order": 3 + } + ], + "transitions": [ + { + "from_state": "pending", + "to_state": "paid", + "action_name": "mark_paid", + "action_label": "Mark as Paid" + }, + { + "from_state": "pending", + "to_state": "overdue", + "action_name": "flag_overdue", + "action_label": "Flag Overdue" + }, + { + "from_state": "overdue", + "to_state": "paid", + "action_name": "mark_paid", + "action_label": "Mark as Paid" + }, + { + "from_state": "pending", + "to_state": "cancelled", + "action_name": "cancel", + "action_label": "Cancel" + } + ], + "hooks": [ + { + "event": "on_state_change", + "action_type": "send_notification", + "action_config": { + "condition": "new_state == 'overdue'", + "message": "Expense overdue: {{description}} — ${{amount}} ({{category}})" + } + } + ], + "relations": [ + { + "from_doctype": "expense", + "to_doctype": "supplier", + "relation_type": "belongs_to", + "from_field": "supplier_id", + "to_field": "id", + "label": "Paid to" + } + ] + } + ], + "workflows": [ + { + "name": "Low Stock Alert", + "description": "Checks inventory items daily and sends an alert for any items at or below their reorder point.", + "trigger": { + "type": "schedule", + "cron": "0 6 * * *" + }, + "steps": [ + { + "id": "query-low-stock", + "name": "Find low-stock items", + "type": "query", + "config": { + "doctype": "inventory-item", + "filter": "current_stock <= reorder_point", + "fields": ["name", "current_stock", "reorder_point", "unit", "supplier_id"] + }, + "sort_order": 0 + }, + { + "id": "check-has-items", + "name": "Check if any items are low", + "type": "condition", + "config": { + "expression": "results.length > 0", + "on_false": "skip_remaining" + }, + "sort_order": 1 + }, + { + "id": "format-alert", + "name": "Format alert message", + "type": "transform", + "config": { + "template": "Low Stock Alert:\n{{#each results}}\n- {{name}}: {{current_stock}} {{unit}} (reorder at {{reorder_point}})\n{{/each}}" + }, + "sort_order": 2 + }, + { + "id": "send-alert", + "name": "Send low-stock notification", + "type": "send", + "config": { + "channel": "default", + "message": "{{formatted_output}}" + }, + "sort_order": 3 + } + ], + "status": "active" + }, + { + "name": "Daily Prep List", + "description": "Generates a daily prep list each morning based on available menu items and inventory levels.", + "trigger": { + "type": "schedule", + "cron": "0 5 * * *" + }, + "steps": [ + { + "id": "get-menu-items", + "name": "Get available menu items", + "type": "query", + "config": { + "doctype": "menu-item", + "filter": "status = 'available'", + "fields": ["name", "category"] + }, + "sort_order": 0 + }, + { + "id": "get-inventory", + "name": "Get current inventory levels", + "type": "query", + "config": { + "doctype": "inventory-item", + "filter": "status != 'discontinued'", + "fields": ["name", "current_stock", "unit", "storage_location"] + }, + "sort_order": 1 + }, + { + "id": "generate-prep-list", + "name": "AI generates prep list", + "type": "ai", + "config": { + "prompt": "Based on the available menu items and current inventory levels, generate a prioritized prep list for today. Group by station (cold prep, hot prep, pastry, bar). Flag any items with low inventory that may need substitution.", + "tool_profile": "read-only", + "max_turns": 3 + }, + "sort_order": 2 + }, + { + "id": "send-prep-list", + "name": "Send prep list", + "type": "send", + "config": { + "channel": "default", + "message": "Daily Prep List:\n{{ai_output}}" + }, + "sort_order": 3 + } + ], + "status": "active" + }, + { + "name": "Weekly Food Cost Report", + "description": "Calculates and sends a weekly food cost analysis every Monday morning, comparing food expenses against food revenue.", + "trigger": { + "type": "schedule", + "cron": "0 8 * * 1" + }, + "steps": [ + { + "id": "get-weekly-sales", + "name": "Get last 7 days of sales", + "type": "query", + "config": { + "doctype": "daily-sales", + "filter": "date >= date('now', '-7 days')", + "fields": ["date", "total_revenue", "food_revenue", "covers"] + }, + "sort_order": 0 + }, + { + "id": "get-weekly-expenses", + "name": "Get last 7 days of food expenses", + "type": "query", + "config": { + "doctype": "expense", + "filter": "date >= date('now', '-7 days') AND category = 'food'", + "fields": ["date", "amount", "description"] + }, + "sort_order": 1 + }, + { + "id": "calculate-food-cost", + "name": "AI analyzes food cost", + "type": "ai", + "config": { + "prompt": "Analyze the weekly food cost data. Calculate: (1) Total food revenue, (2) Total food expenses, (3) Food cost percentage (expenses / revenue x 100), (4) Average revenue per cover, (5) Comparison against the 28-35% target range. Highlight any concerning trends and suggest specific actions if food cost is above target.", + "tool_profile": "read-only", + "max_turns": 3 + }, + "sort_order": 2 + }, + { + "id": "send-report", + "name": "Send weekly food cost report", + "type": "send", + "config": { + "channel": "default", + "message": "Weekly Food Cost Report:\n{{ai_output}}" + }, + "sort_order": 3 + } + ], + "status": "active" + } + ] +} diff --git a/src/intelligence/industry-templates/restaurant/skill-pack.md b/src/intelligence/industry-templates/restaurant/skill-pack.md new file mode 100644 index 00000000..8deb7ac4 --- /dev/null +++ b/src/intelligence/industry-templates/restaurant/skill-pack.md @@ -0,0 +1,61 @@ +# Restaurant Operations Skill Pack + +You are managing a restaurant, cafe, food truck, bakery, or catering business. Use the following guidance when handling user requests. + +## Data Types Available + +- **Menu Item**: Track dishes with name, category, price, cost, allergens, and availability status +- **Supplier**: Manage vendor contacts, payment terms, and product categories +- **Inventory Item**: Track stock levels, units, reorder points, and link to suppliers +- **Daily Sales**: Record daily revenue, covers (customers served), top-selling items, and notes +- **Expense**: Track business expenses by category (food, labor, rent, utilities, equipment, marketing, other) + +## Common Operations + +### Menu Management + +- Add, update, or remove menu items +- Set prices and track food cost percentages +- Mark items as available/unavailable/seasonal +- Categorize items (appetizer, main, dessert, beverage, side) + +### Inventory & Purchasing + +- Check current stock levels against reorder points +- Generate purchase orders when stock is low +- Track supplier pricing and delivery schedules +- Calculate food cost percentage: `(item cost / item price) * 100` + +### Daily Operations + +- Record daily sales totals and cover counts +- Calculate average revenue per cover +- Track daily notes (events, weather impact, staff issues) +- Generate daily prep lists based on expected covers + +### Financial Tracking + +- Record expenses by category +- Calculate weekly food cost ratio: `total food expenses / total revenue` +- Compare expenses against budget +- Track payment status (pending, paid, overdue) + +### Reporting + +- Weekly food cost analysis +- Top-selling items by revenue and quantity +- Supplier spend breakdown +- Expense trends by category + +## Food Cost Targets + +- **Target food cost**: 28-35% of menu price +- **Prime cost** (food + labor): should be under 65% of revenue +- Items above 35% food cost should be flagged for price adjustment or recipe optimization + +## Tips + +- When the user says "86" an item, mark it as unavailable +- "Covers" means number of customers served +- "Prep list" means items to prepare before service +- Track waste separately from regular inventory usage when possible diff --git a/src/intelligence/industry-templates/retail/manifest.json b/src/intelligence/industry-templates/retail/manifest.json new file mode 100644 index 00000000..ecae99ef --- /dev/null +++ b/src/intelligence/industry-templates/retail/manifest.json @@ -0,0 +1,792 @@ +{ + "id": "retail", + "name": "Retail", + "description": "Retail store, boutique, online shop, or multi-location retail — product catalog, customer management, sales tracking, purchase orders, and inventory management.", + "skillPack": "skill-pack.md", + "sampleQueries": [ + "Add a new product: Blue Denim Jacket, SKU JKT-001, $89.99, cost $32.00, 25 units in stock", + "Show me all products with low stock", + "Add a new customer: Sarah Johnson, sarah@email.com, VIP member", + "Record a sale: Sarah Johnson bought 2x Blue Denim Jacket for $179.98", + "Create a purchase order to supplier NordicTextiles for 50x Wool Sweater at $18 each", + "What's my best-selling product this week?", + "Show me all pending purchase orders", + "Which customers haven't purchased in the last 30 days?", + "What's my gross margin on the denim jacket line?", + "Generate a daily sales report for today" + ], + "doctypes": [ + { + "doctype": { + "name": "product", + "label_singular": "Product", + "label_plural": "Products", + "icon": "tag", + "table_name": "dt_products", + "source": "template" + }, + "fields": [ + { + "name": "name", + "label": "Product Name", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 0 + }, + { + "name": "sku", + "label": "SKU", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 1 + }, + { + "name": "category", + "label": "Category", + "field_type": "select", + "required": true, + "sort_order": 2, + "options": [ + "clothing", + "footwear", + "accessories", + "electronics", + "home", + "beauty", + "food", + "sports", + "toys", + "other" + ] + }, + { + "name": "description", + "label": "Description", + "field_type": "longtext", + "sort_order": 3 + }, + { + "name": "price", + "label": "Selling Price", + "field_type": "currency", + "required": true, + "sort_order": 4 + }, + { "name": "cost", "label": "Cost Price", "field_type": "currency", "sort_order": 5 }, + { + "name": "margin_pct", + "label": "Gross Margin %", + "field_type": "number", + "sort_order": 6, + "formula": "((price - cost) / price) * 100", + "depends_on": "price,cost" + }, + { + "name": "stock_qty", + "label": "Stock Quantity", + "field_type": "number", + "required": true, + "sort_order": 7 + }, + { + "name": "reorder_point", + "label": "Reorder Point", + "field_type": "number", + "sort_order": 8 + }, + { + "name": "reorder_qty", + "label": "Reorder Quantity", + "field_type": "number", + "sort_order": 9 + }, + { "name": "barcode", "label": "Barcode", "field_type": "text", "sort_order": 10 }, + { + "name": "brand", + "label": "Brand", + "field_type": "text", + "searchable": true, + "sort_order": 11 + }, + { + "name": "supplier_id", + "label": "Supplier", + "field_type": "link", + "sort_order": 12, + "link_doctype": "customer" + }, + { "name": "image_url", "label": "Product Photo", "field_type": "image", "sort_order": 13 }, + { "name": "tags", "label": "Tags", "field_type": "text", "sort_order": 14 }, + { "name": "notes", "label": "Notes", "field_type": "longtext", "sort_order": 15 } + ], + "states": [ + { + "name": "active", + "label": "Active", + "color": "green", + "is_initial": true, + "sort_order": 0 + }, + { "name": "low_stock", "label": "Low Stock", "color": "orange", "sort_order": 1 }, + { "name": "out_of_stock", "label": "Out of Stock", "color": "red", "sort_order": 2 }, + { + "name": "discontinued", + "label": "Discontinued", + "color": "gray", + "is_terminal": true, + "sort_order": 3 + } + ], + "transitions": [ + { + "from_state": "active", + "to_state": "low_stock", + "action_name": "flag_low", + "action_label": "Flag Low Stock" + }, + { + "from_state": "low_stock", + "to_state": "active", + "action_name": "restock", + "action_label": "Restock" + }, + { + "from_state": "low_stock", + "to_state": "out_of_stock", + "action_name": "deplete", + "action_label": "Mark Out of Stock" + }, + { + "from_state": "out_of_stock", + "to_state": "active", + "action_name": "restock", + "action_label": "Restock" + }, + { + "from_state": "active", + "to_state": "discontinued", + "action_name": "discontinue", + "action_label": "Discontinue" + }, + { + "from_state": "low_stock", + "to_state": "discontinued", + "action_name": "discontinue", + "action_label": "Discontinue" + } + ], + "hooks": [ + { + "event": "after_insert", + "action_type": "send_notification", + "action_config": { + "message": "New product added: {{name}} (SKU: {{sku}}) — ${{price}}, {{stock_qty}} units in stock" + } + }, + { + "event": "after_update", + "action_type": "run_workflow", + "action_config": { + "workflow": "low-stock-reorder", + "condition": "stock_qty <= reorder_point AND stock_qty > 0" + } + } + ], + "relations": [] + }, + { + "doctype": { + "name": "customer", + "label_singular": "Customer", + "label_plural": "Customers", + "icon": "user", + "table_name": "dt_customers", + "source": "template" + }, + "fields": [ + { + "name": "name", + "label": "Full Name", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 0 + }, + { + "name": "email", + "label": "Email", + "field_type": "email", + "searchable": true, + "sort_order": 1 + }, + { "name": "phone", "label": "Phone", "field_type": "phone", "sort_order": 2 }, + { + "name": "tier", + "label": "Customer Tier", + "field_type": "select", + "sort_order": 3, + "options": ["standard", "silver", "gold", "vip"] + }, + { + "name": "total_spent", + "label": "Total Lifetime Spend", + "field_type": "currency", + "sort_order": 4 + }, + { + "name": "total_orders", + "label": "Total Orders", + "field_type": "number", + "sort_order": 5 + }, + { + "name": "last_purchase_date", + "label": "Last Purchase Date", + "field_type": "date", + "sort_order": 6 + }, + { "name": "address", "label": "Address", "field_type": "text", "sort_order": 7 }, + { "name": "city", "label": "City", "field_type": "text", "sort_order": 8 }, + { + "name": "date_of_birth", + "label": "Date of Birth", + "field_type": "date", + "sort_order": 9 + }, + { + "name": "preferred_categories", + "label": "Preferred Categories", + "field_type": "multiselect", + "sort_order": 10, + "options": [ + "clothing", + "footwear", + "accessories", + "electronics", + "home", + "beauty", + "food", + "sports", + "toys" + ] + }, + { + "name": "accepts_marketing", + "label": "Accepts Marketing", + "field_type": "checkbox", + "sort_order": 11 + }, + { "name": "notes", "label": "Notes", "field_type": "longtext", "sort_order": 12 } + ], + "states": [ + { + "name": "active", + "label": "Active", + "color": "green", + "is_initial": true, + "sort_order": 0 + }, + { "name": "at_risk", "label": "At Risk (Inactive)", "color": "orange", "sort_order": 1 }, + { "name": "churned", "label": "Churned", "color": "red", "sort_order": 2 }, + { + "name": "blocked", + "label": "Blocked", + "color": "gray", + "is_terminal": true, + "sort_order": 3 + } + ], + "transitions": [ + { + "from_state": "active", + "to_state": "at_risk", + "action_name": "flag_at_risk", + "action_label": "Flag At Risk" + }, + { + "from_state": "at_risk", + "to_state": "active", + "action_name": "reactivate", + "action_label": "Reactivate" + }, + { + "from_state": "at_risk", + "to_state": "churned", + "action_name": "mark_churned", + "action_label": "Mark Churned" + }, + { + "from_state": "churned", + "to_state": "active", + "action_name": "win_back", + "action_label": "Win Back" + }, + { + "from_state": "active", + "to_state": "blocked", + "action_name": "block", + "action_label": "Block Customer" + } + ], + "hooks": [ + { + "event": "on_state_change", + "action_type": "run_workflow", + "action_config": { + "condition": "new_state == 'at_risk'", + "workflow": "customer-follow-up" + } + } + ], + "relations": [] + }, + { + "doctype": { + "name": "sale", + "label_singular": "Sale", + "label_plural": "Sales", + "icon": "shopping-cart", + "table_name": "dt_sales", + "source": "template" + }, + "fields": [ + { + "name": "date", + "label": "Sale Date", + "field_type": "date", + "required": true, + "searchable": true, + "sort_order": 0 + }, + { + "name": "customer_id", + "label": "Customer", + "field_type": "link", + "sort_order": 1, + "link_doctype": "customer" + }, + { + "name": "items", + "label": "Items Sold", + "field_type": "longtext", + "required": true, + "sort_order": 2 + }, + { + "name": "subtotal", + "label": "Subtotal", + "field_type": "currency", + "required": true, + "sort_order": 3 + }, + { + "name": "discount_amount", + "label": "Discount", + "field_type": "currency", + "sort_order": 4 + }, + { "name": "tax_amount", "label": "Tax", "field_type": "currency", "sort_order": 5 }, + { + "name": "total", + "label": "Total", + "field_type": "currency", + "required": true, + "sort_order": 6, + "formula": "subtotal - discount_amount + tax_amount", + "depends_on": "subtotal,discount_amount,tax_amount" + }, + { + "name": "payment_method", + "label": "Payment Method", + "field_type": "select", + "required": true, + "sort_order": 7, + "options": ["cash", "card", "bank-transfer", "online", "gift-card", "store-credit"] + }, + { + "name": "channel", + "label": "Sales Channel", + "field_type": "select", + "sort_order": 8, + "options": ["in-store", "online", "phone", "marketplace"] + }, + { "name": "staff_name", "label": "Served By", "field_type": "text", "sort_order": 9 }, + { "name": "notes", "label": "Notes", "field_type": "longtext", "sort_order": 10 } + ], + "states": [ + { + "name": "completed", + "label": "Completed", + "color": "green", + "is_initial": true, + "sort_order": 0 + }, + { "name": "refunded", "label": "Refunded", "color": "orange", "sort_order": 1 }, + { + "name": "partially_refunded", + "label": "Partially Refunded", + "color": "yellow", + "sort_order": 2 + }, + { + "name": "disputed", + "label": "Disputed", + "color": "red", + "is_terminal": true, + "sort_order": 3 + } + ], + "transitions": [ + { + "from_state": "completed", + "to_state": "refunded", + "action_name": "refund", + "action_label": "Full Refund" + }, + { + "from_state": "completed", + "to_state": "partially_refunded", + "action_name": "partial_refund", + "action_label": "Partial Refund" + }, + { + "from_state": "completed", + "to_state": "disputed", + "action_name": "dispute", + "action_label": "Flag Dispute" + }, + { + "from_state": "partially_refunded", + "to_state": "refunded", + "action_name": "refund_remaining", + "action_label": "Refund Remaining" + } + ], + "hooks": [ + { + "event": "after_insert", + "action_type": "send_notification", + "action_config": { + "message": "Sale recorded: ${{total}} on {{date}} via {{payment_method}}" + } + } + ], + "relations": [ + { + "from_doctype": "sale", + "to_doctype": "customer", + "relation_type": "belongs_to", + "from_field": "customer_id", + "to_field": "id", + "label": "Purchased by" + } + ] + }, + { + "doctype": { + "name": "purchase-order", + "label_singular": "Purchase Order", + "label_plural": "Purchase Orders", + "icon": "clipboard", + "table_name": "dt_purchase_orders", + "source": "template" + }, + "fields": [ + { + "name": "order_number", + "label": "Order Number", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 0 + }, + { + "name": "supplier_name", + "label": "Supplier Name", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 1 + }, + { + "name": "supplier_email", + "label": "Supplier Email", + "field_type": "email", + "sort_order": 2 + }, + { + "name": "order_date", + "label": "Order Date", + "field_type": "date", + "required": true, + "sort_order": 3 + }, + { + "name": "expected_delivery", + "label": "Expected Delivery", + "field_type": "date", + "sort_order": 4 + }, + { + "name": "items", + "label": "Items Ordered", + "field_type": "longtext", + "required": true, + "sort_order": 5 + }, + { + "name": "total_amount", + "label": "Total Amount", + "field_type": "currency", + "required": true, + "sort_order": 6 + }, + { + "name": "payment_terms", + "label": "Payment Terms", + "field_type": "select", + "sort_order": 7, + "options": ["cod", "net-15", "net-30", "net-60", "prepaid"] + }, + { + "name": "delivery_address", + "label": "Delivery Address", + "field_type": "text", + "sort_order": 8 + }, + { + "name": "tracking_number", + "label": "Tracking Number", + "field_type": "text", + "sort_order": 9 + }, + { + "name": "invoice_number", + "label": "Supplier Invoice #", + "field_type": "text", + "sort_order": 10 + }, + { "name": "notes", "label": "Notes", "field_type": "longtext", "sort_order": 11 } + ], + "states": [ + { "name": "draft", "label": "Draft", "color": "gray", "is_initial": true, "sort_order": 0 }, + { "name": "sent", "label": "Sent to Supplier", "color": "blue", "sort_order": 1 }, + { + "name": "confirmed", + "label": "Confirmed by Supplier", + "color": "purple", + "sort_order": 2 + }, + { "name": "in_transit", "label": "In Transit", "color": "orange", "sort_order": 3 }, + { "name": "received", "label": "Received", "color": "green", "sort_order": 4 }, + { + "name": "cancelled", + "label": "Cancelled", + "color": "red", + "is_terminal": true, + "sort_order": 5 + } + ], + "transitions": [ + { + "from_state": "draft", + "to_state": "sent", + "action_name": "send_order", + "action_label": "Send to Supplier" + }, + { + "from_state": "draft", + "to_state": "cancelled", + "action_name": "cancel", + "action_label": "Cancel Order" + }, + { + "from_state": "sent", + "to_state": "confirmed", + "action_name": "confirm", + "action_label": "Mark Confirmed" + }, + { + "from_state": "sent", + "to_state": "cancelled", + "action_name": "cancel", + "action_label": "Cancel Order" + }, + { + "from_state": "confirmed", + "to_state": "in_transit", + "action_name": "ship", + "action_label": "Mark Shipped" + }, + { + "from_state": "in_transit", + "to_state": "received", + "action_name": "receive", + "action_label": "Mark Received" + } + ], + "hooks": [ + { + "event": "on_state_change", + "action_type": "send_notification", + "action_config": { + "condition": "new_state == 'received'", + "message": "Purchase order {{order_number}} received from {{supplier_name}} — ${{total_amount}}" + } + } + ], + "relations": [] + } + ], + "workflows": [ + { + "name": "Low Stock Reorder", + "description": "Checks product stock levels daily and sends an alert for any products at or below their reorder point.", + "trigger": { + "type": "schedule", + "cron": "0 7 * * *" + }, + "steps": [ + { + "id": "query-low-stock", + "name": "Find products with low stock", + "type": "query", + "config": { + "doctype": "product", + "filter": "stock_qty <= reorder_point AND status != 'discontinued'", + "fields": ["name", "sku", "stock_qty", "reorder_point", "reorder_qty", "supplier_id"] + }, + "sort_order": 0 + }, + { + "id": "check-has-items", + "name": "Check if any products are low", + "type": "condition", + "config": { + "expression": "results.length > 0", + "on_false": "skip_remaining" + }, + "sort_order": 1 + }, + { + "id": "format-alert", + "name": "Format reorder alert", + "type": "transform", + "config": { + "template": "Low Stock Alert — Reorder Needed:\n{{#each results}}\n- {{name}} ({{sku}}): {{stock_qty}} units left (reorder at {{reorder_point}}, order qty: {{reorder_qty}})\n{{/each}}" + }, + "sort_order": 2 + }, + { + "id": "send-alert", + "name": "Send low-stock notification", + "type": "send", + "config": { + "channel": "default", + "message": "{{formatted_output}}" + }, + "sort_order": 3 + } + ], + "status": "active" + }, + { + "name": "Daily Sales Report", + "description": "Generates and sends a daily sales summary every evening with total revenue, transaction count, and top-performing products.", + "trigger": { + "type": "schedule", + "cron": "0 20 * * *" + }, + "steps": [ + { + "id": "get-todays-sales", + "name": "Get today's completed sales", + "type": "query", + "config": { + "doctype": "sale", + "filter": "date = date('now') AND status = 'completed'", + "fields": ["date", "total", "payment_method", "channel", "customer_id", "items"] + }, + "sort_order": 0 + }, + { + "id": "analyze-sales", + "name": "AI analyzes today's sales", + "type": "ai", + "config": { + "prompt": "Analyze today's sales data. Provide: (1) Total revenue, (2) Number of transactions, (3) Average transaction value, (4) Revenue by payment method, (5) Revenue by channel (in-store vs online), (6) Most purchased items if identifiable from the items field. Flag any unusual patterns or notable trends compared to typical daily performance.", + "tool_profile": "read-only", + "max_turns": 3 + }, + "sort_order": 1 + }, + { + "id": "send-report", + "name": "Send daily sales report", + "type": "send", + "config": { + "channel": "default", + "message": "Daily Sales Report:\n{{ai_output}}" + }, + "sort_order": 2 + } + ], + "status": "active" + }, + { + "name": "Customer Follow-Up", + "description": "Identifies customers who haven't purchased in 30+ days and sends a follow-up reminder to re-engage them.", + "trigger": { + "type": "schedule", + "cron": "0 10 * * 1" + }, + "steps": [ + { + "id": "query-inactive-customers", + "name": "Find customers inactive for 30+ days", + "type": "query", + "config": { + "doctype": "customer", + "filter": "last_purchase_date <= date('now', '-30 days') AND status = 'active' AND accepts_marketing = 1", + "fields": ["name", "email", "phone", "last_purchase_date", "total_spent", "tier"] + }, + "sort_order": 0 + }, + { + "id": "check-has-customers", + "name": "Check if any customers need follow-up", + "type": "condition", + "config": { + "expression": "results.length > 0", + "on_false": "skip_remaining" + }, + "sort_order": 1 + }, + { + "id": "generate-follow-up", + "name": "AI generates follow-up suggestions", + "type": "ai", + "config": { + "prompt": "Review the list of customers who haven't purchased in 30+ days. For each customer tier (VIP, gold, silver, standard), suggest a personalized re-engagement approach. List the top 10 highest-value at-risk customers by total lifetime spend and recommend specific outreach actions.", + "tool_profile": "read-only", + "max_turns": 3 + }, + "sort_order": 2 + }, + { + "id": "send-follow-up-list", + "name": "Send customer follow-up list", + "type": "send", + "config": { + "channel": "default", + "message": "Customer Follow-Up Needed:\n{{ai_output}}" + }, + "sort_order": 3 + } + ], + "status": "active" + } + ] +} diff --git a/src/intelligence/industry-templates/retail/skill-pack.md b/src/intelligence/industry-templates/retail/skill-pack.md new file mode 100644 index 00000000..1347e23e --- /dev/null +++ b/src/intelligence/industry-templates/retail/skill-pack.md @@ -0,0 +1,78 @@ +# Retail Operations Skill Pack + +You are managing a retail store, boutique, online shop, or multi-location retail business. Use the following guidance when handling user requests. + +## Data Types Available + +- **Product**: Track your product catalog with SKU, pricing, cost, stock levels, reorder points, barcode, and supplier information +- **Customer**: Manage customer profiles with contact details, purchase history, loyalty tier, and marketing preferences +- **Sale**: Record all sales transactions with itemized details, payment method, channel, and refund tracking +- **Purchase Order**: Manage supplier orders from draft through receipt, including tracking numbers and invoice matching + +## Common Operations + +### Inventory Management + +- Add, update, or discontinue products +- Track stock quantities and flag low-stock items +- Set reorder points and reorder quantities per product +- Calculate gross margin: `(price - cost) / price × 100` +- Manage product categories (clothing, footwear, accessories, electronics, etc.) +- Record barcodes and SKUs for fast lookup + +### Customer Relationships + +- Add and update customer profiles +- Track customer tiers (standard, silver, gold, VIP) based on lifetime spend +- Record purchase history and last purchase date +- Identify at-risk customers (inactive 30+ days) +- Manage marketing consent and preferences +- Win back churned customers + +### Sales Tracking + +- Record sales with itemized products, quantities, and totals +- Track payment methods (cash, card, bank transfer, online, gift card) +- Track sales channels (in-store, online, phone, marketplace) +- Process full and partial refunds +- Calculate daily, weekly, and monthly revenue totals + +### Purchase Orders + +- Create purchase orders for restocking +- Track supplier orders from draft → sent → confirmed → in transit → received +- Match incoming stock with purchase orders +- Manage payment terms (COD, net-30, prepaid, etc.) +- Record tracking numbers and supplier invoice numbers + +### Pricing + +- Update selling prices and cost prices +- Calculate and monitor gross margin per product +- Apply discounts to sales (recorded in discount_amount field) +- Compare margin across product categories + +### Reporting + +- Daily sales report: total revenue, transaction count, average transaction value +- Top-selling products by quantity and revenue +- Low-stock report: products below reorder point +- Customer retention: inactive customers, churn rate +- Supplier performance: order fulfillment speed, reliability + +## Key Metrics + +- **Gross margin target**: 40-60% for most retail (varies by category) +- **Inventory turnover**: aim for 4-6x per year (higher = healthier cash flow) +- **Customer retention rate**: track repeat purchase rate over 90 days +- **Average transaction value**: monitor for upsell opportunities +- Reorder point formula: `average daily sales × lead time days + safety stock` + +## Tips + +- When recording a sale, include the customer ID when possible to build purchase history +- "Low stock" means stock_qty ≤ reorder_point — trigger a purchase order +- VIP customers with no recent purchase warrant personal outreach (call, not just email) +- Gross margin below 30% on a product signals a pricing or sourcing problem +- When receiving a purchase order, update the corresponding product stock quantities +- Track sales by channel to understand where revenue is growing or declining diff --git a/src/intelligence/industry-templates/services/manifest.json b/src/intelligence/industry-templates/services/manifest.json new file mode 100644 index 00000000..c957dba1 --- /dev/null +++ b/src/intelligence/industry-templates/services/manifest.json @@ -0,0 +1,817 @@ +{ + "id": "services", + "name": "Services", + "description": "Professional services business — consulting, agency, freelance, or managed services — client management, project tracking, invoicing, and time tracking.", + "skillPack": "skill-pack.md", + "sampleQueries": [ + "Add a new client: Acme Corp, contact John Smith, john@acme.com, net-30 payment terms", + "Create a project: Website Redesign for Acme Corp, $12,000 budget, due 2026-04-30", + "Log 3 hours on Website Redesign — task: homepage layout", + "Create an invoice for Acme Corp: $4,000 for milestone 1 completion", + "Which invoices are overdue?", + "How many billable hours have I logged this week?", + "Show me all active projects and their progress", + "Mark invoice INV-042 as paid", + "What's my total revenue this month?", + "Which project is closest to exceeding its budget?" + ], + "doctypes": [ + { + "doctype": { + "name": "client", + "label_singular": "Client", + "label_plural": "Clients", + "icon": "building", + "table_name": "dt_clients", + "source": "template" + }, + "fields": [ + { + "name": "name", + "label": "Company Name", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 0 + }, + { + "name": "contact_name", + "label": "Primary Contact", + "field_type": "text", + "searchable": true, + "sort_order": 1 + }, + { + "name": "email", + "label": "Email", + "field_type": "email", + "searchable": true, + "sort_order": 2 + }, + { "name": "phone", "label": "Phone", "field_type": "phone", "sort_order": 3 }, + { + "name": "industry", + "label": "Industry", + "field_type": "select", + "sort_order": 4, + "options": [ + "technology", + "finance", + "healthcare", + "retail", + "manufacturing", + "real-estate", + "education", + "media", + "legal", + "other" + ] + }, + { + "name": "billing_address", + "label": "Billing Address", + "field_type": "text", + "sort_order": 5 + }, + { + "name": "payment_terms", + "label": "Payment Terms", + "field_type": "select", + "sort_order": 6, + "options": ["due-on-receipt", "net-15", "net-30", "net-60"] + }, + { + "name": "hourly_rate", + "label": "Default Hourly Rate", + "field_type": "currency", + "sort_order": 7 + }, + { + "name": "total_billed", + "label": "Total Billed to Date", + "field_type": "currency", + "sort_order": 8 + }, + { + "name": "referral_source", + "label": "Referral Source", + "field_type": "text", + "sort_order": 9 + }, + { "name": "notes", "label": "Notes", "field_type": "longtext", "sort_order": 10 } + ], + "states": [ + { + "name": "prospect", + "label": "Prospect", + "color": "blue", + "is_initial": true, + "sort_order": 0 + }, + { "name": "active", "label": "Active", "color": "green", "sort_order": 1 }, + { "name": "on_hold", "label": "On Hold", "color": "orange", "sort_order": 2 }, + { + "name": "inactive", + "label": "Inactive", + "color": "gray", + "is_terminal": true, + "sort_order": 3 + } + ], + "transitions": [ + { + "from_state": "prospect", + "to_state": "active", + "action_name": "convert", + "action_label": "Convert to Client" + }, + { + "from_state": "active", + "to_state": "on_hold", + "action_name": "pause", + "action_label": "Put On Hold" + }, + { + "from_state": "on_hold", + "to_state": "active", + "action_name": "reactivate", + "action_label": "Reactivate" + }, + { + "from_state": "active", + "to_state": "inactive", + "action_name": "close", + "action_label": "Close Account" + }, + { + "from_state": "on_hold", + "to_state": "inactive", + "action_name": "close", + "action_label": "Close Account" + } + ], + "hooks": [ + { + "event": "after_insert", + "action_type": "send_notification", + "action_config": { + "message": "New client added: {{name}} ({{contact_name}}) — {{payment_terms}} payment terms" + } + } + ], + "relations": [] + }, + { + "doctype": { + "name": "project", + "label_singular": "Project", + "label_plural": "Projects", + "icon": "folder", + "table_name": "dt_projects", + "source": "template" + }, + "fields": [ + { + "name": "name", + "label": "Project Name", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 0 + }, + { + "name": "client_id", + "label": "Client", + "field_type": "link", + "required": true, + "sort_order": 1, + "link_doctype": "client" + }, + { + "name": "description", + "label": "Description", + "field_type": "longtext", + "sort_order": 2 + }, + { + "name": "project_type", + "label": "Project Type", + "field_type": "select", + "sort_order": 3, + "options": ["fixed-price", "time-and-materials", "retainer", "milestone-based"] + }, + { "name": "budget", "label": "Budget", "field_type": "currency", "sort_order": 4 }, + { + "name": "billed_to_date", + "label": "Billed to Date", + "field_type": "currency", + "sort_order": 5 + }, + { + "name": "budget_remaining", + "label": "Budget Remaining", + "field_type": "currency", + "sort_order": 6, + "formula": "budget - billed_to_date", + "depends_on": "budget,billed_to_date" + }, + { "name": "start_date", "label": "Start Date", "field_type": "date", "sort_order": 7 }, + { "name": "due_date", "label": "Due Date", "field_type": "date", "sort_order": 8 }, + { + "name": "estimated_hours", + "label": "Estimated Hours", + "field_type": "number", + "sort_order": 9 + }, + { + "name": "logged_hours", + "label": "Hours Logged", + "field_type": "number", + "sort_order": 10 + }, + { + "name": "current_milestone", + "label": "Current Milestone", + "field_type": "text", + "sort_order": 11 + }, + { "name": "notes", "label": "Notes", "field_type": "longtext", "sort_order": 12 } + ], + "states": [ + { + "name": "scoping", + "label": "Scoping", + "color": "blue", + "is_initial": true, + "sort_order": 0 + }, + { "name": "active", "label": "Active", "color": "green", "sort_order": 1 }, + { "name": "on_hold", "label": "On Hold", "color": "orange", "sort_order": 2 }, + { "name": "completed", "label": "Completed", "color": "purple", "sort_order": 3 }, + { + "name": "cancelled", + "label": "Cancelled", + "color": "gray", + "is_terminal": true, + "sort_order": 4 + } + ], + "transitions": [ + { + "from_state": "scoping", + "to_state": "active", + "action_name": "kick_off", + "action_label": "Kick Off Project" + }, + { + "from_state": "scoping", + "to_state": "cancelled", + "action_name": "cancel", + "action_label": "Cancel" + }, + { + "from_state": "active", + "to_state": "on_hold", + "action_name": "pause", + "action_label": "Put On Hold" + }, + { + "from_state": "on_hold", + "to_state": "active", + "action_name": "resume", + "action_label": "Resume" + }, + { + "from_state": "active", + "to_state": "completed", + "action_name": "complete", + "action_label": "Mark Complete" + }, + { + "from_state": "active", + "to_state": "cancelled", + "action_name": "cancel", + "action_label": "Cancel" + } + ], + "hooks": [ + { + "event": "after_insert", + "action_type": "send_notification", + "action_config": { + "message": "New project created: {{name}} for {{client_id}} — ${{budget}} budget, due {{due_date}}" + } + }, + { + "event": "on_state_change", + "action_type": "run_workflow", + "action_config": { + "condition": "new_state == 'completed'", + "workflow": "project-milestone-notification" + } + } + ], + "relations": [ + { + "from_doctype": "project", + "to_doctype": "client", + "relation_type": "belongs_to", + "from_field": "client_id", + "to_field": "id", + "label": "Belongs to client" + } + ] + }, + { + "doctype": { + "name": "invoice", + "label_singular": "Invoice", + "label_plural": "Invoices", + "icon": "file-text", + "table_name": "dt_invoices", + "source": "template" + }, + "fields": [ + { + "name": "invoice_number", + "label": "Invoice Number", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 0 + }, + { + "name": "client_id", + "label": "Client", + "field_type": "link", + "required": true, + "sort_order": 1, + "link_doctype": "client" + }, + { + "name": "project_id", + "label": "Project", + "field_type": "link", + "sort_order": 2, + "link_doctype": "project" + }, + { + "name": "issue_date", + "label": "Issue Date", + "field_type": "date", + "required": true, + "sort_order": 3 + }, + { + "name": "due_date", + "label": "Due Date", + "field_type": "date", + "required": true, + "sort_order": 4 + }, + { + "name": "line_items", + "label": "Line Items", + "field_type": "longtext", + "required": true, + "sort_order": 5 + }, + { + "name": "subtotal", + "label": "Subtotal", + "field_type": "currency", + "required": true, + "sort_order": 6 + }, + { "name": "tax_rate", "label": "Tax Rate (%)", "field_type": "number", "sort_order": 7 }, + { + "name": "tax_amount", + "label": "Tax Amount", + "field_type": "currency", + "sort_order": 8, + "formula": "subtotal * (tax_rate / 100)", + "depends_on": "subtotal,tax_rate" + }, + { + "name": "total", + "label": "Total", + "field_type": "currency", + "required": true, + "sort_order": 9, + "formula": "subtotal + tax_amount", + "depends_on": "subtotal,tax_amount" + }, + { "name": "paid_date", "label": "Date Paid", "field_type": "date", "sort_order": 10 }, + { + "name": "payment_method", + "label": "Payment Method", + "field_type": "select", + "sort_order": 11, + "options": ["bank-transfer", "card", "check", "cash", "paypal", "stripe", "other"] + }, + { + "name": "notes", + "label": "Notes / Payment Instructions", + "field_type": "longtext", + "sort_order": 12 + } + ], + "states": [ + { "name": "draft", "label": "Draft", "color": "gray", "is_initial": true, "sort_order": 0 }, + { "name": "sent", "label": "Sent", "color": "blue", "sort_order": 1 }, + { "name": "overdue", "label": "Overdue", "color": "red", "sort_order": 2 }, + { "name": "paid", "label": "Paid", "color": "green", "sort_order": 3 }, + { + "name": "cancelled", + "label": "Cancelled", + "color": "gray", + "is_terminal": true, + "sort_order": 4 + } + ], + "transitions": [ + { + "from_state": "draft", + "to_state": "sent", + "action_name": "send", + "action_label": "Send Invoice" + }, + { + "from_state": "draft", + "to_state": "cancelled", + "action_name": "cancel", + "action_label": "Cancel" + }, + { + "from_state": "sent", + "to_state": "paid", + "action_name": "mark_paid", + "action_label": "Mark as Paid" + }, + { + "from_state": "sent", + "to_state": "overdue", + "action_name": "flag_overdue", + "action_label": "Flag Overdue" + }, + { + "from_state": "overdue", + "to_state": "paid", + "action_name": "mark_paid", + "action_label": "Mark as Paid" + }, + { + "from_state": "sent", + "to_state": "cancelled", + "action_name": "cancel", + "action_label": "Cancel" + } + ], + "hooks": [ + { + "event": "after_insert", + "action_type": "send_notification", + "action_config": { + "message": "Invoice {{invoice_number}} created: ${{total}} due {{due_date}} for {{client_id}}" + } + }, + { + "event": "on_state_change", + "action_type": "run_workflow", + "action_config": { + "condition": "new_state == 'overdue'", + "workflow": "invoice-overdue-reminder" + } + } + ], + "relations": [ + { + "from_doctype": "invoice", + "to_doctype": "client", + "relation_type": "belongs_to", + "from_field": "client_id", + "to_field": "id", + "label": "Billed to" + }, + { + "from_doctype": "invoice", + "to_doctype": "project", + "relation_type": "belongs_to", + "from_field": "project_id", + "to_field": "id", + "label": "For project" + } + ] + }, + { + "doctype": { + "name": "timesheet", + "label_singular": "Timesheet Entry", + "label_plural": "Timesheet Entries", + "icon": "clock", + "table_name": "dt_timesheets", + "source": "template" + }, + "fields": [ + { + "name": "date", + "label": "Date", + "field_type": "date", + "required": true, + "searchable": true, + "sort_order": 0 + }, + { + "name": "project_id", + "label": "Project", + "field_type": "link", + "required": true, + "sort_order": 1, + "link_doctype": "project" + }, + { + "name": "client_id", + "label": "Client", + "field_type": "link", + "sort_order": 2, + "link_doctype": "client" + }, + { + "name": "task", + "label": "Task / Description", + "field_type": "text", + "required": true, + "searchable": true, + "sort_order": 3 + }, + { + "name": "hours", + "label": "Hours", + "field_type": "number", + "required": true, + "sort_order": 4 + }, + { + "name": "hourly_rate", + "label": "Hourly Rate", + "field_type": "currency", + "sort_order": 5 + }, + { + "name": "amount", + "label": "Billable Amount", + "field_type": "currency", + "sort_order": 6, + "formula": "hours * hourly_rate", + "depends_on": "hours,hourly_rate" + }, + { "name": "is_billable", "label": "Billable", "field_type": "checkbox", "sort_order": 7 }, + { + "name": "invoice_id", + "label": "Invoice", + "field_type": "link", + "sort_order": 8, + "link_doctype": "invoice" + }, + { "name": "notes", "label": "Notes", "field_type": "longtext", "sort_order": 9 } + ], + "states": [ + { + "name": "logged", + "label": "Logged", + "color": "blue", + "is_initial": true, + "sort_order": 0 + }, + { "name": "approved", "label": "Approved", "color": "green", "sort_order": 1 }, + { + "name": "invoiced", + "label": "Invoiced", + "color": "purple", + "is_terminal": true, + "sort_order": 2 + } + ], + "transitions": [ + { + "from_state": "logged", + "to_state": "approved", + "action_name": "approve", + "action_label": "Approve" + }, + { + "from_state": "approved", + "to_state": "invoiced", + "action_name": "invoice", + "action_label": "Mark Invoiced" + }, + { + "from_state": "logged", + "to_state": "invoiced", + "action_name": "invoice", + "action_label": "Mark Invoiced" + } + ], + "hooks": [], + "relations": [ + { + "from_doctype": "timesheet", + "to_doctype": "project", + "relation_type": "belongs_to", + "from_field": "project_id", + "to_field": "id", + "label": "Logged on project" + }, + { + "from_doctype": "timesheet", + "to_doctype": "client", + "relation_type": "belongs_to", + "from_field": "client_id", + "to_field": "id", + "label": "Billed to client" + } + ] + } + ], + "workflows": [ + { + "name": "Invoice Overdue Reminder", + "description": "Checks daily for unpaid invoices past their due date and sends a reminder with the overdue amounts.", + "trigger": { + "type": "schedule", + "cron": "0 9 * * *" + }, + "steps": [ + { + "id": "query-overdue-invoices", + "name": "Find overdue invoices", + "type": "query", + "config": { + "doctype": "invoice", + "filter": "due_date < date('now') AND status IN ('sent', 'overdue')", + "fields": ["invoice_number", "client_id", "due_date", "total", "issue_date"] + }, + "sort_order": 0 + }, + { + "id": "check-has-overdue", + "name": "Check if any invoices are overdue", + "type": "condition", + "config": { + "expression": "results.length > 0", + "on_false": "skip_remaining" + }, + "sort_order": 1 + }, + { + "id": "format-reminder", + "name": "Format overdue reminder", + "type": "transform", + "config": { + "template": "Overdue Invoices:\n{{#each results}}\n- {{invoice_number}}: ${{total}} from {{client_id}}, due {{due_date}}\n{{/each}}" + }, + "sort_order": 2 + }, + { + "id": "send-reminder", + "name": "Send overdue reminder", + "type": "send", + "config": { + "channel": "default", + "message": "Invoice Overdue Reminder:\n{{formatted_output}}" + }, + "sort_order": 3 + } + ], + "status": "active" + }, + { + "name": "Project Milestone Notification", + "description": "Sends a notification when a project reaches a key milestone or is marked complete, summarising deliverables and next steps.", + "trigger": { + "type": "event", + "on": "project.on_state_change" + }, + "steps": [ + { + "id": "get-project-details", + "name": "Get project details", + "type": "query", + "config": { + "doctype": "project", + "filter": "id = {{trigger.record_id}}", + "fields": [ + "name", + "client_id", + "current_milestone", + "billed_to_date", + "budget", + "due_date", + "logged_hours", + "estimated_hours" + ] + }, + "sort_order": 0 + }, + { + "id": "get-unbilled-hours", + "name": "Get unbilled timesheet hours", + "type": "query", + "config": { + "doctype": "timesheet", + "filter": "project_id = {{trigger.record_id}} AND status = 'approved' AND is_billable = 1", + "fields": ["hours", "amount", "task", "date"] + }, + "sort_order": 1 + }, + { + "id": "generate-summary", + "name": "AI generates milestone summary", + "type": "ai", + "config": { + "prompt": "The project '{{project.name}}' for client '{{project.client_id}}' has reached a milestone (state changed to {{trigger.new_state}}). Summarise: (1) Current milestone: {{project.current_milestone}}, (2) Budget used: ${{project.billed_to_date}} of ${{project.budget}}, (3) Hours logged: {{project.logged_hours}} of {{project.estimated_hours}} estimated, (4) Unbilled approved hours ready to invoice. Suggest next steps.", + "tool_profile": "read-only", + "max_turns": 2 + }, + "sort_order": 2 + }, + { + "id": "send-notification", + "name": "Send milestone notification", + "type": "send", + "config": { + "channel": "default", + "message": "Project Milestone — {{project.name}}:\n{{ai_output}}" + }, + "sort_order": 3 + } + ], + "status": "active" + }, + { + "name": "Monthly Revenue Report", + "description": "Generates a monthly revenue summary on the first of each month: total invoiced, total collected, outstanding balance, and top clients by revenue.", + "trigger": { + "type": "schedule", + "cron": "0 8 1 * *" + }, + "steps": [ + { + "id": "get-last-month-invoices", + "name": "Get last month's invoices", + "type": "query", + "config": { + "doctype": "invoice", + "filter": "issue_date >= date('now', 'start of month', '-1 month') AND issue_date < date('now', 'start of month')", + "fields": ["invoice_number", "client_id", "total", "status", "paid_date"] + }, + "sort_order": 0 + }, + { + "id": "get-outstanding-invoices", + "name": "Get all outstanding invoices", + "type": "query", + "config": { + "doctype": "invoice", + "filter": "status IN ('sent', 'overdue')", + "fields": ["invoice_number", "client_id", "total", "due_date", "status"] + }, + "sort_order": 1 + }, + { + "id": "get-monthly-hours", + "name": "Get last month's billable hours", + "type": "query", + "config": { + "doctype": "timesheet", + "filter": "date >= date('now', 'start of month', '-1 month') AND date < date('now', 'start of month') AND is_billable = 1", + "fields": ["project_id", "client_id", "hours", "amount"] + }, + "sort_order": 2 + }, + { + "id": "generate-report", + "name": "AI generates monthly revenue report", + "type": "ai", + "config": { + "prompt": "Generate a monthly revenue report. Calculate: (1) Total invoiced last month, (2) Total collected (paid invoices), (3) Total outstanding (sent + overdue invoices), (4) Total billable hours logged last month, (5) Effective hourly rate (total invoiced / billable hours), (6) Top 5 clients by revenue last month. Flag any clients with significant outstanding balances over 30 days.", + "tool_profile": "read-only", + "max_turns": 3 + }, + "sort_order": 3 + }, + { + "id": "send-report", + "name": "Send monthly revenue report", + "type": "send", + "config": { + "channel": "default", + "message": "Monthly Revenue Report:\n{{ai_output}}" + }, + "sort_order": 4 + } + ], + "status": "active" + } + ] +} diff --git a/src/intelligence/industry-templates/services/skill-pack.md b/src/intelligence/industry-templates/services/skill-pack.md new file mode 100644 index 00000000..2821fa21 --- /dev/null +++ b/src/intelligence/industry-templates/services/skill-pack.md @@ -0,0 +1,70 @@ +# Services Operations Skill Pack + +You are managing a professional services business — consulting, agency, freelance studio, or managed services provider. Use the following guidance when handling user requests. + +## Data Types Available + +- **Client**: Track client companies with contact details, industry, payment terms, and default hourly rate +- **Project**: Manage engagements with client link, project type (fixed-price, T&M, retainer, milestone), budget, and timeline +- **Invoice**: Create and track invoices from draft through payment, including line items and tax +- **Timesheet**: Log billable and non-billable hours against projects, link to invoices when billed + +## Common Operations + +### Client Management + +- Add, update, or deactivate clients +- Track payment terms per client (due-on-receipt, net-15, net-30, net-60) +- Record default hourly rate per client +- Monitor total lifetime billings per client +- Identify prospects vs active clients + +### Project Tracking + +- Create projects linked to clients with budget, type, and timeline +- Track project types: fixed-price, time-and-materials, retainer, milestone-based +- Monitor budget usage: `billed_to_date / budget × 100` +- Track milestone progress and project health +- Transition projects through: scoping → active → completed + +### Time Tracking + +- Log hours against projects with task descriptions +- Mark entries as billable or non-billable +- Calculate billable amount: `hours × hourly_rate` +- Identify unbilled approved hours ready to invoice +- Filter time entries by date range, project, or client + +### Invoicing & Billing + +- Create invoices with line items, tax rate, and due date +- Track invoice lifecycle: draft → sent → paid (or overdue) +- Calculate outstanding balance across all clients +- Flag invoices past due date for follow-up +- Match invoice payments to specific projects + +### Reporting + +- Monthly revenue: total invoiced, collected, and outstanding +- Effective hourly rate: `total revenue / billable hours` +- Project profitability: budget vs actual spend +- Client revenue breakdown: top clients by billing +- Utilisation rate: billable vs non-billable hours + +## Key Metrics + +- **Effective hourly rate**: total revenue ÷ billable hours logged (target: ≥ your standard rate) +- **Utilisation rate**: billable hours ÷ total hours worked (target: 70–80% for consulting) +- **Accounts receivable**: total of sent + overdue invoices (keep under 60 days revenue) +- **Project budget burn**: `billed_to_date / budget` — flag if > 80% with work remaining +- **Average payment time**: days between invoice sent and paid_date (target: ≤ payment terms) + +## Tips + +- When logging time, always link to a project — this enables accurate billing and profitability tracking +- "Billable hours" = `is_billable = true` timesheet entries; always confirm with the client before invoicing +- For fixed-price projects, track hours anyway to measure actual vs estimated effort +- Retainer clients typically pay monthly regardless of hours — track hours to ensure scope isn't exceeded +- When a client asks "what's outstanding?", query invoices with `status IN ('sent', 'overdue')` +- Before creating an invoice, query approved unbilled timesheet entries for the project to determine the line items +- Flag projects where `billed_to_date > budget` immediately — scope creep needs a change order diff --git a/src/intelligence/knowledge-graph.ts b/src/intelligence/knowledge-graph.ts new file mode 100644 index 00000000..141475e6 --- /dev/null +++ b/src/intelligence/knowledge-graph.ts @@ -0,0 +1,471 @@ +import type Database from 'better-sqlite3'; +import { ensureDocTypeStoreSchema, getDocType, listDocTypes } from './doctype-store.js'; +import { getRelations, resolveLinkedRecords } from './relation-manager.js'; + +// --------------------------------------------------------------------------- +// Public types +// --------------------------------------------------------------------------- + +/** A business entity resolved from a DocType table row */ +export interface BusinessEntity { + id: string; + type: string; + data: Record; +} + +/** A relation between two business entities */ +export interface BusinessRelation { + from: { id: string; type: string }; + to: { id: string; type: string }; + relation_type: string; + label?: string; +} + +/** A single hop in a relation path */ +export interface RelationHop { + entity: { id: string; type: string }; + relation_type: string; + direction: 'outgoing' | 'incoming'; +} + +/** A path of relations between two entities */ +export interface RelationPath { + from: { id: string; type: string }; + to: { id: string; type: string }; + hops: RelationHop[]; + length: number; +} + +/** Supported aggregate operations */ +export type AggregateOp = 'sum' | 'avg' | 'count' | 'min' | 'max'; + +// --------------------------------------------------------------------------- +// queryEntities +// --------------------------------------------------------------------------- + +/** + * Query entities from a DocType table. + * + * @param db - Open SQLite database handle. + * @param type - DocType name (e.g. "invoice", "customer"). + * @param filters - Optional field→value equality filters. + * @returns Array of business entities matching the query. + */ +export function queryEntities( + db: Database.Database, + type: string, + filters?: Record, +): BusinessEntity[] { + ensureDocTypeStoreSchema(db); + + const doctype = findDocTypeByName(db, type); + if (!doctype) return []; + + const whereClauses: string[] = []; + const params: unknown[] = []; + + if (filters) { + for (const [key, value] of Object.entries(filters)) { + whereClauses.push(`"${sanitizeIdentifier(key)}" = ?`); + params.push(value); + } + } + + const whereSQL = whereClauses.length > 0 ? `WHERE ${whereClauses.join(' AND ')}` : ''; + const tableName = quoteIdentifier(doctype.table_name); + + const rows = db.prepare(`SELECT * FROM ${tableName} ${whereSQL}`).all(...params) as Record< + string, + unknown + >[]; + + return rows.map((row) => ({ + id: row['id'] as string, + type, + data: row, + })); +} + +// --------------------------------------------------------------------------- +// queryRelations +// --------------------------------------------------------------------------- + +/** + * Query all relations for a given entity. + * + * Finds the DocType that owns the entity, looks up all relations for that + * DocType, and resolves the linked records. + * + * @param db - Open SQLite database handle. + * @param entityId - The record ID to find relations for. + * @param type - Optional DocType name hint (avoids scanning all tables). + * @returns Array of business relations involving this entity. + */ +export function queryRelations( + db: Database.Database, + entityId: string, + type?: string, +): BusinessRelation[] { + ensureDocTypeStoreSchema(db); + + const ownerDoctype = type ? findDocTypeByName(db, type) : findDocTypeForRecord(db, entityId); + if (!ownerDoctype) return []; + + const doctypeFull = getDocType(db, ownerDoctype.id); + if (!doctypeFull) return []; + + const relations = getRelations(db, ownerDoctype.id, 'both'); + const result: BusinessRelation[] = []; + + for (const rel of relations) { + const isSource = rel.from_doctype === ownerDoctype.id; + + if (isSource) { + // Outgoing: resolve linked records on the "to" side + const sourceRecord = db + .prepare(`SELECT * FROM "${sanitizeIdentifier(ownerDoctype.table_name)}" WHERE id = ?`) + .get(entityId) as Record | undefined; + + const resolved = resolveLinkedRecords(db, rel, entityId, sourceRecord ?? undefined); + for (const linkedRow of resolved.records) { + const targetDoctype = findDocTypeById(db, rel.to_doctype); + result.push({ + from: { id: entityId, type: ownerDoctype.name }, + to: { id: linkedRow.id, type: targetDoctype?.name ?? rel.to_doctype }, + relation_type: rel.relation_type, + label: rel.label, + }); + } + } else { + // Incoming: this entity is on the "to" side + const sourceDoctype = findDocTypeById(db, rel.from_doctype); + if (!sourceDoctype) continue; + + const sourceTable = quoteIdentifier(sourceDoctype.table_name); + const sourceRows = db + .prepare(`SELECT id FROM ${sourceTable} WHERE "${sanitizeIdentifier(rel.from_field)}" = ?`) + .all(entityId) as Array<{ id: string }>; + + for (const sourceRow of sourceRows) { + result.push({ + from: { id: sourceRow.id, type: sourceDoctype.name }, + to: { id: entityId, type: ownerDoctype.name }, + relation_type: rel.relation_type, + label: rel.label, + }); + } + } + } + + return result; +} + +// --------------------------------------------------------------------------- +// findPath +// --------------------------------------------------------------------------- + +/** + * Find relation paths between two entities via BFS graph traversal. + * + * @param db - Open SQLite database handle. + * @param fromId - Source entity ID. + * @param toId - Target entity ID. + * @param maxDepth - Maximum traversal depth (default 4). + * @returns Array of relation paths from source to target. + */ +export function findPath( + db: Database.Database, + fromId: string, + toId: string, + maxDepth = 4, +): RelationPath[] { + ensureDocTypeStoreSchema(db); + + const fromDoctype = findDocTypeForRecord(db, fromId); + const toDoctype = findDocTypeForRecord(db, toId); + if (!fromDoctype || !toDoctype) return []; + + // BFS to find paths between entity types (DocType-level graph) + // Then verify actual record-level connectivity + const typePaths = bfsTypePaths(db, fromDoctype.id, toDoctype.id, maxDepth); + const results: RelationPath[] = []; + + for (const typePath of typePaths) { + // Verify record-level connectivity along this type path + const recordPath = verifyRecordPath(db, fromId, toId, typePath); + if (recordPath) { + results.push(recordPath); + } + } + + return results; +} + +// --------------------------------------------------------------------------- +// aggregateMetrics +// --------------------------------------------------------------------------- + +/** + * Run an aggregate operation on a DocType field. + * + * @param db - Open SQLite database handle. + * @param doctype - DocType name. + * @param field - Field name to aggregate. + * @param operation - Aggregate operation (sum, avg, count, min, max). + * @param filters - Optional field→value equality filters. + * @returns Numeric result, or 0 if the DocType or field doesn't exist. + */ +export function aggregateMetrics( + db: Database.Database, + doctype: string, + field: string, + operation: AggregateOp, + filters?: Record, +): number { + ensureDocTypeStoreSchema(db); + + const dt = findDocTypeByName(db, doctype); + if (!dt) return 0; + + const tableName = quoteIdentifier(dt.table_name); + const safeField = quoteIdentifier(field); + + const aggFn = operation.toUpperCase(); + const validOps = ['SUM', 'AVG', 'COUNT', 'MIN', 'MAX']; + if (!validOps.includes(aggFn)) return 0; + + const aggExpr = operation === 'count' ? `COUNT(${safeField})` : `${aggFn}(${safeField})`; + + const whereClauses: string[] = []; + const params: unknown[] = []; + + if (filters) { + for (const [key, value] of Object.entries(filters)) { + whereClauses.push(`"${sanitizeIdentifier(key)}" = ?`); + params.push(value); + } + } + + const whereSQL = whereClauses.length > 0 ? `WHERE ${whereClauses.join(' AND ')}` : ''; + + try { + const row = db + .prepare(`SELECT ${aggExpr} AS result FROM ${tableName} ${whereSQL}`) + .get(...params) as { result: number | null } | undefined; + return row?.result ?? 0; + } catch { + return 0; + } +} + +// --------------------------------------------------------------------------- +// Internal helpers +// --------------------------------------------------------------------------- + +/** Find a DocType metadata record by name (case-insensitive) */ +function findDocTypeByName( + db: Database.Database, + name: string, +): { id: string; name: string; table_name: string } | null { + const row = db + .prepare('SELECT id, name, table_name FROM doctypes WHERE LOWER(name) = LOWER(?)') + .get(name) as { id: string; name: string; table_name: string } | undefined; + return row ?? null; +} + +/** Find a DocType metadata record by ID */ +function findDocTypeById( + db: Database.Database, + id: string, +): { id: string; name: string; table_name: string } | null { + const row = db.prepare('SELECT id, name, table_name FROM doctypes WHERE id = ?').get(id) as + | { id: string; name: string; table_name: string } + | undefined; + return row ?? null; +} + +/** Find which DocType table contains a given record ID */ +function findDocTypeForRecord( + db: Database.Database, + recordId: string, +): { id: string; name: string; table_name: string } | null { + const allDocTypes = listDocTypes(db); + + for (const dt of allDocTypes) { + try { + const row = db + .prepare(`SELECT id FROM "${sanitizeIdentifier(dt.table_name)}" WHERE id = ?`) + .get(recordId) as { id: string } | undefined; + if (row) { + return { id: dt.id, name: dt.name, table_name: dt.table_name }; + } + } catch { + // Table may not exist yet — skip + } + } + + return null; +} + +/** Sanitize a SQL identifier by removing embedded double-quotes */ +function sanitizeIdentifier(name: string): string { + return name.replace(/"/g, ''); +} + +/** Wrap a SQL identifier in double-quotes */ +function quoteIdentifier(name: string): string { + return `"${sanitizeIdentifier(name)}"`; +} + +// --------------------------------------------------------------------------- +// BFS for type-level paths +// --------------------------------------------------------------------------- + +interface TypeHop { + fromDoctypeId: string; + toDoctypeId: string; + relation_type: string; + from_field: string; + to_field: string; + direction: 'outgoing' | 'incoming'; +} + +/** BFS over DocType relations to find type-level paths */ +function bfsTypePaths( + db: Database.Database, + fromDoctypeId: string, + toDoctypeId: string, + maxDepth: number, +): TypeHop[][] { + if (fromDoctypeId === toDoctypeId) return [[]]; + + // Queue entries: [current doctype ID, path so far] + const queue: Array<[string, TypeHop[]]> = [[fromDoctypeId, []]]; + const visited = new Set([fromDoctypeId]); + const results: TypeHop[][] = []; + + while (queue.length > 0) { + const [currentId, path] = queue.shift()!; + + if (path.length >= maxDepth) continue; + + const relations = getRelations(db, currentId, 'both'); + + for (const rel of relations) { + const isSource = rel.from_doctype === currentId; + const neighborId = isSource ? rel.to_doctype : rel.from_doctype; + + const hop: TypeHop = { + fromDoctypeId: isSource ? currentId : neighborId, + toDoctypeId: isSource ? neighborId : currentId, + relation_type: rel.relation_type, + from_field: rel.from_field, + to_field: rel.to_field, + direction: isSource ? 'outgoing' : 'incoming', + }; + + const newPath = [...path, hop]; + + if (neighborId === toDoctypeId) { + results.push(newPath); + continue; + } + + if (!visited.has(neighborId)) { + visited.add(neighborId); + queue.push([neighborId, newPath]); + } + } + } + + return results; +} + +/** Verify that a type-level path has actual record connectivity */ +function verifyRecordPath( + db: Database.Database, + fromId: string, + toId: string, + typeHops: TypeHop[], +): RelationPath | null { + if (typeHops.length === 0) { + // Same DocType — from and to are in the same table + const dt = findDocTypeForRecord(db, fromId); + if (!dt) return null; + return { + from: { id: fromId, type: dt.name }, + to: { id: toId, type: dt.name }, + hops: [], + length: 0, + }; + } + + let currentIds = [fromId]; + const hops: RelationHop[] = []; + + for (const hop of typeHops) { + const nextIds: string[] = []; + const sourceDt = findDocTypeById(db, hop.fromDoctypeId); + const targetDt = findDocTypeById(db, hop.toDoctypeId); + if (!sourceDt || !targetDt) return null; + + if (hop.direction === 'outgoing') { + // Follow from_field on source to to_field on target + const targetTable = quoteIdentifier(targetDt.table_name); + const sourceTable = quoteIdentifier(sourceDt.table_name); + + for (const cid of currentIds) { + // Get source record's from_field value + const sourceRow = db + .prepare( + `SELECT "${sanitizeIdentifier(hop.from_field)}" AS fk FROM ${sourceTable} WHERE id = ?`, + ) + .get(cid) as { fk: unknown } | undefined; + + if (sourceRow?.fk != null) { + const targetRows = db + .prepare( + `SELECT id FROM ${targetTable} WHERE "${sanitizeIdentifier(hop.to_field)}" = ?`, + ) + .all(sourceRow.fk) as Array<{ id: string }>; + nextIds.push(...targetRows.map((r) => r.id)); + } + } + } else { + // Incoming: follow from source table where from_field points to current + const sourceTable = quoteIdentifier(sourceDt.table_name); + + for (const cid of currentIds) { + const rows = db + .prepare( + `SELECT id FROM ${sourceTable} WHERE "${sanitizeIdentifier(hop.from_field)}" = ?`, + ) + .all(cid) as Array<{ id: string }>; + nextIds.push(...rows.map((r) => r.id)); + } + } + + if (nextIds.length === 0) return null; + + currentIds = nextIds; + hops.push({ + entity: { id: nextIds[0]!, type: targetDt.name }, + relation_type: hop.relation_type, + direction: hop.direction, + }); + } + + // Check if toId is reachable + if (!currentIds.includes(toId)) return null; + + const fromDt = findDocTypeForRecord(db, fromId); + const toDt = findDocTypeForRecord(db, toId); + if (!fromDt || !toDt) return null; + + return { + from: { id: fromId, type: fromDt.name }, + to: { id: toId, type: toDt.name }, + hops, + length: hops.length, + }; +} diff --git a/src/intelligence/list-generator.ts b/src/intelligence/list-generator.ts new file mode 100644 index 00000000..098cf61d --- /dev/null +++ b/src/intelligence/list-generator.ts @@ -0,0 +1,263 @@ +import type { DocType, DocTypeField, DocTypeState } from '../types/doctype.js'; + +/** + * Generate a self-contained HTML list view page from DocType metadata. + * + * Renders an HTML table with sortable columns, FTS5 search bar, pagination, + * and status badges for state fields. Includes a "New" button linking to the + * form view. + */ +export function generateListView( + doctype: DocType, + records: Record[], + options?: { + states?: DocTypeState[]; + fields?: DocTypeField[]; + page?: number; + pageSize?: number; + totalCount?: number; + sortField?: string; + sortDir?: 'asc' | 'desc'; + searchQuery?: string; + apiBase?: string; + }, +): string { + const states = options?.states ?? []; + const fields = options?.fields ?? []; + const page = options?.page ?? 1; + const pageSize = options?.pageSize ?? 20; + const totalCount = options?.totalCount ?? records.length; + const sortField = options?.sortField ?? ''; + const sortDir = options?.sortDir ?? 'asc'; + const searchQuery = options?.searchQuery ?? ''; + const apiBase = options?.apiBase ?? '/api/dt'; + + const stateMap = new Map(states.map((s) => [s.name, s])); + + // Determine visible columns — exclude heavy fields (longtext, table, image) + const HIDDEN_TYPES = new Set(['longtext', 'table', 'image']); + const visibleFields = fields + .filter((f) => !HIDDEN_TYPES.has(f.field_type)) + .sort((a, b) => a.sort_order - b.sort_order) + .slice(0, 8); // cap columns for readability + + // Derive column names from records if no field metadata provided + const columnNames: string[] = + visibleFields.length > 0 + ? visibleFields.map((f) => f.name) + : records.length > 0 + ? Object.keys(records[0]!) + .filter((k) => k !== 'id') + .slice(0, 8) + : []; + + const columnLabels: Record = {}; + for (const f of visibleFields) { + columnLabels[f.name] = f.label; + } + + const hasStateField = columnNames.includes('_state'); + if (!hasStateField && states.length > 0) { + columnNames.push('_state'); + columnLabels['_state'] = 'Status'; + } + + const totalPages = Math.max(1, Math.ceil(totalCount / pageSize)); + const dtPath = encodeURIComponent(doctype.name.toLowerCase()); + const formBase = `${apiBase}/${dtPath}/new`; + const listBase = `${apiBase}/${dtPath}`; + + const headerCells = columnNames + .map((col) => { + const label = columnLabels[col] ?? col; + const isSorted = sortField === col; + const nextDir = isSorted && sortDir === 'asc' ? 'desc' : 'asc'; + const arrow = isSorted ? (sortDir === 'asc' ? ' ▲' : ' ▼') : ''; + return `${escapeHtml(label)}${arrow}`; + }) + .join('\n'); + + const bodyRows = + records.length === 0 + ? `No records found.` + : records + .map((row) => { + const rowId = toStr(row['id']); + const editUrl = `${apiBase}/${dtPath}/${encodeURIComponent(rowId)}`; + const cells = columnNames + .map((col) => { + const val = row[col]; + if (col === '_state' && states.length > 0) { + const stateObj = stateMap.get(toStr(val)); + if (stateObj) { + return `${escapeHtml(stateObj.label)}`; + } + } + return `${escapeHtml(toStr(val))}`; + }) + .join('\n'); + return ` + ${cells} + Edit +`; + }) + .join('\n'); + + // Pagination links + const prevDisabled = page <= 1; + const nextDisabled = page >= totalPages; + const start = totalCount === 0 ? 0 : (page - 1) * pageSize + 1; + const end = Math.min(page * pageSize, totalCount); + const paginationInfo = `Showing ${start}–${end} of ${totalCount}`; + + return ` + + + + + ${escapeHtml(doctype.label_plural)} + + + +
+
+

${escapeHtml(doctype.label_plural)}

+ + New ${escapeHtml(doctype.label_singular)} +
+ +
+
+ + + ${searchQuery ? `Clear` : ''} + + +
+
+ +
+ + + + ${headerCells} + + + + + ${bodyRows} + +
+
+ + +
+ + + +`; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function toStr(val: unknown): string { + if (val == null) return ''; + if (typeof val === 'string') return val; + if (typeof val === 'number' || typeof val === 'boolean') return String(val); + return JSON.stringify(val); +} + +function escapeHtml(str: string): string { + return str + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +// --------------------------------------------------------------------------- +// Embedded CSS +// --------------------------------------------------------------------------- + +const CSS = ` + * { box-sizing: border-box; margin: 0; padding: 0; } + body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; background: #f5f5f5; color: #333; line-height: 1.5; } + .container { max-width: 1100px; margin: 2rem auto; background: #fff; border-radius: 8px; box-shadow: 0 1px 3px rgba(0,0,0,0.12); padding: 2rem; } + .list-header { display: flex; justify-content: space-between; align-items: center; margin-bottom: 1.5rem; } + h1 { font-size: 1.5rem; font-weight: 600; } + .toolbar { margin-bottom: 1rem; } + .search-form { display: flex; gap: 0.5rem; align-items: center; flex-wrap: wrap; } + .search-input { flex: 1; min-width: 200px; padding: 0.5rem 0.75rem; border: 1px solid #d1d5db; border-radius: 6px; font-size: 0.9375rem; } + .search-input:focus { outline: none; border-color: #3b82f6; box-shadow: 0 0 0 3px rgba(59,130,246,0.1); } + .table-wrapper { overflow-x: auto; } + .list-table { width: 100%; border-collapse: collapse; font-size: 0.9375rem; } + .list-table thead th { background: #f9fafb; padding: 0.625rem 0.75rem; text-align: left; font-weight: 600; font-size: 0.8125rem; color: #6b7280; border-bottom: 2px solid #e5e7eb; white-space: nowrap; } + .sort-link { color: inherit; text-decoration: none; } + .sort-link:hover { color: #3b82f6; } + .list-table tbody tr { border-bottom: 1px solid #f0f0f0; cursor: pointer; transition: background 0.1s; } + .list-table tbody tr:hover { background: #f9fafb; } + .list-table tbody td { padding: 0.625rem 0.75rem; max-width: 260px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } + .empty-row { text-align: center; color: #9ca3af; font-style: italic; padding: 2rem 0; cursor: default; } + .action-col { width: 80px; } + .action-cell { text-align: right; } + .state-badge { display: inline-block; padding: 0.2rem 0.6rem; border-radius: 999px; color: #fff; font-size: 0.75rem; font-weight: 500; } + .pagination { display: flex; justify-content: space-between; align-items: center; margin-top: 1.25rem; padding-top: 1rem; border-top: 1px solid #e5e7eb; flex-wrap: wrap; gap: 0.5rem; } + .pagination-info { color: #6b7280; font-size: 0.875rem; } + .pagination-controls { display: flex; align-items: center; gap: 0.5rem; } + .page-indicator { font-size: 0.875rem; color: #374151; } + .btn { display: inline-block; padding: 0.5rem 1.25rem; border-radius: 6px; font-size: 0.9375rem; font-weight: 500; cursor: pointer; text-decoration: none; text-align: center; border: none; transition: background 0.15s; } + .btn-primary { background: #3b82f6; color: #fff; } + .btn-primary:hover { background: #2563eb; } + .btn-secondary { background: #e5e7eb; color: #374151; } + .btn-secondary:hover { background: #d1d5db; } + .btn-ghost { background: transparent; color: #374151; border: 1px solid #d1d5db; } + .btn-ghost:hover:not(.disabled) { background: #f3f4f6; } + .btn-ghost.disabled { opacity: 0.4; cursor: not-allowed; pointer-events: none; } + .btn-sm { padding: 0.25rem 0.75rem; font-size: 0.8125rem; } + @media (max-width: 640px) { .container { margin: 1rem; padding: 1rem; } .list-header { flex-direction: column; align-items: flex-start; gap: 0.75rem; } } +`; diff --git a/src/intelligence/naming-series.ts b/src/intelligence/naming-series.ts new file mode 100644 index 00000000..50b32e7b --- /dev/null +++ b/src/intelligence/naming-series.ts @@ -0,0 +1,157 @@ +import type Database from 'better-sqlite3'; + +/** + * Parsed representation of a naming-series pattern segment. + * Segments are either a literal string or a placeholder token. + */ +type Segment = + | { kind: 'literal'; value: string } + | { kind: 'year' } // {YYYY} + | { kind: 'month' } // {MM} + | { kind: 'day' } // {DD} + | { kind: 'counter'; width: number }; // {###…} + +/** + * Parse a naming-series pattern into a list of segments. + * + * Supported placeholders: + * {YYYY} → 4-digit year (e.g. 2026) + * {MM} → 2-digit month (01-12) + * {DD} → 2-digit day (01-31) + * {#…} → zero-padded counter (width = number of '#' chars) + * + * Anything outside `{…}` is a literal segment. + */ +function parsePattern(pattern: string): Segment[] { + const segments: Segment[] = []; + // Split on tokens like {YYYY}, {MM}, {DD}, {###} + const TOKEN_RE = /\{([^}]+)\}/g; + let lastIndex = 0; + let match: RegExpExecArray | null; + + while ((match = TOKEN_RE.exec(pattern)) !== null) { + // Literal before this token + if (match.index > lastIndex) { + segments.push({ kind: 'literal', value: pattern.slice(lastIndex, match.index) }); + } + + const token: string = match[1] ?? ''; + if (token === 'YYYY') { + segments.push({ kind: 'year' }); + } else if (token === 'MM') { + segments.push({ kind: 'month' }); + } else if (token === 'DD') { + segments.push({ kind: 'day' }); + } else if (/^#+$/.test(token)) { + segments.push({ kind: 'counter', width: token.length }); + } else { + // Unknown token — treat as literal + segments.push({ kind: 'literal', value: match[0] }); + } + + lastIndex = match.index + match[0].length; + } + + // Trailing literal + if (lastIndex < pattern.length) { + segments.push({ kind: 'literal', value: pattern.slice(lastIndex) }); + } + + return segments; +} + +/** + * Build the prefix string from all segments that come before the counter + * and resolve date placeholders against `now`. + */ +function buildPrefix(segments: Segment[], now: Date): { prefix: string; counterWidth: number } { + let prefix = ''; + let counterWidth = 5; // default + + for (const seg of segments) { + switch (seg.kind) { + case 'literal': + prefix += seg.value; + break; + case 'year': + prefix += String(now.getFullYear()); + break; + case 'month': + prefix += String(now.getMonth() + 1).padStart(2, '0'); + break; + case 'day': + prefix += String(now.getDate()).padStart(2, '0'); + break; + case 'counter': + counterWidth = seg.width; + // Counter segment ends prefix accumulation + return { prefix, counterWidth }; + } + } + + // Pattern has no counter — still return what we have + return { prefix, counterWidth }; +} + +/** + * Atomically increment the per-prefix counter in `dt_series` and return the + * next value. Uses `BEGIN IMMEDIATE` (SQLite equivalent of `SELECT FOR UPDATE`) + * to prevent concurrent counter duplication. + */ +function nextCounter(db: Database.Database, prefix: string): number { + // BEGIN IMMEDIATE acquires a reserved lock immediately, blocking other writers + // while still allowing readers. This is the SQLite equivalent of FOR UPDATE. + const run = db.transaction((): number => { + const upsert = db.prepare<[string]>(` + INSERT INTO dt_series (prefix, current_value) + VALUES (?, 1) + ON CONFLICT(prefix) DO UPDATE SET current_value = current_value + 1 + `); + upsert.run(prefix); + + const row = db + .prepare< + [string], + { current_value: number } + >('SELECT current_value FROM dt_series WHERE prefix = ?') + .get(prefix); + + if (row === undefined) { + throw new Error(`dt_series: failed to read counter for prefix "${prefix}"`); + } + return row.current_value; + }); + + // Execute the transaction with BEGIN IMMEDIATE semantics by temporarily + // using the default deferred mode — better-sqlite3 transactions are + // synchronous and exclusive by default on the write path, which is + // sufficient for our use case. + return run(); +} + +/** + * Generate the next number for the given naming-series pattern. + * + * @param db - A better-sqlite3 Database instance (must have `dt_series` table) + * @param pattern - A pattern string such as `"INV-{YYYY}-{#####}"` + * @param now - Optional date override (defaults to `new Date()`) + * @returns A formatted string like `"INV-2026-00042"` + * + * @example + * ```ts + * generateNextNumber(db, 'INV-{YYYY}-{#####}') // → 'INV-2026-00001' + * generateNextNumber(db, 'QUO-{YYYY}-{MM}-{###}') // → 'QUO-2026-03-001' + * ``` + */ +export function generateNextNumber( + db: Database.Database, + pattern: string, + now: Date = new Date(), +): string { + const segments = parsePattern(pattern); + const { prefix, counterWidth } = buildPrefix(segments, now); + + const counter = nextCounter(db, prefix); + + return `${prefix}${String(counter).padStart(counterWidth, '0')}`; +} diff --git a/src/intelligence/pdf-generator.ts b/src/intelligence/pdf-generator.ts new file mode 100644 index 00000000..d4df125b --- /dev/null +++ b/src/intelligence/pdf-generator.ts @@ -0,0 +1,393 @@ +import { createRequire } from 'node:module'; +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import QRCode from 'qrcode'; +/** + * Declarative document definition accepted by pdfmake. + * This is a minimal local definition that mirrors the pdfmake TDocumentDefinitions + * interface. Full types are available via `@types/pdfmake` if needed directly. + */ +export interface TDocumentDefinitions { + /** Page content — array of content nodes */ + content: Content[]; + /** Named style dictionary */ + styles?: Record>; + /** Default style applied to all content */ + defaultStyle?: Record; + /** Page size — e.g. "A4", "LETTER", or { width, height } */ + pageSize?: string | { width: number; height: number }; + /** Page orientation */ + pageOrientation?: 'portrait' | 'landscape'; + /** Page margins [left, top, right, bottom] */ + pageMargins?: [number, number, number, number] | number; + /** Page header (repeated on every page) */ + header?: Content | ((currentPage: number, pageCount: number) => Content); + /** Page footer (repeated on every page) */ + footer?: Content | ((currentPage: number, pageCount: number) => Content); + /** Page background */ + background?: Content | ((currentPage: number) => Content); + /** Watermark */ + watermark?: { text: string; color?: string; opacity?: number; bold?: boolean; italics?: boolean }; + /** Images dictionary */ + images?: Record; + [key: string]: unknown; +} + +/** A content node or array of content nodes in a pdfmake document */ +export type Content = string | Record | Content[]; + +/** A single line item on an invoice */ +export interface InvoiceLineItem { + description: string; + quantity: number; + unitPrice: number; + /** Pre-computed total — defaults to quantity × unitPrice if omitted */ + total?: number; +} + +/** Invoice record metadata */ +export interface InvoiceRecord { + invoiceNumber: string; + date: string; + dueDate?: string; + /** Customer name or company */ + customerName: string; + customerEmail?: string; + customerAddress?: string; + /** Notes displayed at the bottom of the invoice */ + notes?: string; + /** Payment URL included as a clickable link (optional) */ + paymentLink?: string; + /** Tax rate as a decimal, e.g. 0.1 for 10% (optional) */ + taxRate?: number; + currency?: string; +} + +/** Business branding for generated documents */ +export interface InvoiceBranding { + /** Company / business name */ + companyName: string; + /** Address lines shown in the header */ + companyAddress?: string; + /** Contact email shown in the header */ + companyEmail?: string; + /** Primary brand colour as a CSS hex string, e.g. "#1a73e8" */ + primaryColor?: string; +} + +/** Resolved local path to the pdfmake package directory */ +function pdfmakePkgDir(): string { + const req = createRequire(import.meta.url); + return path.dirname(req.resolve('pdfmake/package.json')); +} + +/** + * Generate a QR code as a data URL (base64 PNG). + * + * @param text The text to encode in the QR code (e.g., a payment link) + * @returns A data URL that can be embedded in a PDF image element + */ +export async function generateQrDataUrl(text: string): Promise { + return QRCode.toDataURL(text, { + width: 150, + margin: 1, + color: { + dark: '#000000', + light: '#ffffff', + }, + }); +} + +/** + * Generate a PDF file from a pdfmake declarative document definition. + * + * Writes the result to `/.openbridge/generated/.pdf` + * and returns the absolute path to the created file. + * + * Roboto fonts bundled with pdfmake are used by default so no external + * font assets are required. + * + * @param definition pdfmake document definition + * @param workspacePath Absolute path to the target workspace + * @returns Absolute path of the written PDF + */ +export async function generatePdf( + definition: TDocumentDefinitions, + workspacePath: string, +): Promise { + const outputDir = path.join(workspacePath, '.openbridge', 'generated'); + await fs.mkdir(outputDir, { recursive: true }); + + const outputPath = path.join(outputDir, `${randomUUID()}.pdf`); + + // Dynamic import keeps pdfmake optional at module-load time. + // pdfmake is a CommonJS module without TypeScript declarations for the + // server-side API, so the `any` cast is intentional here. + /* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call */ + const pdfmakeModule = (await import('pdfmake')) as any; + const instance: any = pdfmakeModule.default ?? pdfmakeModule; + + const fontDir = path.join(pdfmakePkgDir(), 'fonts', 'Roboto'); + instance.addFonts({ + Roboto: { + normal: path.join(fontDir, 'Roboto-Regular.ttf'), + bold: path.join(fontDir, 'Roboto-Medium.ttf'), + italics: path.join(fontDir, 'Roboto-Italic.ttf'), + bolditalics: path.join(fontDir, 'Roboto-MediumItalic.ttf'), + }, + }); + + // Ensure a default font is set + const existingDefaultStyle = definition.defaultStyle; + const defWithFont: TDocumentDefinitions = { + ...definition, + defaultStyle: { + font: 'Roboto', + ...existingDefaultStyle, + }, + }; + + const doc: any = instance.createPdf(defWithFont); + await doc.write(outputPath); + /* eslint-enable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call */ + + return outputPath; +} + +// ─── Invoice helper ────────────────────────────────────────────────────────── + +/** Format a number as a currency string */ +function formatCurrency(amount: number, currency = 'USD'): string { + return new Intl.NumberFormat('en-US', { style: 'currency', currency }).format(amount); +} + +/** + * Build a pdfmake `TDocumentDefinitions` object for a professional invoice. + * + * @param record Invoice metadata (number, dates, customer info) + * @param items Line items to display in the items table + * @param branding Business branding (company name, colours) + * @returns pdfmake document definition ready for `generatePdf()` + */ +export async function createInvoicePdfDefinition( + record: InvoiceRecord, + items: InvoiceLineItem[], + branding: InvoiceBranding, +): Promise { + const primaryColor = branding.primaryColor ?? '#1a73e8'; + const currency = record.currency ?? 'USD'; + + // Compute totals + const lineItems = items.map((item) => ({ + ...item, + total: item.total ?? item.quantity * item.unitPrice, + })); + + const subtotal = lineItems.reduce((sum, item) => sum + item.total, 0); + const tax = record.taxRate != null ? subtotal * record.taxRate : 0; + const grandTotal = subtotal + tax; + + // Header row — company info left, invoice metadata right + const headerContent: Content = { + columns: [ + { + stack: [ + { text: branding.companyName, style: 'companyName' }, + ...(branding.companyAddress + ? [{ text: branding.companyAddress, style: 'companyMeta' }] + : []), + ...(branding.companyEmail ? [{ text: branding.companyEmail, style: 'companyMeta' }] : []), + ], + width: '*', + }, + { + stack: [ + { text: 'INVOICE', style: 'invoiceTitle' }, + { text: `#${record.invoiceNumber}`, style: 'invoiceNumber', color: primaryColor }, + { text: `Date: ${record.date}`, style: 'invoiceMeta' }, + ...(record.dueDate ? [{ text: `Due: ${record.dueDate}`, style: 'invoiceMeta' }] : []), + ], + width: 'auto', + alignment: 'right' as const, + }, + ], + marginBottom: 20, + }; + + // Bill-to section + const billToContent: Content = { + stack: [ + { text: 'BILL TO', style: 'sectionHeader', color: primaryColor }, + { text: record.customerName, style: 'customerName' }, + ...(record.customerEmail ? [{ text: record.customerEmail, style: 'customerMeta' }] : []), + ...(record.customerAddress ? [{ text: record.customerAddress, style: 'customerMeta' }] : []), + ], + marginBottom: 20, + }; + + // Line items table body + const tableBody: Content[][] = [ + [ + { text: 'Description', style: 'tableHeader', color: 'white', fillColor: primaryColor }, + { + text: 'Qty', + style: 'tableHeader', + color: 'white', + fillColor: primaryColor, + alignment: 'right' as const, + }, + { + text: 'Unit Price', + style: 'tableHeader', + color: 'white', + fillColor: primaryColor, + alignment: 'right' as const, + }, + { + text: 'Total', + style: 'tableHeader', + color: 'white', + fillColor: primaryColor, + alignment: 'right' as const, + }, + ], + ...lineItems.map((item, idx) => { + const bg = idx % 2 === 0 ? '#f8f9fa' : 'white'; + return [ + { text: item.description, fillColor: bg }, + { text: String(item.quantity), alignment: 'right' as const, fillColor: bg }, + { + text: formatCurrency(item.unitPrice, currency), + alignment: 'right' as const, + fillColor: bg, + }, + { text: formatCurrency(item.total, currency), alignment: 'right' as const, fillColor: bg }, + ]; + }), + ]; + + const itemsTable: Content = { + table: { + headerRows: 1, + widths: ['*', 60, 80, 80], + body: tableBody, + }, + layout: 'lightHorizontalLines', + marginBottom: 20, + }; + + // Totals block (right-aligned) + const totalRows: [unknown, unknown][] = []; + if (record.taxRate != null) { + totalRows.push( + [ + { text: 'Subtotal', alignment: 'right' as const }, + { text: formatCurrency(subtotal, currency), alignment: 'right' as const }, + ], + [ + { + text: `Tax (${(record.taxRate * 100).toFixed(0)}%)`, + alignment: 'right' as const, + }, + { text: formatCurrency(tax, currency), alignment: 'right' as const }, + ], + ); + } + totalRows.push([ + { + text: 'TOTAL', + style: 'totalLabel', + alignment: 'right' as const, + color: primaryColor, + }, + { + text: formatCurrency(grandTotal, currency), + style: 'totalAmount', + alignment: 'right' as const, + color: primaryColor, + }, + ]); + + const totalsSection: Content = { + columns: [ + { text: '', width: '*' }, + { + table: { + widths: [120, 80], + body: totalRows as Content[][], + }, + layout: 'noBorders', + }, + ], + }; + + // Optional footer content + const footerItems: Content[] = []; + if (record.notes) { + footerItems.push( + { text: 'Notes', style: 'sectionHeader', color: primaryColor, marginTop: 20 }, + { text: record.notes, style: 'notes' }, + ); + } + + // Payment section with QR code + const images: Record = {}; + if (record.paymentLink) { + const qrDataUrl = await generateQrDataUrl(record.paymentLink); + images['paymentQr'] = qrDataUrl; + + footerItems.push( + { text: 'Payment', style: 'sectionHeader', color: primaryColor, marginTop: 20 }, + { + columns: [ + { + image: 'paymentQr', + width: 100, + height: 100, + }, + { + stack: [ + { text: 'Scan to Pay', style: 'paymentLabel', color: primaryColor }, + { + text: record.paymentLink, + link: record.paymentLink, + style: 'paymentLink', + color: primaryColor, + }, + ], + width: '*', + margin: [10, 0, 0, 0], + }, + ], + columnGap: 10, + }, + ); + } + + return { + pageSize: 'A4', + pageMargins: [40, 60, 40, 60] as [number, number, number, number], + defaultStyle: { font: 'Roboto', fontSize: 10, lineHeight: 1.3 }, + styles: { + companyName: { fontSize: 16, bold: true, marginBottom: 4 }, + companyMeta: { fontSize: 9, color: '#666666' }, + invoiceTitle: { fontSize: 22, bold: true, color: '#333333', marginBottom: 4 }, + invoiceNumber: { fontSize: 13, bold: true, marginBottom: 4 }, + invoiceMeta: { fontSize: 9, color: '#666666' }, + sectionHeader: { fontSize: 9, bold: true, marginBottom: 4, marginTop: 10 }, + customerName: { fontSize: 11, bold: true, marginBottom: 2 }, + customerMeta: { fontSize: 9, color: '#666666' }, + tableHeader: { fontSize: 10, bold: true }, + totalLabel: { fontSize: 11, bold: true }, + totalAmount: { fontSize: 11, bold: true }, + notes: { fontSize: 9, color: '#555555', italics: true }, + paymentLink: { fontSize: 10, decoration: 'underline' }, + paymentLabel: { fontSize: 9, bold: true, marginBottom: 2 }, + }, + images: Object.keys(images).length > 0 ? images : undefined, + content: ([headerContent, billToContent, itemsTable, totalsSection] as Content[]).concat( + footerItems, + ), + }; +} diff --git a/src/intelligence/processors/csv-processor.ts b/src/intelligence/processors/csv-processor.ts new file mode 100644 index 00000000..d86e81e9 --- /dev/null +++ b/src/intelligence/processors/csv-processor.ts @@ -0,0 +1,172 @@ +/** + * CSV Processor — Extract tables from CSV files + * + * Uses the xlsx package (SheetJS), which handles CSV natively. + * Detects delimiter (comma, semicolon, tab) via first-line heuristic. + * Returns a single table with headers and rows. + */ + +import { readFileSync } from 'node:fs'; +import { createLogger } from '../../core/logger.js'; +import type { ExtractedTable, ProcessorResult } from '../../types/intelligence.js'; + +// Lazy-loaded xlsx module (optional dependency) + +interface XLSXSheet { + [key: string]: unknown; +} + +interface XLSXWorkbook { + SheetNames: string[]; + Sheets: Record; +} + +interface XLSXUtils { + sheet_to_json(ws: XLSXSheet, opts?: Record): unknown[]; +} + +interface XLSXModule { + utils: XLSXUtils; + readFile(path: string, opts?: Record): XLSXWorkbook; +} + +let xlsxModule: XLSXModule | undefined; + +async function getXLSX(): Promise { + if (!xlsxModule) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + xlsxModule = (await import('xlsx')) as any as XLSXModule; + } + return xlsxModule; +} + +const logger = createLogger('csv-processor'); + +/** + * Detect the delimiter used in a CSV file by analyzing the first line. + * Checks for comma, semicolon, and tab; returns the most likely one. + * + * @param filePath - Path to the CSV file + * @returns The detected delimiter character + */ +function detectDelimiter(filePath: string): string { + try { + const content = readFileSync(filePath, 'utf-8'); + const firstLine = content.split('\n')[0] ?? ''; + + if (!firstLine) { + logger.debug('Empty file, defaulting to comma delimiter'); + return ','; + } + + // Count occurrences of each candidate delimiter + const commaCount = (firstLine.match(/,/g) ?? []).length; + const semicolonCount = (firstLine.match(/;/g) ?? []).length; + const tabCount = (firstLine.match(/\t/g) ?? []).length; + + logger.debug({ commaCount, semicolonCount, tabCount }, 'Delimiter counts in first line'); + + // Return the delimiter with the highest count; default to comma if all are zero + if (semicolonCount > commaCount && semicolonCount > tabCount) { + logger.debug('Detected semicolon delimiter'); + return ';'; + } + if (tabCount > commaCount && tabCount > semicolonCount) { + logger.debug('Detected tab delimiter'); + return '\t'; + } + + logger.debug('Defaulting to comma delimiter'); + return ','; + } catch (error) { + logger.warn( + { error: error instanceof Error ? error.message : String(error) }, + 'Error detecting delimiter, defaulting to comma', + ); + return ','; + } +} + +/** + * Process a CSV file and extract structured table data. + * + * @param filePath - Absolute path to the CSV file + * @returns ProcessorResult with a single table, rawText, and metadata + */ +export async function processCsv(filePath: string): Promise { + const XLSX = await getXLSX(); + const delimiter = detectDelimiter(filePath); + + // Read the CSV file using xlsx with the detected delimiter + const workbook = XLSX.readFile(filePath, { delimiter }); + const sheetNames = workbook.SheetNames; + + if (sheetNames.length === 0) { + logger.warn({ filePath }, 'CSV file has no sheets'); + return { + rawText: '', + tables: [], + images: [], + metadata: { delimiter }, + }; + } + + // Use the first sheet (CSV typically has only one) + // sheetNames.length > 0 is guaranteed by the check above + const sheetName = sheetNames[0]; + const worksheet = sheetName ? workbook.Sheets[sheetName] : undefined; + if (!worksheet) { + logger.warn({ filePath }, 'CSV file could not be parsed'); + return { + rawText: '', + tables: [], + images: [], + metadata: { delimiter }, + }; + } + + // Convert sheet to array of arrays (formatted values, no formula strings) + const aoa = XLSX.utils.sheet_to_json(worksheet, { + header: 1, + defval: '', + raw: false, + }) as (string | number | boolean | null)[][]; + + if (aoa.length === 0) { + logger.debug({ filePath }, 'CSV file is empty'); + return { + rawText: '', + tables: [], + images: [], + metadata: { delimiter }, + }; + } + + // First row is treated as headers (aoa.length > 0 checked above) + const firstRow = aoa[0] ?? []; + const headers = firstRow.map((h) => (h !== null && h !== undefined ? String(h) : '')); + + // Remaining rows are data; skip entirely-empty rows + const rows = aoa.slice(1).filter((row) => row.some((cell) => cell !== '' && cell != null)); + + const table: ExtractedTable = { headers, rows }; + + // Build plain-text representation for rawText + const headerLine = headers.join('\t'); + const dataLines = rows.map((row) => + row.map((cell) => (cell !== null && cell !== undefined ? String(cell) : '')).join('\t'), + ); + const rawText = [headerLine, ...dataLines].join('\n'); + + logger.info( + { filePath, delimiter, headerCount: headers.length, rowCount: rows.length }, + 'CSV processing complete', + ); + + return { + rawText, + tables: [table], + images: [], + metadata: { delimiter }, + }; +} diff --git a/src/intelligence/processors/email-processor.ts b/src/intelligence/processors/email-processor.ts new file mode 100644 index 00000000..176b8eb6 --- /dev/null +++ b/src/intelligence/processors/email-processor.ts @@ -0,0 +1,224 @@ +/** + * Email Processor — Extract content and attachments from .eml files + * + * Uses mailparser (simpleParser) to parse RFC 822 email messages. + * Extracts headers (from, to, subject, date), body text, HTML tables, + * and recursively processes attachments via the document processor. + */ + +import { readFile, writeFile, mkdtemp, rm } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { createLogger } from '../../core/logger.js'; +import type { ExtractedTable, ProcessorResult } from '../../types/intelligence.js'; + +const logger = createLogger('email-processor'); + +// Dynamic import to avoid top-level require issues with ESM +interface AddressObject { + value: Array<{ address?: string; name?: string }>; + text: string; +} + +interface ParsedMail { + from?: AddressObject; + to?: AddressObject | AddressObject[]; + subject?: string; + date?: Date; + text?: string; + html?: string | false; + attachments?: Array<{ + filename?: string; + contentType: string; + content: Buffer; + }>; +} + +/** + * Parse HTML tables from an email HTML body. + * Reuses the same regex-based approach as word-processor.ts. + */ +function parseHtmlTables(html: string): ExtractedTable[] { + const tables: ExtractedTable[] = []; + const tableRegex = /]*>([\s\S]*?)<\/table>/gi; + let tableIndex = 0; + + let tableMatch: RegExpExecArray | null; + while ((tableMatch = tableRegex.exec(html)) !== null) { + const tableHtml = tableMatch[1] ?? ''; + const rowRegex = /]*>([\s\S]*?)<\/tr>/gi; + const allRows: string[][] = []; + + let rowMatch: RegExpExecArray | null; + while ((rowMatch = rowRegex.exec(tableHtml)) !== null) { + const rowHtml = rowMatch[1] ?? ''; + const cellRegex = /]*>([\s\S]*?)<\/t[dh]>/gi; + const cells: string[] = []; + + let cellMatch: RegExpExecArray | null; + while ((cellMatch = cellRegex.exec(rowHtml)) !== null) { + const cellText = (cellMatch[1] ?? '') + .replace(/<[^>]+>/g, '') + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/ /g, ' ') + .replace(/"/g, '"') + .trim(); + cells.push(cellText); + } + + if (cells.length > 0) { + allRows.push(cells); + } + } + + if (allRows.length === 0) continue; + + const headers = allRows[0] ?? []; + const rows = allRows.slice(1); + + tables.push({ + sheetName: `Table ${tableIndex + 1}`, + headers, + rows, + }); + + tableIndex++; + } + + return tables; +} + +/** + * Format an address object to a readable string. + */ +function formatAddress(addr: AddressObject | AddressObject[] | undefined): string { + if (!addr) return ''; + const list = Array.isArray(addr) ? addr : [addr]; + return list + .flatMap((a) => a.value) + .map((v) => (v.name ? `${v.name} <${v.address ?? ''}>` : (v.address ?? ''))) + .join(', '); +} + +/** + * Process attachments recursively by writing them to temp buffers + * and calling processDocument() on each. + */ +async function processAttachments( + attachments: ParsedMail['attachments'], +): Promise { + if (!attachments || attachments.length === 0) { + return { rawText: '', tables: [], images: [], metadata: {} }; + } + + // Lazy import to avoid circular dependency at module load time + const { processDocument } = await import('../document-processor.js'); + + const mergedText: string[] = []; + const mergedTables: ExtractedTable[] = []; + + for (const attachment of attachments) { + const filename = attachment.filename ?? `attachment_${Date.now()}`; + const tmpDir = await mkdtemp(join(tmpdir(), 'ob-email-')); + const tmpPath = join(tmpDir, filename); + + try { + await writeFile(tmpPath, attachment.content); + const result = await processDocument(tmpPath); + + if (result.rawText) { + mergedText.push(`--- Attachment: ${filename} ---\n${result.rawText}`); + } + mergedTables.push(...result.tables); + + logger.debug( + { filename, mimeType: attachment.contentType, textLength: result.rawText.length }, + 'Processed email attachment', + ); + } catch (err) { + logger.warn({ filename, err }, 'Failed to process email attachment'); + } finally { + await rm(tmpDir, { recursive: true, force: true }); + } + } + + return { + rawText: mergedText.join('\n\n'), + tables: mergedTables, + images: [], + metadata: { attachmentCount: attachments.length }, + }; +} + +/** + * Process an .eml email file and extract structured content. + * + * @param filePath - Absolute path to the .eml file + * @returns ProcessorResult with email body, HTML tables, attachment content, and headers metadata + */ +export async function processEmail(filePath: string): Promise { + const source = await readFile(filePath); + + // Dynamic import for ESM compatibility + const mailparserMod = (await import('mailparser')) as { + simpleParser: (source: Buffer) => Promise; + }; + const { simpleParser } = mailparserMod; + + const parsed = await simpleParser(source); + + const from = formatAddress(parsed.from); + const to = formatAddress(parsed.to); + const subject = parsed.subject ?? ''; + const date = parsed.date?.toISOString() ?? ''; + + // Build header block for inclusion in rawText + const headerBlock = [ + from ? `From: ${from}` : '', + to ? `To: ${to}` : '', + subject ? `Subject: ${subject}` : '', + date ? `Date: ${date}` : '', + ] + .filter(Boolean) + .join('\n'); + + const bodyText = parsed.text ?? ''; + const html = typeof parsed.html === 'string' ? parsed.html : ''; + + const tables = html ? parseHtmlTables(html) : []; + + // Recursively process attachments + const attachmentResult = await processAttachments(parsed.attachments); + + const rawTextParts = [headerBlock, bodyText, attachmentResult.rawText].filter(Boolean); + const rawText = rawTextParts.join('\n\n'); + + const metadata: Record = { + from, + to, + subject, + date, + hasHtml: html.length > 0, + tableCount: tables.length, + attachmentCount: parsed.attachments?.length ?? 0, + }; + + logger.info( + { + filePath, + subject, + attachmentCount: metadata['attachmentCount'], + tableCount: tables.length, + }, + 'Email processing complete', + ); + + return { + rawText, + tables: [...tables, ...attachmentResult.tables], + images: [], + metadata, + }; +} diff --git a/src/intelligence/processors/excel-processor.ts b/src/intelligence/processors/excel-processor.ts new file mode 100644 index 00000000..c8434d7c --- /dev/null +++ b/src/intelligence/processors/excel-processor.ts @@ -0,0 +1,120 @@ +/** + * Excel Processor — Extract tables and metadata from XLSX/XLS files using SheetJS (xlsx). + * + * Reads each sheet in the workbook, extracts column headers from the first row, + * and returns data rows as structured tables. Formula cells use the computed + * value (.v property) rather than the raw formula string. + */ + +import { createLogger } from '../../core/logger.js'; +import type { ExtractedTable, ProcessorResult } from '../../types/intelligence.js'; + +// Lazy-loaded xlsx module (optional dependency) + +interface XLSXSheet { + [key: string]: unknown; +} + +interface XLSXWorkbook { + SheetNames: string[]; + Sheets: Record; + Props?: Record; +} + +interface XLSXUtils { + sheet_to_json(ws: XLSXSheet, opts?: Record): unknown[]; +} + +interface XLSXModule { + utils: XLSXUtils; + readFile(path: string, opts?: Record): XLSXWorkbook; +} + +let xlsxModule: XLSXModule | undefined; + +async function getXLSX(): Promise { + if (!xlsxModule) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + xlsxModule = (await import('xlsx')) as any as XLSXModule; + } + return xlsxModule; +} + +const logger = createLogger('excel-processor'); + +/** + * Process an Excel file (XLSX/XLS/ODS) and extract structured table data. + * + * @param filePath - Absolute path to the Excel file + * @returns ProcessorResult with structured tables, rawText (tab-delimited), and metadata + */ +export async function processExcel(filePath: string): Promise { + const XLSX = await getXLSX(); + const workbook = XLSX.readFile(filePath); + + const sheetNames = workbook.SheetNames; + const tables: ExtractedTable[] = []; + const rawTextParts: string[] = []; + + for (const sheetName of sheetNames) { + const worksheet = workbook.Sheets[sheetName]; + if (!worksheet) continue; + + // Convert sheet to array of arrays (formatted values, no formula strings) + const aoa = XLSX.utils.sheet_to_json(worksheet, { + header: 1, + defval: '', + raw: false, + }) as (string | number | boolean | null)[][]; + + if (aoa.length === 0) { + logger.debug({ sheetName }, 'Sheet is empty, skipping'); + continue; + } + + // First row is treated as headers (aoa.length > 0 checked above) + const firstRow = aoa[0] ?? []; + const headers = firstRow.map((h) => (h !== null && h !== undefined ? String(h) : '')); + + // Remaining rows are data; skip entirely-empty rows + const rows = aoa.slice(1).filter((row) => row.some((cell) => cell !== '' && cell != null)); + + tables.push({ sheetName, headers, rows }); + + // Build plain-text representation for rawText + const headerLine = headers.join('\t'); + const dataLines = rows.map((row) => + row.map((cell) => (cell !== null && cell !== undefined ? String(cell) : '')).join('\t'), + ); + rawTextParts.push(`[Sheet: ${sheetName}]`, headerLine, ...dataLines); + + logger.debug( + { sheetName, headerCount: headers.length, rowCount: rows.length }, + 'Extracted sheet data', + ); + } + + // Extract workbook-level metadata + const props = workbook.Props ?? {}; + const metadata: Record = { + sheetCount: sheetNames.length, + sheetNames, + }; + if (props['Author']) metadata['author'] = props['Author']; + if (props['Title']) metadata['title'] = props['Title']; + if (props['CreatedDate']) metadata['createdDate'] = JSON.stringify(props['CreatedDate']); + if (props['ModifiedDate']) metadata['modifiedDate'] = JSON.stringify(props['ModifiedDate']); + if (props['Application']) metadata['application'] = props['Application']; + + logger.info( + { filePath, sheetCount: sheetNames.length, tableCount: tables.length }, + 'Excel processing complete', + ); + + return { + rawText: rawTextParts.join('\n'), + tables, + images: [], + metadata, + }; +} diff --git a/src/intelligence/processors/image-processor.ts b/src/intelligence/processors/image-processor.ts new file mode 100644 index 00000000..c67ead2d --- /dev/null +++ b/src/intelligence/processors/image-processor.ts @@ -0,0 +1,239 @@ +/** + * Image Processor — Extract text and data from images via AI vision + OCR + * + * Two extraction paths: + * 1. AI vision: encode image as base64, spawn a read-only worker to describe + * the image and extract visible text, numbers, tables, and business data. + * 2. OCR: use tesseract.js for pure text extraction. + * + * Both paths run concurrently. Results are combined — AI vision provides + * richer descriptions while OCR provides reliable raw text extraction. + */ + +import { readFile, stat } from 'fs/promises'; +import { extname } from 'path'; +import { createLogger } from '../../core/logger.js'; +import type { ProcessorResult } from '../../types/intelligence.js'; + +const logger = createLogger('image-processor'); + +/** Maximum image file size (20 MB) — skip oversized files to avoid memory pressure */ +const MAX_IMAGE_SIZE_BYTES = 20 * 1024 * 1024; + +/** MIME type mapping for common image extensions */ +const EXTENSION_TO_MIME: Record = { + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.bmp': 'image/bmp', + '.tiff': 'image/tiff', + '.tif': 'image/tiff', + '.webp': 'image/webp', +}; + +/** Minimal tesseract.js typings to avoid requiring @types at build time */ +interface TesseractWorker { + recognize(image: Buffer): Promise<{ data: { text: string; confidence: number } }>; + terminate(): Promise; +} + +interface TesseractModule { + createWorker(lang: string): Promise; +} + +/** Minimal AgentRunner typings for optional AI vision path */ +interface AgentRunnerResult { + stdout: string; + exitCode: number; +} + +interface AgentRunnerLike { + spawn(opts: { + prompt: string; + workspacePath: string; + model?: string; + allowedTools?: string[]; + maxTurns?: number; + timeout?: number; + retries?: number; + }): Promise; +} + +/** + * Infer MIME type from file extension. + */ +function getMimeType(filePath: string): string { + const ext = extname(filePath).toLowerCase(); + return EXTENSION_TO_MIME[ext] ?? 'application/octet-stream'; +} + +/** + * OCR path — extract text from image using tesseract.js. + * Returns the recognized text and confidence score. + */ +async function ocrExtract(imageBuffer: Buffer): Promise<{ text: string; confidence: number }> { + let Tesseract: TesseractModule; + try { + const mod = (await import('tesseract.js')) as unknown as { + default?: TesseractModule; + } & TesseractModule; + Tesseract = mod.default ?? mod; + } catch { + throw new Error( + 'tesseract.js is not installed. Run `npm install tesseract.js` to enable image OCR.', + ); + } + + const worker = await Tesseract.createWorker('eng'); + try { + const { data } = await worker.recognize(imageBuffer); + return { text: data.text, confidence: data.confidence }; + } finally { + await worker.terminate(); + } +} + +/** + * AI vision path — encode image as base64 and spawn a read-only worker + * to describe the image and extract visible business data. + */ +async function aiVisionExtract( + imageBuffer: Buffer, + filePath: string, + mimeType: string, +): Promise { + let AgentRunner: new () => AgentRunnerLike; + try { + const mod = (await import('../../core/agent-runner.js')) as { + AgentRunner: new () => AgentRunnerLike; + }; + AgentRunner = mod.AgentRunner; + } catch { + logger.debug('AgentRunner not available, skipping AI vision path'); + return null; + } + + const base64 = imageBuffer.toString('base64'); + const dataUri = `data:${mimeType};base64,${base64}`; + + const prompt = [ + 'Describe this image and extract any text, numbers, tables, or business data visible.', + 'If you see a table, format it as markdown.', + 'If you see text, reproduce it exactly.', + '', + `Image (base64 data URI): ${dataUri}`, + '', + `File: ${filePath}`, + ].join('\n'); + + const runner = new AgentRunner(); + try { + const result = await runner.spawn({ + prompt, + workspacePath: '.', + allowedTools: ['Read', 'Glob', 'Grep'], + maxTurns: 3, + // Sonnet-class models need 90-130s for image analysis (OB-F206) + timeout: 180_000, + retries: 0, + }); + + if (result.exitCode === 0 && result.stdout.trim().length > 0) { + return result.stdout.trim(); + } + + logger.debug( + { exitCode: result.exitCode, outputLen: result.stdout.length }, + 'AI vision worker returned no useful output', + ); + return null; + } catch (err) { + logger.debug({ err, filePath }, 'AI vision extraction failed, falling back to OCR only'); + return null; + } +} + +/** + * Process an image file and extract text content via OCR and optional AI vision. + * + * @param filePath - Absolute path to the image file + * @returns ProcessorResult with rawText, metadata (dimensions, OCR confidence, AI description) + */ +export async function processImage(filePath: string): Promise { + // Validate file size + const fileStat = await stat(filePath); + if (fileStat.size > MAX_IMAGE_SIZE_BYTES) { + throw new Error( + `Image file exceeds ${MAX_IMAGE_SIZE_BYTES / (1024 * 1024)} MB limit: ${fileStat.size} bytes`, + ); + } + + const imageBuffer = await readFile(filePath); + const mimeType = getMimeType(filePath); + + const metadata: Record = { + mimeType, + fileSize: fileStat.size, + }; + + // Run OCR and AI vision concurrently + const [ocrResult, aiDescription] = await Promise.all([ + ocrExtract(imageBuffer).catch((err) => { + logger.warn({ err, filePath }, 'OCR extraction failed'); + metadata['ocrError'] = err instanceof Error ? err.message : String(err); + return null; + }), + aiVisionExtract(imageBuffer, filePath, mimeType).catch((err) => { + logger.debug({ err, filePath }, 'AI vision extraction failed'); + return null; + }), + ]); + + // Build raw text from available results + const textParts: string[] = []; + + if (aiDescription) { + textParts.push(aiDescription); + metadata['aiVisionApplied'] = true; + } + + if (ocrResult) { + const ocrText = ocrResult.text.trim(); + metadata['ocrApplied'] = true; + metadata['ocrConfidence'] = ocrResult.confidence; + + if (ocrText.length > 0) { + // If we already have AI description, append OCR text under a heading + if (aiDescription) { + textParts.push(`\n--- OCR Text ---\n${ocrText}`); + } else { + textParts.push(ocrText); + } + } + } + + const rawText = textParts.join('\n'); + + if (rawText.length === 0) { + logger.warn({ filePath }, 'No text extracted from image via OCR or AI vision'); + metadata['extractionFailed'] = true; + } + + logger.info( + { + filePath, + textLength: rawText.length, + ocrApplied: !!ocrResult, + aiVisionApplied: !!aiDescription, + }, + 'Image processing complete', + ); + + return { + rawText, + tables: [], + images: [{ filePath, mimeType, size: fileStat.size }], + metadata, + }; +} diff --git a/src/intelligence/processors/index.ts b/src/intelligence/processors/index.ts new file mode 100644 index 00000000..d01f3268 --- /dev/null +++ b/src/intelligence/processors/index.ts @@ -0,0 +1,20 @@ +/** + * Processors — Format-specific document processors + * + * Each processor handles a specific file format: + * - pdf-processor.ts — PDF with OCR fallback (OB-1336/OB-1337) + * - excel-processor.ts — XLSX and spreadsheets (OB-1338) + * - csv-processor.ts — CSV files (OB-1339) + * - word-processor.ts — DOCX documents (OB-1340) + * - image-processor.ts — Images with vision AI (OB-1341) + * - email-processor.ts — MIME email messages (OB-1342) + * - structured-processor.ts — JSON and XML documents (OB-1343) + */ + +export { processPdf } from './pdf-processor.js'; +export { processExcel } from './excel-processor.js'; +export { processCsv } from './csv-processor.js'; +export { processWord } from './word-processor.js'; +export { processImage } from './image-processor.js'; +export { processEmail } from './email-processor.js'; +export { processStructured } from './structured-processor.js'; diff --git a/src/intelligence/processors/pdf-processor.ts b/src/intelligence/processors/pdf-processor.ts new file mode 100644 index 00000000..121b7aca --- /dev/null +++ b/src/intelligence/processors/pdf-processor.ts @@ -0,0 +1,210 @@ +/** + * PDF Processor — Extract text and metadata from PDF files using pdf-parse. + * + * Reads a PDF file buffer and extracts: + * - Full plain text from all pages + * - Page count and document metadata (author, title, creator) + * + * OCR fallback: If pdf-parse returns near-empty text (< 50 chars per page), + * pages are rendered to images via Puppeteer and processed with tesseract.js. + */ + +import { readFile } from 'fs/promises'; +import { createLogger } from '../../core/logger.js'; +import type { ProcessorResult } from '../../types/intelligence.js'; + +/** Internal contract matching the pdf-parse v1 result shape. */ +type PdfParseResult = { + numpages: number; + text: string; + info: Record | null | undefined; +}; + +type PdfParseFn = (buffer: Buffer, options?: Record) => Promise; + +const logger = createLogger('pdf-processor'); + +/** Minimum average chars per page before OCR fallback triggers */ +const OCR_THRESHOLD_CHARS_PER_PAGE = 50; + +/** + * Process a PDF file and extract text content and metadata. + * Falls back to OCR via tesseract.js if pdf-parse returns near-empty text. + * + * @param filePath - Absolute path to the PDF file + * @returns ProcessorResult with rawText, empty tables/images, and PDF metadata + */ +export async function processPdf(filePath: string): Promise { + const buffer = await readFile(filePath); + + // Dynamic import allows test mocking via vi.doMock('pdf-parse', …) + const pdfParseModule = await import('pdf-parse'); + const pdfParse = + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + ((pdfParseModule as any).default ?? pdfParseModule) as unknown as PdfParseFn; + + const data = await pdfParse(buffer); + + const metadata: Record = { + pages: data.numpages, + }; + + const info = data.info; + if (info) { + if (typeof info['Author'] === 'string' && info['Author']) { + metadata['author'] = info['Author']; + } + if (typeof info['Title'] === 'string' && info['Title']) { + metadata['title'] = info['Title']; + } + if (typeof info['Creator'] === 'string' && info['Creator']) { + metadata['creator'] = info['Creator']; + } + if (typeof info['Producer'] === 'string' && info['Producer']) { + metadata['producer'] = info['Producer']; + } + if (typeof info['CreationDate'] === 'string' && info['CreationDate']) { + metadata['creationDate'] = info['CreationDate']; + } + } + + const textLength = data.text.trim().length; + const charsPerPage = data.numpages > 0 ? textLength / data.numpages : textLength; + + // If text extraction yielded meaningful content, return it directly + if (charsPerPage >= OCR_THRESHOLD_CHARS_PER_PAGE) { + return { + rawText: data.text, + tables: [], + images: [], + metadata, + }; + } + + // Scanned PDF detected — attempt OCR fallback + logger.info( + { filePath, pages: data.numpages, charsPerPage: Math.round(charsPerPage) }, + 'PDF text extraction yielded near-empty text, attempting OCR fallback', + ); + + try { + const ocrText = await ocrPdfPages(filePath, data.numpages); + metadata['ocrApplied'] = true; + return { + rawText: ocrText, + tables: [], + images: [], + metadata, + }; + } catch (err) { + logger.warn({ err, filePath }, 'OCR fallback failed, returning original (sparse) text'); + metadata['ocrAttempted'] = true; + metadata['ocrError'] = err instanceof Error ? err.message : String(err); + return { + rawText: data.text, + tables: [], + images: [], + metadata, + }; + } +} + +// --------------------------------------------------------------------------- +// OCR helpers — Puppeteer renders PDF pages to images, tesseract.js reads them +// --------------------------------------------------------------------------- + +/** Minimal Puppeteer typings to avoid requiring @types/puppeteer at build time */ +interface PuppeteerPage { + setViewport(opts: { width: number; height: number }): Promise; + goto(url: string, opts: { waitUntil: string }): Promise; + screenshot(opts: { type: 'png'; encoding: 'binary' }): Promise; + close(): Promise; +} + +interface PuppeteerBrowser { + newPage(): Promise; + close(): Promise; +} + +interface PuppeteerModule { + launch(opts: { headless: boolean; args: string[] }): Promise; +} + +/** Minimal tesseract.js typings */ +interface TesseractWorker { + recognize(image: Buffer): Promise<{ data: { text: string } }>; + terminate(): Promise; +} + +interface TesseractModule { + createWorker(lang: string): Promise; +} + +/** + * Render each PDF page to an image via Puppeteer and OCR it with tesseract.js. + * Returns the merged OCR text for all pages. + */ +async function ocrPdfPages(filePath: string, pageCount: number): Promise { + let puppeteer: PuppeteerModule; + try { + const mod = (await import('puppeteer')) as unknown as { + default?: PuppeteerModule; + } & PuppeteerModule; + puppeteer = mod.default ?? mod; + } catch { + throw new Error('Puppeteer is not installed. Run `npm install puppeteer` to enable PDF OCR.'); + } + + let Tesseract: TesseractModule; + try { + const mod = (await import('tesseract.js')) as unknown as { + default?: TesseractModule; + } & TesseractModule; + Tesseract = mod.default ?? mod; + } catch { + throw new Error( + 'tesseract.js is not installed. Run `npm install tesseract.js` to enable PDF OCR.', + ); + } + + const browser = await puppeteer.launch({ + headless: true, + args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu'], + }); + + const worker = await Tesseract.createWorker('eng'); + + try { + const pageTexts: string[] = []; + + for (let pageNum = 1; pageNum <= pageCount; pageNum++) { + const page = await browser.newPage(); + await page.setViewport({ width: 1200, height: 1600 }); + + // pdf.js viewer renders a specific page when given #page=N fragment + const fileUrl = `file://${filePath}#page=${pageNum}`; + await page.goto(fileUrl, { waitUntil: 'networkidle0' }); + + const screenshot = await page.screenshot({ type: 'png', encoding: 'binary' }); + await page.close(); + + const { data } = await worker.recognize(screenshot); + pageTexts.push(data.text); + + logger.debug( + { filePath, page: pageNum, ocrChars: data.text.length }, + 'OCR completed for page', + ); + } + + const mergedText = pageTexts.join('\n\n'); + logger.info( + { filePath, pages: pageCount, totalOcrChars: mergedText.length }, + 'PDF OCR completed successfully', + ); + return mergedText; + } finally { + await worker.terminate(); + await browser.close(); + } +} diff --git a/src/intelligence/processors/structured-processor.ts b/src/intelligence/processors/structured-processor.ts new file mode 100644 index 00000000..13c09202 --- /dev/null +++ b/src/intelligence/processors/structured-processor.ts @@ -0,0 +1,239 @@ +/** + * Structured Processor — Parse JSON and XML into structured tables + * + * For JSON: JSON.parse() + detect schema (array of objects → table format). + * For XML: uses xml2js to parse to JS object, detects repeated elements as tables. + */ + +import { readFile } from 'fs/promises'; +import { createLogger } from '../../core/logger.js'; +import type { ExtractedTable, ProcessorResult } from '../../types/intelligence.js'; + +const logger = createLogger('structured-processor'); + +/** + * Detect if a value is a plain object (not null, not array, not primitive). + */ +function isPlainObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** + * Convert an array of objects into an ExtractedTable. + * Collects all keys across all rows as headers. + */ +function arrayOfObjectsToTable(arr: Record[], name?: string): ExtractedTable { + // Collect union of all keys + const keySet = new Set(); + for (const item of arr) { + for (const key of Object.keys(item)) { + keySet.add(key); + } + } + const headers = Array.from(keySet); + const rows = arr.map((item) => + headers.map((h) => { + const val = item[h]; + if (val === undefined || val === null) return ''; + if (typeof val === 'object') return JSON.stringify(val); + return val; + }), + ); + const table: ExtractedTable = { headers, rows }; + if (name) table.sheetName = name; + return table; +} + +/** + * Recursively walk a parsed JS object and extract tables from repeated elements. + * + * xml2js parses `......` as + * `{ items: { item: [{...}, {...}] } }` — each key whose value is an array + * of objects (or arrays) is treated as a table. + */ +function extractTablesFromObject( + obj: Record, + parentKey = '', + tables: ExtractedTable[] = [], +): ExtractedTable[] { + for (const [key, value] of Object.entries(obj)) { + const label = parentKey ? `${parentKey}.${key}` : key; + + if (Array.isArray(value) && value.length > 0) { + // Array of plain objects → table + const objectItems = value.filter(isPlainObject); + if (objectItems.length > 0) { + tables.push(arrayOfObjectsToTable(objectItems, label)); + // Also recurse into sub-objects + for (const item of objectItems) { + extractTablesFromObject(item, label, tables); + } + continue; + } + + // Array of primitives → single-column table + if (value.every((v) => typeof v !== 'object' || v === null)) { + tables.push({ + sheetName: label, + headers: [key], + rows: value.map((v) => [v === null ? '' : String(v)]), + }); + continue; + } + } + + // Nested object → recurse + if (isPlainObject(value)) { + extractTablesFromObject(value, label, tables); + } + } + return tables; +} + +/** + * Build a plain-text representation of an object recursively. + */ +function objectToText(value: unknown, indent = 0): string { + if (value === null || value === undefined) return ''; + if (typeof value !== 'object') + return value == null ? '' : String(value as string | number | boolean | bigint | symbol); + if (Array.isArray(value)) { + return value.map((v) => objectToText(v, indent)).join('\n'); + } + const obj = value as Record; + return Object.entries(obj) + .map(([k, v]) => { + const prefix = ' '.repeat(indent); + if (typeof v === 'object' && v !== null) { + return `${prefix}${k}:\n${objectToText(v, indent + 1)}`; + } + return `${prefix}${k}: ${v == null ? '' : String(v as string | number | boolean | bigint | symbol)}`; + }) + .join('\n'); +} + +/** + * Process a JSON file. + * If the top-level value is an array of objects, convert to a table. + * Otherwise, recursively extract any nested arrays of objects as tables. + */ +async function processJson(filePath: string): Promise { + const content = await readFile(filePath, 'utf-8'); + let parsed: unknown; + + try { + parsed = JSON.parse(content) as unknown; + } catch (err) { + logger.warn({ filePath, err }, 'Failed to parse JSON file'); + return { + rawText: content, + tables: [], + images: [], + metadata: { parseError: String(err) }, + }; + } + + const tables: ExtractedTable[] = []; + let rawText = ''; + + if (Array.isArray(parsed)) { + const objectItems = parsed.filter(isPlainObject); + if (objectItems.length > 0) { + // Top-level array of objects → primary table + tables.push(arrayOfObjectsToTable(objectItems, 'root')); + // Recurse into each item for nested tables + for (const item of objectItems) { + extractTablesFromObject(item, 'root', tables); + } + } + rawText = objectToText(parsed); + } else if (isPlainObject(parsed)) { + extractTablesFromObject(parsed, '', tables); + rawText = objectToText(parsed); + } else { + // Primitive at top level (string, number, boolean, null) + rawText = parsed == null ? '' : String(parsed as string | number | boolean | bigint | symbol); + } + + logger.info( + { filePath, tableCount: tables.length, textLength: rawText.length }, + 'JSON processing complete', + ); + + return { + rawText, + tables, + images: [], + metadata: { + format: 'json', + topLevelType: Array.isArray(parsed) ? 'array' : typeof parsed, + }, + }; +} + +/** + * Process an XML file using xml2js. + * Detects repeated elements as tables. + */ +async function processXml(filePath: string): Promise { + const content = await readFile(filePath, 'utf-8'); + + // Dynamic import for ESM compatibility + const xml2jsMod = (await import('xml2js')) as { + parseStringPromise: (xml: string, opts?: Record) => Promise; + }; + const { parseStringPromise } = xml2jsMod; + + let parsed: unknown; + try { + parsed = await parseStringPromise(content, { + explicitArray: true, // Always wrap in arrays for consistency + explicitCharkey: false, + mergeAttrs: true, + }); + } catch (err) { + logger.warn({ filePath, err }, 'Failed to parse XML file'); + return { + rawText: content, + tables: [], + images: [], + metadata: { parseError: String(err) }, + }; + } + + const tables: ExtractedTable[] = []; + if (isPlainObject(parsed)) { + extractTablesFromObject(parsed, '', tables); + } + + const rawText = objectToText(parsed); + + logger.info( + { filePath, tableCount: tables.length, textLength: rawText.length }, + 'XML processing complete', + ); + + return { + rawText, + tables, + images: [], + metadata: { format: 'xml' }, + }; +} + +/** + * Process a JSON or XML file and return structured data. + * + * @param filePath - Absolute path to the JSON or XML file + * @param mime - MIME type of the file (e.g. 'application/json', 'application/xml', 'text/xml') + * @returns ProcessorResult with extracted tables and raw text + */ +export async function processStructured(filePath: string, mime: string): Promise { + const isXml = mime.includes('xml') || filePath.endsWith('.xml') || filePath.endsWith('.svg'); + + if (isXml) { + return processXml(filePath); + } + + return processJson(filePath); +} diff --git a/src/intelligence/processors/word-processor.ts b/src/intelligence/processors/word-processor.ts new file mode 100644 index 00000000..169b7267 --- /dev/null +++ b/src/intelligence/processors/word-processor.ts @@ -0,0 +1,145 @@ +/** + * Word Processor — Extract text and tables from DOCX/DOC files using mammoth. + * + * Uses mammoth to: + * - Extract plain text via extractRawText() + * - Extract HTML via convertToHtml() for detecting and parsing tables + * + * HTML tables are parsed into structured ExtractedTable entries. + */ + +import { createLogger } from '../../core/logger.js'; +import type { ExtractedTable, ProcessorResult } from '../../types/intelligence.js'; + +// Lazy-loaded mammoth module (optional dependency) + +interface MammothResult { + value: string; + messages: unknown[]; +} + +interface MammothModule { + extractRawText(options: { path: string }): Promise; + convertToHtml(options: { path: string }): Promise; +} + +let mammothModule: MammothModule | undefined; + +async function getMammoth(): Promise { + if (!mammothModule) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + mammothModule = (await import('mammoth')) as any as MammothModule; + } + return mammothModule; +} + +const logger = createLogger('word-processor'); + +/** + * Parse HTML tables from a mammoth-generated HTML string. + * Returns an array of ExtractedTable objects. + */ +function parseHtmlTables(html: string): ExtractedTable[] { + const tables: ExtractedTable[] = []; + + // Match each ...
block + const tableRegex = /]*>([\s\S]*?)<\/table>/gi; + let tableIndex = 0; + + let tableMatch: RegExpExecArray | null; + while ((tableMatch = tableRegex.exec(html)) !== null) { + const tableHtml = tableMatch[1] ?? ''; + + // Extract all rows + const rowRegex = /]*>([\s\S]*?)<\/tr>/gi; + const allRows: string[][] = []; + + let rowMatch: RegExpExecArray | null; + while ((rowMatch = rowRegex.exec(tableHtml)) !== null) { + const rowHtml = rowMatch[1] ?? ''; + + // Extract cells (th or td) + const cellRegex = /]*>([\s\S]*?)<\/t[dh]>/gi; + const cells: string[] = []; + + let cellMatch: RegExpExecArray | null; + while ((cellMatch = cellRegex.exec(rowHtml)) !== null) { + // Strip inner HTML tags and decode basic entities + const cellText = (cellMatch[1] ?? '') + .replace(/<[^>]+>/g, '') + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/ /g, ' ') + .replace(/"/g, '"') + .trim(); + cells.push(cellText); + } + + if (cells.length > 0) { + allRows.push(cells); + } + } + + if (allRows.length === 0) continue; + + // First row is headers + const headers = allRows[0] ?? []; + const rows = allRows.slice(1); + + tables.push({ + sheetName: `Table ${tableIndex + 1}`, + headers, + rows, + }); + + tableIndex++; + logger.debug( + { tableIndex, headerCount: headers.length, rowCount: rows.length }, + 'Parsed HTML table', + ); + } + + return tables; +} + +/** + * Process a Word document (DOCX/DOC) and extract text and tables. + * + * @param filePath - Absolute path to the Word file + * @returns ProcessorResult with rawText, extracted tables, and metadata + */ +export async function processWord(filePath: string): Promise { + const mammoth = await getMammoth(); + // Extract plain text + const textResult = await mammoth.extractRawText({ path: filePath }); + const rawText = textResult.value; + + if (textResult.messages.length > 0) { + logger.debug({ filePath, messages: textResult.messages }, 'mammoth extractRawText messages'); + } + + // Extract HTML for table detection + const htmlResult = await mammoth.convertToHtml({ path: filePath }); + const html = htmlResult.value; + + if (htmlResult.messages.length > 0) { + logger.debug({ filePath, messages: htmlResult.messages }, 'mammoth convertToHtml messages'); + } + + const tables = parseHtmlTables(html); + + logger.info( + { filePath, textLength: rawText.length, tableCount: tables.length }, + 'Word processing complete', + ); + + return { + rawText, + tables, + images: [], + metadata: { + tableCount: tables.length, + }, + }; +} diff --git a/src/intelligence/query-cache.ts b/src/intelligence/query-cache.ts new file mode 100644 index 00000000..ba5781b8 --- /dev/null +++ b/src/intelligence/query-cache.ts @@ -0,0 +1,236 @@ +import { createHash } from 'node:crypto'; +import type Database from 'better-sqlite3'; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +/** A cached query entry */ +export interface QueryCacheEntry { + id: number; + /** Normalized hash of the question (cache key) */ + questionHash: string; + /** Original question text (for debugging) */ + question: string; + /** Computed answer/aggregate stored as JSON string */ + answer: string; + /** ISO timestamp when the entry was cached */ + cachedAt: string; + /** ISO timestamp when the entry expires */ + expiresAt: string; + /** DocTypes this answer depends on (for targeted invalidation) */ + relatedDocTypes: string[]; +} + +// --------------------------------------------------------------------------- +// Row type for SQLite +// --------------------------------------------------------------------------- + +interface QueryCacheRow { + id: number; + question_hash: string; + question: string; + answer: string; + cached_at: string; + expires_at: string; + related_doc_types: string; +} + +function rowToEntry(row: QueryCacheRow): QueryCacheEntry { + return { + id: row.id, + questionHash: row.question_hash, + question: row.question, + answer: row.answer, + cachedAt: row.cached_at, + expiresAt: row.expires_at, + relatedDocTypes: JSON.parse(row.related_doc_types) as string[], + }; +} + +// --------------------------------------------------------------------------- +// Migration +// --------------------------------------------------------------------------- + +/** + * Ensure the `query_cache` table exists in the given database. + * Safe to call multiple times (uses CREATE TABLE IF NOT EXISTS). + */ +export function ensureQueryCacheTable(db: Database.Database): void { + db.prepare( + `CREATE TABLE IF NOT EXISTS query_cache ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + question_hash TEXT NOT NULL UNIQUE, + question TEXT NOT NULL, + answer TEXT NOT NULL, + cached_at TEXT NOT NULL, + expires_at TEXT NOT NULL, + related_doc_types TEXT NOT NULL DEFAULT '[]' + )`, + ).run(); + db.prepare( + `CREATE INDEX IF NOT EXISTS idx_query_cache_expires ON query_cache (expires_at)`, + ).run(); +} + +// --------------------------------------------------------------------------- +// Question normalisation +// --------------------------------------------------------------------------- + +/** + * Normalise a question to a stable cache key: + * - lower-case + * - collapse whitespace + * - strip leading/trailing punctuation + * Returns a SHA-256 hex digest (64 chars). + */ +export function normalizeQuestion(question: string): string { + const normalized = question + .toLowerCase() + .replace(/\s+/g, ' ') + .trim() + .replace(/^[^a-z0-9]+|[^a-z0-9?!]+$/g, ''); + return createHash('sha256').update(normalized).digest('hex'); +} + +// --------------------------------------------------------------------------- +// CRUD +// --------------------------------------------------------------------------- + +const DEFAULT_TTL_MS = 5 * 60 * 1000; // 5 minutes + +/** + * Store (or overwrite) a cached answer for the given question. + * + * @param db SQLite database instance + * @param question Raw question text + * @param answer Computed answer (will be JSON-serialised if not a string) + * @param relatedDocTypes DocTypes this answer depends on + * @param ttlMs Time-to-live in milliseconds (default: 5 minutes) + * @returns The inserted/replaced row ID + */ +export function cacheAnswer( + db: Database.Database, + question: string, + answer: unknown, + relatedDocTypes: string[] = [], + ttlMs: number = DEFAULT_TTL_MS, +): number { + const hash = normalizeQuestion(question); + const now = new Date(); + const expiresAt = new Date(now.getTime() + ttlMs); + const answerStr = typeof answer === 'string' ? answer : JSON.stringify(answer); + + const result = db + .prepare( + `INSERT INTO query_cache + (question_hash, question, answer, cached_at, expires_at, related_doc_types) + VALUES (@hash, @question, @answer, @cachedAt, @expiresAt, @relatedDocTypes) + ON CONFLICT (question_hash) DO UPDATE SET + question = excluded.question, + answer = excluded.answer, + cached_at = excluded.cached_at, + expires_at = excluded.expires_at, + related_doc_types = excluded.related_doc_types`, + ) + .run({ + hash, + question, + answer: answerStr, + cachedAt: now.toISOString(), + expiresAt: expiresAt.toISOString(), + relatedDocTypes: JSON.stringify(relatedDocTypes), + }); + + return Number(result.lastInsertRowid); +} + +/** + * Retrieve a non-expired cached answer for the given question. + * Returns null if no entry exists or the entry is expired (and prunes it). + */ +export function getCachedAnswer(db: Database.Database, question: string): QueryCacheEntry | null { + const hash = normalizeQuestion(question); + const row = db.prepare('SELECT * FROM query_cache WHERE question_hash = ?').get(hash) as + | QueryCacheRow + | undefined; + + if (!row) return null; + + const now = new Date(); + if (new Date(row.expires_at) <= now) { + // Prune expired entry + db.prepare('DELETE FROM query_cache WHERE question_hash = ?').run(hash); + return null; + } + + return rowToEntry(row); +} + +/** + * Invalidate all cached entries that depend on one or more of the given DocTypes. + * Call this whenever DocType data changes (insert / update / delete). + * + * @returns Number of cache entries removed + */ +export function invalidateByDocTypes(db: Database.Database, docTypes: string[]): number { + if (docTypes.length === 0) return 0; + + // Fetch all rows and filter in JS — SQLite JSON functions require a loaded + // extension which is not guaranteed; a full-table scan on a small cache table + // is acceptable. + const rows = db.prepare('SELECT id, related_doc_types FROM query_cache').all() as Array<{ + id: number; + related_doc_types: string; + }>; + + const toDelete: number[] = []; + for (const row of rows) { + const related = JSON.parse(row.related_doc_types) as string[]; + if (docTypes.some((dt) => related.includes(dt))) { + toDelete.push(row.id); + } + } + + if (toDelete.length === 0) return 0; + + const placeholders = toDelete.map(() => '?').join(','); + const result = db + .prepare(`DELETE FROM query_cache WHERE id IN (${placeholders})`) + .run(...toDelete); + + return result.changes; +} + +/** + * Remove all entries that have already expired. + * + * @returns Number of entries pruned + */ +export function pruneExpiredCache(db: Database.Database): number { + const result = db + .prepare(`DELETE FROM query_cache WHERE expires_at <= ?`) + .run(new Date().toISOString()); + return result.changes; +} + +/** + * Delete a specific cache entry by question. + * + * @returns true if an entry was removed + */ +export function invalidateQuestion(db: Database.Database, question: string): boolean { + const hash = normalizeQuestion(question); + const result = db.prepare('DELETE FROM query_cache WHERE question_hash = ?').run(hash); + return result.changes > 0; +} + +/** + * Flush the entire query cache. + * + * @returns Number of entries removed + */ +export function clearCache(db: Database.Database): number { + const result = db.prepare('DELETE FROM query_cache').run(); + return result.changes; +} diff --git a/src/intelligence/relation-manager.ts b/src/intelligence/relation-manager.ts new file mode 100644 index 00000000..660dbe23 --- /dev/null +++ b/src/intelligence/relation-manager.ts @@ -0,0 +1,211 @@ +import type Database from 'better-sqlite3'; +import { randomUUID } from 'crypto'; +import type { DocTypeRelation } from '../types/doctype.js'; +import { ensureDocTypeStoreSchema } from './doctype-store.js'; + +// --------------------------------------------------------------------------- +// Row type +// --------------------------------------------------------------------------- + +interface DocTypeRelationRow { + id: string; + from_doctype: string; + to_doctype: string; + relation_type: string; + from_field: string; + to_field: string; + label: string | null; +} + +function rowToRelation(row: DocTypeRelationRow): DocTypeRelation { + return { + id: row.id, + from_doctype: row.from_doctype, + to_doctype: row.to_doctype, + relation_type: row.relation_type as DocTypeRelation['relation_type'], + from_field: row.from_field, + to_field: row.to_field, + label: row.label ?? undefined, + }; +} + +// --------------------------------------------------------------------------- +// Result types +// --------------------------------------------------------------------------- + +/** A linked record fetched from a target DocType table */ +export interface LinkedRecord { + id: string; + [key: string]: unknown; +} + +/** Result of resolving linked records for a given relation */ +export interface ResolvedRelation { + relation: DocTypeRelation; + records: LinkedRecord[]; +} + +// --------------------------------------------------------------------------- +// createRelation +// --------------------------------------------------------------------------- + +/** + * Persist a new inter-DocType relation to the `doctype_relations` table. + * Returns the generated relation ID. + */ +export function createRelation( + db: Database.Database, + input: Omit & { id?: string }, +): string { + ensureDocTypeStoreSchema(db); + + const id = input.id ?? randomUUID(); + + db.prepare( + ` + INSERT INTO doctype_relations (id, from_doctype, to_doctype, relation_type, from_field, to_field, label) + VALUES (@id, @from_doctype, @to_doctype, @relation_type, @from_field, @to_field, @label) + `, + ).run({ + id, + from_doctype: input['from_doctype'], + to_doctype: input['to_doctype'], + relation_type: input['relation_type'], + from_field: input['from_field'], + to_field: input['to_field'] ?? 'id', + label: input['label'] ?? null, + }); + + return id; +} + +// --------------------------------------------------------------------------- +// getRelations +// --------------------------------------------------------------------------- + +/** + * Retrieve all relations for a given DocType (either as source or target). + * + * @param doctypeId - The `doctypes.id` to look up. + * @param direction - `'from'` returns relations where this DocType is the owner + * (has_many / belongs_to from this side). + * `'to'` returns relations where this DocType is the target. + * `'both'` (default) returns all. + */ +export function getRelations( + db: Database.Database, + doctypeId: string, + direction: 'from' | 'to' | 'both' = 'both', +): DocTypeRelation[] { + ensureDocTypeStoreSchema(db); + + let rows: DocTypeRelationRow[]; + + if (direction === 'from') { + rows = db + .prepare('SELECT * FROM doctype_relations WHERE from_doctype = ?') + .all(doctypeId) as DocTypeRelationRow[]; + } else if (direction === 'to') { + rows = db + .prepare('SELECT * FROM doctype_relations WHERE to_doctype = ?') + .all(doctypeId) as DocTypeRelationRow[]; + } else { + rows = db + .prepare('SELECT * FROM doctype_relations WHERE from_doctype = ? OR to_doctype = ?') + .all(doctypeId, doctypeId) as DocTypeRelationRow[]; + } + + return rows.map(rowToRelation); +} + +// --------------------------------------------------------------------------- +// resolveLinkedRecords +// --------------------------------------------------------------------------- + +/** + * Resolve the linked records for a `link` field or explicit relation. + * + * For `has_many`: fetches rows from `dt_{to_doctype}` where `from_field` + * (on the child table) equals `sourceRecordId`. + * + * For `belongs_to`: fetches the single row from `dt_{to_doctype}` whose + * `to_field` matches the value stored in `from_field` of `sourceRecord`. + * + * For `many_to_many`: fetches rows via a join table named + * `dt_{from_doctype}___{to_doctype}` (three underscores, alphabetical order). + * The join table must have `from_id` and `to_id` columns. + * + * @param db - Open SQLite database handle. + * @param relation - Relation metadata. + * @param sourceRecordId - ID of the record on the `from_doctype` side. + * @param sourceRecord - Full record object (needed for `belongs_to` look-up). + * @returns Resolved linked records from the target table. + */ +export function resolveLinkedRecords( + db: Database.Database, + relation: DocTypeRelation, + sourceRecordId: string, + sourceRecord?: Record, +): ResolvedRelation { + const toTable = `dt_${relation.to_doctype}`; + + let records: LinkedRecord[] = []; + + switch (relation.relation_type) { + case 'has_many': { + // Child rows that point back to the parent via from_field + records = db + .prepare(`SELECT * FROM "${toTable}" WHERE "${relation.from_field}" = ?`) + .all(sourceRecordId) as LinkedRecord[]; + break; + } + + case 'belongs_to': { + // The foreign-key value lives on the source record + const fkValue = sourceRecord + ? (sourceRecord[relation.from_field] as string | undefined) + : sourceRecordId; + + if (fkValue == null) { + records = []; + } else { + const row = db + .prepare(`SELECT * FROM "${toTable}" WHERE "${relation.to_field}" = ?`) + .get(fkValue) as LinkedRecord | undefined; + records = row ? [row] : []; + } + break; + } + + case 'many_to_many': { + // Derive join table name (sorted alphabetically for determinism) + const parts = [relation.from_doctype, relation.to_doctype].sort(); + const joinTable = `dt_${parts[0]}___${parts[1]}`; + + records = db + .prepare( + `SELECT t.* FROM "${toTable}" t + INNER JOIN "${joinTable}" j ON j.to_id = t."${relation.to_field}" + WHERE j.from_id = ?`, + ) + .all(sourceRecordId) as LinkedRecord[]; + break; + } + } + + return { relation, records }; +} + +// --------------------------------------------------------------------------- +// deleteRelation +// --------------------------------------------------------------------------- + +/** + * Remove a relation by its ID. + * Returns true if a row was deleted. + */ +export function deleteRelation(db: Database.Database, relationId: string): boolean { + ensureDocTypeStoreSchema(db); + const result = db.prepare('DELETE FROM doctype_relations WHERE id = ?').run(relationId); + return result.changes > 0; +} diff --git a/src/intelligence/skill-creator.ts b/src/intelligence/skill-creator.ts new file mode 100644 index 00000000..f448a6d8 --- /dev/null +++ b/src/intelligence/skill-creator.ts @@ -0,0 +1,280 @@ +import type Database from 'better-sqlite3'; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +/** A single step in a completed multi-step task */ +export interface TaskStep { + /** Step index (0-based) */ + index: number; + /** What the step did (e.g., "query orders table", "compute totals") */ + action: string; + /** Tool or integration used (e.g., "sqlite", "whatsapp", "api") */ + integration?: string; + /** DocType involved (e.g., "order", "customer") */ + docType?: string; + /** Whether this step succeeded */ + success: boolean; + /** Duration in milliseconds */ + durationMs?: number; +} + +/** A reusable skill extracted from a successful multi-step task */ +export interface BusinessSkill { + id: number; + /** Short human-readable name (e.g., "weekly-sales-report") */ + name: string; + /** What this skill does */ + description: string; + /** Ordered steps to execute */ + steps: string[]; + /** Integrations required (e.g., ["sqlite", "whatsapp"]) */ + requiredIntegrations: string[]; + /** DocTypes this skill operates on */ + requiredDocTypes: string[]; + /** When this skill was created */ + createdAt: string; + /** Schema version (incremented on structural changes) */ + version: number; + /** Total number of times this skill has been executed */ + usageCount: number; + /** Rolling average success rate (0–1) */ + successRate: number; + /** Rolling average execution duration in milliseconds (null if never run) */ + avgDurationMs: number | null; + /** ISO timestamp of the last execution (null if never run) */ + lastUsed: string | null; +} + +// --------------------------------------------------------------------------- +// Row type for SQLite +// --------------------------------------------------------------------------- + +interface BusinessSkillRow { + id: number; + name: string; + description: string; + steps: string; + required_integrations: string; + required_doc_types: string; + created_at: string; + version: number; + usage_count: number; + success_rate: number; + avg_duration_ms: number | null; + last_used: string | null; +} + +function rowToSkill(row: BusinessSkillRow): BusinessSkill { + return { + id: row.id, + name: row.name, + description: row.description, + steps: JSON.parse(row.steps) as string[], + requiredIntegrations: JSON.parse(row.required_integrations) as string[], + requiredDocTypes: JSON.parse(row.required_doc_types) as string[], + createdAt: row.created_at, + version: row.version ?? 1, + usageCount: row.usage_count ?? 0, + successRate: row.success_rate ?? 0, + avgDurationMs: row.avg_duration_ms ?? null, + lastUsed: row.last_used ?? null, + }; +} + +// --------------------------------------------------------------------------- +// Skill extraction logic +// --------------------------------------------------------------------------- + +const MIN_STEPS = 3; + +/** + * Analyze a completed multi-step task and extract a reusable skill pattern. + * Returns null if the task is too simple (fewer than 3 steps) or if it failed. + */ +export function createSkillFromTask(taskHistory: TaskStep[]): Omit | null { + // Must have at least MIN_STEPS steps + if (taskHistory.length < MIN_STEPS) return null; + + // All steps must have succeeded + if (!taskHistory.every((s) => s.success)) return null; + + // Extract unique integrations and docTypes + const integrations = [ + ...new Set(taskHistory.map((s) => s.integration).filter((v): v is string => !!v)), + ]; + const docTypes = [...new Set(taskHistory.map((s) => s.docType).filter((v): v is string => !!v))]; + + // Build step descriptions + const steps = taskHistory.map((s) => s.action); + + // Generate a kebab-case name from step actions + const name = generateSkillName(steps); + + // Generate a description summarizing the workflow + const description = `${steps.length}-step workflow: ${steps.slice(0, 3).join(' → ')}${steps.length > 3 ? ` → ... (${steps.length} steps total)` : ''}`; + + return { + name, + description, + steps, + requiredIntegrations: integrations, + requiredDocTypes: docTypes, + createdAt: new Date().toISOString(), + version: 1, + usageCount: 0, + successRate: 0, + avgDurationMs: null, + lastUsed: null, + }; +} + +/** + * Generate a kebab-case skill name from step action descriptions. + * Takes the first 3–4 significant words from the first two steps. + */ +function generateSkillName(steps: string[]): string { + const words = steps + .slice(0, 2) + .join(' ') + .toLowerCase() + .replace(/[^a-z0-9\s]/g, '') + .split(/\s+/) + .filter((w) => w.length > 2) + .slice(0, 4); + + if (words.length === 0) return `skill-${Date.now()}`; + return words.join('-'); +} + +// --------------------------------------------------------------------------- +// SQLite CRUD +// --------------------------------------------------------------------------- + +/** + * Store a business skill in the `business_skills` table. + * Returns the inserted row ID. + */ +export function storeSkill(db: Database.Database, skill: Omit): number { + const result = db + .prepare( + `INSERT INTO business_skills + (name, description, steps, required_integrations, required_doc_types, created_at, + version, usage_count, success_rate, avg_duration_ms, last_used) + VALUES (@name, @description, @steps, @required_integrations, @required_doc_types, @created_at, + @version, @usage_count, @success_rate, @avg_duration_ms, @last_used)`, + ) + .run({ + name: skill.name, + description: skill.description, + steps: JSON.stringify(skill.steps), + required_integrations: JSON.stringify(skill.requiredIntegrations), + required_doc_types: JSON.stringify(skill.requiredDocTypes), + created_at: skill.createdAt, + version: skill.version ?? 1, + usage_count: skill.usageCount ?? 0, + success_rate: skill.successRate ?? 0, + avg_duration_ms: skill.avgDurationMs ?? null, + last_used: skill.lastUsed ?? null, + }); + return Number(result.lastInsertRowid); +} + +/** + * Retrieve a business skill by ID. + */ +export function getSkillById(db: Database.Database, id: number): BusinessSkill | null { + const row = db.prepare('SELECT * FROM business_skills WHERE id = ?').get(id) as + | BusinessSkillRow + | undefined; + return row ? rowToSkill(row) : null; +} + +/** + * Retrieve a business skill by name. + */ +export function getSkillByName(db: Database.Database, name: string): BusinessSkill | null { + const row = db.prepare('SELECT * FROM business_skills WHERE name = ?').get(name) as + | BusinessSkillRow + | undefined; + return row ? rowToSkill(row) : null; +} + +/** + * List all business skills, most recent first. + */ +export function listSkills(db: Database.Database, limit = 50): BusinessSkill[] { + const rows = db + .prepare('SELECT * FROM business_skills ORDER BY created_at DESC LIMIT ?') + .all(limit) as BusinessSkillRow[]; + return rows.map(rowToSkill); +} + +/** + * Delete a business skill by ID. + */ +export function deleteSkill(db: Database.Database, id: number): boolean { + const result = db.prepare('DELETE FROM business_skills WHERE id = ?').run(id); + return result.changes > 0; +} + +/** + * Record a skill execution result. + * Increments usage_count, updates success_rate (rolling average), + * updates avg_duration_ms (rolling average), and sets last_used. + * + * Returns false if the skill ID does not exist. + */ +export function recordSkillExecution( + db: Database.Database, + id: number, + success: boolean, + durationMs?: number, +): boolean { + const row = db + .prepare('SELECT usage_count, success_rate, avg_duration_ms FROM business_skills WHERE id = ?') + .get(id) as + | { usage_count: number; success_rate: number; avg_duration_ms: number | null } + | undefined; + + if (!row) return false; + + const newCount = row.usage_count + 1; + // Rolling average: new_avg = old_avg + (new_value - old_avg) / new_count + const newSuccessRate = row.success_rate + ((success ? 1 : 0) - row.success_rate) / newCount; + + let newAvgDuration: number | null = row.avg_duration_ms; + if (durationMs !== undefined) { + if (newAvgDuration === null) { + newAvgDuration = durationMs; + } else { + newAvgDuration = newAvgDuration + (durationMs - newAvgDuration) / newCount; + } + } + + const result = db + .prepare( + `UPDATE business_skills + SET usage_count = ?, success_rate = ?, avg_duration_ms = ?, last_used = ? + WHERE id = ?`, + ) + .run(newCount, newSuccessRate, newAvgDuration, new Date().toISOString(), id); + + return result.changes > 0; +} + +/** + * Return the most effective skills sorted by usage × success_rate (descending). + * Skills with zero executions are included last (score = 0). + */ +export function getTopSkills(db: Database.Database, limit = 10): BusinessSkill[] { + const rows = db + .prepare( + `SELECT * FROM business_skills + ORDER BY (usage_count * success_rate) DESC, created_at DESC + LIMIT ?`, + ) + .all(limit) as BusinessSkillRow[]; + return rows.map(rowToSkill); +} diff --git a/src/intelligence/state-machine.ts b/src/intelligence/state-machine.ts new file mode 100644 index 00000000..566d744b --- /dev/null +++ b/src/intelligence/state-machine.ts @@ -0,0 +1,652 @@ +import type Database from 'better-sqlite3'; +import type { DocType, DocTypeHook, DocTypeTransition } from '../types/doctype.js'; +import { createLogger } from '../core/logger.js'; +import { ensureAuditLogTable, insertAuditEntry } from './audit-log.js'; + +const logger = createLogger('state-machine'); + +// --------------------------------------------------------------------------- +// Result types +// --------------------------------------------------------------------------- + +export interface TransitionResult { + valid: boolean; + toState?: string; + error?: string; +} + +// --------------------------------------------------------------------------- +// Internal row type for reading transitions from SQLite +// --------------------------------------------------------------------------- + +interface TransitionRow { + id: string; + doctype_id: string; + from_state: string; + to_state: string; + action_name: string; + action_label: string; + allowed_roles: string | null; + condition: string | null; +} + +function rowToTransition(row: TransitionRow): DocTypeTransition { + return { + id: row.id, + doctype_id: row.doctype_id, + from_state: row.from_state, + to_state: row.to_state, + action_name: row.action_name, + action_label: row.action_label, + allowed_roles: row.allowed_roles ? (JSON.parse(row.allowed_roles) as string[]) : undefined, + condition: row.condition ?? undefined, + }; +} + +// --------------------------------------------------------------------------- +// Condition expression evaluator (safe — no eval()) +// --------------------------------------------------------------------------- + +/** + * Evaluate a simple condition expression against a record. + * + * Supported syntax: + * - Field references: bare identifier (e.g. `total`, `status`) + * - String literals: single-quoted values (e.g. `'draft'`) + * - Numeric literals: integer or decimal (e.g. `0`, `100.5`) + * - Comparison operators: `==`, `!=`, `>`, `<`, `>=`, `<=` + * - Logical operators: `AND`, `OR` (case-insensitive) + * + * Examples: + * `total > 0` + * `status == 'draft'` + * `items_count > 0 AND total >= 100` + */ +export function evaluateCondition(expression: string, record: Record): boolean { + const trimmed = expression.trim(); + if (!trimmed) return true; + + // Split on OR first (lowest precedence) + const orParts = splitOnLogical(trimmed, 'OR'); + if (orParts.length > 1) { + return orParts.some((part) => evaluateCondition(part, record)); + } + + // Split on AND + const andParts = splitOnLogical(trimmed, 'AND'); + if (andParts.length > 1) { + return andParts.every((part) => evaluateCondition(part, record)); + } + + // Single comparison clause + return evaluateComparison(trimmed, record); +} + +/** + * Split an expression string on a logical keyword (AND / OR) respecting that + * the keyword must appear as a standalone word boundary (not inside a value). + * Returns the original single-element array if the keyword is not found. + */ +function splitOnLogical(expr: string, keyword: 'AND' | 'OR'): string[] { + // Regex: whitespace + keyword + whitespace (case-insensitive, word-bounded) + const re = new RegExp(`\\s+${keyword}\\s+`, 'i'); + const parts = expr.split(re); + return parts.length > 1 ? parts.map((p) => p.trim()) : [expr]; +} + +/** + * Evaluate a single ` ` comparison. + * Returns `true` on unrecognised syntax (fail-open for permissive default). + */ +function evaluateComparison(clause: string, record: Record): boolean { + // Match: + // op: ==, !=, >=, <=, >, < + const COMPARISON_RE = /^(.+?)\s*(==|!=|>=|<=|>|<)\s*(.+)$/; + const match = COMPARISON_RE.exec(clause.trim()); + if (!match) { + // Not a recognised comparison — treat bare field reference as truthy check + const val = resolveToken(clause.trim(), record); + return isTruthy(val); + } + + const left = resolveToken((match[1] ?? '').trim(), record); + const op = match[2] ?? '=='; + const right = resolveToken((match[3] ?? '').trim(), record); + + return compare(left, op, right); +} + +/** + * Resolve a token to its value: a string literal, numeric literal, or field ref. + */ +function resolveToken(token: string, record: Record): unknown { + // Single-quoted string literal + if (token.startsWith("'") && token.endsWith("'")) { + return token.slice(1, -1); + } + + // Numeric literal + if (/^-?\d+(\.\d+)?$/.test(token)) { + return Number(token); + } + + // Boolean literals + if (token === 'true') return true; + if (token === 'false') return false; + if (token === 'null') return null; + + // Field reference + return Object.prototype.hasOwnProperty.call(record, token) ? record[token] : undefined; +} + +function isTruthy(val: unknown): boolean { + if (val === null || val === undefined || val === false || val === 0 || val === '') return false; + return true; +} + +/** + * Safely convert a primitive value to string. Objects are returned as empty string + * to avoid the `[object Object]` default stringification. + */ +function toStringValue(val: unknown): string { + if (val === null || val === undefined) return ''; + if (typeof val === 'string') return val; + if (typeof val === 'number' || typeof val === 'boolean') return String(val); + return ''; +} + +/** + * Compare two values with the given operator. + * Numbers are compared numerically; everything else is compared as strings. + */ +function compare(left: unknown, op: string, right: unknown): boolean { + // Numeric comparison when both sides coerce to finite numbers + if (typeof left === 'number' || typeof right === 'number') { + const l = Number(left); + const r = Number(right); + if (isFinite(l) && isFinite(r)) { + switch (op) { + case '==': + return l === r; + case '!=': + return l !== r; + case '>': + return l > r; + case '<': + return l < r; + case '>=': + return l >= r; + case '<=': + return l <= r; + } + } + } + + // String / equality comparison — only stringify primitives + const l = toStringValue(left); + const r = toStringValue(right); + switch (op) { + case '==': + return l === r; + case '!=': + return l !== r; + case '>': + return l > r; + case '<': + return l < r; + case '>=': + return l >= r; + case '<=': + return l <= r; + default: + return false; + } +} + +// --------------------------------------------------------------------------- +// Transition loader +// --------------------------------------------------------------------------- + +/** + * Load all transitions for a DocType from the metadata tables. + * Requires `doctype_transitions` table to exist (created by ensureDocTypeStoreSchema). + */ +function loadTransitions(db: Database.Database, doctypeId: string): DocTypeTransition[] { + const rows = db + .prepare('SELECT * FROM doctype_transitions WHERE doctype_id = ?') + .all(doctypeId) as TransitionRow[]; + return rows.map(rowToTransition); +} + +// --------------------------------------------------------------------------- +// Core API +// --------------------------------------------------------------------------- + +/** + * Validate whether a state transition is allowed for the given record. + * + * Checks: + * 1. A transition exists from `fromState` via `action` + * 2. The `userRole` (if provided) is in the transition's `allowed_roles` list + * 3. The optional `condition` expression evaluates to true against `record` + * + * @param db - better-sqlite3 Database instance (must have metadata tables) + * @param doctype - DocType definition (used for its `id`) + * @param record - The data record being transitioned (field values as key/value) + * @param fromState - The record's current state name + * @param action - The action being performed (maps to `action_name`) + * @param userRole - Optional role of the user performing the action + * @returns TransitionResult with `valid`, `toState`, or `error` + */ +export function validateTransition( + db: Database.Database, + doctype: DocType, + record: Record, + fromState: string, + action: string, + userRole?: string, +): TransitionResult { + const transitions = loadTransitions(db, doctype.id); + + // 1. Find a matching transition + const transition = transitions.find( + (t) => t.from_state === fromState && t.action_name === action, + ); + + if (!transition) { + return { + valid: false, + error: `No transition found from state "${fromState}" via action "${action}"`, + }; + } + + // 2. Check role + if (transition.allowed_roles && transition.allowed_roles.length > 0) { + if (!userRole || !transition.allowed_roles.includes(userRole)) { + return { + valid: false, + error: `Role "${userRole ?? '(none)'}" is not allowed to perform action "${action}"`, + }; + } + } + + // 3. Evaluate condition + if (transition.condition) { + let conditionMet: boolean; + try { + conditionMet = evaluateCondition(transition.condition, record); + } catch { + return { + valid: false, + error: `Condition evaluation error for expression: "${transition.condition}"`, + }; + } + + if (!conditionMet) { + return { + valid: false, + error: `Transition condition not met: "${transition.condition}"`, + }; + } + } + + return { valid: true, toState: transition.to_state }; +} + +// --------------------------------------------------------------------------- +// Execute transition — full Salesforce-inspired pipeline +// --------------------------------------------------------------------------- + +/** Result returned by executeTransition */ +export interface ExecuteTransitionResult { + success: boolean; + fromState: string; + toState?: string; + error?: string; + /** The updated record after the status change */ + record?: Record; +} + +/** Callback for lifecycle hook execution. Receives the hooks to fire and the current record. */ +export type HookExecutor = ( + hooks: DocTypeHook[], + record: Record, +) => Promise | void; + +/** Callback for triggering dependent workflows after a transition completes. */ +export type WorkflowTrigger = ( + doctypeId: string, + recordId: string, + fromState: string, + toState: string, + action: string, + /** Record state before the transition (used by data triggers). */ + oldRecord: Record, + /** Record state after the transition (used by data triggers). */ + newRecord: Record, +) => Promise | void; + +/** Options for executeTransition */ +export interface ExecuteTransitionOptions { + /** Database instance */ + db: Database.Database; + /** DocType definition */ + doctype: DocType; + /** The data table name for the DocType (e.g. "dt_invoice") */ + tableName: string; + /** The column name that stores the current state (defaults to "status") */ + statusField?: string; + /** The record ID to transition */ + recordId: string; + /** The action being performed */ + action: string; + /** Optional role of the user performing the action */ + userRole?: string; + /** Optional hook executor — called for before/after hooks */ + hookExecutor?: HookExecutor; + /** Optional workflow trigger — called after transition completes */ + workflowTrigger?: WorkflowTrigger; +} + +/** + * Execute a full state transition pipeline: + * + * 1. Load record from the dynamic table + * 2. Validate transition (role, condition checks) + * 3. Fire before-transition hooks + * 4. UPDATE status field + insert audit log entry + * 5. Fire after-transition hooks + * 6. Trigger dependent workflows + * + * Steps 1–5 are wrapped in a SQLite transaction. Step 6 (workflows) runs + * outside the transaction since workflows may perform async/external work. + */ +export async function executeTransition( + opts: ExecuteTransitionOptions, +): Promise { + const { db, doctype, tableName, recordId, action, userRole, hookExecutor, workflowTrigger } = + opts; + const statusField = opts.statusField ?? 'status'; + const qTable = quoteIdent(tableName); + const qStatus = quoteIdent(statusField); + + // Step 1: Load record + logger.debug({ doctypeId: doctype.id, recordId, action }, 'Step 1: Loading record'); + const record = db.prepare(`SELECT * FROM ${qTable} WHERE "id" = ?`).get(recordId) as + | Record + | undefined; + + if (!record) { + logger.warn({ doctypeId: doctype.id, recordId }, 'Record not found'); + return { + success: false, + fromState: '', + error: `Record "${recordId}" not found in table "${tableName}"`, + }; + } + + const rawStatus = record[statusField]; + const fromState = + rawStatus == null || typeof rawStatus === 'object' + ? '' + : String(rawStatus as string | number | boolean); + if (!fromState) { + return { + success: false, + fromState: '', + error: `Record "${recordId}" has no value in status field "${statusField}"`, + }; + } + + // Step 2: Validate transition + logger.debug({ doctypeId: doctype.id, fromState, action }, 'Step 2: Validating transition'); + const validation = validateTransition(db, doctype, record, fromState, action, userRole); + if (!validation.valid) { + logger.info( + { doctypeId: doctype.id, fromState, action, error: validation.error }, + 'Transition validation failed', + ); + return { success: false, fromState, error: validation.error }; + } + + const toState = validation.toState!; + + // Load hooks for before/after firing + const allHooks = loadHooks(db, doctype.id); + const beforeHooks = allHooks.filter((h) => h.event === 'before_transition' && h.enabled); + const afterHooks = allHooks.filter((h) => h.event === 'after_transition' && h.enabled); + + // Ensure audit log table exists before entering the transaction (exec not allowed inside transactions) + ensureAuditLogTable(db); + + // Steps 3–5 in a transaction + const runTransaction = db.transaction(() => { + // Step 3: Fire before-hooks (synchronous within transaction) + if (beforeHooks.length > 0) { + logger.debug( + { doctypeId: doctype.id, hookCount: beforeHooks.length }, + 'Step 3: Firing before-transition hooks', + ); + if (hookExecutor) { + // hookExecutor may be sync or async; within transaction we call it synchronously + const result = hookExecutor(beforeHooks, record); + // If it returns a promise inside a transaction, it won't actually await — + // callers providing async hookExecutors should ensure before-hooks are sync + if (result && typeof result === 'object' && 'then' in result) { + logger.warn( + 'hookExecutor returned a Promise inside transaction — before-hooks should be synchronous', + ); + } + } + } else { + logger.debug('Step 3: No before-transition hooks to fire'); + } + + // Step 4: UPDATE status + audit log + logger.debug( + { doctypeId: doctype.id, recordId, fromState, toState }, + 'Step 4: Updating status and writing audit log', + ); + + const now = new Date().toISOString(); + db.prepare(`UPDATE ${qTable} SET ${qStatus} = ?, "updated_at" = ? WHERE "id" = ?`).run( + toState, + now, + recordId, + ); + + // Write transition-specific audit table + ensureTransitionAuditTable(db); + db.prepare( + `INSERT INTO doctype_transition_audit + (doctype_id, record_id, from_state, to_state, action, user_role, transitioned_at) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + ).run(doctype.id, recordId, fromState, toState, action, userRole ?? null, now); + + // Write to the shared dt_audit_log + insertAuditEntry(db, { + doctype: doctype.name, + record_id: recordId, + event: 'transition', + old_value: fromState, + new_value: toState, + changed_by: userRole ?? null, + changed_at: now, + }); + + // Step 5: Fire after-hooks (synchronous within transaction) + // Re-read record with updated status for after-hooks + const updatedRecord = db + .prepare(`SELECT * FROM ${qTable} WHERE "id" = ?`) + .get(recordId) as Record; + + if (afterHooks.length > 0) { + logger.debug( + { doctypeId: doctype.id, hookCount: afterHooks.length }, + 'Step 5: Firing after-transition hooks', + ); + if (hookExecutor) { + const result = hookExecutor(afterHooks, updatedRecord); + if (result && typeof result === 'object' && 'then' in result) { + logger.warn( + 'hookExecutor returned a Promise inside transaction — after-hooks should be synchronous', + ); + } + } + } else { + logger.debug('Step 5: No after-transition hooks to fire'); + } + + return updatedRecord; + }); + + let updatedRecord: Record; + try { + updatedRecord = runTransaction(); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + logger.error({ err, doctypeId: doctype.id, recordId, action }, 'Transaction failed'); + return { success: false, fromState, error: `Transaction failed: ${message}` }; + } + + // Step 6: Trigger dependent workflows (outside transaction) + if (workflowTrigger) { + logger.debug( + { doctypeId: doctype.id, fromState, toState, action }, + 'Step 6: Triggering dependent workflows', + ); + try { + await workflowTrigger( + doctype.id, + recordId, + fromState, + toState, + action, + record, + updatedRecord, + ); + } catch (err) { + // Workflow failures are non-fatal — log but don't fail the transition + logger.error( + { err, doctypeId: doctype.id, recordId, action }, + 'Workflow trigger failed (non-fatal)', + ); + } + } else { + logger.debug('Step 6: No workflow trigger configured'); + } + + logger.info( + { doctypeId: doctype.id, recordId, fromState, toState, action }, + 'Transition executed successfully', + ); + + return { success: true, fromState, toState, record: updatedRecord }; +} + +// --------------------------------------------------------------------------- +// Audit table schema +// --------------------------------------------------------------------------- + +let auditTableCreated = false; + +function ensureTransitionAuditTable(db: Database.Database): void { + if (auditTableCreated) return; + db.exec(` + CREATE TABLE IF NOT EXISTS doctype_transition_audit ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + doctype_id TEXT NOT NULL, + record_id TEXT NOT NULL, + from_state TEXT NOT NULL, + to_state TEXT NOT NULL, + action TEXT NOT NULL, + user_role TEXT, + transitioned_at TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_transition_audit_record + ON doctype_transition_audit(doctype_id, record_id); + `); + auditTableCreated = true; +} + +/** Reset the audit table creation flag (for testing) */ +export function resetAuditTableFlag(): void { + auditTableCreated = false; +} + +// --------------------------------------------------------------------------- +// Hook loader +// --------------------------------------------------------------------------- + +interface HookRow { + id: string; + doctype_id: string; + event: string; + action_type: string; + action_config: string; + sort_order: number; + enabled: number; +} + +function loadHooks(db: Database.Database, doctypeId: string): DocTypeHook[] { + const rows = db + .prepare('SELECT * FROM doctype_hooks WHERE doctype_id = ? ORDER BY sort_order') + .all(doctypeId) as HookRow[]; + return rows.map((row) => ({ + id: row.id, + doctype_id: row.doctype_id, + event: row.event, + action_type: row.action_type as DocTypeHook['action_type'], + action_config: JSON.parse(row.action_config) as Record, + sort_order: row.sort_order, + enabled: row.enabled === 1, + })); +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** Quote a SQLite identifier (used locally to avoid collision with table-builder's private fn) */ +function quoteIdent(name: string): string { + return `"${name.replace(/"/g, '""')}"`; +} + +// --------------------------------------------------------------------------- +// List available actions +// --------------------------------------------------------------------------- + +/** + * List all valid actions available from `currentState` for the given record and role. + * Useful for building UI action buttons or validating user input. + */ +export function listAvailableActions( + db: Database.Database, + doctype: DocType, + record: Record, + currentState: string, + userRole?: string, +): Array<{ action_name: string; action_label: string; to_state: string }> { + const transitions = loadTransitions(db, doctype.id); + + return transitions + .filter((t) => { + if (t.from_state !== currentState) return false; + if (t.allowed_roles && t.allowed_roles.length > 0) { + if (!userRole || !t.allowed_roles.includes(userRole)) return false; + } + if (t.condition) { + try { + return evaluateCondition(t.condition, record); + } catch { + return false; + } + } + return true; + }) + .map((t) => ({ + action_name: t.action_name, + action_label: t.action_label, + to_state: t.to_state, + })); +} diff --git a/src/intelligence/table-builder.ts b/src/intelligence/table-builder.ts new file mode 100644 index 00000000..e022bde0 --- /dev/null +++ b/src/intelligence/table-builder.ts @@ -0,0 +1,313 @@ +import type { DocType, DocTypeField, FieldType } from '../types/doctype.js'; + +/** + * Returns true if `formula` is a safe SQLite expression suitable for a GENERATED column. + * Rejects: semicolons (statement separator) and DDL/DML keywords that could indicate injection. + */ +export function isValidSQLiteExpression(formula: string): boolean { + if (formula.includes(';')) return false; + const BANNED = /\b(DROP|ALTER|CREATE|INSERT|UPDATE|DELETE|ATTACH|DETACH|PRAGMA)\b/i; + return !BANNED.test(formula); +} + +/** Maps DocType field types to SQLite column type affinity */ +const FIELD_TYPE_MAP: Record = { + text: 'TEXT', + longtext: 'TEXT', + email: 'TEXT', + phone: 'TEXT', + url: 'TEXT', + select: 'TEXT', + multiselect: 'TEXT', + link: 'TEXT', + image: 'TEXT', + date: 'TEXT', + datetime: 'TEXT', + number: 'REAL', + currency: 'REAL', + checkbox: 'INTEGER', + table: 'TEXT', +}; + +/** + * Generates a `CREATE TABLE` DDL statement for a DocType's data table. + * + * Standard columns: id (PK), created_at, updated_at, created_by + * User-defined columns: one per DocTypeField, typed from FIELD_TYPE_MAP + */ +export function buildCreateTableDDL(doctype: DocType, fields: DocTypeField[]): string { + const tableName = doctype.table_name; + + const columnDefs: string[] = [ + 'id TEXT NOT NULL PRIMARY KEY', + 'created_at TEXT NOT NULL', + 'updated_at TEXT NOT NULL', + 'created_by TEXT NOT NULL', + ]; + + for (const field of fields) { + const sqliteType = FIELD_TYPE_MAP[field.field_type] ?? 'TEXT'; + if (field.formula != null) { + if (!isValidSQLiteExpression(field.formula)) { + throw new Error( + `Field "${field.name}": formula contains disallowed SQL keywords or characters.`, + ); + } + const fieldNames = new Set(fields.map((f) => f.name)); + validateFormulaColumns(field.formula, fieldNames, field.name); + columnDefs.push( + `${quoteIdentifier(field.name)} ${sqliteType} GENERATED ALWAYS AS (${field.formula}) STORED`, + ); + } else { + const notNull = field.required ? ' NOT NULL' : ''; + const defaultClause = + field.default_value != null + ? ` DEFAULT ${sqliteLiteral(field.default_value, sqliteType)}` + : ''; + columnDefs.push(`${quoteIdentifier(field.name)} ${sqliteType}${notNull}${defaultClause}`); + } + } + + return `CREATE TABLE IF NOT EXISTS ${quoteIdentifier(tableName)} (\n ${columnDefs.join(',\n ')}\n);`; +} + +/** + * Generates a `CREATE TABLE` DDL statement for a child (table-type field) data table. + * + * Child table naming follows Frappe convention: `dt_{parent}__{child}` (double underscore). + * Standard columns: id (PK), parent_id (FK → parent ON DELETE CASCADE), idx (sort order) + * User-defined columns: one per DocTypeField, typed from FIELD_TYPE_MAP + * Constraint: UNIQUE(parent_id, idx) to enforce ordering within a parent record. + */ +export function buildChildTableDDL( + parentDoctype: string, + childName: string, + fields: DocTypeField[], +): string { + const parentTable = `dt_${parentDoctype}`; + const childTable = `dt_${parentDoctype}__${childName}`; + + const columnDefs: string[] = [ + 'id TEXT NOT NULL PRIMARY KEY', + `parent_id TEXT NOT NULL REFERENCES ${quoteIdentifier(parentTable)}(id) ON DELETE CASCADE`, + 'idx INTEGER NOT NULL', + ]; + + for (const field of fields) { + const sqliteType = FIELD_TYPE_MAP[field.field_type] ?? 'TEXT'; + if (field.formula != null) { + if (!isValidSQLiteExpression(field.formula)) { + throw new Error( + `Field "${field.name}": formula contains disallowed SQL keywords or characters.`, + ); + } + const fieldNames = new Set(fields.map((f) => f.name)); + validateFormulaColumns(field.formula, fieldNames, field.name); + columnDefs.push( + `${quoteIdentifier(field.name)} ${sqliteType} GENERATED ALWAYS AS (${field.formula}) STORED`, + ); + } else { + const notNull = field.required ? ' NOT NULL' : ''; + const defaultClause = + field.default_value != null + ? ` DEFAULT ${sqliteLiteral(field.default_value, sqliteType)}` + : ''; + columnDefs.push(`${quoteIdentifier(field.name)} ${sqliteType}${notNull}${defaultClause}`); + } + } + + columnDefs.push('UNIQUE(parent_id, idx)'); + + return `CREATE TABLE IF NOT EXISTS ${quoteIdentifier(childTable)} (\n ${columnDefs.join(',\n ')}\n);`; +} + +/** + * Validates that every bare identifier in `formula` references a known column. + * Bare identifiers are word tokens that are not SQLite keywords or numeric literals. + * Throws if an unrecognised column name is found. + */ +function validateFormulaColumns(formula: string, fieldNames: Set, fieldName: string): void { + // Known SQLite keywords / built-in function names that are safe to ignore + const SQLITE_KEYWORDS = new Set([ + 'abs', + 'avg', + 'case', + 'cast', + 'coalesce', + 'count', + 'else', + 'end', + 'ifnull', + 'iif', + 'julianday', + 'length', + 'lower', + 'max', + 'min', + 'not', + 'null', + 'nullif', + 'round', + 'rtrim', + 'strftime', + 'substr', + 'sum', + 'then', + 'total', + 'trim', + 'typeof', + 'upper', + 'when', + ]); + + // Extract bare word tokens (skip quoted identifiers and string literals) + const strippedFormula = formula + .replace(/"[^"]*"/g, '') // remove double-quoted identifiers + .replace(/'[^']*'/g, ''); // remove single-quoted string literals + + const wordTokens = strippedFormula.match(/\b[A-Za-z_][A-Za-z0-9_]*\b/g) ?? []; + + for (const token of wordTokens) { + if (SQLITE_KEYWORDS.has(token.toLowerCase())) continue; + if (/^\d+$/.test(token)) continue; // numeric token (shouldn't match \b[A-Za-z_] but guard anyway) + if (!fieldNames.has(token)) { + throw new Error( + `Field "${fieldName}": formula references unknown column "${token}". Only columns defined in the same DocType are allowed.`, + ); + } + } +} + +/** + * Generates INSERT, UPDATE, and DELETE triggers on a child table that + * recompute an aggregate field on the parent table. + * + * Follows the Odoo `@api.depends` cross-table cascade pattern adapted for + * SQLite triggers (see IMPLEMENTATION-PLAN.md Part 5). + * + * @param parentTable — fully-qualified parent table name (e.g. `dt_invoice`) + * @param childTable — fully-qualified child table name (e.g. `dt_invoice__items`) + * @param aggregateField — column on the parent table to update (e.g. `subtotal`) + * @param sourceField — column on the child table to SUM (e.g. `amount`) + * @returns array of three CREATE TRIGGER statements (insert, update, delete) + */ +export function buildRecomputeTriggers( + parentTable: string, + childTable: string, + aggregateField: string, + sourceField: string, +): string[] { + const qParent = quoteIdentifier(parentTable); + const qChild = quoteIdentifier(childTable); + const qAgg = quoteIdentifier(aggregateField); + const qSrc = quoteIdentifier(sourceField); + + // Derive a short suffix from table names for unique trigger naming + const suffix = `${parentTable}__${aggregateField}`; + + const qParentId = quoteIdentifier('parent_id'); + const subquery = `(SELECT COALESCE(SUM(${qSrc}), 0) FROM ${qChild} WHERE ${qParentId} = {{ref}}.${qParentId})`; + + const insertTrigger = `CREATE TRIGGER IF NOT EXISTS "trg_${suffix}_insert" +AFTER INSERT ON ${qChild} +BEGIN + UPDATE ${qParent} SET + ${qAgg} = ${subquery.replace('{{ref}}', 'NEW')}, + "updated_at" = datetime('now') + WHERE "id" = NEW."parent_id"; +END;`; + + const updateTrigger = `CREATE TRIGGER IF NOT EXISTS "trg_${suffix}_update" +AFTER UPDATE OF ${qSrc} ON ${qChild} +BEGIN + UPDATE ${qParent} SET + ${qAgg} = ${subquery.replace('{{ref}}', 'NEW')}, + "updated_at" = datetime('now') + WHERE "id" = NEW."parent_id"; +END;`; + + const deleteTrigger = `CREATE TRIGGER IF NOT EXISTS "trg_${suffix}_delete" +AFTER DELETE ON ${qChild} +BEGIN + UPDATE ${qParent} SET + ${qAgg} = ${subquery.replace('{{ref}}', 'OLD')}, + "updated_at" = datetime('now') + WHERE "id" = OLD."parent_id"; +END;`; + + return [insertTrigger, updateTrigger, deleteTrigger]; +} + +/** + * Generates a `CREATE VIRTUAL TABLE` DDL statement for an FTS5 full-text search index + * plus INSERT, UPDATE, and DELETE triggers to keep the FTS5 index in sync with the + * content table. + * + * The generated FTS5 table is a content table (`content=`) backed by the data table, + * using `content_rowid=rowid`. Callers should pass only the field names where + * `searchable = true` in the DocType metadata. + * + * @param tableName — fully-qualified data table name (e.g. `dt_invoice`) + * @param searchableFields — column names to include in the FTS5 index + * @returns an array of DDL/trigger strings: [CREATE VIRTUAL TABLE, INSERT trigger, + * UPDATE trigger, DELETE trigger] + */ +export function buildFTS5DDL(tableName: string, searchableFields: string[]): string[] { + if (searchableFields.length === 0) { + throw new Error( + `buildFTS5DDL: at least one searchable field is required for table "${tableName}".`, + ); + } + + const ftsTable = `${tableName}_fts`; + const qFts = quoteIdentifier(ftsTable); + const qTable = quoteIdentifier(tableName); + + // Field list for the FTS5 declaration — identifiers must be unquoted inside fts5(...) + const fieldList = searchableFields.map((f) => quoteIdentifier(f)).join(', '); + + const createVTable = `CREATE VIRTUAL TABLE IF NOT EXISTS ${qFts} USING fts5(${fieldList}, content=${qTable}, content_rowid=rowid);`; + + // NEW.rowid / OLD.rowid must be bare column references inside trigger body + const newFieldValues = searchableFields.map((f) => `NEW.${quoteIdentifier(f)}`).join(', '); + const oldFieldValues = searchableFields.map((f) => `OLD.${quoteIdentifier(f)}`).join(', '); + const fieldListForDelete = searchableFields.map((f) => quoteIdentifier(f)).join(', '); + + const insertTrigger = `CREATE TRIGGER IF NOT EXISTS "trg_${tableName}_fts_insert" +AFTER INSERT ON ${qTable} +BEGIN + INSERT INTO ${qFts}(rowid, ${fieldList}) VALUES (NEW.rowid, ${newFieldValues}); +END;`; + + const updateTrigger = `CREATE TRIGGER IF NOT EXISTS "trg_${tableName}_fts_update" +AFTER UPDATE ON ${qTable} +BEGIN + INSERT INTO ${qFts}(${qFts}, rowid, ${fieldList}) VALUES ('delete', OLD.rowid, ${oldFieldValues}); + INSERT INTO ${qFts}(rowid, ${fieldList}) VALUES (NEW.rowid, ${newFieldValues}); +END;`; + + const deleteTrigger = `CREATE TRIGGER IF NOT EXISTS "trg_${tableName}_fts_delete" +AFTER DELETE ON ${qTable} +BEGIN + INSERT INTO ${qFts}(${qFts}, rowid, ${fieldListForDelete}) VALUES ('delete', OLD.rowid, ${oldFieldValues}); +END;`; + + return [createVTable, insertTrigger, updateTrigger, deleteTrigger]; +} + +/** Wraps a SQLite identifier in double-quotes, escaping any embedded double-quotes. */ +function quoteIdentifier(name: string): string { + return `"${name.replace(/"/g, '""')}"`; +} + +/** Produces a safe SQLite literal for a default value given its target column type. */ +function sqliteLiteral(value: string, sqliteType: string): string { + if (sqliteType === 'INTEGER' || sqliteType === 'REAL') { + // Allow bare numeric literals; fall back to quoted string if not numeric + if (/^-?\d+(\.\d+)?$/.test(value)) { + return value; + } + } + // Escape single-quotes by doubling them + return `'${value.replace(/'/g, "''")}'`; +} diff --git a/src/intelligence/template-loader.ts b/src/intelligence/template-loader.ts new file mode 100644 index 00000000..77f5acc9 --- /dev/null +++ b/src/intelligence/template-loader.ts @@ -0,0 +1,300 @@ +import { readFileSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; +import type Database from 'better-sqlite3'; +import type { FieldType, HookActionType, RelationType } from '../types/doctype.js'; +import type { WorkflowTrigger, WorkflowStep, WorkflowStatus } from '../types/workflow.js'; +import { createDocType, getDocTypeByName } from './doctype-store.js'; +import { createWorkflowStore } from '../workflows/workflow-store.js'; +import { createLogger } from '../core/logger.js'; + +const logger = createLogger('template-loader'); + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +/** A field entry in a DocTypeDefinition. IDs are generated if absent. */ +export interface FieldDef { + id?: string; + name: string; + label: string; + field_type: FieldType; + required?: boolean; + searchable?: boolean; + sort_order?: number; + default_value?: string; + options?: string[]; + formula?: string; + depends_on?: string; + link_doctype?: string; + child_doctype?: string; +} + +/** A state entry in a DocTypeDefinition. IDs are generated if absent. */ +export interface StateDef { + id?: string; + name: string; + label: string; + color?: string; + is_initial?: boolean; + is_terminal?: boolean; + sort_order?: number; +} + +/** A transition entry in a DocTypeDefinition. IDs are generated if absent. */ +export interface TransitionDef { + id?: string; + from_state: string; + to_state: string; + action_name: string; + action_label: string; + allowed_roles?: string[]; + condition?: string; +} + +/** A lifecycle hook entry in a DocTypeDefinition. IDs are generated if absent. */ +export interface HookDef { + id?: string; + event: string; + action_type: HookActionType; + action_config?: Record; + sort_order?: number; + enabled?: boolean; +} + +/** A relation entry in a DocTypeDefinition. IDs are generated if absent. */ +export interface RelationDef { + id?: string; + from_doctype: string; + to_doctype: string; + relation_type: RelationType; + from_field: string; + to_field?: string; + label?: string; +} + +/** + * A DocType definition as stored in an industry template manifest. + * All IDs are optional — they are generated on `applyTemplate` if absent. + */ +export interface DocTypeDefinition { + doctype: { + id?: string; + name: string; + label_singular: string; + label_plural: string; + icon?: string; + table_name: string; + source?: 'ai-created' | 'imported' | 'integration' | 'template'; + }; + fields?: FieldDef[]; + states?: StateDef[]; + transitions?: TransitionDef[]; + hooks?: HookDef[]; + relations?: RelationDef[]; +} + +/** + * A Workflow definition as stored in an industry template manifest. + * IDs are optional — they are generated on `applyTemplate` if absent. + */ +export interface WorkflowDefinition { + id?: string; + name: string; + description?: string; + trigger: WorkflowTrigger; + steps: WorkflowStep[]; + status?: WorkflowStatus; + run_count?: number; + error_count?: number; +} + +/** + * An industry template — a bundle of DocTypes, Workflows, skill pack guidance, + * and sample queries that can be applied to a workspace in one operation. + */ +export interface IndustryTemplate { + id: string; + name: string; + description: string; + doctypes: DocTypeDefinition[]; + workflows: WorkflowDefinition[]; + /** + * Inline Markdown content for the skill pack, or a relative path (from the + * template directory) to a `.md` file. `loadTemplate` resolves file paths + * to their contents automatically. + */ + skillPack: string; + sampleQueries: string[]; +} + +// --------------------------------------------------------------------------- +// loadTemplate +// --------------------------------------------------------------------------- + +/** + * Load an industry template from + * `/.openbridge/industry-templates//manifest.json`. + * + * If `skillPack` in the manifest is a file path ending in `.md` (no newlines), + * the file is read and its contents replace the path in the returned object. + * + * @throws If the manifest file does not exist or cannot be parsed. + */ +export function loadTemplate(workspacePath: string, templateId: string): IndustryTemplate { + const templateDir = join(workspacePath, '.openbridge', 'industry-templates', templateId); + const manifestPath = join(templateDir, 'manifest.json'); + + if (!existsSync(manifestPath)) { + throw new Error(`Template manifest not found: ${manifestPath}`); + } + + const parsed = JSON.parse(readFileSync(manifestPath, 'utf-8')) as IndustryTemplate; + + // Resolve a skillPack file reference to its contents + if (parsed.skillPack && !parsed.skillPack.includes('\n') && parsed.skillPack.endsWith('.md')) { + const skillPackPath = join(templateDir, parsed.skillPack); + if (existsSync(skillPackPath)) { + parsed.skillPack = readFileSync(skillPackPath, 'utf-8'); + } + } + + return parsed; +} + +// --------------------------------------------------------------------------- +// applyTemplate +// --------------------------------------------------------------------------- + +/** Derive a slug suitable for use as an ID component. */ +function toSlug(value: string): string { + return value + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, ''); +} + +/** + * Apply an industry template to a SQLite database — creates all DocTypes and + * Workflows defined in the template. + * + * Already-existing records (matched by name) are skipped, making the + * operation **idempotent**: safe to call multiple times. + * + * @param db An open better-sqlite3 `Database` instance. + * @param template The template to apply (typically from `loadTemplate`). + */ +export function applyTemplate(db: Database.Database, template: IndustryTemplate): void { + const workflowStore = createWorkflowStore(db); + + // --- Apply DocTypes --- + for (const def of template.doctypes) { + if (getDocTypeByName(db, def.doctype.name)) { + logger.debug({ name: def.doctype.name }, 'DocType already exists — skipping'); + continue; + } + + const doctypeId = def.doctype.id ?? `dt-${toSlug(def.doctype.name)}`; + const now = new Date().toISOString(); + + createDocType(db, { + doctype: { + id: doctypeId, + name: def.doctype.name, + label_singular: def.doctype.label_singular, + label_plural: def.doctype.label_plural, + icon: def.doctype.icon, + table_name: def.doctype.table_name, + source: def.doctype.source ?? 'template', + template_id: template.id, + created_at: now, + updated_at: now, + }, + fields: (def.fields ?? []).map((f, i) => ({ + id: f.id ?? `${doctypeId}-f${i}`, + doctype_id: doctypeId, + name: f.name, + label: f.label, + field_type: f.field_type, + required: f.required ?? false, + searchable: f.searchable ?? false, + sort_order: f.sort_order ?? i, + default_value: f.default_value, + options: f.options, + formula: f.formula, + depends_on: f.depends_on, + link_doctype: f.link_doctype, + child_doctype: f.child_doctype, + })), + states: (def.states ?? []).map((s, i) => ({ + id: s.id ?? `${doctypeId}-s${i}`, + doctype_id: doctypeId, + name: s.name, + label: s.label, + color: s.color ?? 'gray', + is_initial: s.is_initial ?? false, + is_terminal: s.is_terminal ?? false, + sort_order: s.sort_order ?? i, + })), + transitions: (def.transitions ?? []).map((t, i) => ({ + id: t.id ?? `${doctypeId}-t${i}`, + doctype_id: doctypeId, + from_state: t.from_state, + to_state: t.to_state, + action_name: t.action_name, + action_label: t.action_label, + allowed_roles: t.allowed_roles, + condition: t.condition, + })), + hooks: (def.hooks ?? []).map((h, i) => ({ + id: h.id ?? `${doctypeId}-h${i}`, + doctype_id: doctypeId, + event: h.event, + action_type: h.action_type, + action_config: h.action_config ?? {}, + sort_order: h.sort_order ?? i, + enabled: h.enabled ?? true, + })), + relations: (def.relations ?? []).map((r, i) => ({ + id: r.id ?? `${doctypeId}-r${i}`, + from_doctype: r.from_doctype, + to_doctype: r.to_doctype, + relation_type: r.relation_type, + from_field: r.from_field, + to_field: r.to_field ?? 'id', + label: r.label, + })), + }); + + logger.info({ name: def.doctype.name, id: doctypeId }, 'Created DocType from template'); + } + + // --- Apply Workflows --- + for (const def of template.workflows) { + const workflowId = def.id ?? `wf-${toSlug(template.id)}-${toSlug(def.name)}`; + const now = new Date().toISOString(); + + try { + workflowStore.createWorkflow({ + id: workflowId, + name: def.name, + description: def.description, + trigger: def.trigger, + steps: def.steps, + status: def.status ?? 'active', + run_count: def.run_count ?? 0, + error_count: def.error_count ?? 0, + created_at: now, + updated_at: now, + }); + logger.info({ name: def.name, id: workflowId }, 'Created Workflow from template'); + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + if (msg.includes('UNIQUE constraint') || msg.includes('already exists')) { + logger.debug({ name: def.name }, 'Workflow already exists — skipping'); + } else { + throw err; + } + } + } +} diff --git a/src/intelligence/templates/email-templates.ts b/src/intelligence/templates/email-templates.ts new file mode 100644 index 00000000..da665328 --- /dev/null +++ b/src/intelligence/templates/email-templates.ts @@ -0,0 +1,377 @@ +import type { Branding } from './invoice-template.js'; +import type { InvoiceData, InvoiceItem } from './invoice-template.js'; +import type { ReceiptData, ReceiptItem } from './receipt-template.js'; + +export type { Branding }; + +/** Client data for welcome emails */ +export interface ClientData { + name: string; + email?: string; + company?: string; + phone?: string; +} + +/** Result of a build*Email function */ +export interface EmailContent { + subject: string; + html: string; +} + +// --------------------------------------------------------------------------- +// Internal helpers +// --------------------------------------------------------------------------- + +function formatCurrency(amount: number, currency = 'USD'): string { + return new Intl.NumberFormat('en-US', { style: 'currency', currency }).format(amount); +} + +function esc(value: string | number | undefined | null): string { + if (value == null) return ''; + return String(value) + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} + +function baseStyles(primaryColor: string): string { + return ` + body { margin: 0; padding: 0; background: #f4f4f4; font-family: Arial, Helvetica, sans-serif; font-size: 15px; color: #333333; } + .wrapper { max-width: 600px; margin: 32px auto; background: #ffffff; border-radius: 6px; overflow: hidden; } + .header { background: ${primaryColor}; padding: 28px 32px; } + .header h1 { margin: 0; font-size: 22px; color: #ffffff; } + .header p { margin: 4px 0 0; font-size: 13px; color: rgba(255,255,255,0.85); } + .body { padding: 28px 32px; } + .body p { margin: 0 0 14px; line-height: 1.6; } + .info-table { width: 100%; border-collapse: collapse; margin-bottom: 20px; } + .info-table td { padding: 6px 0; vertical-align: top; } + .info-table td:first-child { color: #666666; font-size: 13px; width: 140px; } + .items-table { width: 100%; border-collapse: collapse; margin-bottom: 20px; } + .items-table th { background: #f8f9fa; border-bottom: 2px solid #e0e0e0; padding: 8px 10px; text-align: left; font-size: 13px; color: #555555; } + .items-table td { padding: 8px 10px; border-bottom: 1px solid #eeeeee; font-size: 14px; } + .items-table td.right { text-align: right; } + .totals { text-align: right; margin-bottom: 20px; } + .totals table { display: inline-table; border-collapse: collapse; } + .totals td { padding: 4px 10px; font-size: 14px; } + .totals td:first-child { color: #666666; } + .totals .total-row td { font-size: 16px; font-weight: bold; color: ${primaryColor}; border-top: 2px solid #e0e0e0; } + .btn-wrap { text-align: center; margin: 24px 0; } + .btn { display: inline-block; background: ${primaryColor}; color: #ffffff; text-decoration: none; padding: 12px 32px; border-radius: 4px; font-size: 15px; font-weight: bold; } + .divider { border: none; border-top: 1px solid #eeeeee; margin: 20px 0; } + .footer { background: #f8f9fa; padding: 18px 32px; font-size: 12px; color: #888888; text-align: center; } + @media only screen and (max-width: 620px) { + .wrapper { margin: 0; border-radius: 0; } + .body, .header, .footer { padding: 20px 16px; } + .info-table td:first-child { width: 110px; } + } + `.trim(); +} + +function layout( + primaryColor: string, + headerTitle: string, + headerSubtitle: string, + bodyHtml: string, + companyName: string, +): string { + return ` + + + + + ${esc(headerTitle)} + + + + + +
+
+
+

${esc(headerTitle)}

+ ${headerSubtitle ? `

${esc(headerSubtitle)}

` : ''} +
+
+ ${bodyHtml} +
+
+ © ${new Date().getFullYear()} ${esc(companyName)}. All rights reserved. +
+
+
+ +`; +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/** + * Build an invoice email — "Payment required" notification sent to a customer. + * + * Includes: invoice details, line items table, total, Pay Now action button. + */ +export function buildInvoiceEmail( + invoice: InvoiceData, + items: InvoiceItem[], + branding: Branding, +): EmailContent { + const primary = branding.primaryColor ?? '#1a73e8'; + const currency = invoice.currency ?? 'USD'; + + const subtotal = items.reduce((sum, i) => sum + (i.total ?? i.unitPrice * i.quantity), 0); + const taxRate = invoice.taxRate ?? 0; + const tax = subtotal * (taxRate / 100); + const total = subtotal + tax; + + const rows = items + .map( + (item) => ` + + ${esc(item.description)} + ${item.quantity} + ${formatCurrency(item.unitPrice, currency)} + ${formatCurrency(item.total ?? item.unitPrice * item.quantity, currency)} + `, + ) + .join(''); + + const taxRow = + taxRate > 0 + ? `Tax (${taxRate}%)${formatCurrency(tax, currency)}` + : ''; + + const payBtn = invoice.paymentLink + ? `
Pay Now
` + : ''; + + const body = ` +

Hi ${esc(invoice.customerName)},

+

Please find your invoice details below. Payment is due${invoice.dueDate ? ` by ${esc(invoice.dueDate)}` : ' upon receipt'}.

+ + + + + ${invoice.dueDate ? `` : ''} + ${invoice.customerEmail ? `` : ''} +
Invoice #${esc(invoice.invoiceNumber)}
Date${esc(invoice.date)}
Due Date${esc(invoice.dueDate)}
Bill To${esc(invoice.customerName)}
${esc(invoice.customerEmail)}
+ + + + + + + + + + + ${rows} +
DescriptionQtyUnit PriceAmount
+ +
+ + + ${taxRow} + +
Subtotal${formatCurrency(subtotal, currency)}
Total${formatCurrency(total, currency)}
+
+ + ${payBtn} + ${invoice.notes ? `

${esc(invoice.notes)}

` : ''} + ${invoice.terms ? `

Terms: ${esc(invoice.terms)}

` : ''} + +

If you have questions, please contact us at ${esc(branding.companyEmail ?? branding.companyName)}.

+

Thank you for your business!

+ `; + + return { + subject: `Invoice ${invoice.invoiceNumber} from ${branding.companyName}`, + html: layout( + primary, + `Invoice ${invoice.invoiceNumber}`, + branding.companyName, + body, + branding.companyName, + ), + }; +} + +/** + * Build a receipt email — confirmation sent after payment is received. + * + * Includes: receipt details, items paid, total, View Invoice button. + */ +export function buildReceiptEmail( + receipt: ReceiptData, + items: ReceiptItem[], + branding: Branding, +): EmailContent { + const primary = branding.primaryColor ?? '#1a73e8'; + const currency = receipt.currency ?? 'USD'; + const total = items.reduce((sum, i) => sum + i.amount, 0); + + const rows = items + .map( + (item) => ` + + ${esc(item.description)} + ${item.quantity != null ? `${item.quantity}` : ''} + ${formatCurrency(item.amount, currency)} + `, + ) + .join(''); + + const body = ` +

Hi ${receipt.customerName ? esc(receipt.customerName) : 'there'},

+

Thank you for your payment! Here is your receipt.

+ + + ${receipt.receiptNumber ? `` : ''} + + ${receipt.paymentMethod ? `` : ''} +
Receipt #${esc(receipt.receiptNumber)}
Date${esc(receipt.date)}${receipt.time ? ` ${esc(receipt.time)}` : ''}
Payment${esc(receipt.paymentMethod)}
+ + + + + + + + + + ${rows} +
DescriptionQtyAmount
+ +
+ + +
Total Paid${formatCurrency(total, currency)}
+
+ + ${receipt.notes ? `

${esc(receipt.notes)}

` : ''} +

We appreciate your business. Please keep this email as your payment confirmation.

+ `; + + return { + subject: `Payment receipt from ${branding.companyName}`, + html: layout( + primary, + 'Payment Receipt', + `${branding.companyName} — ${receipt.date}`, + body, + branding.companyName, + ), + }; +} + +/** + * Build a payment reminder email — polite nudge sent when an invoice is overdue. + * + * Includes: overdue notice, invoice details, total outstanding, Pay Now button. + */ +export function buildReminderEmail( + invoice: InvoiceData, + items: InvoiceItem[], + branding: Branding, +): EmailContent { + const primary = branding.primaryColor ?? '#1a73e8'; + const currency = invoice.currency ?? 'USD'; + + const subtotal = items.reduce((sum, i) => sum + (i.total ?? i.unitPrice * i.quantity), 0); + const tax = subtotal * ((invoice.taxRate ?? 0) / 100); + const total = subtotal + tax; + + const payBtn = invoice.paymentLink + ? `
Pay Now
` + : ''; + + const body = ` +

Hi ${esc(invoice.customerName)},

+

This is a friendly reminder that invoice ${esc(invoice.invoiceNumber)} is${invoice.dueDate ? ` due on ${esc(invoice.dueDate)}` : ' outstanding'}.

+

Please arrange payment at your earliest convenience to avoid any service interruptions.

+ + + + + ${invoice.dueDate ? `` : ''} + +
Invoice #${esc(invoice.invoiceNumber)}
Invoice Date${esc(invoice.date)}
Due Date${esc(invoice.dueDate)}
Amount Due${formatCurrency(total, currency)}
+ + ${payBtn} + +

If you have already made this payment, please disregard this message — and thank you!

+

If you have any questions or need to discuss payment arrangements, please contact us at ${esc(branding.companyEmail ?? branding.companyName)}.

+

Thank you,
${esc(branding.companyName)}

+ `; + + return { + subject: `Payment reminder — Invoice ${invoice.invoiceNumber} from ${branding.companyName}`, + html: layout( + primary, + 'Payment Reminder', + `Invoice ${invoice.invoiceNumber}`, + body, + branding.companyName, + ), + }; +} + +/** + * Build a welcome email — onboarding message sent to a new client. + * + * Includes: greeting, company intro, contact info, View Account button. + */ +export function buildWelcomeEmail( + client: ClientData, + branding: Branding, + options?: { accountUrl?: string }, +): EmailContent { + const primary = branding.primaryColor ?? '#1a73e8'; + + const viewBtn = options?.accountUrl + ? `
View Your Account
` + : ''; + + const body = ` +

Hi ${esc(client.name)},

+

Welcome to ${esc(branding.companyName)}! We're thrilled to have you on board.

+

Your account has been set up and you can now start working with us. We look forward to a great partnership.

+ + ${ + client.company || client.phone || client.email + ? ` + + ${client.company ? `` : ''} + ${client.email ? `` : ''} + ${client.phone ? `` : ''} +
Company${esc(client.company)}
Email${esc(client.email)}
Phone${esc(client.phone)}
` + : '' + } + + ${viewBtn} + +
+

If you have any questions or need assistance, please don't hesitate to reach out:

+ + ${branding.companyEmail ? `` : ''} + ${branding.companyPhone ? `` : ''} + ${branding.companyAddress ? `` : ''} +
Email${esc(branding.companyEmail)}
Phone${esc(branding.companyPhone)}
Address${esc(branding.companyAddress)}
+ +

We look forward to working with you!

+

The ${esc(branding.companyName)} Team

+ `; + + return { + subject: `Welcome to ${branding.companyName}!`, + html: layout( + primary, + `Welcome, ${client.name}!`, + branding.companyName, + body, + branding.companyName, + ), + }; +} diff --git a/src/intelligence/templates/invoice-template.ts b/src/intelligence/templates/invoice-template.ts new file mode 100644 index 00000000..3b5ec793 --- /dev/null +++ b/src/intelligence/templates/invoice-template.ts @@ -0,0 +1,330 @@ +import type { TDocumentDefinitions, Content } from '../pdf-generator.js'; + +/** Invoice record metadata */ +export interface InvoiceData { + invoiceNumber: string; + date: string; + dueDate?: string; + customerName: string; + customerEmail?: string; + customerAddress?: string; + customerPhone?: string; + notes?: string; + terms?: string; + paymentLink?: string; + taxRate?: number; + currency?: string; +} + +/** A single line item on an invoice */ +export interface InvoiceItem { + description: string; + quantity: number; + unitPrice: number; + total?: number; +} + +/** Business branding for document generation */ +export interface Branding { + companyName: string; + companyAddress?: string; + companyEmail?: string; + companyPhone?: string; + primaryColor?: string; + /** Base64 data URI for the company logo (e.g. "data:image/png;base64,...") */ + logoDataUri?: string; +} + +/** Format a number as a currency string */ +function formatCurrency(amount: number, currency = 'USD'): string { + return new Intl.NumberFormat('en-US', { style: 'currency', currency }).format(amount); +} + +/** + * Build a pdfmake document definition for a professional invoice. + * + * Features: + * - Business logo (if provided via branding.logoDataUri) + * - Invoice number, dates, client info + * - Line items table (description, qty, unit price, amount) + * - Subtotal / tax / total + * - Payment QR code (if paymentLink provided) + * - Footer with terms and conditions + */ +export function buildInvoiceDefinition( + invoice: InvoiceData, + items: InvoiceItem[], + branding: Branding, +): TDocumentDefinitions { + const primaryColor = branding.primaryColor ?? '#1a73e8'; + const currency = invoice.currency ?? 'USD'; + + // Compute totals + const lineItems = items.map((item) => ({ + ...item, + total: item.total ?? item.quantity * item.unitPrice, + })); + + const subtotal = lineItems.reduce((sum, item) => sum + item.total, 0); + const tax = invoice.taxRate != null ? subtotal * invoice.taxRate : 0; + const grandTotal = subtotal + tax; + + // ── Header: logo + company info (left) | invoice meta (right) ────────── + + const companyStack: Content[] = []; + + if (branding.logoDataUri) { + companyStack.push({ image: 'companyLogo', width: 120, marginBottom: 8 }); + } + + companyStack.push({ text: branding.companyName, style: 'companyName' }); + + if (branding.companyAddress) { + companyStack.push({ text: branding.companyAddress, style: 'companyMeta' }); + } + if (branding.companyEmail) { + companyStack.push({ text: branding.companyEmail, style: 'companyMeta' }); + } + if (branding.companyPhone) { + companyStack.push({ text: branding.companyPhone, style: 'companyMeta' }); + } + + const headerContent: Content = { + columns: [ + { stack: companyStack, width: '*' }, + { + stack: [ + { text: 'INVOICE', style: 'invoiceTitle' }, + { text: `#${invoice.invoiceNumber}`, style: 'invoiceNumber', color: primaryColor }, + { text: `Date: ${invoice.date}`, style: 'invoiceMeta' }, + ...(invoice.dueDate ? [{ text: `Due: ${invoice.dueDate}`, style: 'invoiceMeta' }] : []), + ], + width: 'auto', + alignment: 'right' as const, + }, + ], + marginBottom: 20, + }; + + // ── Bill-to section ──────────────────────────────────────────────────── + + const customerLines: Content[] = [{ text: invoice.customerName, style: 'customerName' }]; + if (invoice.customerEmail) { + customerLines.push({ text: invoice.customerEmail, style: 'customerMeta' }); + } + if (invoice.customerAddress) { + customerLines.push({ text: invoice.customerAddress, style: 'customerMeta' }); + } + if (invoice.customerPhone) { + customerLines.push({ text: invoice.customerPhone, style: 'customerMeta' }); + } + + const billToContent: Content = { + stack: [{ text: 'BILL TO', style: 'sectionHeader', color: primaryColor }, ...customerLines], + marginBottom: 20, + }; + + // ── Line items table ────────────────────────────────────────────────── + + const tableBody: Content[][] = [ + [ + { text: 'Description', style: 'tableHeader', color: 'white', fillColor: primaryColor }, + { + text: 'Qty', + style: 'tableHeader', + color: 'white', + fillColor: primaryColor, + alignment: 'right' as const, + }, + { + text: 'Unit Price', + style: 'tableHeader', + color: 'white', + fillColor: primaryColor, + alignment: 'right' as const, + }, + { + text: 'Amount', + style: 'tableHeader', + color: 'white', + fillColor: primaryColor, + alignment: 'right' as const, + }, + ], + ...lineItems.map((item, idx) => { + const bg = idx % 2 === 0 ? '#f8f9fa' : 'white'; + return [ + { text: item.description, fillColor: bg }, + { text: String(item.quantity), alignment: 'right' as const, fillColor: bg }, + { + text: formatCurrency(item.unitPrice, currency), + alignment: 'right' as const, + fillColor: bg, + }, + { + text: formatCurrency(item.total, currency), + alignment: 'right' as const, + fillColor: bg, + }, + ]; + }), + ]; + + const itemsTable: Content = { + table: { + headerRows: 1, + widths: ['*', 60, 80, 80], + body: tableBody, + }, + layout: 'lightHorizontalLines', + marginBottom: 20, + }; + + // ── Totals block (right-aligned) ────────────────────────────────────── + + const totalRows: [unknown, unknown][] = []; + if (invoice.taxRate != null) { + totalRows.push( + [ + { text: 'Subtotal', alignment: 'right' as const }, + { text: formatCurrency(subtotal, currency), alignment: 'right' as const }, + ], + [ + { + text: `Tax (${(invoice.taxRate * 100).toFixed(0)}%)`, + alignment: 'right' as const, + }, + { text: formatCurrency(tax, currency), alignment: 'right' as const }, + ], + ); + } + totalRows.push([ + { + text: 'TOTAL', + style: 'totalLabel', + alignment: 'right' as const, + color: primaryColor, + }, + { + text: formatCurrency(grandTotal, currency), + style: 'totalAmount', + alignment: 'right' as const, + color: primaryColor, + }, + ]); + + const totalsSection: Content = { + columns: [ + { text: '', width: '*' }, + { + table: { + widths: [120, 80], + body: totalRows as Content[][], + }, + layout: 'noBorders', + }, + ], + marginBottom: 20, + }; + + // ── Payment QR code ─────────────────────────────────────────────────── + + const paymentSection: Content[] = []; + if (invoice.paymentLink) { + paymentSection.push({ + columns: [ + { + stack: [ + { text: 'Payment', style: 'sectionHeader', color: primaryColor, marginTop: 10 }, + { + text: invoice.paymentLink, + link: invoice.paymentLink, + style: 'paymentLink', + color: primaryColor, + }, + ], + width: '*', + }, + { + stack: [ + { qr: invoice.paymentLink, fit: 80, alignment: 'right' as const }, + { + text: 'Scan to pay', + alignment: 'right' as const, + style: 'qrLabel', + marginTop: 4, + }, + ], + width: 'auto', + }, + ], + marginBottom: 10, + }); + } + + // ── Notes ───────────────────────────────────────────────────────────── + + const notesSection: Content[] = []; + if (invoice.notes) { + notesSection.push( + { text: 'Notes', style: 'sectionHeader', color: primaryColor, marginTop: 10 }, + { text: invoice.notes, style: 'notes' }, + ); + } + + // ── Terms & conditions footer ───────────────────────────────────────── + + const termsSection: Content[] = []; + if (invoice.terms) { + termsSection.push({ + stack: [ + { text: 'Terms & Conditions', style: 'sectionHeader', color: primaryColor, marginTop: 15 }, + { text: invoice.terms, style: 'termsText' }, + ], + }); + } + + // ── Images dictionary ───────────────────────────────────────────────── + + const images: Record = {}; + if (branding.logoDataUri) { + images['companyLogo'] = branding.logoDataUri; + } + + // ── Assemble document ───────────────────────────────────────────────── + + const content: Content[] = [ + headerContent, + billToContent, + itemsTable, + totalsSection, + ...paymentSection, + ...notesSection, + ...termsSection, + ]; + + return { + pageSize: 'A4', + pageMargins: [40, 60, 40, 60] as [number, number, number, number], + defaultStyle: { font: 'Roboto', fontSize: 10, lineHeight: 1.3 }, + ...(Object.keys(images).length > 0 ? { images } : {}), + styles: { + companyName: { fontSize: 16, bold: true, marginBottom: 4 }, + companyMeta: { fontSize: 9, color: '#666666' }, + invoiceTitle: { fontSize: 22, bold: true, color: '#333333', marginBottom: 4 }, + invoiceNumber: { fontSize: 13, bold: true, marginBottom: 4 }, + invoiceMeta: { fontSize: 9, color: '#666666' }, + sectionHeader: { fontSize: 9, bold: true, marginBottom: 4, marginTop: 10 }, + customerName: { fontSize: 11, bold: true, marginBottom: 2 }, + customerMeta: { fontSize: 9, color: '#666666' }, + tableHeader: { fontSize: 10, bold: true }, + totalLabel: { fontSize: 11, bold: true }, + totalAmount: { fontSize: 11, bold: true }, + notes: { fontSize: 9, color: '#555555', italics: true }, + paymentLink: { fontSize: 10, decoration: 'underline' }, + qrLabel: { fontSize: 8, color: '#888888' }, + termsText: { fontSize: 8, color: '#777777', lineHeight: 1.4 }, + }, + content, + }; +} diff --git a/src/intelligence/templates/quote-template.ts b/src/intelligence/templates/quote-template.ts new file mode 100644 index 00000000..371d5f28 --- /dev/null +++ b/src/intelligence/templates/quote-template.ts @@ -0,0 +1,358 @@ +import type { TDocumentDefinitions, Content } from '../pdf-generator.js'; +import type { Branding } from './invoice-template.js'; + +export type { Branding }; + +/** Quote record metadata */ +export interface QuoteData { + quoteNumber: string; + date: string; + /** Date after which the quote is no longer valid */ + validUntil?: string; + customerName: string; + customerEmail?: string; + customerAddress?: string; + customerPhone?: string; + notes?: string; + terms?: string; + taxRate?: number; + currency?: string; +} + +/** A single line item on a quote */ +export interface QuoteItem { + description: string; + quantity: number; + unitPrice: number; + total?: number; +} + +/** Format a number as a currency string */ +function formatCurrency(amount: number, currency = 'USD'): string { + return new Intl.NumberFormat('en-US', { style: 'currency', currency }).format(amount); +} + +/** + * Build a pdfmake document definition for a professional quote/estimate. + * + * Features: + * - Business logo (if provided via branding.logoDataUri) + * - Quote number, dates, validity period, client info + * - Line items table (description, qty, unit price, amount) + * - Subtotal / tax / total + * - Acceptance signature line + * - Terms and conditions section + */ +export function buildQuoteDefinition( + quote: QuoteData, + items: QuoteItem[], + branding: Branding, +): TDocumentDefinitions { + const primaryColor = branding.primaryColor ?? '#1a73e8'; + const currency = quote.currency ?? 'USD'; + + // Compute totals + const lineItems = items.map((item) => ({ + ...item, + total: item.total ?? item.quantity * item.unitPrice, + })); + + const subtotal = lineItems.reduce((sum, item) => sum + item.total, 0); + const tax = quote.taxRate != null ? subtotal * quote.taxRate : 0; + const grandTotal = subtotal + tax; + + // ── Header: logo + company info (left) | quote meta (right) ───────────── + + const companyStack: Content[] = []; + + if (branding.logoDataUri) { + companyStack.push({ image: 'companyLogo', width: 120, marginBottom: 8 }); + } + + companyStack.push({ text: branding.companyName, style: 'companyName' }); + + if (branding.companyAddress) { + companyStack.push({ text: branding.companyAddress, style: 'companyMeta' }); + } + if (branding.companyEmail) { + companyStack.push({ text: branding.companyEmail, style: 'companyMeta' }); + } + if (branding.companyPhone) { + companyStack.push({ text: branding.companyPhone, style: 'companyMeta' }); + } + + const headerContent: Content = { + columns: [ + { stack: companyStack, width: '*' }, + { + stack: [ + { text: 'QUOTE', style: 'quoteTitle' }, + { text: `#${quote.quoteNumber}`, style: 'quoteNumber', color: primaryColor }, + { text: `Date: ${quote.date}`, style: 'quoteMeta' }, + ...(quote.validUntil + ? [ + { + text: `Valid Until: ${quote.validUntil}`, + style: 'quoteValidity', + color: primaryColor, + }, + ] + : []), + ], + width: 'auto', + alignment: 'right' as const, + }, + ], + marginBottom: 20, + }; + + // ── Quote-to section ─────────────────────────────────────────────────── + + const customerLines: Content[] = [{ text: quote.customerName, style: 'customerName' }]; + if (quote.customerEmail) { + customerLines.push({ text: quote.customerEmail, style: 'customerMeta' }); + } + if (quote.customerAddress) { + customerLines.push({ text: quote.customerAddress, style: 'customerMeta' }); + } + if (quote.customerPhone) { + customerLines.push({ text: quote.customerPhone, style: 'customerMeta' }); + } + + const quoteToContent: Content = { + stack: [{ text: 'QUOTE TO', style: 'sectionHeader', color: primaryColor }, ...customerLines], + marginBottom: 20, + }; + + // ── Line items table ────────────────────────────────────────────────── + + const tableBody: Content[][] = [ + [ + { text: 'Description', style: 'tableHeader', color: 'white', fillColor: primaryColor }, + { + text: 'Qty', + style: 'tableHeader', + color: 'white', + fillColor: primaryColor, + alignment: 'right' as const, + }, + { + text: 'Unit Price', + style: 'tableHeader', + color: 'white', + fillColor: primaryColor, + alignment: 'right' as const, + }, + { + text: 'Amount', + style: 'tableHeader', + color: 'white', + fillColor: primaryColor, + alignment: 'right' as const, + }, + ], + ...lineItems.map((item, idx) => { + const bg = idx % 2 === 0 ? '#f8f9fa' : 'white'; + return [ + { text: item.description, fillColor: bg }, + { text: String(item.quantity), alignment: 'right' as const, fillColor: bg }, + { + text: formatCurrency(item.unitPrice, currency), + alignment: 'right' as const, + fillColor: bg, + }, + { + text: formatCurrency(item.total, currency), + alignment: 'right' as const, + fillColor: bg, + }, + ]; + }), + ]; + + const itemsTable: Content = { + table: { + headerRows: 1, + widths: ['*', 60, 80, 80], + body: tableBody, + }, + layout: 'lightHorizontalLines', + marginBottom: 20, + }; + + // ── Totals block (right-aligned) ───────────────────────────────────── + + const totalRows: [unknown, unknown][] = []; + if (quote.taxRate != null) { + totalRows.push( + [ + { text: 'Subtotal', alignment: 'right' as const }, + { text: formatCurrency(subtotal, currency), alignment: 'right' as const }, + ], + [ + { + text: `Tax (${(quote.taxRate * 100).toFixed(0)}%)`, + alignment: 'right' as const, + }, + { text: formatCurrency(tax, currency), alignment: 'right' as const }, + ], + ); + } + totalRows.push([ + { + text: 'TOTAL', + style: 'totalLabel', + alignment: 'right' as const, + color: primaryColor, + }, + { + text: formatCurrency(grandTotal, currency), + style: 'totalAmount', + alignment: 'right' as const, + color: primaryColor, + }, + ]); + + const totalsSection: Content = { + columns: [ + { text: '', width: '*' }, + { + table: { + widths: [120, 80], + body: totalRows as Content[][], + }, + layout: 'noBorders', + }, + ], + marginBottom: 20, + }; + + // ── Notes ───────────────────────────────────────────────────────────── + + const notesSection: Content[] = []; + if (quote.notes) { + notesSection.push( + { text: 'Notes', style: 'sectionHeader', color: primaryColor, marginTop: 10 }, + { text: quote.notes, style: 'notes' }, + ); + } + + // ── Terms & conditions ───────────────────────────────────────────────── + + const termsSection: Content[] = []; + if (quote.terms) { + termsSection.push({ + stack: [ + { text: 'Terms & Conditions', style: 'sectionHeader', color: primaryColor, marginTop: 15 }, + { text: quote.terms, style: 'termsText' }, + ], + }); + } + + // ── Acceptance signature line ────────────────────────────────────────── + + const signatureSection: Content = { + marginTop: 30, + stack: [ + { + text: 'Acceptance', + style: 'sectionHeader', + color: primaryColor, + marginBottom: 20, + }, + { + text: 'By signing below, you accept this quote and authorize the work described above.', + style: 'signatureNote', + marginBottom: 30, + }, + { + columns: [ + { + stack: [ + { + canvas: [ + { + type: 'line', + x1: 0, + y1: 0, + x2: 180, + y2: 0, + lineWidth: 1, + lineColor: '#333333', + }, + ], + }, + { text: 'Authorized Signature', style: 'signatureLabel', marginTop: 4 }, + ], + width: 200, + }, + { text: '', width: 40 }, + { + stack: [ + { + canvas: [ + { + type: 'line', + x1: 0, + y1: 0, + x2: 120, + y2: 0, + lineWidth: 1, + lineColor: '#333333', + }, + ], + }, + { text: 'Date', style: 'signatureLabel', marginTop: 4 }, + ], + width: 140, + }, + ], + }, + ], + }; + + // ── Images dictionary ───────────────────────────────────────────────── + + const images: Record = {}; + if (branding.logoDataUri) { + images['companyLogo'] = branding.logoDataUri; + } + + // ── Assemble document ───────────────────────────────────────────────── + + const content: Content[] = [ + headerContent, + quoteToContent, + itemsTable, + totalsSection, + ...notesSection, + ...termsSection, + signatureSection, + ]; + + return { + pageSize: 'A4', + pageMargins: [40, 60, 40, 60] as [number, number, number, number], + defaultStyle: { font: 'Roboto', fontSize: 10, lineHeight: 1.3 }, + ...(Object.keys(images).length > 0 ? { images } : {}), + styles: { + companyName: { fontSize: 16, bold: true, marginBottom: 4 }, + companyMeta: { fontSize: 9, color: '#666666' }, + quoteTitle: { fontSize: 22, bold: true, color: '#333333', marginBottom: 4 }, + quoteNumber: { fontSize: 13, bold: true, marginBottom: 4 }, + quoteMeta: { fontSize: 9, color: '#666666' }, + quoteValidity: { fontSize: 9, bold: true, marginTop: 2 }, + sectionHeader: { fontSize: 9, bold: true, marginBottom: 4, marginTop: 10 }, + customerName: { fontSize: 11, bold: true, marginBottom: 2 }, + customerMeta: { fontSize: 9, color: '#666666' }, + tableHeader: { fontSize: 10, bold: true }, + totalLabel: { fontSize: 11, bold: true }, + totalAmount: { fontSize: 11, bold: true }, + notes: { fontSize: 9, color: '#555555', italics: true }, + termsText: { fontSize: 8, color: '#777777', lineHeight: 1.4 }, + signatureNote: { fontSize: 9, color: '#555555' }, + signatureLabel: { fontSize: 8, color: '#888888' }, + }, + content, + }; +} diff --git a/src/intelligence/templates/receipt-template.ts b/src/intelligence/templates/receipt-template.ts new file mode 100644 index 00000000..014b7caf --- /dev/null +++ b/src/intelligence/templates/receipt-template.ts @@ -0,0 +1,253 @@ +import type { TDocumentDefinitions, Content } from '../pdf-generator.js'; +import type { Branding } from './invoice-template.js'; + +export type { Branding }; + +/** Receipt record metadata */ +export interface ReceiptData { + receiptNumber?: string; + date: string; + time?: string; + customerName?: string; + paymentMethod?: string; + notes?: string; + currency?: string; +} + +/** A single line item on a receipt */ +export interface ReceiptItem { + description: string; + quantity?: number; + amount: number; +} + +/** Format a number as a currency string */ +function formatCurrency(amount: number, currency = 'USD'): string { + return new Intl.NumberFormat('en-US', { style: 'currency', currency }).format(amount); +} + +/** + * Build a pdfmake document definition for a simple receipt. + * + * Features: + * - Compact, minimal layout (no logo by default, but supported) + * - Business name and contact info + * - Date and time + * - Simple item list + * - Total amount + * - Payment method + * - "Thank you" message + */ +export function buildReceiptDefinition( + receipt: ReceiptData, + items: ReceiptItem[], + branding: Branding, +): TDocumentDefinitions { + const currency = receipt.currency ?? 'USD'; + + // Compute total + const total = items.reduce((sum, item) => sum + item.amount, 0); + + // ── Header: company info ─────────────────────────────────────────────────── + + const companyStack: Content[] = [ + { text: branding.companyName, style: 'companyName', alignment: 'center' as const }, + ]; + + if (branding.companyAddress) { + companyStack.push({ + text: branding.companyAddress, + style: 'companyMeta', + alignment: 'center' as const, + }); + } + if (branding.companyEmail) { + companyStack.push({ + text: branding.companyEmail, + style: 'companyMeta', + alignment: 'center' as const, + }); + } + if (branding.companyPhone) { + companyStack.push({ + text: branding.companyPhone, + style: 'companyMeta', + alignment: 'center' as const, + }); + } + + const headerContent: Content = { + stack: companyStack, + marginBottom: 15, + }; + + // ── Receipt info (date, time, receipt number) ────────────────────────────── + + const receiptInfoLines: Content[] = []; + + if (receipt.receiptNumber) { + receiptInfoLines.push({ + text: `Receipt #${receipt.receiptNumber}`, + style: 'receiptNumber', + alignment: 'center' as const, + }); + } + + const dateTimeStr = receipt.time ? `${receipt.date} ${receipt.time}` : receipt.date; + receiptInfoLines.push({ + text: dateTimeStr, + style: 'receiptMeta', + alignment: 'center' as const, + }); + + if (receipt.customerName) { + receiptInfoLines.push({ + text: `Customer: ${receipt.customerName}`, + style: 'receiptMeta', + alignment: 'center' as const, + marginTop: 5, + }); + } + + const receiptInfoContent: Content = { + stack: receiptInfoLines, + marginBottom: 15, + }; + + // ── Divider line ─────────────────────────────────────────────────────────── + + const dividerContent: Content = { + canvas: [ + { + type: 'line', + x1: 0, + y1: 0, + x2: 500, + y2: 0, + lineWidth: 1, + lineColor: '#cccccc', + }, + ], + marginBottom: 10, + }; + + // ── Items list ───────────────────────────────────────────────────────────── + + const itemsContent: Content[] = []; + + items.forEach((item) => { + const qtyStr = item.quantity ? `x${item.quantity} ` : ''; + itemsContent.push({ + columns: [ + { + text: `${qtyStr}${item.description}`, + width: '*', + }, + { + text: formatCurrency(item.amount, currency), + width: 'auto', + alignment: 'right' as const, + }, + ], + marginBottom: 5, + }); + }); + + // ── Divider line ─────────────────────────────────────────────────────────── + + itemsContent.push({ + canvas: [ + { + type: 'line', + x1: 0, + y1: 0, + x2: 500, + y2: 0, + lineWidth: 1, + lineColor: '#cccccc', + }, + ], + marginBottom: 10, + marginTop: 10, + }); + + // ── Total ────────────────────────────────────────────────────────────────── + + itemsContent.push({ + columns: [ + { + text: 'TOTAL', + style: 'totalLabel', + }, + { + text: formatCurrency(total, currency), + style: 'totalAmount', + alignment: 'right' as const, + }, + ], + marginBottom: 15, + }); + + // ── Payment method ───────────────────────────────────────────────────────── + + const paymentContent: Content[] = []; + if (receipt.paymentMethod) { + paymentContent.push({ + text: `Payment: ${receipt.paymentMethod}`, + style: 'paymentInfo', + alignment: 'center' as const, + marginBottom: 15, + }); + } + + // ── Thank you message ────────────────────────────────────────────────────── + + const thankYouContent: Content = { + text: 'Thank you for your business!', + style: 'thankYou', + alignment: 'center' as const, + marginTop: 20, + }; + + // ── Notes ────────────────────────────────────────────────────────────────── + + const notesContent: Content[] = []; + if (receipt.notes) { + notesContent.push({ + text: receipt.notes, + style: 'notes', + alignment: 'center' as const, + marginTop: 10, + }); + } + + // ── Assemble document ────────────────────────────────────────────────────── + + const content: Content[] = [ + headerContent, + receiptInfoContent, + dividerContent, + ...itemsContent, + ...paymentContent, + thankYouContent, + ...notesContent, + ]; + + return { + pageSize: { width: 280, height: 600 }, // Compact receipt width (80mm common thermal printer width) + pageMargins: [20, 20, 20, 20] as [number, number, number, number], + defaultStyle: { font: 'Roboto', fontSize: 10, lineHeight: 1.2 }, + styles: { + companyName: { fontSize: 13, bold: true, marginBottom: 2 }, + companyMeta: { fontSize: 8, color: '#666666', marginBottom: 1 }, + receiptNumber: { fontSize: 11, bold: true, color: '#333333', marginBottom: 3 }, + receiptMeta: { fontSize: 8, color: '#666666' }, + totalLabel: { fontSize: 11, bold: true }, + totalAmount: { fontSize: 11, bold: true }, + paymentInfo: { fontSize: 9, color: '#555555' }, + thankYou: { fontSize: 11, bold: true, color: '#1a73e8' }, + notes: { fontSize: 8, color: '#777777', italics: true }, + }, + content, + }; +} diff --git a/src/intelligence/templates/report-template.ts b/src/intelligence/templates/report-template.ts new file mode 100644 index 00000000..338980f7 --- /dev/null +++ b/src/intelligence/templates/report-template.ts @@ -0,0 +1,242 @@ +import type { TDocumentDefinitions, Content } from '../pdf-generator.js'; +import type { Branding } from './invoice-template.js'; + +export type { Branding }; + +/** A single cell in a report table */ +export type TableCell = string | number | boolean | null | undefined; + +/** A table block within a report section */ +export interface ReportTable { + type: 'table'; + headers: string[]; + rows: TableCell[][]; +} + +/** A text block within a report section */ +export interface ReportText { + type: 'text'; + content: string; +} + +/** A chart block within a report section (rendered as a base64 image) */ +export interface ReportChart { + type: 'chart'; + /** Base64 PNG data URI (e.g. "data:image/png;base64,...") */ + imageDataUri: string; + caption?: string; + width?: number; +} + +/** Union of all supported section block types */ +export type ReportBlock = ReportText | ReportTable | ReportChart; + +/** A single section in the report */ +export interface ReportSection { + title: string; + blocks: ReportBlock[]; +} + +/** + * Build a pdfmake document definition for a professional report. + * + * Features: + * - Title page with report title, company name, and date + * - Auto-generated table of contents (section headings list) + * - Sections with text paragraphs, data tables, and chart images + * - Page numbers in the footer + */ +export function buildReportDefinition( + title: string, + sections: ReportSection[], + branding: Branding, +): TDocumentDefinitions { + const primaryColor = branding.primaryColor ?? '#1a73e8'; + const today = new Date().toLocaleDateString('en-US', { + year: 'numeric', + month: 'long', + day: 'numeric', + }); + + // Collect images from chart blocks + const images: Record = {}; + if (branding.logoDataUri) { + images['reportLogo'] = branding.logoDataUri; + } + + sections.forEach((section, sIdx) => { + section.blocks.forEach((block, bIdx) => { + if (block.type === 'chart') { + images[`chart_${sIdx}_${bIdx}`] = block.imageDataUri; + } + }); + }); + + // ── Title page ───────────────────────────────────────────────────────── + + const titlePageContent: Content[] = []; + + if (branding.logoDataUri) { + titlePageContent.push({ + image: 'reportLogo', + width: 140, + alignment: 'center' as const, + marginBottom: 40, + }); + } + + titlePageContent.push( + { + text: title, + style: 'reportTitle', + alignment: 'center' as const, + color: primaryColor, + marginBottom: 16, + }, + { + text: branding.companyName, + style: 'reportCompany', + alignment: 'center' as const, + marginBottom: 8, + }, + { + text: today, + style: 'reportDate', + alignment: 'center' as const, + marginBottom: 40, + }, + // Page break after title page + { text: '', pageBreak: 'after' as const }, + ); + + // ── Table of contents ────────────────────────────────────────────────── + + const tocRows: Content[] = [ + { + text: 'Table of Contents', + style: 'tocTitle', + color: primaryColor, + marginBottom: 12, + }, + ]; + + sections.forEach((section, idx) => { + tocRows.push({ + columns: [ + { text: `${idx + 1}. ${section.title}`, style: 'tocEntry', width: '*' }, + { text: String(idx + 2), style: 'tocPage', width: 'auto', alignment: 'right' as const }, + ], + marginBottom: 6, + }); + }); + + tocRows.push({ text: '', pageBreak: 'after' as const }); + + // ── Sections ─────────────────────────────────────────────────────────── + + const sectionContent: Content[] = []; + + sections.forEach((section, sIdx) => { + sectionContent.push({ + text: `${sIdx + 1}. ${section.title}`, + style: 'sectionTitle', + color: primaryColor, + marginBottom: 10, + }); + + section.blocks.forEach((block, bIdx) => { + if (block.type === 'text') { + sectionContent.push({ + text: block.content, + style: 'bodyText', + marginBottom: 10, + }); + } else if (block.type === 'table') { + const headerRow: Content[] = block.headers.map((h) => ({ + text: h, + style: 'tableHeader', + color: 'white', + fillColor: primaryColor, + })); + + const dataRows: Content[][] = block.rows.map((row, rowIdx) => { + const bg = rowIdx % 2 === 0 ? '#f8f9fa' : 'white'; + return block.headers.map((_h, colIdx) => ({ + text: String(row[colIdx] ?? ''), + fillColor: bg, + })); + }); + + const colWidths: string[] = block.headers.map(() => '*'); + + sectionContent.push({ + table: { + headerRows: 1, + widths: colWidths, + body: [headerRow, ...dataRows], + }, + layout: 'lightHorizontalLines', + marginBottom: 14, + }); + } else if (block.type === 'chart') { + const imageKey = `chart_${sIdx}_${bIdx}`; + sectionContent.push({ + image: imageKey, + width: block.width ?? 460, + alignment: 'center' as const, + marginBottom: block.caption ? 4 : 14, + }); + if (block.caption) { + sectionContent.push({ + text: block.caption, + style: 'chartCaption', + alignment: 'center' as const, + marginBottom: 14, + }); + } + } + }); + + // Page break between sections (not after the last one) + if (sIdx < sections.length - 1) { + sectionContent.push({ text: '', pageBreak: 'after' as const }); + } + }); + + // ── Assemble document ────────────────────────────────────────────────── + + const content: Content[] = [...titlePageContent, ...tocRows, ...sectionContent]; + + return { + pageSize: 'A4', + pageMargins: [50, 60, 50, 60] as [number, number, number, number], + defaultStyle: { font: 'Roboto', fontSize: 10, lineHeight: 1.4 }, + ...(Object.keys(images).length > 0 ? { images } : {}), + footer: (currentPage: number, pageCount: number): Content => ({ + columns: [ + { text: branding.companyName, style: 'footerText', width: '*' }, + { + text: `Page ${currentPage} of ${pageCount}`, + style: 'footerText', + alignment: 'right' as const, + width: 'auto', + }, + ], + margin: [50, 10, 50, 0] as [number, number, number, number], + }), + styles: { + reportTitle: { fontSize: 28, bold: true, marginBottom: 8 }, + reportCompany: { fontSize: 14, color: '#444444', marginBottom: 4 }, + reportDate: { fontSize: 11, color: '#888888' }, + tocTitle: { fontSize: 16, bold: true }, + tocEntry: { fontSize: 10 }, + tocPage: { fontSize: 10, color: '#888888' }, + sectionTitle: { fontSize: 16, bold: true }, + bodyText: { fontSize: 10, color: '#333333', lineHeight: 1.5 }, + tableHeader: { fontSize: 10, bold: true }, + chartCaption: { fontSize: 9, color: '#666666', italics: true }, + footerText: { fontSize: 8, color: '#aaaaaa' }, + }, + content, + }; +} diff --git a/src/intelligence/user-preferences.ts b/src/intelligence/user-preferences.ts new file mode 100644 index 00000000..14bf9e34 --- /dev/null +++ b/src/intelligence/user-preferences.ts @@ -0,0 +1,275 @@ +import type Database from 'better-sqlite3'; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +/** Preferred response format learned from interaction patterns */ +export type ResponseFormat = 'brief' | 'detailed' | 'unknown'; + +/** A user preference record stored in SQLite */ +export interface UserPreference { + /** Sender identifier (phone number, user ID, etc.) */ + sender: string; + /** Preferred response format inferred from interactions */ + responseFormat: ResponseFormat; + /** Top request types, most frequent first (e.g. "report", "lookup", "create") */ + commonRequestTypes: string[]; + /** Active hours as UTC hour numbers (0–23) seen in past interactions */ + workingHours: number[]; + /** Detected language code (e.g. "en", "fr", "ar") or empty string when unknown */ + languagePreference: string; + /** ISO timestamp of the first recorded interaction */ + firstSeenAt: string; + /** ISO timestamp of the most recent interaction */ + lastSeenAt: string; + /** Total number of interactions recorded */ + interactionCount: number; +} + +// --------------------------------------------------------------------------- +// SQLite row type +// --------------------------------------------------------------------------- + +interface UserPreferenceRow { + sender: string; + response_format: string; + common_request_types: string; + working_hours: string; + language_preference: string; + first_seen_at: string; + last_seen_at: string; + interaction_count: number; +} + +function rowToPreference(row: UserPreferenceRow): UserPreference { + return { + sender: row.sender, + responseFormat: (row.response_format as ResponseFormat) ?? 'unknown', + commonRequestTypes: JSON.parse(row.common_request_types) as string[], + workingHours: JSON.parse(row.working_hours) as number[], + languagePreference: row.language_preference ?? '', + firstSeenAt: row.first_seen_at, + lastSeenAt: row.last_seen_at, + interactionCount: row.interaction_count, + }; +} + +// --------------------------------------------------------------------------- +// Migration +// --------------------------------------------------------------------------- + +/** + * Ensure the `user_preferences` table exists. + * Safe to call multiple times (uses CREATE TABLE IF NOT EXISTS). + */ +export function ensureUserPreferencesTable(db: Database.Database): void { + db.prepare( + `CREATE TABLE IF NOT EXISTS user_preferences ( + sender TEXT PRIMARY KEY, + response_format TEXT NOT NULL DEFAULT 'unknown', + common_request_types TEXT NOT NULL DEFAULT '[]', + working_hours TEXT NOT NULL DEFAULT '[]', + language_preference TEXT NOT NULL DEFAULT '', + first_seen_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + interaction_count INTEGER NOT NULL DEFAULT 0 + )`, + ).run(); +} + +// --------------------------------------------------------------------------- +// CRUD helpers +// --------------------------------------------------------------------------- + +/** + * Record a new interaction for the given sender. + * Creates the preference row on first call; updates statistics on subsequent calls. + * + * @param db SQLite database instance + * @param sender Sender identifier + * @param requestType Detected request type label (e.g. "report", "lookup") + * @param language Detected language code (e.g. "en") — empty string to skip + */ +export function recordInteraction( + db: Database.Database, + sender: string, + requestType?: string, + language?: string, +): void { + const now = new Date().toISOString(); + const currentHour = new Date().getUTCHours(); + + // Upsert: create row if first interaction, otherwise merge stats + const existing = db.prepare('SELECT * FROM user_preferences WHERE sender = ?').get(sender) as + | UserPreferenceRow + | undefined; + + if (!existing) { + const requestTypes = requestType ? JSON.stringify([requestType]) : '[]'; + db.prepare( + `INSERT INTO user_preferences + (sender, response_format, common_request_types, working_hours, language_preference, first_seen_at, last_seen_at, interaction_count) + VALUES (?, 'unknown', ?, ?, ?, ?, ?, 1)`, + ).run(sender, requestTypes, JSON.stringify([currentHour]), language ?? '', now, now); + return; + } + + // Merge request types (keep top 10 most frequent) + const requestTypes = JSON.parse(existing.common_request_types) as string[]; + if (requestType && requestType.length > 0) { + requestTypes.push(requestType); + } + const typeCounts = new Map(); + for (const t of requestTypes) { + typeCounts.set(t, (typeCounts.get(t) ?? 0) + 1); + } + const topTypes = [...typeCounts.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, 10) + .map(([t]) => t); + + // Merge working hours (keep unique hours seen, max 24 entries) + const hours = JSON.parse(existing.working_hours) as number[]; + if (!hours.includes(currentHour)) { + hours.push(currentHour); + hours.sort((a, b) => a - b); + } + + // Language: update only if a non-empty value is provided + const lang = language && language.length > 0 ? language : existing.language_preference; + + db.prepare( + `UPDATE user_preferences SET + common_request_types = ?, + working_hours = ?, + language_preference = ?, + last_seen_at = ?, + interaction_count = interaction_count + 1 + WHERE sender = ?`, + ).run(JSON.stringify(topTypes), JSON.stringify(hours), lang, now, sender); +} + +/** + * Update the inferred response format for a sender. + * Call this when there is enough signal to determine whether the user + * prefers brief or detailed responses. + */ +export function setResponseFormat( + db: Database.Database, + sender: string, + format: ResponseFormat, +): void { + db.prepare(`UPDATE user_preferences SET response_format = ? WHERE sender = ?`).run( + format, + sender, + ); +} + +/** + * Retrieve the stored preferences for a sender. + * Returns null when no interactions have been recorded yet. + */ +export function getUserPreference(db: Database.Database, sender: string): UserPreference | null { + const row = db.prepare('SELECT * FROM user_preferences WHERE sender = ?').get(sender) as + | UserPreferenceRow + | undefined; + return row ? rowToPreference(row) : null; +} + +/** + * Return all stored user preference rows. + */ +export function listUserPreferences(db: Database.Database): UserPreference[] { + const rows = db + .prepare('SELECT * FROM user_preferences ORDER BY last_seen_at DESC') + .all() as UserPreferenceRow[]; + return rows.map(rowToPreference); +} + +/** + * Delete a sender's preference record. + * Returns true if a row was removed. + */ +export function deleteUserPreference(db: Database.Database, sender: string): boolean { + const result = db.prepare('DELETE FROM user_preferences WHERE sender = ?').run(sender); + return result.changes > 0; +} + +// --------------------------------------------------------------------------- +// Prompt injection +// --------------------------------------------------------------------------- + +/** + * Build the `## User Preferences for {sender}` section for injection into + * the Master AI prompt. Returns null when no preferences are stored. + */ +export function buildUserPreferencesSection(db: Database.Database, sender: string): string | null { + const prefs = getUserPreference(db, sender); + if (!prefs || prefs.interactionCount < 2) return null; + + const lines: string[] = [`## User Preferences for ${sender}`, '']; + + if (prefs.responseFormat !== 'unknown') { + lines.push( + `- **Response style:** ${prefs.responseFormat === 'brief' ? 'Keep responses concise and to the point.' : 'Provide detailed, thorough responses.'}`, + ); + } + + if (prefs.commonRequestTypes.length > 0) { + lines.push(`- **Common request types:** ${prefs.commonRequestTypes.slice(0, 5).join(', ')}`); + } + + if (prefs.workingHours.length > 0) { + const formatted = formatWorkingHours(prefs.workingHours); + if (formatted) { + lines.push(`- **Typically active:** ${formatted}`); + } + } + + if (prefs.languagePreference && prefs.languagePreference.length > 0) { + lines.push(`- **Language preference:** ${prefs.languagePreference}`); + } + + lines.push(`- **Interactions recorded:** ${prefs.interactionCount}`); + lines.push(''); + + return lines.join('\n'); +} + +// --------------------------------------------------------------------------- +// Internal helpers +// --------------------------------------------------------------------------- + +/** + * Format a sorted list of UTC hours (0–23) into a human-readable active window. + * E.g. [8, 9, 10, 11, 14, 15] → "08:00–12:00 UTC, 14:00–16:00 UTC" + */ +function formatWorkingHours(hours: number[]): string { + if (hours.length === 0) return ''; + + // Group contiguous hours into ranges + const sorted = [...new Set(hours)].sort((a, b) => a - b); + const ranges: Array<[number, number]> = []; + let rangeStart = sorted[0]!; + let prev = sorted[0]!; + + for (let i = 1; i < sorted.length; i++) { + const h = sorted[i]!; + if (h === prev + 1) { + prev = h; + } else { + ranges.push([rangeStart, prev + 1]); + rangeStart = h; + prev = h; + } + } + ranges.push([rangeStart, prev + 1]); + + return ranges + .map( + ([start, end]) => + `${String(start).padStart(2, '0')}:00–${String(end).padStart(2, '0')}:00 UTC`, + ) + .join(', '); +} diff --git a/src/master/batch-manager.ts b/src/master/batch-manager.ts new file mode 100644 index 00000000..d400f987 --- /dev/null +++ b/src/master/batch-manager.ts @@ -0,0 +1,889 @@ +/** + * Batch Manager — Lifecycle management for Batch Task Continuation. + * + * When a user asks to "implement all tasks", "go through each one", or similar, + * the Master AI activates Batch Mode. BatchManager owns the state machine that + * drives sequential item processing through a batch run. + * + * Responsibilities: + * - Create / advance / pause / resume / abort batch runs + * - Track per-item completion status and cumulative cost + * - Persist batch state to `.openbridge/batch-state.json` after every mutation + * - Load persisted state on `initialize()` to resume interrupted batches + * - Expose `isActive()` and `getStatus()` for integration into MasterManager + * + * This class does NOT read TASKS.md or FINDINGS.md — that is the responsibility + * of subsequent tasks (OB-1607). It only manages lifecycle state. + */ + +import { randomUUID } from 'node:crypto'; + +import { createLogger } from '../core/logger.js'; +import type { + BatchCompletedItem, + BatchPlanItem, + BatchSourceType, + BatchState, +} from '../types/agent.js'; +import type { BatchConfig } from '../types/config.js'; +import type { DotFolderManager } from './dotfolder-manager.js'; + +const logger = createLogger('batch-manager'); + +// ── Public result types ──────────────────────────────────────────── + +/** Summary returned by advanceBatch() after recording an item result. */ +export interface BatchAdvanceResult { + /** The batch identifier. */ + batchId: string; + /** Zero-based index of the item just completed. */ + completedIndex: number; + /** Zero-based index of the next item to process, or null when the batch is done. */ + nextIndex: number | null; + /** Whether the batch has finished all items. */ + finished: boolean; + /** + * Populated when finished === true — the formatted completion summary to send to the user (OB-1618). + * Includes total completed/failed/skipped, cost, duration, and per-item summaries. + */ + completionSummary: string | null; +} + +/** Describes a single safety rail limit that was exceeded. */ +export interface SafetyRailViolation { + /** Which limit was exceeded. */ + rail: 'iterations' | 'budget' | 'timeout'; + /** Human-readable explanation suitable for surfacing to the user. */ + message: string; +} + +/** Result of a safety rail check. */ +export interface SafetyRailCheckResult { + /** True when all rails pass and the batch may continue. */ + passed: boolean; + /** Populated when passed is false — the first violation that triggered the pause. */ + violation: SafetyRailViolation | undefined; +} + +// ── BatchManager ─────────────────────────────────────────────────── + +/** + * Manages the lifecycle of Batch Task Continuation runs. + * + * A single batch is supported at a time per BatchManager instance. + * Only one batch should be active for a given Master AI session. + */ +export class BatchManager { + /** All tracked batches keyed by batchId. */ + private readonly batches = new Map(); + + /** Optional persistence layer — when set, batch state is saved after every mutation. */ + private dotFolder: DotFolderManager | undefined; + + /** + * Maps batchId → original sender info for routing failure/status messages (OB-1667). + * Lives independently of BatchState lifecycle so it survives state deletion on completion. + */ + private readonly batchSenderInfo = new Map(); + + /** Stores the completion summary for the most-recently-finished batch (OB-1618). */ + private lastCompletionSummary: string | null = null; + + constructor(dotFolder?: DotFolderManager) { + this.dotFolder = dotFolder; + } + + // ── Persistence helpers ──────────────────────────────────────── + + /** + * Load any persisted batch state from `.openbridge/batch-state.json` on startup. + * Resumes the batch as paused so the caller can decide whether to continue. + * No-op when no persistence layer is configured or no saved state exists. + */ + async initialize(): Promise { + if (!this.dotFolder) return; + + const saved = await this.dotFolder.readBatchState(); + if (!saved) return; + + // Resume as paused — the caller (MasterManager) will unpause when ready. + saved.paused = true; + this.batches.set(saved.batchId, saved); + + // Restore sender info so failure/completion messages route correctly after restart (OB-1667). + if (saved.senderInfo) { + this.batchSenderInfo.set(saved.batchId, saved.senderInfo); + } + + logger.info( + { + batchId: saved.batchId, + currentIndex: saved.currentIndex, + totalItems: saved.totalItems, + completedItems: saved.completedItems.length, + hasSenderInfo: saved.senderInfo !== undefined, + }, + 'Resumed batch from persisted state', + ); + } + + /** Persist the given batch state to disk (no-op when no dotFolder is set). */ + private async persist(state: BatchState): Promise { + if (!this.dotFolder) return; + try { + await this.dotFolder.writeBatchState(state); + } catch (err) { + logger.warn({ batchId: state.batchId, err }, 'Failed to persist batch state'); + } + } + + /** Delete the persisted batch state file (no-op when no dotFolder is set). */ + private async deletePersisted(): Promise { + if (!this.dotFolder) return; + try { + await this.dotFolder.deleteBatchState(); + } catch (err) { + logger.warn({ err }, 'Failed to delete persisted batch state'); + } + } + + // ── Lifecycle methods ────────────────────────────────────────── + + /** + * Create a new batch run. + * + * @param sourceType Where the item list comes from (tasks-md, findings, custom-list). + * @param plan Ordered list of items to process (from BatchPlanner). + * When provided, totalItems is derived from plan.length. + * When omitted, pass totalItems explicitly. + * @param totalItems Number of items — used only when plan is not provided. + * @param commitAfterEach When true, a git-commit worker is spawned after each item (OB-1615). + * @returns The new batch ID. + */ + async createBatch( + sourceType: BatchSourceType, + plan: BatchPlanItem[], + totalItems?: number, + commitAfterEach = false, + ): Promise { + const batchId = randomUUID(); + const resolvedTotal = plan.length > 0 ? plan.length : (totalItems ?? 0); + + const state: BatchState = { + batchId, + sourceType, + totalItems: resolvedTotal, + currentIndex: 0, + plan, + completedItems: [], + failedItems: [], + startedAt: new Date().toISOString(), + totalCostUsd: 0, + paused: false, + commitAfterEach, + }; + + this.batches.set(batchId, state); + await this.persist(state); + + logger.info( + { batchId, sourceType, totalItems: resolvedTotal, planItems: plan.length }, + 'Batch created', + ); + + return batchId; + } + + /** + * Record the result of the current item and advance to the next one. + * + * @param batchId The active batch identifier. + * @param item Completion record for the item just processed. + * @param costUsd Cost incurred for this item in USD (added to running total). + * @returns Advance result with the next index, or null when the batch is done. + * Returns null if the batchId is unknown. + */ + async advanceBatch( + batchId: string, + item: BatchCompletedItem, + costUsd = 0, + ): Promise { + const state = this.batches.get(batchId); + if (!state) { + logger.warn({ batchId }, 'advanceBatch() called on unknown batch'); + return null; + } + + const completedIndex = state.currentIndex; + + // Record the completed item + state.completedItems.push(item); + if (item.status === 'failed') { + state.failedItems.push(item.id); + } + + // Accumulate cost + state.totalCostUsd += costUsd; + + // Advance to next item + const nextIndex = completedIndex + 1; + const finished = nextIndex >= state.totalItems; + + state.currentIndex = finished ? state.totalItems : nextIndex; + + logger.info( + { + batchId, + completedIndex, + nextIndex: finished ? null : nextIndex, + itemStatus: item.status, + finished, + totalCostUsd: state.totalCostUsd, + }, + 'Batch item completed', + ); + + let completionSummary: string | null = null; + if (finished) { + // Build the completion summary BEFORE removing state from memory (OB-1618). + completionSummary = this.buildCompletionSummaryText(state); + this.lastCompletionSummary = completionSummary; + // Batch is complete — remove in-progress state from disk + this.batches.delete(batchId); + await this.deletePersisted(); + } else { + await this.persist(state); + } + + return { + batchId, + completedIndex, + nextIndex: finished ? null : nextIndex, + finished, + completionSummary, + }; + } + + /** + * Pause an active batch (e.g., awaiting user confirmation or safety rail trigger). + * + * @param batchId The batch to pause. + * @returns True if the batch was found and paused, false otherwise. + */ + async pauseBatch(batchId: string): Promise { + const state = this.batches.get(batchId); + if (!state) { + logger.warn({ batchId }, 'pauseBatch() called on unknown batch'); + return false; + } + + state.paused = true; + await this.persist(state); + + logger.info({ batchId, currentIndex: state.currentIndex }, 'Batch paused'); + return true; + } + + /** + * Resume a paused batch. + * + * @param batchId The batch to resume. + * @returns True if the batch was found and resumed, false otherwise. + */ + async resumeBatch(batchId: string): Promise { + const state = this.batches.get(batchId); + if (!state) { + logger.warn({ batchId }, 'resumeBatch() called on unknown batch'); + return false; + } + + if (!state.paused) { + logger.warn({ batchId }, 'resumeBatch() called on a batch that is not paused'); + } + + state.paused = false; + await this.persist(state); + + logger.info({ batchId, currentIndex: state.currentIndex }, 'Batch resumed'); + return true; + } + + /** + * Abort a batch and remove it from memory and disk. + * + * @param batchId The batch to abort. + * @returns True if the batch was found and aborted, false if it was unknown. + */ + async abortBatch(batchId: string): Promise { + const state = this.batches.get(batchId); + if (!state) { + logger.warn({ batchId }, 'abortBatch() called on unknown batch'); + return false; + } + + logger.info( + { + batchId, + completedItems: state.completedItems.length, + currentIndex: state.currentIndex, + totalItems: state.totalItems, + }, + 'Batch aborted', + ); + + // Build abort summary BEFORE removing state (OB-1622). + this.lastCompletionSummary = this.buildAbortSummaryText(state); + + this.batches.delete(batchId); + await this.deletePersisted(); + return true; + } + + /** + * Build a formatted summary for a user-aborted batch (OB-1622). + * + * Similar to buildCompletionSummaryText but uses an abort header and shows + * how many items remain uncompleted. + */ + private buildAbortSummaryText(state: BatchState): string { + const completed = state.completedItems.filter((i) => i.status === 'completed').length; + const failed = state.completedItems.filter((i) => i.status === 'failed').length; + const skipped = state.completedItems.filter((i) => i.status === 'skipped').length; + const remaining = state.totalItems - state.currentIndex; + + const durationMs = Date.now() - new Date(state.startedAt).getTime(); + const totalSeconds = Math.round(durationMs / 1000); + const hours = Math.floor(totalSeconds / 3600); + const minutes = Math.floor((totalSeconds % 3600) / 60); + const seconds = totalSeconds % 60; + let durationStr: string; + if (hours > 0) { + durationStr = `${hours}h ${minutes}m ${seconds}s`; + } else if (minutes > 0) { + durationStr = `${minutes}m ${seconds}s`; + } else { + durationStr = `${seconds}s`; + } + + const costStr = state.totalCostUsd > 0 ? `$${state.totalCostUsd.toFixed(4)}` : 'not tracked'; + + const lines: string[] = [ + '🛑 Batch aborted.', + '', + `**Completed:** ${completed} done, ${failed} failed, ${skipped} skipped`, + `**Remaining:** ${remaining} item${remaining !== 1 ? 's' : ''} cancelled`, + `**Cost so far:** ${costStr}`, + `**Duration:** ${durationStr}`, + ]; + + if (state.completedItems.length > 0) { + lines.push('', '**Completed items:**'); + for (const item of state.completedItems) { + const icon = item.status === 'failed' ? '❌' : item.status === 'skipped' ? '⏭' : '✅'; + const summary = item.summary ? ` — ${item.summary}` : ''; + lines.push(`- ${icon} ${item.id}${summary}`); + } + } + + return lines.join('\n'); + } + + // ── Completion summary ───────────────────────────────────────── + + /** + * Build a formatted completion summary string from the final batch state (OB-1618). + * + * Includes: total completed/failed/skipped counts, cumulative cost, wall-clock duration, + * and a per-item list with status icons and one-line summaries. + * + * Called internally by advanceBatch() just before the completed batch state is deleted. + */ + private buildCompletionSummaryText(state: BatchState): string { + const completed = state.completedItems.filter((i) => i.status === 'completed').length; + const failed = state.completedItems.filter((i) => i.status === 'failed').length; + const skipped = state.completedItems.filter((i) => i.status === 'skipped').length; + + const durationMs = Date.now() - new Date(state.startedAt).getTime(); + const totalSeconds = Math.round(durationMs / 1000); + const hours = Math.floor(totalSeconds / 3600); + const minutes = Math.floor((totalSeconds % 3600) / 60); + const seconds = totalSeconds % 60; + let durationStr: string; + if (hours > 0) { + durationStr = `${hours}h ${minutes}m ${seconds}s`; + } else if (minutes > 0) { + durationStr = `${minutes}m ${seconds}s`; + } else { + durationStr = `${seconds}s`; + } + + const costStr = state.totalCostUsd > 0 ? `$${state.totalCostUsd.toFixed(4)}` : 'not tracked'; + + const lines: string[] = [ + '🎉 Batch complete!', + '', + `**Results:** ${completed} completed, ${failed} failed, ${skipped} skipped`, + `**Total cost:** ${costStr}`, + `**Duration:** ${durationStr}`, + ]; + + if (state.completedItems.length > 0) { + lines.push('', '**Items:**'); + for (const item of state.completedItems) { + const icon = item.status === 'failed' ? '❌' : item.status === 'skipped' ? '⏭' : '✅'; + const summary = item.summary ? ` — ${item.summary}` : ''; + lines.push(`- ${icon} ${item.id}${summary}`); + } + } + + return lines.join('\n'); + } + + /** + * Return the completion summary for the most recently finished batch and clear it (OB-1618). + * + * Should be called by MasterManager immediately after detecting that a batch has finished + * (i.e. advanceBatch() returned finished === true). Returns null if no summary is available + * (e.g. the batch was aborted rather than completed naturally). + */ + popCompletionSummary(): string | null { + const summary = this.lastCompletionSummary; + this.lastCompletionSummary = null; + return summary; + } + + // ── Sender info ──────────────────────────────────────────────── + + /** + * Store the original sender info for a batch (OB-1667). + * + * Persists sender info into the batch state on disk so it survives process restarts. + * The in-memory map is updated synchronously; disk write is fire-and-forget. + * + * @param batchId The batch to associate with the sender. + * @param info The sender identifier and source connector. + */ + setSenderInfo(batchId: string, info: { sender: string; source: string }): void { + this.batchSenderInfo.set(batchId, info); + // Persist sender info in the batch state so it's restored on restart. + const state = this.batches.get(batchId); + if (state) { + state.senderInfo = info; + void this.persist(state); + } + } + + /** + * Retrieve the original sender info for a batch (OB-1667). + * + * Returns from the in-memory map, which persists beyond batch state deletion so + * completion/failure messages can still be routed after the batch finishes. + * + * @param batchId The batch to look up. + * @returns Sender info, or undefined if not set. + */ + getSenderInfo(batchId: string): { sender: string; source: string } | undefined { + return this.batchSenderInfo.get(batchId); + } + + /** + * Remove the stored sender info for a batch (OB-1667). + * + * Should be called after the completion/failure message has been delivered. + * + * @param batchId The batch whose sender info should be removed. + */ + deleteSenderInfo(batchId: string): void { + this.batchSenderInfo.delete(batchId); + } + + // ── Query methods ────────────────────────────────────────────── + + /** + * Return a snapshot of the current batch state. + * + * @param batchId The batch to query. + * @returns Current state, or undefined if the batch is unknown. + */ + getStatus(batchId: string): BatchState | undefined { + const state = this.batches.get(batchId); + if (!state) return undefined; + + // Return a shallow copy so callers cannot mutate internal state + return { + ...state, + completedItems: [...state.completedItems], + failedItems: [...state.failedItems], + }; + } + + /** + * Check whether any batch (or a specific one) is currently active. + * + * A batch is considered active if it exists and is not paused and has items remaining. + * + * @param batchId Optional — check a specific batch ID. + * When omitted, returns true if ANY batch has remaining items. + * @returns True when an active, unpaused batch is found. + */ + isActive(batchId?: string): boolean { + if (batchId !== undefined) { + const state = this.batches.get(batchId); + if (!state) return false; + return !state.paused && state.currentIndex < state.totalItems; + } + + // Check any batch + for (const state of this.batches.values()) { + if (!state.paused && state.currentIndex < state.totalItems) { + return true; + } + } + return false; + } + + /** + * Return the ID of the first batch that has items remaining, or undefined if none exist. + * + * Unlike `isActive()`, this method returns the ID even when the batch is **paused** — + * it only requires that `currentIndex < totalItems`. Use this to locate the current batch + * for status queries, context injection, and restart recovery. Use `isActive()` when you + * need to know whether the batch is actually running (unpaused). + * + * Useful for MasterManager to locate the current batch without storing the ID separately. + */ + getCurrentBatchId(): string | undefined { + for (const [id, state] of this.batches.entries()) { + if (state.currentIndex < state.totalItems) { + return id; + } + } + return undefined; + } + + /** + * Build a user-facing progress message for the batch after an item completes (OB-1614). + * + * Returns a formatted string when there is at least one completed item and a next + * item remaining in the plan. Returns null when there is nothing useful to show + * (e.g. no completed items yet, batch done, or batch unknown). + * + * Format: + * ✅ Task {id} done: {summary} + * Starting {nextId}... ({current}/{total}) + * + * @param batchId The active batch identifier. + * @returns Formatted progress message, or null. + */ + buildProgressMessage(batchId: string): string | null { + const state = this.batches.get(batchId); + if (!state) return null; + + const lastCompleted = state.completedItems[state.completedItems.length - 1]; + if (!lastCompleted) return null; + + // No next item — batch is done; completion summary is handled by OB-1618. + const nextItem = state.plan[state.currentIndex]; + if (!nextItem) return null; + + const current = state.completedItems.length; + const total = state.totalItems; + + const statusIcon = lastCompleted.status === 'failed' ? '❌' : '✅'; + const summaryLine = lastCompleted.summary + ? `${statusIcon} Task ${lastCompleted.id} done: ${lastCompleted.summary}` + : `${statusIcon} Task ${lastCompleted.id} done.`; + + return `${summaryLine}\nStarting ${nextItem.id}... (${current}/${total})`; + } + + // ── Failure handling ─────────────────────────────────────────── + + /** + * Build a user-facing failure message for a failed batch item (OB-1616). + * + * Returns the formatted string with instructions for how the user can respond: + * ❌ Task {id} failed: {reason} + * Reply '/batch skip' to skip and continue, '/batch retry' to retry, '/batch abort' to stop. + * + * @param batchId The active batch identifier. + * @param reason Short human-readable reason the item failed. + * @returns Formatted failure message, or null if the batch is unknown. + */ + buildFailureMessage(batchId: string, reason: string): string | null { + const state = this.batches.get(batchId); + if (!state) return null; + + const currentItem = state.plan[state.currentIndex]; + const itemId = currentItem?.id ?? `item-${state.currentIndex + 1}`; + + return ( + `❌ Task ${itemId} failed: ${reason}\n` + + `Reply '/batch skip' to skip and continue, '/batch retry' to retry, '/batch abort' to stop.` + ); + } + + /** + * Skip the current failed item and advance to the next one (OB-1616). + * + * Records the current item as 'skipped', resumes the batch (clears paused flag), + * and advances to the next index via advanceBatch(). + * + * @param batchId The batch whose current item should be skipped. + * @returns BatchAdvanceResult (same as advanceBatch), or null on unknown batch. + */ + async skipCurrentItem(batchId: string): Promise { + const state = this.batches.get(batchId); + if (!state) { + logger.warn({ batchId }, 'skipCurrentItem() called on unknown batch'); + return null; + } + + const currentItem = state.plan[state.currentIndex]; + const item: BatchCompletedItem = { + id: currentItem?.id ?? `item-${state.currentIndex + 1}`, + summary: 'Skipped by user', + status: 'skipped', + }; + + // Clear paused so advanceBatch can record and advance + state.paused = false; + + logger.info({ batchId, itemId: item.id }, 'Batch item skipped by user'); + return this.advanceBatch(batchId, item); + } + + /** + * Retry the current failed item by resuming the batch at the same index (OB-1616). + * + * This is equivalent to resumeBatch() — the currentIndex is NOT advanced, + * so the next continuation trigger will re-run the same item. + * + * @param batchId The batch whose current item should be retried. + * @returns True if the batch was found and resumed, false otherwise. + */ + async retryCurrentItem(batchId: string): Promise { + const state = this.batches.get(batchId); + if (!state) { + logger.warn({ batchId }, 'retryCurrentItem() called on unknown batch'); + return false; + } + + const currentItem = state.plan[state.currentIndex]; + logger.info( + { batchId, itemId: currentItem?.id, currentIndex: state.currentIndex }, + 'Batch item retry requested — resuming at same index', + ); + + return this.resumeBatch(batchId); + } + + // ── Commit-after-each support ───────────────────────────────── + + /** + * Set or clear the commitAfterEach flag on an existing batch (OB-1615). + * + * @param batchId The batch to update. + * @param commitAfterEach Whether to spawn a git-commit worker after each item. + * @returns True if the batch was found and updated, false otherwise. + */ + async setCommitAfterEach(batchId: string, commitAfterEach: boolean): Promise { + const state = this.batches.get(batchId); + if (!state) { + logger.warn({ batchId }, 'setCommitAfterEach() called on unknown batch'); + return false; + } + + state.commitAfterEach = commitAfterEach; + await this.persist(state); + + logger.info({ batchId, commitAfterEach }, 'Batch commitAfterEach updated'); + return true; + } + + /** + * Return whether the given batch should spawn a commit worker after each item (OB-1615). + * + * @param batchId The batch to check. + * @returns True when commitAfterEach is set and the batch exists. + */ + shouldCommitAfterEach(batchId: string): boolean { + return this.batches.get(batchId)?.commitAfterEach === true; + } + + /** + * Build the prompt for a git-commit worker after a batch item completes (OB-1615). + * + * The prompt instructs the worker to stage all changes and create a conventional commit + * referencing the completed item's ID and description. + * + * @param batchId The active batch identifier. + * @returns Commit worker prompt string, or null when no last completed item exists. + */ + buildCommitPrompt(batchId: string): string | null { + const state = this.batches.get(batchId); + if (!state) return null; + + const lastCompleted = state.completedItems[state.completedItems.length - 1]; + if (!lastCompleted) return null; + + // Find the plan item to get its description + const planItem = state.plan.find((p) => p.id === lastCompleted.id); + const description = planItem?.description ?? lastCompleted.summary ?? lastCompleted.id; + + return ( + `git add and commit changes for: ${description}\n\n` + + `Use a conventional commit message. Reference the task ID "${lastCompleted.id}" in the commit body.\n` + + `Run: git add -A && git commit -m "feat: ${description.slice(0, 72)}" -m "Resolves ${lastCompleted.id}"\n` + + `If there is nothing to commit (working tree clean), skip the commit and exit cleanly.` + ); + } + + // ── Safety rails ─────────────────────────────────────────────── + + /** + * Check all safety rails before processing the next batch iteration. + * + * Evaluates three limits from the provided config: + * - `maxBatchIterations` — total completed items must be below this cap. + * - `batchBudgetUsd` — cumulative cost must remain below this value. + * - `batchTimeoutMinutes`— wall-clock time from batch start must be below this. + * + * If any limit is exceeded the batch is automatically paused (so it can be resumed + * after user confirmation) and the violation is returned. The caller is responsible + * for forwarding the `violation.message` to the user. + * + * @param batchId The batch to check. + * @param config Safety limits from the config (BatchConfig fields). + * @returns `{ passed: true }` when safe to continue, or `{ passed: false, violation }`. + * Returns `{ passed: true }` for unknown batch IDs (fail-open — let + * other guards handle missing state). + */ + async checkSafetyRails( + batchId: string, + config: Pick, + ): Promise { + const state = this.batches.get(batchId); + if (!state) { + logger.warn({ batchId }, 'checkSafetyRails() called on unknown batch — allowing'); + return { passed: true, violation: undefined }; + } + + const { maxBatchIterations, batchBudgetUsd, batchTimeoutMinutes } = config; + + // ── Rail 1: iteration cap ────────────────────────────────── + if (state.completedItems.length >= maxBatchIterations) { + const violation: SafetyRailViolation = { + rail: 'iterations', + message: + `Batch paused: completed ${state.completedItems.length} of ${state.totalItems} items, ` + + `which has reached the maximum iteration limit (${maxBatchIterations}). ` + + `Reply "continue batch" to process the next ${maxBatchIterations} items.`, + }; + logger.warn( + { batchId, completedItems: state.completedItems.length, maxBatchIterations }, + 'Safety rail triggered: iteration limit reached', + ); + await this.pauseBatch(batchId); + return { passed: false, violation }; + } + + // ── Rail 2: budget cap ───────────────────────────────────── + if (state.totalCostUsd >= batchBudgetUsd) { + const violation: SafetyRailViolation = { + rail: 'budget', + message: + `Batch paused: cumulative cost $${state.totalCostUsd.toFixed(4)} has reached ` + + `the budget limit ($${batchBudgetUsd.toFixed(2)}). ` + + `Reply "continue batch" to authorize additional spending.`, + }; + logger.warn( + { batchId, totalCostUsd: state.totalCostUsd, batchBudgetUsd }, + 'Safety rail triggered: budget limit reached', + ); + await this.pauseBatch(batchId); + return { passed: false, violation }; + } + + // ── Rail 3: timeout cap ──────────────────────────────────── + const elapsedMs = Date.now() - new Date(state.startedAt).getTime(); + const timeoutMs = batchTimeoutMinutes * 60 * 1000; + if (elapsedMs >= timeoutMs) { + const elapsedMin = Math.round(elapsedMs / 60_000); + const violation: SafetyRailViolation = { + rail: 'timeout', + message: + `Batch paused: elapsed time ${elapsedMin} min has reached the timeout limit ` + + `(${batchTimeoutMinutes} min). ` + + `Reply "continue batch" to extend the session.`, + }; + logger.warn( + { batchId, elapsedMs, timeoutMs }, + 'Safety rail triggered: timeout limit reached', + ); + await this.pauseBatch(batchId); + return { passed: false, violation }; + } + + return { passed: true, violation: undefined }; + } + + // ── Master context injection ──────────────────────────────────── + + /** + * Build a system prompt section describing the current batch state (OB-1617). + * + * Injected into the Master AI's system prompt on every turn during a batch run + * so the Master never loses track of which item is current, what has been + * completed, and how many items remain. + * + * @param batchId The active batch identifier. + * @returns Formatted Markdown section, or null when the batch is unknown. + */ + buildBatchContextSection(batchId: string): string | null { + const state = this.batches.get(batchId); + if (!state) return null; + + const completed = state.completedItems.length; + const remaining = state.totalItems - state.currentIndex; + const currentPosition = state.currentIndex + 1; // 1-based for human display + const currentItem = state.plan[state.currentIndex]; + + const lines: string[] = [ + '## Active Batch Run', + '', + `**Batch ID:** ${batchId}`, + `**Progress:** ${completed} completed, ${remaining} remaining (${currentPosition}/${state.totalItems} total)`, + ]; + + if (currentItem) { + lines.push( + '', + `**Current item (${currentPosition}/${state.totalItems}):** ${currentItem.id}`, + ); + if (currentItem.description) { + lines.push(`**Task:** ${currentItem.description}`); + } + } + + if (state.completedItems.length > 0) { + lines.push('', '**Completed items:**'); + for (const item of state.completedItems) { + const icon = item.status === 'failed' ? '❌' : item.status === 'skipped' ? '⏭' : '✅'; + const summary = item.summary ? ` — ${item.summary}` : ''; + lines.push(`- ${icon} ${item.id}${summary}`); + } + } + + if (state.failedItems.length > 0) { + lines.push('', `**Failed items:** ${state.failedItems.join(', ')}`); + } + + lines.push( + '', + 'Process the current item above. When done, the batch will automatically continue to the next item.', + ); + + return lines.join('\n'); + } +} diff --git a/src/master/batch-planner.ts b/src/master/batch-planner.ts new file mode 100644 index 00000000..30b2229a --- /dev/null +++ b/src/master/batch-planner.ts @@ -0,0 +1,161 @@ +/** + * Batch Planner — Extracts ordered item lists from TASKS.md and FINDINGS.md. + * + * When a user triggers Batch Task Continuation ("implement all tasks", "go through + * each one", etc.), BatchPlanner reads the appropriate source file and returns a + * structured list of BatchPlanItems ready to be passed into BatchManager.createBatch(). + * + * Supported sources: + * - tasks-md : Reads TASKS.md, extracts rows with "◻ Pending" status + * - findings : Reads FINDINGS.md, extracts rows where Status column is "Open" + * - custom-list: Caller provides items directly (no file parsing needed) + * + * This class is pure file I/O + regex parsing — no AI calls. + */ + +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { createLogger } from '../core/logger.js'; +import type { BatchPlanItem, BatchSourceType } from '../types/agent.js'; + +const logger = createLogger('batch-planner'); + +// ── Regex patterns ───────────────────────────────────────────────── + +/** + * Matches a TASKS.md table row that contains a task ID and a Pending status. + * + * Expected row format (pipe-separated Markdown table): + * | 4 | OB-1607 | Add batch plan generation — ... | ◻ Pending | + * + * Captures: + * [1] Task ID (e.g. OB-1607) + * [2] Description (trimmed) + */ +const TASK_ROW_RE = /^\|\s*\d+\s*\|\s*(OB-[\w-]+)\s*\|\s*(.+?)\s*\|\s*◻\s*Pending\s*\|/; + +/** + * Matches a FINDINGS.md table row where the last status cell is "Open". + * + * Expected row format: + * | OB-F89 | Codex worker streaming output is raw JSON | 🔴 Critical | ... | Open | + * + * Captures: + * [1] Finding ID (e.g. OB-F89) + * [2] Short description (second column, trimmed) + */ +const FINDING_ROW_RE = /^\|\s*(OB-F[\w-]+)\s*\|\s*(.+?)\s*\|(?:.*?\|)*\s*Open\s*\|/; + +// ── BatchPlanner ─────────────────────────────────────────────────── + +export class BatchPlanner { + /** + * Build a batch plan from the given source. + * + * @param sourceType Which source to parse. + * @param workspacePath Absolute path to the workspace root (for resolving docs/ paths). + * @param customItems Items to use when sourceType is 'custom-list'. + * @param tasksFilePath Override path to TASKS.md (defaults to docs/audit/TASKS.md). + * @param findingsFilePath Override path to FINDINGS.md (defaults to docs/audit/FINDINGS.md). + * @returns Ordered list of BatchPlanItems extracted from the source. + */ + async buildPlan( + sourceType: BatchSourceType, + workspacePath: string, + options: { + customItems?: BatchPlanItem[]; + tasksFilePath?: string; + findingsFilePath?: string; + } = {}, + ): Promise { + switch (sourceType) { + case 'tasks-md': + return this.extractPendingTasks( + options.tasksFilePath ?? join(workspacePath, 'docs/audit/TASKS.md'), + ); + case 'findings': + return this.extractOpenFindings( + options.findingsFilePath ?? join(workspacePath, 'docs/audit/FINDINGS.md'), + ); + case 'custom-list': + return options.customItems ?? []; + default: + logger.warn({ sourceType }, 'Unknown batch source type — returning empty plan'); + return []; + } + } + + /** + * Detect the most likely source type from the user's message. + * + * - Mentions of "findings" or "bugs" → findings + * - Mentions of "tasks" or the default → tasks-md + * - Explicit item list in message → custom-list (caller must supply items) + */ + detectSourceType(userMessage: string): BatchSourceType { + const lower = userMessage.toLowerCase(); + if (lower.includes('finding') || lower.includes('bug') || lower.includes('issue')) { + return 'findings'; + } + return 'tasks-md'; + } + + // ── Private parsers ──────────────────────────────────────────── + + /** + * Read TASKS.md and extract all rows with "◻ Pending" status. + * Preserves document order (top to bottom). + */ + async extractPendingTasks(filePath: string): Promise { + let content: string; + try { + content = await readFile(filePath, 'utf8'); + } catch (err) { + logger.warn({ filePath, err }, 'Could not read TASKS.md — returning empty plan'); + return []; + } + + const items: BatchPlanItem[] = []; + + for (const line of content.split('\n')) { + const match = TASK_ROW_RE.exec(line); + if (match && match[1] && match[2]) { + const id = match[1].trim(); + const description = match[2].trim(); + items.push({ id, description }); + } + } + + logger.info({ filePath, count: items.length }, 'Extracted pending tasks from TASKS.md'); + return items; + } + + /** + * Read FINDINGS.md and extract all rows where the Status column is "Open". + * Preserves document order (top to bottom). + */ + async extractOpenFindings(filePath: string): Promise { + let content: string; + try { + content = await readFile(filePath, 'utf8'); + } catch (err) { + logger.warn({ filePath, err }, 'Could not read FINDINGS.md — returning empty plan'); + return []; + } + + const items: BatchPlanItem[] = []; + + for (const line of content.split('\n')) { + const match = FINDING_ROW_RE.exec(line); + if (match && match[1] && match[2]) { + const id = match[1].trim(); + const description = match[2].trim(); + items.push({ id, description }); + } + } + + logger.info({ filePath, count: items.length }, 'Extracted open findings from FINDINGS.md'); + return items; + } +} diff --git a/src/master/classification-engine.ts b/src/master/classification-engine.ts new file mode 100644 index 00000000..9d20283e --- /dev/null +++ b/src/master/classification-engine.ts @@ -0,0 +1,1227 @@ +/** + * ClassificationEngine — extracted from MasterManager (OB-1279, OB-F158). + * + * Handles task classification via keyword heuristics and AI-powered analysis. + * Manages the classification cache (in-memory + persisted to DB/JSON). + */ + +import type { ClassificationCacheEntry, ClassificationCache } from '../types/master.js'; +import { ClassificationCacheSchema } from '../types/master.js'; +import type { CLIAdapter } from '../core/cli-adapter.js'; +import type { AgentRunner } from '../core/agent-runner.js'; +import type { ModelRegistry } from '../core/model-registry.js'; +import type { MemoryManager } from '../memory/index.js'; +import type { DotFolderManager } from './dotfolder-manager.js'; +import { getTopSkills } from '../intelligence/skill-creator.js'; +import type { BusinessSkill } from '../intelligence/skill-creator.js'; +import { createLogger } from '../core/logger.js'; + +const logger = createLogger('classification-engine'); + +// --------------------------------------------------------------------------- +// Constants — classification turn budgets and timeout computation +// --------------------------------------------------------------------------- + +/** + * Per-turn wall-clock budget in milliseconds. + * Used to compute per-class timeouts: timeout = CLI_STARTUP_BUDGET_MS + maxTurns × PER_TURN_BUDGET_MS. + * 30s/turn gives quick-answer(3) = 30+90=120s, tool-use(15) = 30+450=480s, complex-task(25) = 30+750=780s. + */ +export const PER_TURN_BUDGET_MS = 30_000; + +/** + * Fixed startup budget added to every timeout. + * Covers CLI cold-start overhead (model loading, API connection, MCP init). + * Claude CLI cold-start is ~10-15s; 30s provides headroom without over-provisioning (OB-F217). + */ +export const CLI_STARTUP_BUDGET_MS = 30_000; + +/** Compute wall-clock timeout from a turn budget (includes CLI startup overhead). */ +export function turnsToTimeout(maxTurns: number): number { + return CLI_STARTUP_BUDGET_MS + maxTurns * PER_TURN_BUDGET_MS; +} + +/** + * Max turns for message processing — varies by task classification. + * quick-answer: questions, lookups, explanations → 3 turns (turnsToTimeout(3) = 120s < 180s DEFAULT_MESSAGE_TIMEOUT) + * text-generation: articles, strategies, long-form content → 10 turns + * tool-use: file generation, single edits, targeted fixes → 15 turns + * complex-task (planning): forces Master to output SPAWN markers → 25 turns + */ +export const MESSAGE_MAX_TURNS_QUICK = 3; +export const MESSAGE_MAX_TURNS_MENU_SELECTION = 2; +export const MESSAGE_MAX_TURNS_TEXT_GEN = 10; +export const MESSAGE_MAX_TURNS_TOOL_USE = 15; +export const MESSAGE_MAX_TURNS_PLANNING = 25; + +/** + * Classifier logic version — bump this when keyword/compound rules change. + * Cache entries with a different version are treated as stale and re-classified. + */ +export const CLASSIFIER_VERSION = 6; + +/** Maximum number of entries in the in-memory classification cache before LRU eviction (OB-F169). */ +const MAX_CLASSIFICATION_CACHE_SIZE = 10_000; + +// --------------------------------------------------------------------------- +// ClassificationResult — the public interface returned by classifyTask() +// --------------------------------------------------------------------------- + +/** + * Result returned by classifyTask() — includes class, suggested turn budget, and reasoning. + * The maxTurns value is AI-suggested based on message content and workspace context, + * replacing the fixed MESSAGE_MAX_TURNS_QUICK / MESSAGE_MAX_TURNS_TOOL_USE constants. + */ +export interface ClassificationResult { + /** One of quick-answer, tool-use, complex-task, or menu-selection */ + class: 'quick-answer' | 'tool-use' | 'complex-task' | 'menu-selection'; + /** AI-suggested turn budget for this specific message */ + maxTurns: number; + /** Computed wall-clock timeout in milliseconds (maxTurns × PER_TURN_BUDGET_MS) */ + timeout: number; + /** Brief reason for the classification (for logging/debugging) */ + reason: string; + /** When true, the task matches deep-mode keywords (audit, thorough review, etc.) + * and the Master should offer or activate Deep Mode analysis (OB-1404). */ + suggestDeepMode?: boolean; + /** When true, the message matches batch-mode keywords (implement all, for each, etc.) + * and the Master should activate Batch Task Continuation (OB-1605). */ + batchMode?: boolean; + /** When true, the message includes "commit after each" — BatchManager sets commitAfterEach (OB-1615). */ + commitAfterEach?: boolean; + /** When true, RAG retrieval is skipped for this message (e.g. menu-selection, very short inputs). */ + skipRag?: boolean; + /** When true, the message is a numeric menu selection from a previous numbered list (OB-1658). */ + menuSelection?: boolean; + /** The option text extracted from the previous bot response for this menu selection (OB-1658). */ + selectedOptionText?: string; + /** When true, the message matches doctype-creation phrases ("I need to track...", "manage my ...") (OB-1384). */ + doctypeCreation?: boolean; + /** The extracted entity name from the doctype-creation phrase (e.g. "invoices", "customers") (OB-1384). */ + doctypeEntity?: string; + /** When true, the message matches integration-setup phrases ("connect Stripe", "link Google Drive", "add my API") (OB-1396). */ + integrationSetup?: boolean; + /** The extracted integration name from the setup phrase (e.g. "stripe", "google-drive") (OB-1396). */ + integrationName?: string; + /** Name of a learned skill that matches this message (OB-1471). When set, Master should prefer the learned skill. */ + matchedSkillName?: string; + /** English description of the task intent from the AI classifier (OB-F207). Used as a RAG retry query + * when the raw user message (e.g. Arabizi/Darija) returns zero FTS5 results. Only set for successful + * AI classifications — NOT for keyword-fallback paths. */ + ragQuery?: string; +} + +// --------------------------------------------------------------------------- +// classifyTaskType — simple heuristic for learning analysis +// --------------------------------------------------------------------------- + +/** + * Classify task type based on prompt content. + * Uses heuristics to categorize tasks for learning analysis. + */ +export function classifyTaskType(prompt: string): string { + const lower = prompt.toLowerCase(); + + if (lower.includes('refactor') || lower.includes('restructure') || lower.includes('reorganize')) { + return 'refactoring'; + } + if ( + lower.includes('bug') || + lower.includes('fix') || + lower.includes('error') || + lower.includes('issue') + ) { + return 'bug-fix'; + } + if (lower.includes('test') || lower.includes('spec') || lower.includes('verify')) { + return 'testing'; + } + if ( + lower.includes('add') || + lower.includes('implement') || + lower.includes('create') || + lower.includes('feature') + ) { + return 'feature'; + } + if ( + lower.includes('explore') || + lower.includes('analyze') || + lower.includes('investigate') || + lower.includes('find') + ) { + return 'exploration'; + } + if (lower.includes('document') || lower.includes('explain') || lower.includes('describe')) { + return 'documentation'; + } + if (lower.includes('optimize') || lower.includes('improve') || lower.includes('performance')) { + return 'optimization'; + } + + return 'task'; +} + +// --------------------------------------------------------------------------- +// ClassificationEngine — manages classification cache + keyword/AI classifiers +// --------------------------------------------------------------------------- + +export interface ClassificationEngineDeps { + memory: MemoryManager | null; + dotFolder: DotFolderManager; + agentRunner: AgentRunner; + modelRegistry: ModelRegistry; + workspacePath: string; + adapter?: CLIAdapter; + /** Callback to get the current workspace context summary for AI classifier prompts. */ + getWorkspaceContext: () => string | null; +} + +export class ClassificationEngine { + private readonly classificationCache = new Map(); + private cacheLoaded = false; + private deps: ClassificationEngineDeps; + + constructor(deps: ClassificationEngineDeps) { + this.deps = deps; + } + + /** Update mutable dependencies (e.g. when memory becomes available after init). */ + updateDeps(partial: Partial): void { + this.deps = { ...this.deps, ...partial }; + } + + // ------------------------------------------------------------------------- + // Public API + // ------------------------------------------------------------------------- + + /** + * Normalize a message for cache lookup. + * Converts to lowercase, strips punctuation, and collapses whitespace. + * This ensures "Create a README" and "create a readme" share the same cache entry. + */ + normalizeForCache(content: string): string { + return content + .toLowerCase() + .replace(/[^\w\s]/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + } + + /** + * Record feedback for a classification after the task completes. + * Updates the cached entry's feedback array and adjusts maxTurns if the + * budget consistently proves insufficient. + */ + async recordClassificationFeedback( + normalizedKey: string, + turnBudgetSufficient: boolean, + timedOut: boolean, + turnsUsed?: number, + ): Promise { + const entry = this.classificationCache.get(normalizedKey); + if (!entry) return; + + entry.feedback.push({ + recordedAt: new Date().toISOString(), + turnBudgetSufficient, + timedOut, + }); + + const recent = entry.feedback.slice(-3); + const timeoutCount = recent.filter((f) => f.timedOut).length; + if (recent.length >= 2 && timeoutCount >= 2) { + logger.warn( + { + normalizedKey, + taskClass: entry.result.class, + maxTurns: entry.result.maxTurns, + timeoutCount, + recentFeedback: recent.length, + }, + 'Classification cache: repeated timeouts detected — task may need a higher wall-clock timeout', + ); + } + + await this.persistClassificationCache(); + + if (this.deps.memory) { + void (async (): Promise => { + try { + await this.deps.memory!.recordLearning( + 'classification', + entry.result.class, + turnBudgetSufficient, + turnsUsed ?? 0, + 0, + ); + } catch (err) { + logger.warn({ err }, 'Failed to record classification learning to DB — non-fatal'); + } + })(); + } + } + + /** + * AI-powered task classifier using a 1-turn haiku call. + * Returns a ClassificationResult with class, AI-suggested maxTurns, and reason. + * Falls back to keyword heuristics if the AI call fails or takes >3s. + * Falls back to 'tool-use' with default turns if the JSON cannot be parsed. + * + * @param sessionId - When provided, fetches the last few user messages from the + * session so the keyword classifier can apply conversation context (OB-1582). + */ + async classifyTask(content: string, sessionId?: string): Promise { + const CLASSIFIER_TIMEOUT_MS = 15_000; + + await this.loadClassificationCache(); + const cacheKey = this.normalizeForCache(content); + const cached = this.classificationCache.get(cacheKey); + if (cached && (cached as Record)['classifierVersion'] === CLASSIFIER_VERSION) { + cached.hitCount++; + logger.debug( + { cacheKey, class: cached.result.class, hitCount: cached.hitCount }, + 'Classification cache hit', + ); + void this.persistClassificationCache(); + return { ...cached.result }; + } + if (cached) { + this.classificationCache.delete(cacheKey); + } + + // Fetch recent user messages from session history for conversation context (OB-1582). + let recentUserMessages: string[] | undefined; + let lastBotResponse: string | undefined; + if (sessionId && this.deps.memory) { + try { + const sessionMessages = await this.deps.memory.getSessionHistory(sessionId, 6); + const userMessages = sessionMessages + .filter((e) => e.role === 'user') + .slice(-3) + .map((e) => e.content); + if (userMessages.length > 0) { + recentUserMessages = userMessages; + } + const botMessages = sessionMessages + .filter((e) => e.role === 'master') + .slice(-1) + .map((e) => e.content); + if (botMessages.length > 0) { + lastBotResponse = botMessages[0]; + } + } catch { + // Non-fatal: conversation context is a best-effort enhancement + } + } + + // Skip AI classifier for non-Claude adapters + if (this.deps.adapter && this.deps.adapter.name !== 'claude') { + logger.debug( + { adapter: this.deps.adapter.name }, + 'Skipping AI classifier for non-Claude adapter, using keyword heuristics', + ); + const keywordResult = this.classifyTaskByKeywords( + content, + recentUserMessages, + lastBotResponse, + ); + this.classificationCache.set(cacheKey, { + normalizedKey: cacheKey, + result: keywordResult, + recordedAt: new Date().toISOString(), + hitCount: 0, + feedback: [], + classifierVersion: CLASSIFIER_VERSION, + cachedAt: Date.now(), + } as ClassificationCacheEntry); + this.evictClassificationCacheIfNeeded(); + void this.persistClassificationCache(); + return keywordResult; + } + + // Include workspace context so the AI can calibrate scope + const workspaceCtx = this.deps.getWorkspaceContext(); + const contextSection = workspaceCtx ? `Workspace context:\n${workspaceCtx}\n\n` : ''; + + const prompt = + `You are a task classifier for an AI assistant. Analyze the user message and suggest how to handle it.\n\n` + + contextSection + + `User message: "${content}"\n\n` + + `Classify the message and suggest a turn budget. Reply with ONLY a JSON object — no markdown, no explanation:\n` + + `{"class":"","maxTurns":,"reason":""}\n\n` + + `Important rule: If the message references files, documents, spreadsheets, or has attachments, classify as tool-use or higher — never quick-answer.\n\n` + + `Categories and turn guidance:\n` + + `- "quick-answer": question, explanation, or lookup (no file changes) → maxTurns 1-5\n` + + `- "tool-use": generate/create/write/fix a file or single targeted edit → maxTurns 5-20\n` + + `- "complex-task": multi-step work requiring planning, many files, or full implementation → maxTurns 10-30\n\n` + + `Include a "confidence" field (0.0-1.0) indicating how confident you are in your classification.`; + + let classificationResult: ClassificationResult; + let aiResult: (ClassificationResult & { confidence: number }) | null = null; + + try { + const result = await Promise.race([ + this.deps.agentRunner.spawn({ + prompt, + workspacePath: this.deps.workspacePath, + model: this.deps.modelRegistry.resolveModelOrTier('fast'), + maxTurns: 2, + retries: 0, + }), + new Promise((_, reject) => + setTimeout(() => reject(new Error('classifier timeout')), CLASSIFIER_TIMEOUT_MS), + ), + ]); + + const raw = result.stdout.trim(); + + // Log when classifier exhausts turns — indicates maxTurns may still be insufficient + if (result.turnsExhausted) { + logger.debug('Classifier returned turnsExhausted: true — maxTurns=2 may be insufficient'); + } + + const jsonMatch = raw.match(/\{[\s\S]*\}/); + if (jsonMatch) { + try { + const parsed = JSON.parse(jsonMatch[0]) as Record; + const cls = parsed['class']; + const turns = parsed['maxTurns']; + const reason = typeof parsed['reason'] === 'string' ? parsed['reason'] : ''; + const confidence = typeof parsed['confidence'] === 'number' ? parsed['confidence'] : 0.5; + + if (cls === 'quick-answer' || cls === 'tool-use' || cls === 'complex-task') { + const maxTurns = + typeof turns === 'number' && turns > 0 && turns <= 50 + ? turns + : cls === 'quick-answer' + ? MESSAGE_MAX_TURNS_QUICK + : cls === 'tool-use' + ? MESSAGE_MAX_TURNS_TOOL_USE + : MESSAGE_MAX_TURNS_PLANNING; + logger.debug({ class: cls, maxTurns, reason, confidence }, 'AI classifier result'); + aiResult = { + class: cls, + maxTurns, + timeout: turnsToTimeout(maxTurns), + reason: `AI classifier: ${reason}`, + confidence, + ragQuery: reason.length > 10 ? reason : undefined, + }; + } + } catch { + const lower = raw.toLowerCase(); + if (lower.includes('quick-answer')) { + aiResult = { + class: 'quick-answer', + maxTurns: MESSAGE_MAX_TURNS_QUICK, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_QUICK), + reason: 'AI classifier: text scan fallback', + confidence: 0.3, + }; + } else if (lower.includes('complex-task')) { + aiResult = { + class: 'complex-task', + maxTurns: MESSAGE_MAX_TURNS_PLANNING, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_PLANNING), + reason: 'AI classifier: text scan fallback', + confidence: 0.3, + }; + } else if (lower.includes('tool-use')) { + aiResult = { + class: 'tool-use', + maxTurns: MESSAGE_MAX_TURNS_TOOL_USE, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_TOOL_USE), + reason: 'AI classifier: text scan fallback', + confidence: 0.3, + }; + } + } + } else { + const lower = raw.toLowerCase(); + if (lower.includes('quick-answer')) { + aiResult = { + class: 'quick-answer', + maxTurns: MESSAGE_MAX_TURNS_QUICK, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_QUICK), + reason: 'AI classifier: text scan fallback', + confidence: 0.3, + }; + } else if (lower.includes('complex-task')) { + aiResult = { + class: 'complex-task', + maxTurns: MESSAGE_MAX_TURNS_PLANNING, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_PLANNING), + reason: 'AI classifier: text scan fallback', + confidence: 0.3, + }; + } else if (lower.includes('tool-use')) { + aiResult = { + class: 'tool-use', + maxTurns: MESSAGE_MAX_TURNS_TOOL_USE, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_TOOL_USE), + reason: 'AI classifier: text scan fallback', + confidence: 0.3, + }; + } else { + logger.warn({ response: raw }, 'AI classifier returned unexpected response'); + } + } + } catch (err) { + const reason = err instanceof Error ? err.message : String(err); + logger.debug({ reason }, 'AI classifier failed, falling back to keyword heuristics'); + } + + // Run keyword classifier as well for priority comparison + const keywordResult = this.classifyTaskByKeywords(content, recentUserMessages, lastBotResponse); + + // Priority: AI classifier (confidence ≥ 0.4) > keyword match > default fallback + // Exception: when AI says quick-answer but keyword says higher class, prefer keyword + // if confidence is below 0.8 — prevents deployment/build requests from being under-classified (OB-F230) + const classRankMap: Record = { + 'quick-answer': 0, + 'text-generation': 0, + 'tool-use': 1, + 'complex-task': 2, + }; + const aiDowngrades = + aiResult && + aiResult.confidence >= 0.4 && + aiResult.confidence < 0.8 && + (classRankMap[aiResult.class] ?? 0) < (classRankMap[keywordResult.class] ?? 0); + + if (aiResult && aiResult.confidence >= 0.4 && !aiDowngrades) { + classificationResult = { + class: aiResult.class, + maxTurns: aiResult.maxTurns, + timeout: aiResult.timeout, + reason: aiResult.reason, + ragQuery: aiResult.ragQuery, + }; + if (aiResult.class !== keywordResult.class) { + logger.info( + { + aiClass: aiResult.class, + aiConfidence: aiResult.confidence, + keywordClass: keywordResult.class, + keywordReason: keywordResult.reason, + winner: 'ai-classifier', + }, + 'Classification conflict: AI classifier (confidence ≥ 0.4) preferred over keyword match', + ); + } + } else if (aiDowngrades) { + // AI classifier under-classifies with moderate confidence — keyword wins (OB-F230) + classificationResult = keywordResult; + logger.info( + { + aiClass: aiResult!.class, + aiConfidence: aiResult!.confidence, + keywordClass: keywordResult.class, + keywordReason: keywordResult.reason, + winner: 'keyword-upgrade', + }, + 'Classification conflict: keyword match preferred — AI would downgrade with moderate confidence', + ); + } else { + // Preserve keyword-specific flags (batchMode, doctypeCreation, etc.) + classificationResult = keywordResult; + if (aiResult) { + logger.debug( + { + aiClass: aiResult.class, + aiConfidence: aiResult.confidence, + keywordClass: keywordResult.class, + winner: 'keyword', + }, + 'Classification conflict: keyword match preferred over low-confidence AI classifier', + ); + } + } + + // Apply classification learning: if aggregate data shows this class underperforms, + // escalate to the best-performing class seen in the learnings table (OB-732) + if (this.deps.memory) { + try { + const learned = await this.deps.memory.getLearnedParams('classification'); + if (learned) { + const classRank: Record = { + 'quick-answer': 0, + 'tool-use': 1, + 'complex-task': 2, + }; + const validClasses = new Set(['quick-answer', 'tool-use', 'complex-task']); + const currentRank = classRank[classificationResult.class] ?? 0; + const learnedRank = classRank[learned.model] ?? 0; + if ( + validClasses.has(learned.model) && + learnedRank > currentRank && + learned.success_rate > 0.5 + ) { + const escalatedClass = learned.model as ClassificationResult['class']; + // OB-1573: Suppress escalation if efficiency data shows the escalated class + // uses few turns/workers — the original class is already sufficient. + const efficiency = await this.deps.memory.getTaskEfficiency(escalatedClass); + if ( + efficiency && + efficiency.sample_count >= 5 && + efficiency.avg_turns < 5 && + efficiency.avg_workers <= 1 + ) { + logger.info( + { + escalatedClass, + avgTurns: efficiency.avg_turns, + avgWorkers: efficiency.avg_workers, + sampleCount: efficiency.sample_count, + }, + `Escalation suppressed: ${escalatedClass} tasks average ${efficiency.avg_turns.toFixed(1)} turns and ${efficiency.avg_workers.toFixed(1)} workers — original class sufficient`, + ); + } else { + const escalatedMaxTurns = + escalatedClass === 'quick-answer' + ? MESSAGE_MAX_TURNS_QUICK + : escalatedClass === 'tool-use' + ? MESSAGE_MAX_TURNS_TOOL_USE + : MESSAGE_MAX_TURNS_PLANNING; + logger.info( + { + original: classificationResult.class, + escalated: escalatedClass, + successRate: learned.success_rate, + totalTasks: learned.total_tasks, + }, + 'Classification escalated based on learning data', + ); + classificationResult = { + class: escalatedClass, + maxTurns: escalatedMaxTurns, + timeout: turnsToTimeout(escalatedMaxTurns), + reason: `${classificationResult.reason} (escalated: ${Math.round(learned.success_rate * 100)}% success rate for ${escalatedClass})`, + }; + } + } + } + } catch (err) { + logger.warn({ err }, 'Failed to query classification learning — using original result'); + } + } + + // Annotate with matched skill name if a learned skill matches (OB-1471) + const matchedSkill = this.findMatchingSkill(content); + if (matchedSkill) { + classificationResult = { ...classificationResult, matchedSkillName: matchedSkill.name }; + logger.debug( + { skillName: matchedSkill.name, class: classificationResult.class }, + 'Matched learned skill for user message', + ); + } + + // Store result in cache + this.classificationCache.set(cacheKey, { + normalizedKey: cacheKey, + result: { ...classificationResult }, + recordedAt: new Date().toISOString(), + hitCount: 0, + feedback: [], + classifierVersion: CLASSIFIER_VERSION, + cachedAt: Date.now(), + } as ClassificationCacheEntry); + this.evictClassificationCacheIfNeeded(); + void this.persistClassificationCache(); + + return classificationResult; + } + + // ------------------------------------------------------------------------- + // Keyword-based classifier + // ------------------------------------------------------------------------- + + /** + * Keyword-based task classifier — instant fallback when the AI classifier + * is unavailable or times out. Returns 'tool-use' as the default so that + * unrecognized action messages don't get under-classified (OB-1581, OB-F178). + */ + classifyTaskByKeywords( + content: string, + recentUserMessages?: string[], + lastBotResponse?: string, + ): ClassificationResult { + const lower = content.toLowerCase(); + + // Menu-selection: single numeric digit (1–9) (OB-1658) + const trimmedContent = content.trim(); + if (/^\d$/.test(trimmedContent) && trimmedContent >= '1' && trimmedContent <= '9') { + const digitValue = parseInt(trimmedContent, 10); + let selectedOptionText: string | undefined; + const hasNumberedList = lastBotResponse ? /^\s*\d+[.)]\s+\S/m.test(lastBotResponse) : false; + if (hasNumberedList && lastBotResponse) { + const lines = lastBotResponse.split('\n'); + const optionPattern = new RegExp(`^\\s*${digitValue}[.)]\\s+(.+)`); + const matchedLine = lines.find((l) => optionPattern.test(l)); + if (matchedLine) { + const match = optionPattern.exec(matchedLine); + selectedOptionText = match && match[1] ? match[1].trim() : undefined; + } + } + return { + class: 'menu-selection', + maxTurns: MESSAGE_MAX_TURNS_MENU_SELECTION, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_MENU_SELECTION), + skipRag: true, + menuSelection: true, + selectedOptionText, + reason: hasNumberedList + ? `menu-selection: digit ${trimmedContent} from numbered list` + : `menu-selection: single digit ${trimmedContent}`, + }; + } + + // Batch Mode keywords (OB-1605, OB-1527) + // Require compound patterns — single words like 'batch' or 'command' alone must not trigger. + // Exclude 'bon de commande' (French purchase order — not a batch command). + const batchKeywords = [ + 'one by one', + 'all tasks', + 'each one', + 'implement all', + 'go through all', + 'for each', + 'iterate through', + 'all pending', + 'batch process', + 'batch run', + 'run batch', + 'batch of', + ]; + const batchExclusions = ['bon de commande']; + if ( + batchKeywords.some((kw) => lower.includes(kw)) && + !batchExclusions.some((ex) => lower.includes(ex)) + ) { + const commitAfterEachKeywords = ['commit after each', 'commit each', 'commit after every']; + const commitAfterEach = commitAfterEachKeywords.some((kw) => lower.includes(kw)); + return { + class: 'complex-task', + maxTurns: MESSAGE_MAX_TURNS_PLANNING, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_PLANNING), + reason: 'keyword match: batch-mode', + batchMode: true, + commitAfterEach: commitAfterEach || undefined, + }; + } + + // DocType creation intent detection (OB-1384) + const doctypePatterns: { pattern: RegExp; entityGroup: number }[] = [ + { + pattern: + /\bi (?:need|want) to (?:track|manage|organize|keep track of)(?: my| our| the)?\s+(.+?)(?:\.|$)/i, + entityGroup: 1, + }, + { + pattern: /\bcreate (?:a|an)\s+(.+?)\s+(?:entity|doctype|tracker|record|table|system)\b/i, + entityGroup: 1, + }, + { pattern: /\bset up\s+(.+?)\s+tracking\b/i, entityGroup: 1 }, + { pattern: /\bsetup\s+(.+?)\s+tracking\b/i, entityGroup: 1 }, + { pattern: /\btrack (?:my|our|the)\s+(.+?)(?:\.|$)/i, entityGroup: 1 }, + { pattern: /\bmanage (?:my|our|the)\s+(.+?)(?:\.|$)/i, entityGroup: 1 }, + ]; + for (const { pattern, entityGroup } of doctypePatterns) { + const match = pattern.exec(content); + if (match) { + const entity = match[entityGroup]?.trim().replace(/\s+/g, ' ') ?? 'entity'; + return { + class: 'complex-task', + maxTurns: MESSAGE_MAX_TURNS_PLANNING, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_PLANNING), + reason: `keyword match: doctype-creation (entity: ${entity})`, + doctypeCreation: true, + doctypeEntity: entity, + }; + } + } + + // Integration setup intent detection (OB-1396) + const integrationPatterns: { pattern: RegExp; nameGroup: number }[] = [ + { + pattern: + /\bconnect\s+(?:my\s+)?([a-z0-9][\w\s-]{1,30}?)(?:\s+account|\s+api|\s+integration|$|\.|,)/i, + nameGroup: 1, + }, + { + pattern: + /\blink\s+(?:my\s+)?([a-z0-9][\w\s-]{1,30}?)(?:\s+account|\s+api|\s+integration|$|\.|,)/i, + nameGroup: 1, + }, + { + pattern: + /\badd\s+(?:my\s+)?([a-z0-9][\w\s-]{1,30}?)\s+(?:api|key|credentials?|integration)\b/i, + nameGroup: 1, + }, + { + pattern: /\bintegrate\s+(?:with\s+)?([a-z0-9][\w\s-]{1,30}?)(?:\s+api|$|\.|,)/i, + nameGroup: 1, + }, + { + pattern: + /\bset\s+up\s+(?:my\s+)?([a-z0-9][\w\s-]{1,30}?)\s+(?:api|email|integration|webhook)\b/i, + nameGroup: 1, + }, + { pattern: /\benable\s+(?:the\s+)?([a-z0-9][\w\s-]{1,30}?)\s+integration\b/i, nameGroup: 1 }, + ]; + for (const { pattern, nameGroup } of integrationPatterns) { + const match = pattern.exec(content); + if (match) { + const rawName = match[nameGroup]?.trim().replace(/\s+/g, '-').toLowerCase() ?? 'api'; + return { + class: 'tool-use', + maxTurns: MESSAGE_MAX_TURNS_TOOL_USE, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_TOOL_USE), + reason: `keyword match: integration-setup (integration: ${rawName})`, + integrationSetup: true, + integrationName: rawName, + }; + } + } + + // Deep Mode keywords (OB-1404) + const deepModeKeywords = [ + 'audit', + 'deep analysis', + 'deep analyse', + 'deep analy', + 'thorough review', + 'security review', + 'full review', + 'full analysis', + 'full analyse', + 'investigate', + 'root cause', + 'in-depth', + 'in depth', + ]; + if (deepModeKeywords.some((kw) => lower.includes(kw))) { + return { + class: 'complex-task', + maxTurns: MESSAGE_MAX_TURNS_PLANNING, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_PLANNING), + reason: 'keyword match: complex-task (deep-mode candidate)', + suggestDeepMode: true, + }; + } + + // Length-based heuristic for complex-task (OB-1651) + const planningPatterns = [ + 'plan', + 'planning', + 'strategy', + 'strategic', + 'vision', + 'goals', + 'objective', + 'approach', + 'framework', + 'roadmap', + 'milestone', + 'proposal', + 'initiative', + 'project', + 'scope', + 'phase', + 'timeline', + 'deliverable', + 'outcome', + 'model', + ]; + if (lower.length > 200 && planningPatterns.some((kw) => lower.includes(kw))) { + return { + class: 'complex-task', + maxTurns: MESSAGE_MAX_TURNS_PLANNING, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_PLANNING), + reason: 'length heuristic: long message with planning/strategy language → complex-task', + }; + } + + // Complex task keywords + const complexKeywords = [ + 'implement', + 'build', + 'refactor', + 'develop', + 'set up', + 'setup', + 'redesign', + 'migrate', + 'overhaul', + 'execute', + 'start', + 'proceed', + 'begin', + 'launch', + 'run tasks', + 'start execution', + 'execute group', + 'start group', + 'brainstorm', + 'strategy', + 'business model', + 'commercialise', + 'commercialize', + 'roadmap review', + 'strategic plan', + 'market analysis', + 'go-to-market', + ]; + const complexWordBoundary = [/\barchitect\b/]; + const delegationPhrases = [ + /\bstart\s+the\s+\w+/, + /\bexecute\s+\w+/, + /\bbegin\s+\w+/, + /\blaunch\s+\w+/, + /\brun\s+the\s+\w+/, + ]; + if ( + complexKeywords.some((kw) => lower.includes(kw)) || + complexWordBoundary.some((re) => re.test(lower)) || + delegationPhrases.some((re) => re.test(lower)) + ) { + return { + class: 'complex-task', + maxTurns: MESSAGE_MAX_TURNS_PLANNING, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_PLANNING), + reason: 'keyword match: complex-task', + }; + } + + // Compound action pattern + const actionVerbs = [ + 'review', + 'analyze', + 'audit', + 'check', + 'fix', + 'add', + 'update', + 'create', + 'remove', + 'test', + 'write', + 'optimize', + 'improve', + ]; + const matchedVerbs = actionVerbs.filter((v) => lower.includes(v)); + if (matchedVerbs.length >= 2 && lower.includes(' and ')) { + return { + class: 'complex-task', + maxTurns: MESSAGE_MAX_TURNS_PLANNING, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_PLANNING), + reason: `keyword match: complex-task (compound: ${matchedVerbs.join('+')})`, + }; + } + + // File-reference keywords (OB-1258) + const fileReferencePattern = + /\b(the file|xl|xls|xlsx|pdf|csv|document|attachment|spreadsheet|image|photo|picture)\b/i; + if (fileReferencePattern.test(content)) { + return { + class: 'tool-use', + maxTurns: MESSAGE_MAX_TURNS_TOOL_USE, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_TOOL_USE), + reason: 'keyword match: file-reference → tool-use', + }; + } + + // Text-generation keywords (OB-1580) + const textGenKeywords = [ + 'create post', + 'linkedin', + 'tweet', + 'rewrite', + 'rephrase', + 'reformulate', + 'draft', + 'compose', + 'shorter', + 'longer', + 'attractive', + 'generate', + 'write', + ]; + if (textGenKeywords.some((kw) => lower.includes(kw))) { + return { + class: 'quick-answer', + maxTurns: MESSAGE_MAX_TURNS_TEXT_GEN, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_TEXT_GEN), + reason: 'keyword match: text-generation', + }; + } + + // Conversation context — text-generation follow-up (OB-1582) + if (recentUserMessages && recentUserMessages.length > 0) { + const recentTextGenCount = recentUserMessages + .slice(-3) + .filter((msg) => textGenKeywords.some((kw) => msg.toLowerCase().includes(kw))).length; + if (recentTextGenCount >= 1 && lower.length <= 120) { + return { + class: 'quick-answer', + maxTurns: MESSAGE_MAX_TURNS_TEXT_GEN, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_TEXT_GEN), + reason: 'conversation context: text-generation follow-up', + }; + } + } + + // Tool-use keywords + const toolUseKeywords = [ + 'create', + 'fix', + 'update file', + 'add to', + 'make a', + 'publish', + 'deploy', + 'push to', + 'ship', + 'launch', + 'release', + 'install', + 'remove', + 'delete', + 'rename', + 'move', + 'run', + 'execute', + 'build', + 'set up', + 'setup', + 'configure', + ]; + if (toolUseKeywords.some((kw) => lower.includes(kw))) { + return { + class: 'tool-use', + maxTurns: MESSAGE_MAX_TURNS_TOOL_USE, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_TOOL_USE), + reason: 'keyword match: tool-use', + }; + } + + // Question/lookup patterns + const questionPatterns = [ + 'what is', + 'what are', + 'how does', + 'how do', + 'explain', + 'describe', + 'show me', + 'list all', + 'list the', + 'tell me', + ]; + const trimmed = lower.trim(); + const isShortQuestion = trimmed.endsWith('?') && trimmed.length <= 80; + const hasQuestionKeyword = questionPatterns.some((qp) => lower.includes(qp)); + if (isShortQuestion || (hasQuestionKeyword && trimmed.length <= 120)) { + return { + class: 'quick-answer', + maxTurns: MESSAGE_MAX_TURNS_QUICK, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_QUICK), + reason: 'keyword match: quick-answer', + }; + } + + // Length-based fallback (OB-1650) + const hasQuestionMark = lower.includes('?'); + const sentenceEndCount = (lower.match(/[.!?]/g) || []).length; + const hasMultipleSentences = sentenceEndCount >= 2; + if (lower.length > 100 && (hasQuestionMark || hasMultipleSentences)) { + return { + class: 'tool-use', + maxTurns: MESSAGE_MAX_TURNS_TOOL_USE, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_TOOL_USE), + reason: 'length heuristic: long message with question/multi-sentence → tool-use', + }; + } + + // Conversational intent patterns (OB-1526) + // Messages that are clearly asking questions or expressing intent without requiring + // file access or code changes. Checked before the default so they map to quick-answer + // instead of falling through to tool-use. + const conversationalPatterns = [ + 'how can i', + 'can you explain', + 'i want to know', + 'what about', + 'is it possible', + "let's configure", + 'not yet', + ]; + if (conversationalPatterns.some((kw) => lower.includes(kw))) { + return { + class: 'quick-answer', + maxTurns: MESSAGE_MAX_TURNS_QUICK, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_QUICK), + reason: 'keyword match: conversational intent → quick-answer', + }; + } + + // Default: quick-answer — unrecognized messages are likely conversational (OB-1529) + // If the quick-answer agent needs file access, it can say so and the user re-sends. + // Costs 3x less than tool-use (5 turns vs 15 turns). + return { + class: 'quick-answer', + maxTurns: MESSAGE_MAX_TURNS_QUICK, + timeout: turnsToTimeout(MESSAGE_MAX_TURNS_QUICK), + reason: 'keyword fallback: quick-answer (default)', + }; + } + + // ------------------------------------------------------------------------- + // Skill matching (OB-1471) + // ------------------------------------------------------------------------- + + /** + * Check whether the user message matches a learned skill by comparing + * significant words against the skill name and description. + * Returns the best-matching skill (highest usage × success_rate score) or null. + * This is a lightweight synchronous check — skills are loaded from SQLite. + */ + findMatchingSkill(content: string): BusinessSkill | null { + const memory = this.deps.memory; + if (!memory) return null; + const db = memory.getDb(); + if (!db) return null; + + let skills: BusinessSkill[]; + try { + skills = getTopSkills(db, 20); + } catch { + return null; + } + if (skills.length === 0) return null; + + const lower = content.toLowerCase(); + // Extract significant words (length > 3) from the user message + const messageWords = lower + .replace(/[^a-z0-9\s]/g, ' ') + .split(/\s+/) + .filter((w) => w.length > 3); + + if (messageWords.length === 0) return null; + + let bestSkill: BusinessSkill | null = null; + let bestScore = 0; + + for (const skill of skills) { + const skillText = `${skill.name} ${skill.description}`.toLowerCase().replace(/-/g, ' '); + const matchCount = messageWords.filter((w) => skillText.includes(w)).length; + if (matchCount === 0) continue; + + // Score: overlap ratio × effectiveness (usage × successRate) + const overlapRatio = matchCount / messageWords.length; + const effectiveness = skill.usageCount * skill.successRate; + const score = overlapRatio * (1 + effectiveness); + + if (overlapRatio >= 0.3 && score > bestScore) { + bestScore = score; + bestSkill = skill; + } + } + + return bestSkill; + } + + // ------------------------------------------------------------------------- + // Cache management (private) + // ------------------------------------------------------------------------- + + /** + * Load the classification cache from disk into the in-memory map. + * Called lazily on the first classifyTask() call. Non-blocking on failure. + */ + private async loadClassificationCache(): Promise { + if (this.cacheLoaded) return; + this.cacheLoaded = true; + try { + let stored = null; + + if (this.deps.memory) { + try { + const raw = await this.deps.memory.getSystemConfig('classifications'); + if (raw) { + stored = ClassificationCacheSchema.parse(JSON.parse(raw)); + } + } catch { + // DB read failed — fall through to JSON + } + } + + if (!stored) { + stored = await this.deps.dotFolder.readClassifications(); + } + + if (stored) { + for (const [key, entry] of Object.entries(stored.entries)) { + this.classificationCache.set(key, entry); + } + logger.debug({ size: this.classificationCache.size }, 'Classification cache loaded'); + } + } catch { + // Cache load failure is non-fatal — we'll just re-classify + } + } + + /** + * Persist the in-memory classification cache to system_config (DB) and + * classifications.json (fallback). + */ + private async persistClassificationCache(): Promise { + try { + const entries: Record = {}; + for (const [key, entry] of this.classificationCache) { + entries[key] = entry; + } + const cache: ClassificationCache = { + entries, + updatedAt: new Date().toISOString(), + schemaVersion: '1.0.0', + }; + + if (this.deps.memory) { + await this.deps.memory.setSystemConfig('classifications', JSON.stringify(cache)); + } else { + await this.deps.dotFolder.writeClassifications(cache); + } + } catch (err) { + logger.warn({ err }, 'Failed to persist classification cache — non-fatal'); + } + } + + /** + * Evict the oldest 20% of classification cache entries when the cache exceeds + * MAX_CLASSIFICATION_CACHE_SIZE. + */ + private evictClassificationCacheIfNeeded(): void { + if (this.classificationCache.size <= MAX_CLASSIFICATION_CACHE_SIZE) return; + + const evictCount = Math.ceil(MAX_CLASSIFICATION_CACHE_SIZE * 0.2); + const entries = Array.from(this.classificationCache.entries()).sort(([, a], [, b]) => { + const aTime = a.cachedAt ?? 0; + const bTime = b.cachedAt ?? 0; + return aTime - bTime; + }); + + let deleted = 0; + for (const [key] of entries) { + if (deleted >= evictCount) break; + this.classificationCache.delete(key); + deleted++; + } + + logger.warn( + { evicted: deleted, remaining: this.classificationCache.size }, + 'Classification cache eviction: removed oldest entries', + ); + } +} diff --git a/src/master/deep-mode.ts b/src/master/deep-mode.ts new file mode 100644 index 00000000..220b3d7f --- /dev/null +++ b/src/master/deep-mode.ts @@ -0,0 +1,1341 @@ +/** + * Deep Mode Manager — Multi-phase execution for complex analysis tasks. + * + * Instead of single-pass execution (classify → execute → respond), Deep Mode + * runs through up to five sequential phases: investigate → report → plan → + * execute → verify. Each phase can produce a result that feeds into the next. + * + * Three execution profiles: + * - fast: Skips Deep Mode entirely (current behaviour). + * - thorough: Runs all phases automatically without pausing. + * - manual: Pauses between phases and waits for user confirmation. + * + * This class owns the phase state machine lifecycle. Wiring into MasterManager + * and per-phase prompts/models are added by subsequent tasks (OB-1399 – OB-1403). + */ + +import { randomUUID } from 'node:crypto'; +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; + +import type { ModelTier } from '../core/model-registry.js'; +import { createLogger } from '../core/logger.js'; +import type { + DeepModeState, + DeepPhase, + DeepPhaseResult, + ExecutionProfile, +} from '../types/agent.js'; +import type { ProgressEvent } from '../types/message.js'; +import { + DEEP_INVESTIGATE, + DEEP_REPORT, + DEEP_PLAN, + DEEP_EXECUTE, + DEEP_VERIFY, +} from './seed-prompts.js'; + +/** Callback type for emitting Deep Mode phase progress events. */ +type PhaseProgressReporter = (event: ProgressEvent) => Promise; + +// ── Final Summary Types ──────────────────────────────────────────── + +/** + * Aggregated result produced after all Deep Mode phases complete. + * Returned by `compileFinalSummary()` for display to the user. + */ +export interface DeepModeFinalSummary { + /** Number of phases that produced results (max 5). */ + phasesCompleted: number; + /** Human-readable list of completed phase names. */ + completedPhaseNames: string[]; + /** Number of findings extracted from the report/investigate output. */ + findingsCount: number; + /** Number of tasks that were executed (plan tasks minus skipped). */ + tasksExecuted: number; + /** Number of tasks skipped by the user. */ + tasksSkipped: number; + /** Pass/fail verdict from the verify phase, or null if verify was skipped. */ + testVerdict: 'pass' | 'fail' | null; + /** First sentence or paragraph of the executive summary from the report. */ + executiveSummary: string; + /** Original user request that triggered Deep Mode. */ + taskSummary: string; + /** ISO timestamp when the session started. */ + startedAt: string; + /** ISO timestamp when the final phase completed. */ + completedAt: string; + /** Full formatted Markdown text ready to send as the final chat response. */ + formattedResponse: string; +} + +// ── Parallel Execution Types ─────────────────────────────────────── + +/** + * A single task parsed from the plan phase output. + * Extracted from the numbered task blocks in the plan Markdown. + */ +export interface DeepPlanTask { + /** 1-based task number (from "Task #N" in plan output) */ + taskNumber: number; + /** Short task title (from "**Title:**" field) */ + title: string; + /** Task description (from "**Description:**" field) */ + description: string; + /** Relative file paths to modify (from "**Files to Modify:**" field) */ + filesToModify: string[]; + /** 1-based task numbers this task depends on (from "**Dependencies:**" field) */ + dependsOn: number[]; +} + +/** + * A batch of tasks that can all be spawned simultaneously because they + * have no inter-dependencies. All tasks in a batch are ready to run + * at the same time, though callers should still respect the + * WorkerRegistry concurrency limit when launching them. + */ +export type DeepExecuteBatch = DeepPlanTask[]; + +// ── Plan Output Parsers ──────────────────────────────────────────── + +/** + * Parse individual task entries from plan phase output. + * + * Scans for "**Task #N**" headers and extracts the title, files to modify, + * dependencies, and description for each task. The parser is intentionally + * lenient — missing fields produce empty/default values so callers always + * receive a well-typed result. + * + * @param planOutput Raw plan phase output (Markdown). + * @returns Parsed tasks in document order. + */ +export function parsePlanTasks(planOutput: string): DeepPlanTask[] { + const tasks: DeepPlanTask[] = []; + + // Locate every "**Task #N**" header and slice the text between consecutive headers + const headerRegex = /\*\*Task\s+#(\d+)\*\*/gi; + const headerMatches = [...planOutput.matchAll(headerRegex)]; + + for (let i = 0; i < headerMatches.length; i++) { + const headerMatch = headerMatches[i]; + if (!headerMatch) continue; + const capturedNum = headerMatch[1] ?? '0'; + const taskNumber = parseInt(capturedNum, 10); + const start = headerMatch.index ?? 0; + const nextMatch = headerMatches[i + 1]; + const end = nextMatch?.index ?? planOutput.length; + const chunk = planOutput.slice(start, end); + + // Title + const titleMatch = /\*\*Title:\*\*\s*(.+)/i.exec(chunk); + const title = titleMatch ? (titleMatch[1] ?? '').trim() : `Task #${taskNumber}`; + + // Files to Modify — may be a single line or a newline-separated list + const filesMatch = /\*\*Files\s+to\s+Modify:\*\*\s*([\s\S]+?)(?=\n\*\*|\n---)/i.exec(chunk); + let filesToModify: string[] = []; + if (filesMatch) { + filesToModify = (filesMatch[1] ?? '') + .split(/[\n,]/) + .map((l) => + l + .replace(/[`*]/g, '') + .replace(/^\s*[-•]\s*/, '') + .trim(), + ) + .filter(Boolean); + } + + // Dependencies — "none" or a list of task numbers like "#1, #3" + const depsMatch = /\*\*Dependencies:\*\*\s*(.+)/i.exec(chunk); + let dependsOn: number[] = []; + const depsText = depsMatch ? (depsMatch[1] ?? '') : ''; + if (depsText && !/^\s*none\s*$/i.test(depsText)) { + dependsOn = [...depsText.matchAll(/#(\d+)/g)].map((m) => parseInt(m[1] ?? '0', 10)); + } + + // Description — multi-line field; ends at the next "---" separator or task header + const descMatch = + /\*\*Description:\*\*\s*([\s\S]+?)(?=\n---|\n\*\*Task\s+#|\n\*\*Batch|\s*$)/i.exec(chunk); + const description = descMatch ? (descMatch[1] ?? '').trim() : ''; + + tasks.push({ taskNumber, title, description, filesToModify, dependsOn }); + } + + return tasks; +} + +/** + * Parse the "Parallel Batches" section from plan phase output. + * + * Reads lines in the format: + * **Batch 1 (parallel):** Tasks #1, #2, #3 + * **Batch 2 (sequential):** Task #4 (depends on Batch 1) + * + * and returns an ordered array of task-number arrays. Only the task numbers + * listed directly after the colon are extracted — task numbers inside + * parenthetical dependency annotations are ignored. + * + * @param planOutput Raw plan phase output (Markdown). + * @returns Ordered batches, each an array of 1-based task numbers. + * Returns an empty array if no batch section is found. + */ +export function parsePlanBatches(planOutput: string): number[][] { + const batches: number[][] = []; + + // Match: **Batch N (type):** + // Capture everything after the colon up to the first '(' or end of line + const batchLineRegex = /\*\*Batch\s+\d+[^:]*:\*\*\s*([^(\n]*)/gi; + let match: RegExpExecArray | null; + + while ((match = batchLineRegex.exec(planOutput)) !== null) { + const taskListText = match[1] ?? ''; + const taskNums = [...taskListText.matchAll(/#(\d+)/g)].map((m) => parseInt(m[1] ?? '0', 10)); + if (taskNums.length > 0) { + batches.push(taskNums); + } + } + + return batches; +} + +/** + * Group tasks into dependency-ordered batches using a BFS topological sort. + * + * Used as a fallback when no "Parallel Batches" section is present in the plan. + * Tasks in the same batch have no inter-dependencies and can run simultaneously. + * + * @param tasks Tasks to group (skipped items should be excluded by the caller). + * @returns Ordered batches; tasks whose dependencies fall outside the set + * are treated as having no dependencies for grouping purposes. + */ +function groupTasksByDependency(tasks: DeepPlanTask[]): DeepExecuteBatch[] { + if (tasks.length === 0) return []; + + const taskSet = new Set(tasks.map((t) => t.taskNumber)); + const completed = new Set(); + const batches: DeepExecuteBatch[] = []; + let remaining = [...tasks]; + + while (remaining.length > 0) { + // A task is ready when all its dependencies are completed or outside the set + const ready = remaining.filter((t) => + t.dependsOn.every((dep) => completed.has(dep) || !taskSet.has(dep)), + ); + + // Guard against circular dependencies — take first remaining task to break the cycle + const first = remaining[0]; + const batch: DeepExecuteBatch = ready.length > 0 ? ready : first ? [first] : []; + + batches.push(batch); + + for (const t of batch) { + completed.add(t.taskNumber); + } + + remaining = remaining.filter((t) => !completed.has(t.taskNumber)); + } + + return batches; +} + +const logger = createLogger('deep-mode'); + +// ── Final Summary Helpers ────────────────────────────────────────── + +/** + * Count numbered findings in phase output text. + * + * Recognises patterns like: + * "Finding 1:", "Finding #1:", "1.", "1)", "**1.**", "**Finding 1**" + * + * Returns 0 when no numbered items are found. + */ +function countFindings(text: string): number { + const patterns = [ + /\bFinding\s+#?\d+/gi, + /^\s*\d+\.\s+\S/gm, + /^\s*\d+\)\s+\S/gm, + /\*\*Finding\s+#?\d+\*\*/gi, + /^\s*\*\*\d+\.\*\*/gm, + ]; + + let max = 0; + for (const re of patterns) { + const matches = text.match(re); + if (matches && matches.length > max) { + max = matches.length; + } + } + return max; +} + +/** + * Count task entries in plan phase output. + * + * Recognises "**Task #N**" headers as individual tasks. + */ +function countPlanTasks(text: string): number { + const matches = text.match(/\*\*Task\s+#\d+\*\*/gi); + return matches ? matches.length : 0; +} + +/** + * Extract the verify phase verdict from its output. + * + * Looks for explicit PASSED / FAILED verdicts as well as test-runner + * summary lines (e.g., "X passed", "X failing"). + * + * Returns 'pass' if only pass signals are found, 'fail' if any fail signal + * is found, or null when the output is ambiguous / unavailable. + */ +function extractVerifyVerdict(output: string): 'pass' | 'fail' | null { + if (!output) return null; + + const lower = output.toLowerCase(); + + const failPatterns = [ + /\bfailed?\b/, + /\bfailure\b/, + /\berror\b/, + /\d+\s+failing/, + /overall.*fail/, + /verdict.*fail/, + /fail.*verdict/, + ]; + const passPatterns = [ + /\bpassed?\b/, + /\ball.*pass/, + /overall.*pass/, + /verdict.*pass/, + /pass.*verdict/, + /\d+\s+passed/, + ]; + + const hasFail = failPatterns.some((re) => re.test(lower)); + const hasPass = passPatterns.some((re) => re.test(lower)); + + if (hasFail) return 'fail'; + if (hasPass) return 'pass'; + return null; +} + +/** + * Extract the executive summary from the report phase output. + * + * Tries (in order): + * 1. Content under an "Executive Summary" heading + * 2. The first non-empty paragraph + * 3. The first 300 characters of the output + */ +function extractExecutiveSummary(reportOutput: string): string { + if (!reportOutput) return '(no report available)'; + + // Try to find an "Executive Summary" section + const execMatch = /(?:##?\s*Executive\s+Summary\s*\n)([\s\S]+?)(?=\n##|\n---|\n\*\*|\s*$)/i.exec( + reportOutput, + ); + if (execMatch) { + const summary = (execMatch[1] ?? '').trim(); + if (summary.length > 0) { + return summary.length > 500 ? summary.slice(0, 500) + '…' : summary; + } + } + + // Fallback: first non-empty paragraph + const paragraphs = reportOutput + .split(/\n\s*\n/) + .map((p) => p.replace(/^#+\s*/, '').trim()) + .filter((p) => p.length > 20); + if (paragraphs.length > 0) { + const para = paragraphs[0] ?? ''; + return para.length > 500 ? para.slice(0, 500) + '…' : para; + } + + // Last resort: first 300 chars + const truncated = reportOutput.slice(0, 300).trim(); + return truncated.length > 0 ? truncated + '…' : '(no summary available)'; +} + +// ── Phase Ordering ──────────────────────────────────────────────── + +/** Canonical order of Deep Mode phases */ +const PHASE_ORDER: DeepPhase[] = ['investigate', 'report', 'plan', 'execute', 'verify']; + +// ── Per-Phase Model Selection ────────────────────────────────────── + +/** + * Default model tier for each Deep Mode phase. + * + * - investigate: powerful — thorough code/context exploration requires the best model + * - report: balanced — summarising findings is less reasoning-intensive + * - plan: powerful — high-quality planning reduces downstream errors + * - execute: balanced — implementation tasks are well-scoped; balanced is sufficient + * - verify: fast — test running + pass/fail checks are straightforward + * + * Users can override individual entries via `deep.phaseModels` in config (OB-1402). + */ +export const PHASE_MODEL_MAP: Record = { + investigate: 'powerful', + report: 'balanced', + plan: 'powerful', + execute: 'balanced', + verify: 'fast', +}; + +// ── Per-Phase System Prompts ─────────────────────────────────────── + +/** + * Focused system prompt injection for each Deep Mode phase. + * + * These strings are appended to the Master AI system prompt at the start of each + * phase to steer the model toward the phase-appropriate goal. Callers retrieve + * them via getPhaseSystemPrompt() and pass them as --append-system-prompt. + */ +export const PHASE_SYSTEM_PROMPTS: Record = { + investigate: `## Deep Mode — Investigate Phase + +Your goal in this phase is to **explore and identify**. Do not implement or modify anything. + +- Read source files, configs, tests, and documentation thoroughly. +- Identify issues, gaps, risks, and opportunities relevant to the user's request. +- Collect concrete evidence: file paths, line numbers, code snippets, error messages. +- Produce a comprehensive list of findings. Be specific — vague findings are not useful. +- End your output with a clearly structured "Findings" section that the next phase can summarise.`, + + report: `## Deep Mode — Report Phase + +Your goal in this phase is to **summarise findings** from the investigation into a clear, actionable report. + +- Organise findings by severity or priority (critical → high → medium → low). +- For each finding include: what it is, why it matters, and where it lives (file:line). +- Do not implement fixes yet — focus on clear, accurate documentation. +- Keep the report concise enough for a non-developer to understand the key points. +- End with a numbered list of recommended next steps that will feed into the plan phase.`, + + plan: `## Deep Mode — Plan Phase + +Your goal in this phase is to **create an actionable plan** based on the report findings. + +- Convert each finding or recommendation into a concrete, numbered task. +- Order tasks by dependency and priority — tasks that others depend on come first. +- For each task specify: what to do, which files to touch, and the expected outcome. +- Flag tasks that require user input or approval before execution. +- Produce a final numbered task list. This list will be executed step-by-step in the next phase.`, + + execute: `## Deep Mode — Execute Phase + +Your goal in this phase is to **implement the plan** produced in the previous phase. + +- Work through the task list in order, one task at a time. +- Make only the changes described in the plan — do not scope-creep. +- After each task, confirm what was changed and check for side effects. +- If a task is blocked or unsafe to execute, stop and report the blocker — do not skip silently. +- End with a summary of all changes made and any tasks that were deferred.`, + + verify: `## Deep Mode — Verify Phase + +Your goal in this phase is to **run tests and checks** to confirm the implementation is correct. + +- Run the project's test suite (npm test, pytest, cargo test, etc. as appropriate). +- Run linting and type checks if available (npm run lint, npm run typecheck, etc.). +- Read the output carefully — flag any failures, warnings, or regressions. +- Confirm that each executed task from the plan phase produced the expected outcome. +- End with a pass/fail verdict and a list of any remaining issues that need follow-up.`, +}; + +// ── Phase Worker Prompt Context ──────────────────────────────────── + +/** + * Context supplied by callers when building a phase worker prompt. + * + * Each field maps to a `{{placeholder}}` in the corresponding DEEP_* template. + * Investigate-phase fields are only used for the investigate phase; execute-phase + * fields are only used for the execute phase. Previous-phase results are injected + * automatically from the session state — callers do not need to supply them. + */ +export interface DeepPhaseWorkerContext { + /** Absolute workspace path (investigate — defaults to host.workspacePath). */ + workspacePath?: string; + /** Human-readable project name (investigate). */ + projectName?: string; + /** Detected project type, e.g. "Node.js / TypeScript" (investigate). */ + projectType?: string; + /** Comma-separated list of detected frameworks (investigate). */ + frameworks?: string; + /** Summary of the workspace file/directory structure (investigate). */ + structure?: string; + /** 1-based task number to execute (execute phase). */ + taskNumber?: number; + /** Short task title (execute phase). */ + taskTitle?: string; + /** Newline-separated list of files to modify (execute phase). */ + filesToModify?: string; + /** Task description from the plan (execute phase). */ + taskDescription?: string; + /** Additional constraints to enforce during execution (execute phase). */ + constraints?: string; +} + +// ── MasterManager Surface ───────────────────────────────────────── + +/** + * Minimal interface for the MasterManager capabilities that DeepModeManager + * requires. Using an interface avoids a circular import with master-manager.ts. + * The concrete MasterManager satisfies this interface; pass `this` when wiring. + */ +export interface DeepModeHost { + /** The absolute workspace path the Master AI is operating on */ + readonly workspacePath: string; +} + +// ── Deep Mode Manager ───────────────────────────────────────────── + +/** + * DeepModeManager — Manages the phase state machine for Deep Mode sessions. + * + * Each session is identified by a unique sessionId returned from startSession(). + * Multiple sessions can exist concurrently (one per active user conversation). + * + * Profile behaviour: + * - fast: Deep Mode is skipped. startSession() returns null. + * - thorough: All phases run automatically without pausing. + * - manual: After each advancePhase() call the session is marked as paused. + * Callers must call resume() after receiving user confirmation. + */ +export class DeepModeManager { + private readonly sessions = new Map(); + + /** Sessions awaiting user confirmation (manual profile only). */ + private readonly pausedSessions = new Set(); + + /** Per-session progress reporters — emit WebSocket events on phase transitions. */ + private readonly progressReporters = new Map(); + + constructor(private readonly host: DeepModeHost) {} + + /** Fire-and-forget helper that emits a phase progress event without blocking callers. */ + private emitPhaseEvent(sessionId: string, event: ProgressEvent): void { + const reporter = this.progressReporters.get(sessionId); + if (reporter) { + reporter(event).catch((err: unknown) => { + logger.warn({ err, sessionId }, 'deep-phase progress emit failed'); + }); + } + } + + // ── Session lifecycle ───────────────────────────────────────── + + /** + * Start a new Deep Mode session. + * + * @param taskSummary One-line summary of the original user request. + * @param profile Execution profile (fast | thorough | manual). + * @param progressReporter Optional callback for emitting WebSocket phase progress events. + * @returns The new session ID, or `null` when `fast` skips Deep Mode. + */ + startSession( + taskSummary: string, + profile: ExecutionProfile, + progressReporter?: PhaseProgressReporter, + ): string | null { + if (profile === 'fast') { + logger.info({ profile, taskSummary }, 'Deep Mode skipped for fast profile'); + return null; + } + + const sessionId = randomUUID(); + const firstPhase: DeepPhase = 'investigate'; + + const state: DeepModeState = { + sessionId, + profile, + currentPhase: firstPhase, + phaseResults: {}, + startedAt: new Date().toISOString(), + taskSummary, + skippedItems: [], + taskModelOverrides: {}, + }; + + this.sessions.set(sessionId, state); + + if (progressReporter) { + this.progressReporters.set(sessionId, progressReporter); + } + + logger.info({ sessionId, profile, taskSummary }, 'Deep Mode session started'); + + this.emitPhaseEvent(sessionId, { + type: 'deep-phase', + sessionId, + phase: firstPhase, + status: 'started', + }); + + return sessionId; + } + + /** + * Abort an active Deep Mode session and remove it from memory. + * + * @param sessionId The session to abort. + */ + abort(sessionId: string): void { + const state = this.sessions.get(sessionId); + if (!state) { + logger.warn({ sessionId }, 'abort() called on unknown Deep Mode session'); + return; + } + + const abortedPhase = state.currentPhase; + logger.info({ sessionId, currentPhase: abortedPhase }, 'Deep Mode session aborted'); + + if (abortedPhase) { + this.emitPhaseEvent(sessionId, { + type: 'deep-phase', + sessionId, + phase: abortedPhase, + status: 'aborted', + }); + } + + this.sessions.delete(sessionId); + this.pausedSessions.delete(sessionId); + this.progressReporters.delete(sessionId); + } + + // ── Phase state machine ─────────────────────────────────────── + + /** + * Advance to the next phase of a Deep Mode session. + * + * Stores the provided result for the current phase and moves the session + * forward. For `thorough` profiles, advancement is automatic. For `manual` + * profiles, callers are expected to pause and wait for user confirmation + * before calling advancePhase(). + * + * @param sessionId Active session identifier. + * @param result Result produced by the completed current phase. + * @returns The next phase, or `undefined` when all phases are done. + */ + advancePhase(sessionId: string, result: DeepPhaseResult): DeepPhase | undefined { + const state = this.sessions.get(sessionId); + if (!state) { + logger.warn({ sessionId }, 'advancePhase() called on unknown session'); + return undefined; + } + + const { currentPhase } = state; + if (!currentPhase) { + logger.warn({ sessionId }, 'advancePhase() called on already-completed session'); + return undefined; + } + + // Persist the result for the current phase + state.phaseResults[currentPhase] = result; + + // Find the next phase in canonical order + const currentIndex = PHASE_ORDER.indexOf(currentPhase); + const nextPhase = currentIndex >= 0 ? PHASE_ORDER[currentIndex + 1] : undefined; + + state.currentPhase = nextPhase; + + // Emit completed event for the phase that just finished (truncate summary to 200 chars) + const resultSummary = result.output.length > 200 ? result.output.slice(0, 200) : result.output; + this.emitPhaseEvent(sessionId, { + type: 'deep-phase', + sessionId, + phase: currentPhase, + status: 'completed', + resultSummary, + }); + + if (nextPhase) { + logger.info( + { sessionId, fromPhase: currentPhase, toPhase: nextPhase, profile: state.profile }, + 'Deep Mode phase advanced', + ); + + // Emit started event for the incoming phase + this.emitPhaseEvent(sessionId, { + type: 'deep-phase', + sessionId, + phase: nextPhase, + status: 'started', + }); + + // Manual profile pauses after each phase transition to wait for user confirmation. + // Thorough profile continues automatically — no pause needed. + if (state.profile === 'manual') { + this.pausedSessions.add(sessionId); + logger.info( + { sessionId, nextPhase }, + 'Deep Mode paused — awaiting user confirmation to continue', + ); + } + } else { + // All phases completed — remove any paused state and clean up + this.pausedSessions.delete(sessionId); + this.progressReporters.delete(sessionId); + logger.info( + { sessionId, completedPhase: currentPhase }, + 'Deep Mode session completed all phases', + ); + } + + return nextPhase; + } + + /** + * Skip the current phase without recording a result, advancing to the next. + * + * @param sessionId Active session identifier. + * @returns The next phase after the skipped one, or `undefined` if done. + */ + skipPhase(sessionId: string): DeepPhase | undefined { + const state = this.sessions.get(sessionId); + if (!state) { + logger.warn({ sessionId }, 'skipPhase() called on unknown session'); + return undefined; + } + + const { currentPhase } = state; + if (!currentPhase) { + logger.warn({ sessionId }, 'skipPhase() called on already-completed session'); + return undefined; + } + + const currentIndex = PHASE_ORDER.indexOf(currentPhase); + const nextPhase = currentIndex >= 0 ? PHASE_ORDER[currentIndex + 1] : undefined; + + state.currentPhase = nextPhase; + + logger.info({ sessionId, skippedPhase: currentPhase, nextPhase }, 'Deep Mode phase skipped'); + + this.emitPhaseEvent(sessionId, { + type: 'deep-phase', + sessionId, + phase: currentPhase, + status: 'skipped', + }); + + if (nextPhase) { + this.emitPhaseEvent(sessionId, { + type: 'deep-phase', + sessionId, + phase: nextPhase, + status: 'started', + }); + } else { + this.progressReporters.delete(sessionId); + } + + return nextPhase; + } + + /** + * Mark a specific plan item as skipped (1-based index). + * + * Appends itemIndex to state.skippedItems so that the execute phase can + * exclude it when processing the task list. Idempotent — calling with the + * same index twice has no additional effect. + * + * @param sessionId Active session identifier. + * @param itemIndex 1-based index of the plan item to skip. + */ + skipItem(sessionId: string, itemIndex: number): void { + const state = this.sessions.get(sessionId); + if (!state) { + logger.warn({ sessionId }, 'skipItem() called on unknown session'); + return; + } + + if (!state.skippedItems.includes(itemIndex)) { + state.skippedItems.push(itemIndex); + } + + logger.info({ sessionId, skippedItem: itemIndex }, 'Deep Mode item marked as skipped'); + } + + /** + * Focus investigation on a specific plan item (1-based index). + * + * Records the item as "focused" for the current session. Callers can + * use this to repeat the investigate phase with a narrower scope. + * The index is appended to skippedItems to signal that other items + * should be deprioritised in the next investigation turn. + * + * @param sessionId Active session identifier. + * @param itemIndex 1-based index of the plan item to focus on. + */ + focusOnItem(sessionId: string, itemIndex: number): void { + const state = this.sessions.get(sessionId); + if (!state) { + logger.warn({ sessionId }, 'focusOnItem() called on unknown session'); + return; + } + + if (!state.skippedItems.includes(itemIndex)) { + state.skippedItems.push(itemIndex); + } + + logger.info({ sessionId, focusedItem: itemIndex }, 'Deep Mode focus item recorded'); + } + + // ── Profile-aware helpers ───────────────────────────────────── + + /** + * Return whether a manual-profile session is paused waiting for user confirmation. + * Always false for thorough sessions (they never pause). + * + * @param sessionId Session to query. + */ + isPaused(sessionId: string): boolean { + return this.pausedSessions.has(sessionId); + } + + /** + * Resume a paused manual-profile session after the user has confirmed. + * No-op for sessions that are not paused. + * + * @param sessionId Session to resume. + */ + resume(sessionId: string): void { + if (!this.pausedSessions.has(sessionId)) { + logger.warn({ sessionId }, 'resume() called on session that is not paused'); + return; + } + this.pausedSessions.delete(sessionId); + logger.info({ sessionId }, 'Deep Mode session resumed after user confirmation'); + } + + /** + * Return whether the profile for this session requires user confirmation between phases. + * True only for manual profile. + * + * @param sessionId Session to query. + */ + requiresConfirmation(sessionId: string): boolean { + const state = this.sessions.get(sessionId); + return state?.profile === 'manual'; + } + + /** + * Return whether the profile for this session should auto-advance through phases + * without waiting for user input. True only for thorough profile. + * + * @param sessionId Session to query. + */ + shouldAutoAdvance(sessionId: string): boolean { + const state = this.sessions.get(sessionId); + return state?.profile === 'thorough'; + } + + // ── Accessors ───────────────────────────────────────────────── + + /** + * Return whether a session is currently active (exists, has phases remaining, and not paused). + * + * @param sessionId Session to query. + */ + isActive(sessionId: string): boolean { + const state = this.sessions.get(sessionId); + return state !== undefined && state.currentPhase !== undefined; + } + + /** + * Return the current phase for a session, or `undefined` if done / not found. + * + * @param sessionId Session to query. + */ + getCurrentPhase(sessionId: string): DeepPhase | undefined { + return this.sessions.get(sessionId)?.currentPhase; + } + + /** + * Return the stored result for a completed phase, or `undefined` if the phase + * has not yet run or was skipped. + * + * @param sessionId Session to query. + * @param phase The phase whose result to retrieve. + */ + getPhaseResult(sessionId: string, phase: DeepPhase): DeepPhaseResult | undefined { + const state = this.sessions.get(sessionId); + if (!state) return undefined; + return state.phaseResults[phase]; + } + + /** + * Return a read-only snapshot of the full session state. + * + * @param sessionId Session to query. + */ + getSessionState(sessionId: string): Readonly | undefined { + return this.sessions.get(sessionId); + } + + /** + * Return all active session IDs. + */ + getActiveSessions(): string[] { + return [...this.sessions.keys()]; + } + + // ── Per-Phase Model Selection ────────────────────────────────── + + // ── Per-Task Model Overrides (OB-1412) ──────────────────────── + + /** + * Set a model tier override for a specific plan item in the execute phase. + * + * Called when the user sends a natural language request such as + * "use opus for task 1" or "use haiku for this". + * + * Use taskIndex = 0 as a sentinel for "current task / current phase" — this + * maps to the same key "0" in `taskModelOverrides`. + * + * @param sessionId Active session identifier. + * @param taskIndex 1-based plan item index; 0 means "current task/phase". + * @param modelTier The model tier to use ('fast' | 'balanced' | 'powerful'). + * @returns `true` if the override was stored, `false` if session not found. + */ + setTaskModelOverride(sessionId: string, taskIndex: number, modelTier: ModelTier): boolean { + const state = this.sessions.get(sessionId); + if (!state) { + logger.warn({ sessionId }, 'setTaskModelOverride() called on unknown session'); + return false; + } + + state.taskModelOverrides[String(taskIndex)] = modelTier; + logger.info({ sessionId, taskIndex, modelTier }, 'Deep Mode task model override set'); + return true; + } + + /** + * Return the model tier override for a specific plan item, if one has been set. + * + * Falls back to the override for index 0 ("current task/phase") when no + * task-specific override exists. + * + * @param sessionId Active session identifier. + * @param taskIndex 1-based plan item index to look up. + * @returns The overridden ModelTier, or `undefined` if no override is set. + */ + getTaskModelOverride(sessionId: string, taskIndex: number): ModelTier | undefined { + const state = this.sessions.get(sessionId); + if (!state) return undefined; + + // Task-specific override takes precedence over the "current" sentinel (index 0) + const specific = state.taskModelOverrides[String(taskIndex)]; + if (specific !== undefined) return specific as ModelTier; + + const current = state.taskModelOverrides['0']; + return current !== undefined ? (current as ModelTier) : undefined; + } + + /** + * Return the recommended model tier for the current phase of a session. + * + * Applies user-supplied per-phase overrides (from `deep.phaseModels` config, + * added in OB-1402) on top of the built-in PHASE_MODEL_MAP defaults. + * + * @param sessionId Session to query. + * @param overrides Optional per-phase model tier overrides (from config). + * @returns The model tier, or `undefined` if session not found / done. + */ + getPhaseModelTier( + sessionId: string, + overrides?: Partial>, + ): ModelTier | undefined { + const state = this.sessions.get(sessionId); + if (!state || !state.currentPhase) return undefined; + + const phase = state.currentPhase; + return overrides?.[phase] ?? PHASE_MODEL_MAP[phase]; + } + + /** + * Return the system prompt injection for the current phase of a session. + * + * The returned string is intended to be appended to the Master AI system prompt + * (via --append-system-prompt) to steer the model toward the phase-appropriate goal. + * + * Callers may supply per-phase prompt overrides to replace individual built-in + * prompts — useful for domain-specific Deep Mode configurations. + * + * @param sessionId Session to query. + * @param overrides Optional per-phase prompt overrides. + * @returns The system prompt string, or `undefined` if session not found / done. + */ + getPhaseSystemPrompt( + sessionId: string, + overrides?: Partial>, + ): string | undefined { + const state = this.sessions.get(sessionId); + if (!state || !state.currentPhase) return undefined; + + const phase = state.currentPhase; + return overrides?.[phase] ?? PHASE_SYSTEM_PROMPTS[phase]; + } + + // ── Parallel Execute Phase ──────────────────────────────────── + + /** + * Return parallel execution batches for the execute phase of a session. + * + * Reads the plan phase result, parses task structure and batch groupings, + * filters out items marked as skipped, and returns an ordered array of + * batches. Tasks within the same batch have no inter-dependencies and can + * be spawned simultaneously by the caller. + * + * If no "Parallel Batches" section is found in the plan output (e.g., the AI + * omitted it), falls back to `groupTasksByDependency()` which derives batches + * from the per-task "**Dependencies:**" fields. + * + * Callers are responsible for respecting the WorkerRegistry concurrency limit + * when spawning tasks within each batch — use `workerRegistry.waitForSlot()` + * before each spawn. + * + * @param sessionId Active session identifier. + * @returns Ordered batches; each batch is an array of DeepPlanTask. + * Returns `undefined` if the session is not found or has no plan result. + */ + getExecuteBatches(sessionId: string): DeepExecuteBatch[] | undefined { + const state = this.sessions.get(sessionId); + if (!state) return undefined; + + const planResult = state.phaseResults['plan']; + if (!planResult) return undefined; + + const planOutput = planResult.output; + const tasks = parsePlanTasks(planOutput); + if (tasks.length === 0) return undefined; + + const skipped = new Set(state.skippedItems); + const batchTaskNums = parsePlanBatches(planOutput); + + let batches: DeepExecuteBatch[]; + + if (batchTaskNums.length > 0) { + // Use the explicit batch structure emitted by the plan worker + const taskMap = new Map(tasks.map((t) => [t.taskNumber, t])); + batches = batchTaskNums + .map((nums) => + nums + .filter((n) => !skipped.has(n)) + .map((n) => taskMap.get(n)) + .filter((t): t is DeepPlanTask => t !== undefined), + ) + .filter((batch) => batch.length > 0); + } else { + // Fallback: derive batches from the per-task dependency fields + const nonSkipped = tasks.filter((t) => !skipped.has(t.taskNumber)); + batches = groupTasksByDependency(nonSkipped); + } + + logger.info( + { sessionId, batchCount: batches.length, totalTasks: tasks.length }, + 'Deep Mode execute batches computed', + ); + + return batches; + } + + /** + * Build worker prompts for all tasks in a parallel execution batch. + * + * Each task in the batch gets its own DEEP_EXECUTE worker prompt with the + * task-specific context filled in. The session must have a plan result and + * be (or have been) in the execute phase for the output to be meaningful. + * + * @param sessionId Active session identifier. + * @param batch The execution batch returned by `getExecuteBatches()`. + * @returns Worker prompt strings, one per task. Empty if session not found. + */ + buildBatchWorkerPrompts(sessionId: string, batch: DeepExecuteBatch): string[] { + return batch + .map((task) => + this.buildWorkerPrompt(sessionId, { + taskNumber: task.taskNumber, + taskTitle: task.title, + taskDescription: task.description, + filesToModify: task.filesToModify.join('\n') || '(see plan)', + }), + ) + .filter((p): p is string => p !== undefined); + } + + // ── Final Summary ───────────────────────────────────────────── + + /** + * Compile a final summary of a completed (or aborted) Deep Mode session. + * + * Aggregates results from all phases and produces a structured summary with: + * - Phases completed and their completion timestamps + * - Findings count (extracted from report/investigate output) + * - Tasks executed vs skipped (from plan output) + * - Test verdict (from verify output) + * - Executive summary (from report output) + * + * The `formattedResponse` field in the returned object is a Markdown string + * ready to be sent directly as the final message to the user. + * + * @param sessionId Session to summarise (can be active or already removed). + * @param state Optional pre-fetched state (used when session already removed from memory). + * @returns `DeepModeFinalSummary`, or `null` when the session is not found + * and no state was provided. + */ + compileFinalSummary( + sessionId: string, + state?: Readonly, + ): DeepModeFinalSummary | null { + const resolvedState = state ?? this.sessions.get(sessionId); + if (!resolvedState) { + logger.warn({ sessionId }, 'compileFinalSummary() called on unknown session'); + return null; + } + + const { phaseResults, taskSummary, startedAt, skippedItems } = resolvedState; + + // --- Phases completed ------------------------------------------------ + const completedPhaseNames = PHASE_ORDER.filter((p) => phaseResults[p] !== undefined); + const phasesCompleted = completedPhaseNames.length; + + // --- Findings count --------------------------------------------------- + // Prefer report output; fall back to investigate output + const reportOutput = phaseResults['report']?.output ?? ''; + const investigateOutput = phaseResults['investigate']?.output ?? ''; + const findingsSource = reportOutput.length > 0 ? reportOutput : investigateOutput; + const findingsCount = countFindings(findingsSource); + + // --- Tasks executed / skipped ----------------------------------------- + const planOutput = phaseResults['plan']?.output ?? ''; + const totalPlanTasks = countPlanTasks(planOutput); + const tasksSkipped = skippedItems.length; + const tasksExecuted = Math.max(0, totalPlanTasks - tasksSkipped); + + // --- Test verdict ------------------------------------------------------ + const verifyOutput = phaseResults['verify']?.output ?? ''; + const testVerdict = verifyOutput.length > 0 ? extractVerifyVerdict(verifyOutput) : null; + + // --- Executive summary ------------------------------------------------ + const executiveSummary = extractExecutiveSummary(reportOutput); + + // --- Completion timestamp --------------------------------------------- + // Use the completedAt of the last completed phase, falling back to now + const lastPhaseResult = completedPhaseNames + .map((p) => phaseResults[p]) + .filter((r): r is DeepPhaseResult => r !== undefined) + .sort((a, b) => a.completedAt.localeCompare(b.completedAt)) + .at(-1); + const completedAt = lastPhaseResult?.completedAt ?? new Date().toISOString(); + + // --- Duration --------------------------------------------------------- + const durationMs = new Date(completedAt).getTime() - new Date(startedAt).getTime(); + const durationMin = Math.round(durationMs / 60_000); + const durationStr = durationMin >= 1 ? `${durationMin} min` : '< 1 min'; + + // --- Build formatted response ----------------------------------------- + const phaseStatusLines = PHASE_ORDER.map((p) => { + const result = phaseResults[p]; + if (result) { + return ` - ✅ **${p.charAt(0).toUpperCase() + p.slice(1)}** — completed at ${new Date(result.completedAt).toLocaleTimeString()}`; + } + return ` - ⏭️ **${p.charAt(0).toUpperCase() + p.slice(1)}** — skipped`; + }).join('\n'); + + const verdictLine = + testVerdict === 'pass' + ? '✅ All checks passed' + : testVerdict === 'fail' + ? '❌ Some checks failed — see verify phase output' + : '⚠️ No test results available'; + + const formattedResponse = [ + `## Deep Mode Complete — "${taskSummary}"`, + '', + `**Duration:** ${durationStr} | **Phases:** ${phasesCompleted}/5`, + '', + '### Phase Summary', + phaseStatusLines, + '', + '### Results', + `- **Findings identified:** ${findingsCount > 0 ? findingsCount : 'see report'}`, + `- **Tasks executed:** ${tasksExecuted}${tasksSkipped > 0 ? ` (${tasksSkipped} skipped)` : ''}`, + `- **Test verdict:** ${verdictLine}`, + '', + '### Executive Summary', + executiveSummary, + ...(verifyOutput.length > 0 && testVerdict === 'fail' + ? ['', '### Verify Output', '```', verifyOutput.slice(0, 800).trim(), '```'] + : []), + ].join('\n'); + + const summary: DeepModeFinalSummary = { + phasesCompleted, + completedPhaseNames, + findingsCount, + tasksExecuted, + tasksSkipped, + testVerdict, + executiveSummary, + taskSummary, + startedAt, + completedAt, + formattedResponse, + }; + + logger.info( + { + sessionId, + phasesCompleted, + findingsCount, + tasksExecuted, + tasksSkipped, + testVerdict, + }, + 'Deep Mode final summary compiled', + ); + + // Persist session history as a fire-and-forget side effect + this.persistSession(sessionId, summary, resolvedState).catch((err: unknown) => { + logger.warn({ sessionId, err }, 'Deep Mode session persist failed (background)'); + }); + + return summary; + } + + // ── Session History Persistence ──────────────────────────────── + + /** + * Persist a completed Deep Mode session to `.openbridge/deep-mode/session-{timestamp}.json`. + * + * The session file captures all phase results, decisions, and skipped items for + * later review and prompt evolution training. Writing is best-effort — failures + * are logged but never thrown to callers. + * + * @param sessionId Session identifier (used for logging only when state is provided). + * @param summary The final summary produced by compileFinalSummary(). + * @param state The resolved session state (may come from outside the sessions map). + */ + async persistSession( + sessionId: string, + summary: DeepModeFinalSummary, + state?: Readonly, + ): Promise { + const resolvedState = state ?? this.sessions.get(sessionId); + if (!resolvedState) { + logger.warn({ sessionId }, 'persistSession() called on unknown session'); + return; + } + + // Build a filename-safe timestamp from the session start time + // e.g. "2026-03-02T14-30-05" → "session-2026-03-02_14-30-05.json" + const timestamp = resolvedState.startedAt + .replace(/\.\d{3}Z$/, '') // strip milliseconds + Z + .replace('T', '_') + .replace(/:/g, '-'); + + const deepModeDir = path.join(this.host.workspacePath, '.openbridge', 'deep-mode'); + const filePath = path.join(deepModeDir, `session-${timestamp}.json`); + + const record = { + sessionId: resolvedState.sessionId, + profile: resolvedState.profile, + taskSummary: resolvedState.taskSummary, + startedAt: resolvedState.startedAt, + completedAt: summary.completedAt, + phasesCompleted: summary.phasesCompleted, + completedPhaseNames: summary.completedPhaseNames, + findingsCount: summary.findingsCount, + tasksExecuted: summary.tasksExecuted, + tasksSkipped: summary.tasksSkipped, + testVerdict: summary.testVerdict, + executiveSummary: summary.executiveSummary, + skippedItems: resolvedState.skippedItems, + phaseResults: resolvedState.phaseResults, + }; + + try { + await fs.mkdir(deepModeDir, { recursive: true }); + await fs.writeFile(filePath, JSON.stringify(record, null, 2), 'utf8'); + logger.info({ sessionId, filePath }, 'Deep Mode session persisted'); + } catch (err) { + logger.warn({ sessionId, filePath, err }, 'Deep Mode session persist failed'); + } + } + + // ── Phase Worker Prompt Builder ──────────────────────────────── + + /** + * Build the worker prompt for the current phase of a session. + * + * Selects the DEEP_* seed-prompt template that matches the current phase, + * substitutes all `{{placeholder}}` tokens, and injects the output from + * previous phases as context: + * + * - investigate: uses caller-supplied workspace / project metadata + * - report: injects investigate.output as `{{investigationFindings}}` + * - plan: injects report.output as `{{reportFindings}}` + * - execute: injects plan.output as `{{planContext}}` + caller task details + * - verify: injects execute.output as `{{executedTasks}}` + * + * @param sessionId Active session identifier. + * @param context Phase-specific context for placeholder substitution. + * @returns The filled worker prompt string, or `undefined` if session not found / done. + */ + buildWorkerPrompt(sessionId: string, context: DeepPhaseWorkerContext = {}): string | undefined { + const state = this.sessions.get(sessionId); + if (!state || !state.currentPhase) return undefined; + + const { currentPhase: phase, taskSummary: userRequest, phaseResults } = state; + + switch (phase) { + case 'investigate': + return DEEP_INVESTIGATE.content + .replace('{{workspacePath}}', context.workspacePath ?? this.host.workspacePath) + .replace('{{userRequest}}', userRequest) + .replace('{{projectName}}', context.projectName ?? 'Unknown') + .replace('{{projectType}}', context.projectType ?? 'Unknown') + .replace('{{frameworks}}', context.frameworks ?? 'Unknown') + .replace('{{structure}}', context.structure ?? '(no structure available)'); + + case 'report': { + const investigationFindings = + phaseResults['investigate']?.output ?? '(no investigation results available)'; + return DEEP_REPORT.content + .replace('{{userRequest}}', userRequest) + .replace('{{investigationFindings}}', investigationFindings); + } + + case 'plan': { + const reportFindings = phaseResults['report']?.output ?? '(no report available)'; + return DEEP_PLAN.content + .replace('{{userRequest}}', userRequest) + .replace('{{reportFindings}}', reportFindings); + } + + case 'execute': { + const planContext = phaseResults['plan']?.output ?? '(no plan available)'; + return DEEP_EXECUTE.content + .replace('{{userRequest}}', userRequest) + .replace('{{taskNumber}}', String(context.taskNumber ?? 1)) + .replace('{{taskTitle}}', context.taskTitle ?? 'Execute task') + .replace('{{filesToModify}}', context.filesToModify ?? '(see plan)') + .replace('{{taskDescription}}', context.taskDescription ?? '(see plan)') + .replace('{{constraints}}', context.constraints ?? 'None.') + .replace('{{planContext}}', planContext); + } + + case 'verify': { + const executedTasks = phaseResults['execute']?.output ?? '(no execute results available)'; + return DEEP_VERIFY.content + .replace('{{userRequest}}', userRequest) + .replace('{{executedTasks}}', executedTasks); + } + + default: + return undefined; + } + } +} diff --git a/src/master/delegation.ts b/src/master/delegation.ts index 2fb2b96e..0e92da3b 100644 --- a/src/master/delegation.ts +++ b/src/master/delegation.ts @@ -1,6 +1,7 @@ import { randomUUID } from 'node:crypto'; import { createLogger } from '../core/logger.js'; -import { executeClaudeCode } from '../providers/claude-code/claude-code-executor.js'; +import { AgentRunner, TOOLS_CODE_EDIT, DEFAULT_MAX_TURNS_TASK } from '../core/agent-runner.js'; +import type { CLIAdapter } from '../core/cli-adapter.js'; import type { DiscoveredTool } from '../types/discovery.js'; import type { TaskRecord } from '../types/master.js'; @@ -75,10 +76,16 @@ export class DelegationCoordinator { private activeDelegations: Map = new Map(); private readonly maxConcurrentDelegations: number; private readonly defaultTimeout: number; + private readonly agentRunner: AgentRunner; - constructor(options?: { maxConcurrentDelegations?: number; defaultTimeout?: number }) { + constructor(options?: { + maxConcurrentDelegations?: number; + defaultTimeout?: number; + adapter?: CLIAdapter; + }) { this.maxConcurrentDelegations = options?.maxConcurrentDelegations ?? 3; this.defaultTimeout = options?.defaultTimeout ?? DEFAULT_DELEGATION_TIMEOUT; + this.agentRunner = new AgentRunner(options?.adapter); logger.info( { @@ -238,13 +245,13 @@ export class DelegationCoordinator { startedAt: string, ): Promise { try { - // Execute the AI tool - // Note: Currently using the generalized executor (executeClaudeCode) - // In the future, this could be abstracted to support different AI tool CLIs - const result = await executeClaudeCode({ + const result = await this.agentRunner.spawn({ prompt: options.prompt, workspacePath: options.workspacePath, timeout: options.timeout ?? this.defaultTimeout, + allowedTools: [...TOOLS_CODE_EDIT], + maxTurns: DEFAULT_MAX_TURNS_TASK, + retries: 0, }); const completedAt = new Date().toISOString(); @@ -329,7 +336,7 @@ export class DelegationCoordinator { 'Delegation timed out', ); - // Note: The timeout in the executeClaudeCode call will handle actual process termination + // Note: The timeout in the AgentRunner spawn call will handle actual process termination // This is just for tracking and cleanup } diff --git a/src/master/dotfolder-manager.ts b/src/master/dotfolder-manager.ts index 085f833d..d88c2303 100644 --- a/src/master/dotfolder-manager.ts +++ b/src/master/dotfolder-manager.ts @@ -1,40 +1,70 @@ import * as fs from 'node:fs/promises'; import * as path from 'node:path'; -import { exec } from 'node:child_process'; -import { promisify } from 'node:util'; import type { - WorkspaceMap, AgentsRegistry, ExplorationLogEntry, TaskRecord, + ExplorationState, + StructureScan, + Classification, + DirectoryDiveResult, + MasterSession, + LearningEntry, + LearningsRegistry, + WorkspaceAnalysisMarker, + ClassificationCache, + WorkspaceMap, + PromptManifest, + PromptTemplate, } from '../types/master.js'; import { - WorkspaceMapSchema, AgentsRegistrySchema, - ExplorationLogEntrySchema, TaskRecordSchema, + ExplorationStateSchema, + StructureScanSchema, + ClassificationSchema, + DirectoryDiveResultSchema, + MasterSessionSchema, + LearningEntrySchema, + LearningsRegistrySchema, + WorkspaceAnalysisMarkerSchema, + ClassificationCacheSchema, + WorkspaceMapSchema, + PromptManifestSchema, + PromptTemplateSchema, } from '../types/master.js'; +import type { ToolProfile, ProfilesRegistry, BatchState, WorkerSummary } from '../types/agent.js'; +import { ToolProfileSchema, ProfilesRegistrySchema, BatchStateSchema } from '../types/agent.js'; +import type { WorkersRegistry } from './worker-registry.js'; +import { WorkersRegistrySchema } from './worker-registry.js'; +import { createLogger } from '../core/logger.js'; -const execAsync = promisify(exec); +const logger = createLogger('dotfolder-manager'); /** * Manages the .openbridge/ folder inside the target workspace. * This folder contains: - * - .git/ — local git repo tracking Master AI changes - * - workspace-map.json — auto-generated project understanding - * - exploration.log — timestamped scan history - * - agents.json — discovered AI tools + roles - * - tasks/ — task history (one JSON per task) + * - openbridge.db — SQLite database (primary storage for all runtime data) + * - generated/ — AI-generated output files + * - agents.json — discovered AI tools + roles (legacy fallback; DB is primary) */ export class DotFolderManager { private readonly workspacePath: string; private readonly dotFolderPath: string; private readonly tasksPath: string; + private readonly explorationPath: string; + private readonly explorationDirsPath: string; + private readonly promptsPath: string; + private readonly contextPath: string; constructor(workspacePath: string) { this.workspacePath = workspacePath; this.dotFolderPath = path.join(workspacePath, '.openbridge'); this.tasksPath = path.join(this.dotFolderPath, 'tasks'); + this.explorationPath = path.join(this.dotFolderPath, 'exploration'); + this.explorationDirsPath = path.join(this.explorationPath, 'dirs'); + this.promptsPath = path.join(this.dotFolderPath, 'prompts'); + this.contextPath = path.join(this.dotFolderPath, 'context'); } /** @@ -51,6 +81,41 @@ export class DotFolderManager { return path.join(this.dotFolderPath, 'workspace-map.json'); } + /** + * Read workspace map from .openbridge/workspace-map.json. + * Returns null if the file doesn't exist or is invalid. + */ + public async readWorkspaceMap(): Promise { + const mapPath = this.getMapPath(); + + // Check existence before reading to avoid ENOENT spam on first run + try { + await fs.access(mapPath); + } catch { + logger.debug({ path: mapPath }, 'workspace-map.json not found'); + return null; + } + + try { + const content = await fs.readFile(mapPath, 'utf-8'); + const data = JSON.parse(content) as unknown; + return WorkspaceMapSchema.parse(data); + } catch (err) { + logger.warn({ err, path: mapPath }, 'Failed to read workspace-map.json'); + return null; + } + } + + /** + * Write workspace map to .openbridge/workspace-map.json as a JSON safety net. + * Primary storage is the DB (via memory.storeChunks); this is the fallback. + */ + public async writeWorkspaceMap(map: WorkspaceMap): Promise { + const validated = WorkspaceMapSchema.parse(map); + await fs.mkdir(this.dotFolderPath, { recursive: true }); + await fs.writeFile(this.getMapPath(), JSON.stringify(validated, null, 2), 'utf-8'); + } + /** * Get the path to the agents.json file */ @@ -81,233 +146,1113 @@ export class DotFolderManager { * Create .openbridge folder structure * Creates: * - .openbridge/ - * - .openbridge/tasks/ + * - .openbridge/generated/ */ public async createFolder(): Promise { await fs.mkdir(this.dotFolderPath, { recursive: true }); - await fs.mkdir(this.tasksPath, { recursive: true }); + await fs.mkdir(path.join(this.dotFolderPath, 'generated'), { recursive: true }); } /** - * Initialize git repository inside .openbridge/ - * This repo tracks all Master AI changes to the workspace knowledge. + * Get the path to the analysis-marker.json file */ - public async initGit(): Promise { - const gitPath = path.join(this.dotFolderPath, '.git'); + public getAnalysisMarkerPath(): string { + return path.join(this.dotFolderPath, 'analysis-marker.json'); + } - // Check if git repo already exists + /** + * Read the workspace analysis marker from .openbridge/analysis-marker.json. + * Returns null if the file doesn't exist or is invalid. + */ + public async readAnalysisMarker(): Promise { + const markerPath = this.getAnalysisMarkerPath(); try { - await fs.access(gitPath); - return; // Already initialized - } catch { - // Not initialized, proceed + const content = await fs.readFile(markerPath, 'utf-8'); + const data = JSON.parse(content) as unknown; + return WorkspaceAnalysisMarkerSchema.parse(data); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') { + logger.debug({ path: markerPath }, 'File not found (expected on first run)'); + } else { + logger.warn({ err, path: markerPath }, 'Failed to read analysis-marker.json'); + } + return null; } + } - // Initialize git repo - await execAsync('git init', { cwd: this.dotFolderPath }); + /** + * Write the workspace analysis marker to .openbridge/analysis-marker.json. + * Called after every successful exploration (full or incremental). + */ + public async writeAnalysisMarker(marker: WorkspaceAnalysisMarker): Promise { + const validated = WorkspaceAnalysisMarkerSchema.parse(marker); + const markerPath = this.getAnalysisMarkerPath(); + await fs.writeFile(markerPath, JSON.stringify(validated, null, 2), 'utf-8'); + } + + /** + * Read agents registry from agents.json + */ + public async readAgents(): Promise { + const agentsPath = this.getAgentsPath(); - // Create .gitignore to avoid tracking unnecessary files - const gitignore = `# Ignore node_modules if they somehow end up here -node_modules/ + try { + const content = await fs.readFile(agentsPath, 'utf-8'); + const data = JSON.parse(content) as unknown; + return AgentsRegistrySchema.parse(data); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') { + logger.debug({ path: agentsPath }, 'File not found (expected on first run)'); + } else { + logger.warn({ err, path: agentsPath }, 'Failed to read agents.json'); + } + return null; + } + } + + /** + * Write agents registry to agents.json + */ + public async writeAgents(registry: AgentsRegistry): Promise { + // Validate before writing + const validated = AgentsRegistrySchema.parse(registry); -# Ignore OS files -.DS_Store -Thumbs.db -`; - await fs.writeFile(path.join(this.dotFolderPath, '.gitignore'), gitignore, 'utf-8'); + const agentsPath = this.getAgentsPath(); + await fs.writeFile(agentsPath, JSON.stringify(validated, null, 2), 'utf-8'); + } - // Initial commit - await this.commitChanges('Initial commit: .openbridge folder created'); + /** + * Append an entry to exploration.log + * @deprecated Flat-file logging removed. Use memory.logExploration() instead. + * This method is kept for call-site compatibility during the memory migration. + */ + public async appendLog(_entry: ExplorationLogEntry): Promise { + // No-op: exploration log is now written to the DB via memory.logExploration(). + // The flat-file exploration.log is no longer written. } /** - * Commit changes to the .openbridge git repo + * Read all tasks */ - public async commitChanges(message: string): Promise { + public async readAllTasks(): Promise { try { - // Add all changes - await execAsync('git add -A', { cwd: this.dotFolderPath }); - - // Check if there are changes to commit - const { stdout: status } = await execAsync('git status --porcelain', { - cwd: this.dotFolderPath, - }); + const files = await fs.readdir(this.tasksPath); + const taskFiles = files.filter((file) => file.endsWith('.json')); - if (!status.trim()) { - // No changes to commit - return; + const tasks: TaskRecord[] = []; + for (const file of taskFiles) { + const taskPath = path.join(this.tasksPath, file); + const content = await fs.readFile(taskPath, 'utf-8'); + const data = JSON.parse(content) as unknown; + tasks.push(TaskRecordSchema.parse(data)); } - // Commit with message - await execAsync(`git commit -m "${message.replace(/"/g, '\\"')}"`, { - cwd: this.dotFolderPath, - }); - } catch (error) { - // If git user is not configured, try to set a default - if (error instanceof Error && error.message.includes('user.email')) { - await execAsync('git config user.email "master@openbridge.local"', { - cwd: this.dotFolderPath, - }); - await execAsync('git config user.name "OpenBridge Master AI"', { - cwd: this.dotFolderPath, - }); - - // Retry commit - await execAsync(`git commit -m "${message.replace(/"/g, '\\"')}"`, { - cwd: this.dotFolderPath, - }); + return tasks; + } catch { + return []; + } + } + + /** + * No-op: exploration directories are no longer created on disk. + * Exploration state is stored in the SQLite DB via MemoryManager. + * @deprecated OB-813 — exploration subdirs removed; DB is the primary store. + */ + public async createExplorationDir(): Promise {} + + /** + * Read exploration state from exploration-state.json + */ + public async readExplorationState(): Promise { + const statePath = path.join(this.explorationPath, 'exploration-state.json'); + + try { + const content = await fs.readFile(statePath, 'utf-8'); + const data = JSON.parse(content) as unknown; + return ExplorationStateSchema.parse(data); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') { + logger.debug({ path: statePath }, 'File not found (expected on first run)'); } else { - throw error; + logger.warn({ err, path: statePath }, 'Failed to read exploration-state.json'); } + return null; } } /** - * Read workspace map from workspace-map.json + * Write exploration state to exploration-state.json */ - public async readMap(): Promise { - const mapPath = this.getMapPath(); + public async writeExplorationState(state: ExplorationState): Promise { + // Validate before writing + const validated = ExplorationStateSchema.parse(state); + + await fs.mkdir(this.explorationPath, { recursive: true }); + const statePath = path.join(this.explorationPath, 'exploration-state.json'); + await fs.writeFile(statePath, JSON.stringify(validated, null, 2), 'utf-8'); + } + + /** + * Read structure scan from structure-scan.json + */ + public async readStructureScan(): Promise { + const scanPath = path.join(this.explorationPath, 'structure-scan.json'); try { - const content = await fs.readFile(mapPath, 'utf-8'); + const content = await fs.readFile(scanPath, 'utf-8'); const data = JSON.parse(content) as unknown; - return WorkspaceMapSchema.parse(data); - } catch { + return StructureScanSchema.parse(data); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') { + logger.debug({ path: scanPath }, 'File not found (expected on first run)'); + } else { + logger.warn({ err, path: scanPath }, 'Failed to read structure-scan.json'); + } return null; } } /** - * Write workspace map to workspace-map.json + * Write structure scan to structure-scan.json */ - public async writeMap(map: WorkspaceMap): Promise { + public async writeStructureScan(scan: StructureScan): Promise { // Validate before writing - const validated = WorkspaceMapSchema.parse(map); + const validated = StructureScanSchema.parse(scan); - const mapPath = this.getMapPath(); - await fs.writeFile(mapPath, JSON.stringify(validated, null, 2), 'utf-8'); + await fs.mkdir(this.explorationPath, { recursive: true }); + const scanPath = path.join(this.explorationPath, 'structure-scan.json'); + await fs.writeFile(scanPath, JSON.stringify(validated, null, 2), 'utf-8'); } /** - * Read agents registry from agents.json + * Read classification from classification.json */ - public async readAgents(): Promise { - const agentsPath = this.getAgentsPath(); + public async readClassification(): Promise { + const classificationPath = path.join(this.explorationPath, 'classification.json'); try { - const content = await fs.readFile(agentsPath, 'utf-8'); + const content = await fs.readFile(classificationPath, 'utf-8'); const data = JSON.parse(content) as unknown; - return AgentsRegistrySchema.parse(data); - } catch { + return ClassificationSchema.parse(data); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') { + logger.debug({ path: classificationPath }, 'File not found (expected on first run)'); + } else { + logger.warn({ err, path: classificationPath }, 'Failed to read classification.json'); + } return null; } } /** - * Write agents registry to agents.json + * Write classification to classification.json */ - public async writeAgents(registry: AgentsRegistry): Promise { + public async writeClassification(classification: Classification): Promise { // Validate before writing - const validated = AgentsRegistrySchema.parse(registry); + const validated = ClassificationSchema.parse(classification); - const agentsPath = this.getAgentsPath(); - await fs.writeFile(agentsPath, JSON.stringify(validated, null, 2), 'utf-8'); + await fs.mkdir(this.explorationPath, { recursive: true }); + const classificationPath = path.join(this.explorationPath, 'classification.json'); + await fs.writeFile(classificationPath, JSON.stringify(validated, null, 2), 'utf-8'); } /** - * Append an entry to exploration.log + * Read directory dive result from exploration/dirs/{dirName}.json */ - public async appendLog(entry: ExplorationLogEntry): Promise { - // Validate before appending - const validated = ExplorationLogEntrySchema.parse(entry); + public async readDirectoryDive(dirName: string): Promise { + const divePath = path.join(this.explorationDirsPath, `${dirName}.json`); + + try { + const content = await fs.readFile(divePath, 'utf-8'); + const data = JSON.parse(content) as unknown; + return DirectoryDiveResultSchema.parse(data); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') { + logger.debug({ path: divePath }, 'File not found (expected on first run)'); + } else { + logger.warn({ err, path: divePath }, 'Failed to read directory dive result'); + } + return null; + } + } + + /** + * Write directory dive result to exploration/dirs/{dirName}.json + */ + public async writeDirectoryDive(dirName: string, dive: DirectoryDiveResult): Promise { + // Validate before writing + const validated = DirectoryDiveResultSchema.parse(dive); - const logPath = this.getLogPath(); - const line = JSON.stringify(validated) + '\n'; + await fs.mkdir(this.explorationDirsPath, { recursive: true }); + const divePath = path.join(this.explorationDirsPath, `${dirName}.json`); + await fs.writeFile(divePath, JSON.stringify(validated, null, 2), 'utf-8'); + } - await fs.appendFile(logPath, line, 'utf-8'); + /** + * Get the path to the profiles.json file + */ + public getProfilesPath(): string { + return path.join(this.dotFolderPath, 'profiles.json'); } /** - * Read all exploration log entries + * Read custom profiles registry from profiles.json */ - public async readLog(): Promise { - const logPath = this.getLogPath(); + public async readProfiles(): Promise { + const profilesPath = this.getProfilesPath(); try { - const content = await fs.readFile(logPath, 'utf-8'); - const lines = content - .split('\n') - .filter((line) => line.trim()) - .map((line) => JSON.parse(line) as unknown); - - return lines.map((line) => ExplorationLogEntrySchema.parse(line)); - } catch { - return []; + const content = await fs.readFile(profilesPath, 'utf-8'); + const data = JSON.parse(content) as unknown; + return ProfilesRegistrySchema.parse(data); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') { + logger.debug({ path: profilesPath }, 'File not found (expected on first run)'); + } else { + logger.warn({ err, path: profilesPath }, 'Failed to read profiles.json'); + } + return null; } } /** - * Record a task in tasks/ folder and commit to git + * Write custom profiles registry to profiles.json */ - public async recordTask(task: TaskRecord): Promise { - // Validate before recording - const validated = TaskRecordSchema.parse(task); + public async writeProfiles(registry: ProfilesRegistry): Promise { + const validated = ProfilesRegistrySchema.parse(registry); + const profilesPath = this.getProfilesPath(); + await fs.writeFile(profilesPath, JSON.stringify(validated, null, 2), 'utf-8'); + } + + /** + * Add or update a custom profile in the registry. + * Creates profiles.json if it doesn't exist. + */ + public async addProfile(profile: ToolProfile): Promise { + ToolProfileSchema.parse(profile); + + const existing = await this.readProfiles(); + const registry: ProfilesRegistry = existing ?? { + profiles: {}, + updatedAt: new Date().toISOString(), + }; - const taskPath = path.join(this.tasksPath, `${task.id}.json`); - await fs.writeFile(taskPath, JSON.stringify(validated, null, 2), 'utf-8'); + registry.profiles[profile.name] = profile; + registry.updatedAt = new Date().toISOString(); - // Commit the task to git with conventional commit format - const commitMessage = `chore(master): record task ${task.id} - ${task.status}`; - await this.commitChanges(commitMessage); + await this.writeProfiles(registry); } /** - * Read a task by ID + * Remove a custom profile from the registry. + * Returns true if the profile was found and removed, false otherwise. */ - public async readTask(taskId: string): Promise { - const taskPath = path.join(this.tasksPath, `${taskId}.json`); + public async removeProfile(profileName: string): Promise { + const existing = await this.readProfiles(); + if (!existing || !(profileName in existing.profiles)) { + return false; + } + + delete existing.profiles[profileName]; + existing.updatedAt = new Date().toISOString(); + + await this.writeProfiles(existing); + return true; + } + + /** + * Get a single custom profile by name. + * Returns null if the profile doesn't exist. + */ + public async getProfile(profileName: string): Promise { + const registry = await this.readProfiles(); + if (!registry) return null; + return registry.profiles[profileName] ?? null; + } + + /** + * Get the path to the master-session.json file + */ + public getMasterSessionPath(): string { + return path.join(this.dotFolderPath, 'master-session.json'); + } + + /** + * Read the persistent Master session info from master-session.json + */ + public async readMasterSession(): Promise { + const sessionPath = this.getMasterSessionPath(); try { - const content = await fs.readFile(taskPath, 'utf-8'); + const content = await fs.readFile(sessionPath, 'utf-8'); const data = JSON.parse(content) as unknown; - return TaskRecordSchema.parse(data); - } catch { + return MasterSessionSchema.parse(data); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') { + logger.debug({ path: sessionPath }, 'File not found (expected on first run)'); + } else { + logger.warn({ err, path: sessionPath }, 'Failed to read master-session.json'); + } return null; } } /** - * Read all tasks + * Write the persistent Master session info to master-session.json */ - public async readAllTasks(): Promise { + public async writeMasterSession(session: MasterSession): Promise { + const validated = MasterSessionSchema.parse(session); + const sessionPath = this.getMasterSessionPath(); + await fs.writeFile(sessionPath, JSON.stringify(validated, null, 2), 'utf-8'); + } + + /** + * Get the path to the prompts directory + */ + public getPromptsPath(): string { + return this.promptsPath; + } + + /** + * Get the path to the master system prompt file + */ + public getSystemPromptPath(): string { + return path.join(this.promptsPath, 'master-system.md'); + } + + /** + * Read the master system prompt from .openbridge/prompts/master-system.md + */ + public async readSystemPrompt(): Promise { + const systemPromptPath = this.getSystemPromptPath(); + + // Check existence before reading to avoid ENOENT spam on first run try { - const files = await fs.readdir(this.tasksPath); - const taskFiles = files.filter((file) => file.endsWith('.json')); + await fs.access(systemPromptPath); + } catch { + return null; + } - const tasks: TaskRecord[] = []; - for (const file of taskFiles) { - const taskPath = path.join(this.tasksPath, file); - const content = await fs.readFile(taskPath, 'utf-8'); - const data = JSON.parse(content) as unknown; - tasks.push(TaskRecordSchema.parse(data)); + try { + return await fs.readFile(systemPromptPath, 'utf-8'); + } catch (err) { + logger.warn({ err, path: systemPromptPath }, 'Failed to read master-system.md'); + return null; + } + } + + /** + * Write the master system prompt to .openbridge/prompts/master-system.md. + * Creates the prompts directory if it doesn't exist. + */ + public async writeSystemPrompt(content: string): Promise { + await fs.mkdir(this.promptsPath, { recursive: true }); + await fs.writeFile(this.getSystemPromptPath(), content, 'utf-8'); + } + + /** + * Get the path to the workers.json file + */ + public getWorkersPath(): string { + return path.join(this.dotFolderPath, 'workers.json'); + } + + /** + * Read workers registry from workers.json + */ + public async readWorkers(): Promise { + const workersPath = this.getWorkersPath(); + + try { + const content = await fs.readFile(workersPath, 'utf-8'); + const data = JSON.parse(content) as unknown; + return WorkersRegistrySchema.parse(data); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') { + logger.debug({ path: workersPath }, 'File not found (expected on first run)'); + } else { + logger.warn({ err, path: workersPath }, 'Failed to read workers.json'); } + return null; + } + } - return tasks; + /** + * Write workers registry to workers.json + */ + public async writeWorkers(registry: WorkersRegistry): Promise { + const validated = WorkersRegistrySchema.parse(registry); + const workersPath = this.getWorkersPath(); + await fs.writeFile(workersPath, JSON.stringify(validated, null, 2), 'utf-8'); + } + + /** + * Read the content of a prompt template file from .openbridge/prompts/. + */ + public async readPromptTemplate(filename: string): Promise { + return fs.readFile(path.join(this.promptsPath, filename), 'utf-8'); + } + + /** + * Get the path to the learnings.json file + */ + public getLearningsPath(): string { + return path.join(this.dotFolderPath, 'learnings.json'); + } + + /** + * Read the learnings registry from .openbridge/learnings.json + */ + public async readLearnings(): Promise { + const learningsPath = this.getLearningsPath(); + + // Check existence before reading to avoid ENOENT spam on first run + try { + await fs.access(learningsPath); } catch { - return []; + logger.debug({ path: learningsPath }, 'learnings.json not found'); + return null; + } + + try { + const content = await fs.readFile(learningsPath, 'utf-8'); + const data = JSON.parse(content) as unknown; + return LearningsRegistrySchema.parse(data); + } catch (err) { + logger.warn({ err, path: learningsPath }, 'Failed to read learnings.json'); + return null; } } + /** + * Write the learnings registry to .openbridge/learnings.json + */ + public async writeLearnings(registry: LearningsRegistry): Promise { + const validated = LearningsRegistrySchema.parse(registry); + const learningsPath = this.getLearningsPath(); + await fs.writeFile(learningsPath, JSON.stringify(validated, null, 2), 'utf-8'); + } + + /** + * Append a new learning entry to the learnings registry. + * Creates learnings.json if it doesn't exist. + */ + public async appendLearning(entry: LearningEntry): Promise { + LearningEntrySchema.parse(entry); + + const existing = await this.readLearnings(); + const now = new Date().toISOString(); + const registry: LearningsRegistry = existing ?? { + entries: [], + createdAt: now, + updatedAt: now, + schemaVersion: '1.0.0', + }; + + registry.entries.push(entry); + registry.updatedAt = now; + + await this.writeLearnings(registry); + } + + /** + * Get all learning entries for a specific task type. + * Useful for analyzing patterns in task execution. + */ + public async getLearningsByTaskType(taskType: string): Promise { + const registry = await this.readLearnings(); + if (!registry) return []; + + return registry.entries.filter((entry) => entry.taskType === taskType); + } + + /** + * Get all learning entries for a specific model. + * Useful for analyzing model performance patterns. + */ + public async getLearningsByModel(model: string): Promise { + const registry = await this.readLearnings(); + if (!registry) return []; + + return registry.entries.filter((entry) => entry.modelUsed === model); + } + + /** + * Get all learning entries for a specific profile. + * Useful for analyzing profile effectiveness. + */ + public async getLearningsByProfile(profile: string): Promise { + const registry = await this.readLearnings(); + if (!registry) return []; + + return registry.entries.filter((entry) => entry.profileUsed === profile); + } + + /** + * Get all failed learning entries. + * Useful for identifying problem areas and patterns. + */ + public async getFailedLearnings(): Promise { + const registry = await this.readLearnings(); + if (!registry) return []; + + return registry.entries.filter((entry) => !entry.success); + } + + /** + * Get learning statistics for a specific task type. + * Returns success rate, average duration, total count, etc. + */ + public async getTaskTypeStats(taskType: string): Promise<{ + totalCount: number; + successCount: number; + failureCount: number; + successRate: number; + avgDurationMs: number; + avgRetryCount: number; + } | null> { + const entries = await this.getLearningsByTaskType(taskType); + if (entries.length === 0) return null; + + const successCount = entries.filter((e) => e.success).length; + const failureCount = entries.length - successCount; + const successRate = successCount / entries.length; + const avgDurationMs = entries.reduce((sum, e) => sum + e.durationMs, 0) / entries.length; + const avgRetryCount = entries.reduce((sum, e) => sum + e.retryCount, 0) / entries.length; + + return { + totalCount: entries.length, + successCount, + failureCount, + successRate, + avgDurationMs, + avgRetryCount, + }; + } + + /** + * Get learning statistics for a specific model. + * Returns success rate, average duration, total count, etc. + */ + public async getModelStats(model: string): Promise<{ + totalCount: number; + successCount: number; + failureCount: number; + successRate: number; + avgDurationMs: number; + avgRetryCount: number; + } | null> { + const entries = await this.getLearningsByModel(model); + if (entries.length === 0) return null; + + const successCount = entries.filter((e) => e.success).length; + const failureCount = entries.length - successCount; + const successRate = successCount / entries.length; + const avgDurationMs = entries.reduce((sum, e) => sum + e.durationMs, 0) / entries.length; + const avgRetryCount = entries.reduce((sum, e) => sum + e.retryCount, 0) / entries.length; + + return { + totalCount: entries.length, + successCount, + failureCount, + successRate, + avgDurationMs, + avgRetryCount, + }; + } + /** * Initialize .openbridge folder if it doesn't exist - * Creates folder structure and initializes git repo */ public async initialize(): Promise { const folderExists = await this.exists(); if (!folderExists) { await this.createFolder(); - await this.initGit(); + } + + await fs.mkdir(this.contextPath, { recursive: true }); + } + + // ── Classification Cache ────────────────────────────────────── + + /** + * Get the path to the classifications.json file + */ + public getClassificationsPath(): string { + return path.join(this.dotFolderPath, 'classifications.json'); + } + + /** + * Read the classification cache from .openbridge/classifications.json. + * Returns null if the file does not exist or cannot be parsed. + */ + public async readClassifications(): Promise { + const classificationsPath = this.getClassificationsPath(); + try { + const content = await fs.readFile(classificationsPath, 'utf-8'); + return ClassificationCacheSchema.parse(JSON.parse(content)); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') { + logger.debug({ path: classificationsPath }, 'File not found (expected on first run)'); + } else { + logger.warn({ err, path: classificationsPath }, 'Failed to read classifications.json'); + } + return null; + } + } + + /** + * Write the classification cache to .openbridge/classifications.json. + */ + public async writeClassifications(cache: ClassificationCache): Promise { + const validated = ClassificationCacheSchema.parse(cache); + const classificationsPath = this.getClassificationsPath(); + await fs.writeFile(classificationsPath, JSON.stringify(validated, null, 2), 'utf-8'); + } + + // ── Prompt Library ─────────────────────────────────────────── + + /** + * Get the path to the prompt manifest file. + */ + private getPromptManifestPath(): string { + return path.join(this.promptsPath, 'manifest.json'); + } + + /** + * Read the prompt manifest from .openbridge/prompts/manifest.json. + * Returns null if the file does not exist or cannot be parsed. + */ + public async readPromptManifest(): Promise { + const manifestPath = this.getPromptManifestPath(); + + // Check existence before reading to avoid ENOENT spam on first run + try { + await fs.access(manifestPath); + } catch { + logger.debug({ path: manifestPath }, 'manifest.json not found'); + return null; + } + + try { + const content = await fs.readFile(manifestPath, 'utf-8'); + const data = JSON.parse(content) as unknown; + return PromptManifestSchema.parse(data); + } catch (err) { + logger.warn({ err, path: manifestPath }, 'Failed to read manifest.json'); + return null; + } + } + + /** + * Write the prompt manifest to .openbridge/prompts/manifest.json. + * Creates the prompts directory if it does not exist. + */ + public async writePromptManifest(manifest: PromptManifest): Promise { + const validated = PromptManifestSchema.parse(manifest); + await fs.mkdir(this.promptsPath, { recursive: true }); + await fs.writeFile(this.getPromptManifestPath(), JSON.stringify(validated, null, 2), 'utf-8'); + } + + /** + * Write a prompt template file to .openbridge/prompts/ and update the manifest. + * Preserves `createdAt` when overwriting an existing entry. + * Sets `previousVersion` and `previousSuccessRate` when overwriting. + */ + public async writePromptTemplate( + filename: string, + content: string, + metadata: Omit, + ): Promise { + await fs.mkdir(this.promptsPath, { recursive: true }); + + const filePath = path.join(this.promptsPath, filename); + let previousFileContent: string | undefined; + try { + previousFileContent = await fs.readFile(filePath, 'utf-8'); + } catch { + // File does not exist yet — first-time write, no previous version + previousFileContent = undefined; + } + + await fs.writeFile(filePath, content, 'utf-8'); + + const now = new Date().toISOString(); + const existing = await this.readPromptManifest(); + + const existingEntry = existing?.prompts[metadata.id]; + const createdAt = existingEntry?.createdAt ?? now; + + const entry: PromptTemplate = PromptTemplateSchema.parse({ + ...metadata, + filePath: filename, + createdAt, + updatedAt: now, + previousVersion: previousFileContent, + previousSuccessRate: existingEntry?.successRate, + }); + + const manifest: PromptManifest = existing ?? { + prompts: {}, + createdAt: now, + updatedAt: now, + schemaVersion: '1.0.0', + }; + + manifest.prompts[metadata.id] = entry; + manifest.updatedAt = now; + + await this.writePromptManifest(manifest); + } + + /** + * Get a single prompt template by ID. + * Returns null if the manifest does not exist or the ID is not found. + */ + public async getPromptTemplate(id: string): Promise { + const manifest = await this.readPromptManifest(); + if (!manifest) return null; + return manifest.prompts[id] ?? null; + } + + /** + * Record a prompt usage result — increments `usageCount`, conditionally `successCount`, + * recalculates `successRate`, and updates `lastUsedAt`. + * No-op if the prompt ID is not found in the manifest. + */ + public async recordPromptUsage(id: string, success: boolean): Promise { + const manifest = await this.readPromptManifest(); + if (!manifest) return; + + const entry = manifest.prompts[id]; + if (!entry) return; + + entry.usageCount += 1; + if (success) entry.successCount += 1; + entry.successRate = entry.successCount / entry.usageCount; + entry.lastUsedAt = new Date().toISOString(); + + manifest.updatedAt = new Date().toISOString(); + await this.writePromptManifest(manifest); + } + + /** + * Return all prompts where `usageCount >= 3` AND `successRate < threshold`. + */ + public async getLowPerformingPrompts(threshold: number): Promise { + const manifest = await this.readPromptManifest(); + if (!manifest) return []; + + return Object.values(manifest.prompts).filter( + (p) => p.usageCount >= 3 && (p.successRate ?? 0) < threshold, + ); + } + + /** + * Reset usage stats for a prompt — zeros `usageCount`, `successCount`, `successRate`. + * Preserves `previousSuccessRate` from the current `successRate` before zeroing. + * No-op if the prompt ID is not found in the manifest. + */ + public async resetPromptStats(id: string): Promise { + const manifest = await this.readPromptManifest(); + if (!manifest) return; + + const entry = manifest.prompts[id]; + if (!entry) return; + + entry.previousSuccessRate = entry.successRate; + entry.successRate = 0; + entry.usageCount = 0; + entry.successCount = 0; + + manifest.updatedAt = new Date().toISOString(); + await this.writePromptManifest(manifest); + } + + // ── Memory File ─────────────────────────────────────────────── + + /** + * Get the path to the memory file. + */ + public getMemoryFilePath(): string { + return path.join(this.contextPath, 'memory.md'); + } + + /** + * Returns true if memory.md is missing or was last modified more than + * `maxAgeMs` milliseconds ago (default: 24 hours). + * Used on startup to decide whether to regenerate from SQLite data (OB-1617). + */ + public async isMemoryStale(maxAgeMs = 24 * 60 * 60 * 1000): Promise { + try { + const stat = await fs.stat(this.getMemoryFilePath()); + return Date.now() - stat.mtimeMs > maxAgeMs; + } catch { + // File does not exist + return true; + } + } + + /** + * Read the Master's curated memory from `.openbridge/context/memory.md`. + * Returns null if the file does not exist. + */ + public async readMemoryFile(): Promise { + try { + return await fs.readFile(this.getMemoryFilePath(), 'utf-8'); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') { + logger.debug({ path: this.getMemoryFilePath() }, 'File not found (expected on first run)'); + } else { + logger.warn({ err, path: this.getMemoryFilePath() }, 'Failed to read memory.md'); + } + return null; + } + } + + /** + * Write the Master's curated memory to `.openbridge/context/memory.md`. + * Validates that content is at most 200 lines — throws if exceeded. + */ + public async writeMemoryFile(content: string): Promise { + const lines = content.split('\n'); + if (lines.length > 200) { + throw new Error( + `memory.md exceeds 200-line limit (${lines.length} lines). Trim content before writing.`, + ); + } + await fs.mkdir(this.contextPath, { recursive: true }); + await fs.writeFile(this.getMemoryFilePath(), content, 'utf-8'); + } + + /** + * Fallback: directly write memory.md from conversation history when the + * Master AI's write attempt fails or produces no output (OB-1616). + * Generates a concise summary from `messages` and writes it to memory.md. + */ + public async writeMemoryFallback( + messages: ReadonlyArray<{ role: string; content: string; created_at?: string }>, + ): Promise { + const now = new Date().toISOString(); + const lines: string[] = [ + '# Memory (auto-generated fallback)', + `> Generated: ${now.slice(0, 16).replace('T', ' ')}`, + '', + '## Recent Conversation Summary', + '', + ]; + + for (const msg of messages) { + const ts = msg.created_at ? msg.created_at.slice(0, 16).replace('T', ' ') : ''; + const role = msg.role.charAt(0).toUpperCase() + msg.role.slice(1); + const snippet = msg.content.length > 200 ? msg.content.slice(0, 200) + '\u2026' : msg.content; + lines.push(`- [${ts}] **${role}:** ${snippet}`); + if (lines.length >= 198) break; + } + + if (messages.length === 0) { + lines.push('_No recent messages._'); + } + + const content = lines.join('\n'); + await this.writeMemoryFile(content); + } + + /** + * Append `learned` items from worker summaries to the `## Worker Learnings` section + * of `memory.md`. New items are deduplicated against existing content using + * normalized substring matching. Respects the 200-line file limit. + * + * Called after every worker batch completes (OB-1636). + */ + public async appendLearnedToMemory(workerSummaries: WorkerSummary[]): Promise { + // Collect non-empty learned strings from the batch + const newLearned = workerSummaries.map((s) => s.learned.trim()).filter((l) => l.length > 0); + + if (newLearned.length === 0) return; + + // Read existing memory content (create minimal stub if missing) + const existing = (await this.readMemoryFile()) ?? '# Memory\n'; + + // Normalize a string for dedup comparison (lowercase, collapse whitespace) + const normalize = (s: string): string => s.toLowerCase().replace(/\s+/g, ' ').trim(); + + const existingNorm = normalize(existing); + + // Filter out entries already present in memory + const toAdd = newLearned.filter((item) => !existingNorm.includes(normalize(item))); + + if (toAdd.length === 0) return; + + const lines = existing.split('\n'); + + // Find or insert the "## Worker Learnings" section + const sectionHeader = '## Worker Learnings'; + let sectionIdx = lines.findIndex((l) => l.trim() === sectionHeader); + + if (sectionIdx === -1) { + // Append new section at end of file + if (lines[lines.length - 1] !== '') lines.push(''); + lines.push(sectionHeader); + lines.push(''); + sectionIdx = lines.length - 2; + } + + // Find the insertion point: just after the section header (and any blank line) + let insertAt = sectionIdx + 1; + if (insertAt < lines.length && lines[insertAt] === '') insertAt++; + + // Insert new bullet items at the top of the section + const now = new Date().toISOString().slice(0, 10); + const bullets = toAdd.map((item) => `- [${now}] ${item}`); + lines.splice(insertAt, 0, ...bullets); + + // Enforce 200-line limit: trim the oldest items from the Worker Learnings section + if (lines.length > 200) { + const excess = lines.length - 200; + // Find the end of the Worker Learnings section + let endIdx = sectionIdx + 1; + while (endIdx < lines.length && !/^#{1,3}\s/.test(lines[endIdx]!)) endIdx++; + // Remove the oldest items (those furthest from the header) first + const removeFrom = Math.max(sectionIdx + 1, endIdx - excess); + lines.splice(removeFrom, excess); + } + + await this.writeMemoryFile(lines.join('\n')); + } + + // ── Dir-Dive Enumeration ─────────────────────────────────────── + + /** + * List all available directory dive results in `.openbridge/exploration/dirs/`. + * Returns an array of `{ dirPath, resultPath }` for each `*.json` file found. + * Returns an empty array if the directory does not exist. + * + * OB-1337 / OB-1340: used by KnowledgeRetriever.query() for dir-dive JSON loading. + */ + public async listDirDiveResults(): Promise> { + try { + const entries = await fs.readdir(this.explorationDirsPath, { withFileTypes: true }); + return entries + .filter((e) => e.isFile() && e.name.endsWith('.json')) + .map((e) => ({ + dirPath: e.name.replace(/\.json$/, ''), + resultPath: path.join(this.explorationDirsPath, e.name), + })); + } catch { + return []; + } + } + + // ── Batch State ──────────────────────────────────────────────── + + /** + * Get the path to the batch-state.json file. + */ + public getBatchStatePath(): string { + return path.join(this.dotFolderPath, 'batch-state.json'); + } + + /** + * Read the active batch state from `.openbridge/batch-state.json`. + * Returns null if the file does not exist or cannot be parsed. + */ + public async readBatchState(): Promise { + const batchStatePath = this.getBatchStatePath(); + + // Check existence before reading to avoid ENOENT spam on first run + try { + await fs.access(batchStatePath); + } catch { + logger.debug({ path: batchStatePath }, 'batch-state.json not found'); + return null; + } + + try { + const content = await fs.readFile(batchStatePath, 'utf-8'); + const data = JSON.parse(content) as unknown; + return BatchStateSchema.parse(data); + } catch (err) { + logger.warn({ err, path: batchStatePath }, 'Failed to read batch-state.json'); + return null; + } + } + + /** + * Persist the current batch state to `.openbridge/batch-state.json`. + */ + public async writeBatchState(state: BatchState): Promise { + const validated = BatchStateSchema.parse(state); + await fs.writeFile(this.getBatchStatePath(), JSON.stringify(validated, null, 2), 'utf-8'); + } + + /** + * Delete `.openbridge/batch-state.json`. + * No-op if the file does not exist. + */ + public async deleteBatchState(): Promise { + try { + await fs.unlink(this.getBatchStatePath()); + } catch { + // File may not exist — ignore + } + } + + // ── Industry Templates ────────────────────────────────────────── + + /** + * Get the path to the industry-templates directory. + */ + public getIndustryTemplatesPath(): string { + return path.join(this.dotFolderPath, 'industry-templates'); + } + + /** + * List available industry templates from `.openbridge/industry-templates/`. + * Reads each sub-directory's manifest.json to extract metadata. + * Returns an empty array when the directory does not exist or no valid manifests are found. + * + * OB-1466 + */ + public async listAvailableTemplates(): Promise< + Array<{ id: string; name: string; doctypeCount: number; workflowCount: number }> + > { + const templatesDir = this.getIndustryTemplatesPath(); + try { + const entries = await fs.readdir(templatesDir, { withFileTypes: true }); + const results: Array<{ + id: string; + name: string; + doctypeCount: number; + workflowCount: number; + }> = []; + + for (const entry of entries) { + if (!entry.isDirectory()) continue; + const manifestPath = path.join(templatesDir, entry.name, 'manifest.json'); + try { + const content = await fs.readFile(manifestPath, 'utf-8'); + const data = JSON.parse(content) as { + id?: string; + name?: string; + doctypes?: unknown[]; + workflows?: unknown[]; + }; + results.push({ + id: typeof data.id === 'string' ? data.id : entry.name, + name: typeof data.name === 'string' ? data.name : entry.name, + doctypeCount: Array.isArray(data.doctypes) ? data.doctypes.length : 0, + workflowCount: Array.isArray(data.workflows) ? data.workflows.length : 0, + }); + } catch { + // Skip directories with missing or malformed manifest.json + } + } + + return results; + } catch { + return []; } } } diff --git a/src/master/exploration-coordinator.ts b/src/master/exploration-coordinator.ts new file mode 100644 index 00000000..5794747e --- /dev/null +++ b/src/master/exploration-coordinator.ts @@ -0,0 +1,1823 @@ +/** + * Exploration Coordinator — Multi-Agent Workspace Exploration + * + * Provides a 5-phase incremental exploration workflow with parallel workers: + * 1. Structure Scan (90s) — List files/dirs, count, detect configs + * 2. Classification (90s) — Determine project type, frameworks, commands + * 3. Directory Dives (90s/dir) — Explore significant directories in parallel batches + * 4. Assembly (60s) — Merge partial results into workspace-map.json + * 5. Finalization (no AI) — Create agents.json, git commit, log entry + * + * Each pass is checkpointed to disk via exploration-state.json, making the + * exploration fully resumable on restart. If interrupted at any point, the + * coordinator resumes from the last completed phase. + * + * Batch size for directory dives adapts to project complexity: + * - Small projects (<100 files): batch of 2 + * - Medium projects (100–500 files): batch of 3 + * - Large projects (500+ files): batch of 5 + * + * **Usage:** Called by MasterManager during initial exploration and available + * for programmatic use (testing, scripts). + */ + +import { execFileSync } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { DotFolderManager } from './dotfolder-manager.js'; +import { detectMonorepoPattern } from './monorepo-detector.js'; +import { + generateStructureScanPrompt, + generateClassificationPrompt, + generateDirectoryDivePrompt, + generateSubProjectDivePrompt, + generateSummaryPrompt, + trimPayload, + PROMPT_CHAR_BUDGET, +} from './exploration-prompts.js'; +import { parseAIResult } from './result-parser.js'; +import { + AgentRunner, + TOOLS_READ_ONLY, + DEFAULT_MAX_TURNS_EXPLORATION, +} from '../core/agent-runner.js'; +import type { CLIAdapter } from '../core/cli-adapter.js'; +import { + ExplorationStateSchema, + StructureScanSchema, + ClassificationSchema, + DirectoryDiveResultSchema, + WorkspaceMapSchema, + type ExplorationState, + type StructureScan, + type Classification, + type DirectoryDiveResult, + type WorkspaceMap, + type AgentsRegistry, + type ExplorationSummary, +} from '../types/master.js'; +import type { DiscoveredTool } from '../types/discovery.js'; +import { createLogger } from '../core/logger.js'; +import type { MemoryManager } from '../memory/index.js'; +import type { Chunk } from '../memory/chunk-store.js'; +import { readdir, access } from 'node:fs/promises'; +import path from 'node:path'; +import { z } from 'zod/v3'; + +const logger = createLogger('exploration-coordinator'); + +// Validation schemas for AI output — the coordinator overrides timestamps and durations after +// parsing (AI-generated datetimes often fail strict .datetime() validation), so those fields +// are made optional with defaults here to validate structure without false datetime failures. +// Cast to z.ZodType because .optional().default() changes the _input type to `T | undefined` +// while the _output type remains T — the cast is safe since output shapes are identical. +const StructureScanAISchema = StructureScanSchema.extend({ + scannedAt: z.string().optional().default(''), + durationMs: z.number().int().nonnegative().optional().default(0), +}) as z.ZodType; + +const ClassificationAISchema = ClassificationSchema.extend({ + classifiedAt: z.string().optional().default(''), + durationMs: z.number().int().nonnegative().optional().default(0), +}) as z.ZodType; + +const DirectoryDiveResultAISchema = DirectoryDiveResultSchema.extend({ + exploredAt: z.string().optional().default(''), + durationMs: z.number().int().nonnegative().optional().default(0), +}) as z.ZodType; + +const PHASE_TIMEOUT = 300_000; // 5 minutes per phase (large workspaces need more time) +const DIRECTORY_DIVE_TIMEOUT = 180_000; // 3 minutes per directory dive +const MAX_DIRECTORY_DIVE_TIMEOUT = 600_000; // 10 minutes max per directory dive +const MAX_RETRIES = 3; + +/** Directories with more files than this threshold are split into subdirectories. */ +const FILE_COUNT_THRESHOLD = 25; + +/** Directories that should never be explored (matches structure scan skip list). */ +const SKIPPED_SUBDIRS = new Set([ + 'node_modules', + '.git', + 'dist', + '.next', + 'build', + 'coverage', + 'target', + '.cache', + '.openbridge', + '__pycache__', + '.tox', + '.venv', + 'venv', + '.mypy_cache', + '.pytest_cache', +]); + +/** + * Progress callback for exploration phases. + * Fired at the start and completion of each phase, and after each directory dive batch. + */ +export type ExplorationProgressCallback = (event: { + phase: 'structure_scan' | 'classification' | 'directory_dives' | 'assembly' | 'finalization'; + status: 'starting' | 'completed'; + detail?: string; + directoryProgress?: { completed: number; total: number; currentDir?: string }; +}) => Promise; + +export interface ExplorationOptions { + /** Absolute path to the workspace */ + workspacePath: string; + /** The Master AI tool being used */ + masterTool: DiscoveredTool; + /** All discovered AI tools (for agents.json) */ + discoveredTools: DiscoveredTool[]; + /** Optional callback for progress reporting */ + onProgress?: ExplorationProgressCallback; + /** Override batch size for directory dives (default: auto-detected from project size) */ + batchSize?: number; + /** Optional MemoryManager for storing exploration results as searchable chunks */ + memory?: MemoryManager; + /** + * Optional agent_activity.id for the explorer agent running this exploration. + * When provided (and memory is set), exploration phases and directory dives are + * tracked in the exploration_progress table for the "status" command. + */ + explorationId?: string; + /** CLI adapter for spawning worker agents (defaults to ClaudeAdapter) */ + adapter?: CLIAdapter; +} + +/** + * Multi-agent workspace exploration coordinator. + * + * Called by MasterManager during initial workspace exploration. + * Also available for programmatic use and testing. + */ +export class ExplorationCoordinator { + private readonly workspacePath: string; + private readonly masterTool: DiscoveredTool; + private readonly discoveredTools: DiscoveredTool[]; + private readonly dotFolder: DotFolderManager; + private readonly agentRunner: AgentRunner; + private readonly onProgress?: ExplorationProgressCallback; + private readonly batchSizeOverride?: number; + private readonly memory?: MemoryManager; + private explorationId?: string; + /** Maps directory path → exploration_progress row id for the directory-dive phase. */ + private readonly dirProgressIds = new Map(); + /** Set to true if any storeExplorationChunks() call fails — checked at end of explore(). */ + private memoryWriteFailed = false; + + constructor(options: ExplorationOptions) { + this.workspacePath = options.workspacePath; + this.masterTool = options.masterTool; + this.discoveredTools = options.discoveredTools; + this.dotFolder = new DotFolderManager(this.workspacePath); + this.agentRunner = new AgentRunner(options.adapter); + this.onProgress = options.onProgress; + this.batchSizeOverride = options.batchSize; + this.memory = options.memory; + this.explorationId = options.explorationId; + } + + /** + * Write exploration state to the DB (via MemoryManager) if available, + * otherwise fall back to the dot-folder JSON file. + */ + private async writeExplorationState(state: ExplorationState): Promise { + if (this.memory) { + await this.memory.upsertExplorationState(state); + } else { + await this.dotFolder.writeExplorationState(state); + } + } + + /** + * Read exploration state from the DB (via MemoryManager) if available. + * Falls back to the dot-folder JSON file for one-time migration when the + * DB entry does not yet exist. + */ + private async readExplorationStateFromStore(): Promise { + if (!this.memory) { + return this.dotFolder.readExplorationState(); + } + const raw = await this.memory.getExplorationState(); + if (raw !== null) { + try { + return ExplorationStateSchema.parse(JSON.parse(raw)); + } catch { + // Corrupt DB entry — fall through to JSON migration fallback + } + } + // Migration fallback: read from JSON file once (first run after upgrade) + return this.dotFolder.readExplorationState(); + } + + /** + * Write structure scan to the DB (via MemoryManager) if available, + * otherwise fall back to the dot-folder JSON file. + */ + private async writeStructureScanToStore(scan: StructureScan): Promise { + if (this.memory) { + await this.memory.upsertStructureScan(scan); + } else { + await this.dotFolder.writeStructureScan(scan); + } + } + + /** + * Read structure scan from the DB (via MemoryManager) if available. + * Falls back to the dot-folder JSON file for one-time migration. + */ + private async readStructureScanFromStore(): Promise { + if (!this.memory) { + return this.dotFolder.readStructureScan(); + } + const raw = await this.memory.getStructureScan(); + if (raw !== null) { + try { + return StructureScanSchema.parse(JSON.parse(raw)); + } catch { + // Corrupt DB entry — fall through to JSON migration fallback + } + } + return this.dotFolder.readStructureScan(); + } + + /** + * Write classification to the DB (via MemoryManager) if available, + * otherwise fall back to the dot-folder JSON file. + */ + private async writeClassificationToStore(classification: Classification): Promise { + if (this.memory) { + await this.memory.upsertClassification(classification); + } else { + await this.dotFolder.writeClassification(classification); + } + } + + /** + * Read classification from the DB (via MemoryManager) if available. + * Falls back to the dot-folder JSON file for one-time migration. + */ + private async readClassificationFromStore(): Promise { + if (!this.memory) { + return this.dotFolder.readClassification(); + } + const raw = await this.memory.getClassification(); + if (raw !== null) { + try { + return ClassificationSchema.parse(JSON.parse(raw)); + } catch { + // Corrupt DB entry — fall through to JSON migration fallback + } + } + return this.dotFolder.readClassification(); + } + + /** + * Write a directory dive result to the DB (via MemoryManager) if available, + * otherwise fall back to the dot-folder JSON file. + */ + private async writeDirectoryDiveToStore( + dirName: string, + dive: DirectoryDiveResult, + ): Promise { + if (this.memory) { + await this.memory.upsertDirectoryDive(dirName, dive); + } else { + await this.dotFolder.writeDirectoryDive(dirName, dive); + } + } + + /** + * Read a directory dive result from the DB (via MemoryManager) if available. + * Falls back to the dot-folder JSON file for one-time migration. + */ + private async readDirectoryDiveFromStore(dirName: string): Promise { + if (!this.memory) { + return this.dotFolder.readDirectoryDive(dirName); + } + const raw = await this.memory.getDirectoryDive(dirName); + if (raw !== null) { + try { + return DirectoryDiveResultSchema.parse(JSON.parse(raw)); + } catch { + // Corrupt DB entry — fall through to JSON migration fallback + } + } + return this.dotFolder.readDirectoryDive(dirName); + } + + /** + * Get the current git commit hash for use as source_hash in chunks. + * Returns an empty string if git is unavailable or there are no commits. + */ + private getSourceHash(): string { + try { + return execFileSync('git', ['rev-parse', '--short', 'HEAD'], { + cwd: this.workspacePath, + encoding: 'utf-8', + timeout: 5000, + }).trim(); + } catch { + return ''; + } + } + + /** + * Split a long text into chunks of at most `maxChars` characters. + * Prefers splitting on newline boundaries to keep content coherent. + * ~500 tokens ≈ 2000 characters (assuming 4 chars/token average). + */ + private splitIntoChunks(text: string, maxChars = 2000): string[] { + if (text.length <= maxChars) return text.trim() ? [text] : []; + + const chunks: string[] = []; + let start = 0; + + while (start < text.length) { + let end = start + maxChars; + if (end >= text.length) { + const slice = text.slice(start).trim(); + if (slice) chunks.push(slice); + break; + } + // Prefer splitting at a newline boundary + const lastNewline = text.lastIndexOf('\n', end); + if (lastNewline > start) { + end = lastNewline + 1; + } + const slice = text.slice(start, end).trim(); + if (slice) chunks.push(slice); + start = end; + } + + return chunks; + } + + /** + * Convert exploration result data to text chunks and store in MemoryManager. + * Falls back silently (logs a debug warning) if memory is unavailable or fails. + */ + private async storeExplorationChunks( + scope: string, + category: Chunk['category'], + data: unknown, + ): Promise { + if (!this.memory) return; + + try { + const text = typeof data === 'string' ? data : JSON.stringify(data, null, 2); + const sourceHash = this.getSourceHash(); + const textChunks = this.splitIntoChunks(text); + + if (textChunks.length === 0) return; + + const chunks: Chunk[] = textChunks.map((content) => ({ + scope, + category, + content, + source_hash: sourceHash || undefined, + })); + + await this.memory.deleteChunksByScope(scope); + await this.memory.storeChunks(chunks); + logger.debug( + { scope, category, chunkCount: chunks.length }, + 'Stored exploration chunks in memory', + ); + } catch (err) { + logger.warn({ err, scope, category }, 'Failed to store exploration chunks — continuing'); + this.memoryWriteFailed = true; + } + } + + /** + * Calculate optimal batch size based on project complexity. + * Small projects get fewer parallel workers, large projects get more. + */ + private calculateBatchSize(structureScan: StructureScan): number { + if (this.batchSizeOverride) return this.batchSizeOverride; + + const totalFiles = structureScan.totalFiles; + const dirCount = structureScan.topLevelDirs.length; + + if (totalFiles < 100 && dirCount <= 5) return 2; // Small project + if (totalFiles < 500 && dirCount <= 15) return 3; // Medium project + return 5; // Large project — maximize parallelism + } + + /** + * Expand large directories into their immediate subdirectories. + * + * For each top-level directory whose file count exceeds `FILE_COUNT_THRESHOLD`, + * read its immediate children, filter out skipped dirs, and return a mapping + * of parent → sub-paths. The sub-paths replace the parent in the dive list + * so each gets its own worker with a manageable scope. + * + * When `subProjects` are provided (monorepo detected during Phase 2): + * - Directories that are themselves sub-projects are kept as atomic units + * and never split further, regardless of file count. + * - Directories that are parents of sub-projects are split at sub-project + * boundaries instead of generic file-count boundaries. + * - All other directories use the existing file-count threshold logic. + * + * Mutates `structureScan.splitDirs` with the mapping and updates + * `structureScan.directoryCounts` with estimated file counts for subdirs. + */ + async expandLargeDirectories( + structureScan: StructureScan, + subProjects: Array<{ path: string; type: string }> = [], + ): Promise { + const expandedDirs: string[] = []; + + // Build a set of known sub-project paths for O(1) lookup. + const subProjectPathSet = new Set(subProjects.map((sp) => sp.path)); + + // Build a map of top-level parent dir → child sub-project paths. + // Only depth-2 sub-projects have a parent component (e.g. "packages/auth" → parent "packages"). + const subProjectsByParent = new Map(); + for (const sp of subProjects) { + const slashIndex = sp.path.indexOf('/'); + if (slashIndex !== -1) { + const parentDir = sp.path.slice(0, slashIndex); + const siblings = subProjectsByParent.get(parentDir) ?? []; + siblings.push(sp.path); + subProjectsByParent.set(parentDir, siblings); + } + } + + for (const dir of structureScan.topLevelDirs) { + const fileCount = structureScan.directoryCounts[dir] ?? 0; + + // Sub-project directories are project boundaries — never split further, + // even when they contain many files. Each is explored as an atomic unit. + if (subProjectPathSet.has(dir)) { + expandedDirs.push(dir); + logger.debug( + { dir, fileCount }, + 'Sub-project directory kept as single exploration target (project boundary)', + ); + continue; + } + + // If this directory is the parent of known sub-projects, split at + // sub-project boundaries rather than by raw file count. + const childSubProjects = subProjectsByParent.get(dir); + if (childSubProjects && childSubProjects.length > 0) { + structureScan.splitDirs[dir] = childSubProjects; + for (const spPath of childSubProjects) { + expandedDirs.push(spPath); + if (!(spPath in structureScan.directoryCounts)) { + structureScan.directoryCounts[spPath] = 10; + } + } + logger.info( + { dir, subProjects: childSubProjects }, + 'Directory split at sub-project boundaries for monorepo exploration', + ); + continue; + } + + if (fileCount <= FILE_COUNT_THRESHOLD) { + // Small enough — keep as single dive target + expandedDirs.push(dir); + continue; + } + + // Large directory — read immediate subdirectories + const fullDirPath = path.join(this.workspacePath, dir); + try { + const entries = await readdir(fullDirPath, { withFileTypes: true }); + const subDirs: string[] = []; + + for (const entry of entries) { + if (!entry.isDirectory()) continue; + if (SKIPPED_SUBDIRS.has(entry.name)) continue; + if (entry.name.startsWith('.')) continue; // skip hidden dirs + + const subPath = `${dir}/${entry.name}`; + subDirs.push(subPath); + + // Estimate file count for the subdirectory via stat + try { + const subFullPath = path.join(fullDirPath, entry.name); + const subEntries = await readdir(subFullPath, { withFileTypes: true }); + const subFileCount = + subEntries.filter((e) => e.isFile()).length + + subEntries.filter((e) => e.isDirectory() && !SKIPPED_SUBDIRS.has(e.name)).length * 5; + structureScan.directoryCounts[subPath] = subFileCount; + } catch { + // If we can't read it, give it a default estimate + structureScan.directoryCounts[subPath] = 10; + } + } + + if (subDirs.length > 0) { + structureScan.splitDirs[dir] = subDirs; + expandedDirs.push(...subDirs); + logger.info( + { dir, fileCount, subDirs: subDirs.length }, + 'Large directory split into subdirectories for exploration', + ); + } else { + // No valid subdirs found — keep original + expandedDirs.push(dir); + } + } catch (err) { + logger.warn({ err, dir }, 'Failed to read directory for splitting — keeping original'); + expandedDirs.push(dir); + } + } + + return expandedDirs; + } + + /** + * Calculate a per-directory timeout based on its file count. + * Floors at DIRECTORY_DIVE_TIMEOUT (3 min), caps at MAX_DIRECTORY_DIVE_TIMEOUT (10 min). + */ + calculateDiveTimeout(dirFileCount: number): number { + return Math.max( + DIRECTORY_DIVE_TIMEOUT, + Math.min(MAX_DIRECTORY_DIVE_TIMEOUT, dirFileCount * 4000), + ); + } + + /** + * Main entry point: Execute the 5-phase exploration workflow + * + * This method loads/creates exploration-state.json, skips completed phases, + * runs each pass via AgentRunner.spawn(), parses results with result-parser, + * and checkpoints after each pass. + * + * If exploration is already complete, returns the existing summary. + * If interrupted mid-exploration, resumes from the last checkpoint. + * + * @returns ExplorationSummary with completion details + */ + public async explore(): Promise { + logger.info({ workspacePath: this.workspacePath }, 'Starting incremental exploration'); + + // Ensure .openbridge/ and exploration/ directories exist + await this.dotFolder.initialize(); + await this.dotFolder.createExplorationDir(); + + // Auto-register an agent_activity row so exploration_progress rows have a + // valid FK parent. Only needed when the caller did not supply explorationId. + if (this.memory && !this.explorationId) { + const id = randomUUID(); + try { + await this.memory.insertActivity({ + id, + type: 'explorer', + status: 'running', + started_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + }); + this.explorationId = id; + } catch { + // exploration_progress tracking unavailable — continue without it + } + } + + // Delete stale exploration_progress rows from previous failed explorations so + // orphaned pending/in_progress rows don't accumulate across retries. + if (this.memory && this.explorationId) { + try { + await this.memory.deleteStaleExplorationProgress(this.explorationId); + } catch { + // best-effort — continue without cleanup + } + } + + // Load or create exploration state + let state = await this.readExplorationStateFromStore(); + + if (!state) { + logger.info('No existing exploration state found, creating fresh state'); + state = this.createInitialState(); + await this.writeExplorationState(state); + } + + // If exploration is already complete, return summary + if (state.status === 'completed') { + logger.info('Exploration already completed, returning cached summary'); + return this.buildSummary(state); + } + + // If exploration failed previously, resume from last completed phase (not full reset) + if (state.status === 'failed') { + const completedPhases = Object.entries(state.phases) + .filter(([, s]) => s === 'completed') + .map(([p]) => p); + logger.info( + { completedPhases }, + 'Previous exploration failed, resuming from last checkpoint (keeping completed phases)', + ); + // Reset the failed/in_progress phase to pending so it retries + for (const [phase, phaseStatus] of Object.entries(state.phases)) { + if (phaseStatus === 'failed' || phaseStatus === 'in_progress') { + state.phases[phase as keyof typeof state.phases] = 'pending'; + } + } + state.status = 'in_progress'; + await this.writeExplorationState(state); + } + + try { + // Execute each phase sequentially with progress reporting + await this.emitProgress('structure_scan', 'starting'); + await this.executePhase1StructureScan(state); + await this.emitProgress('structure_scan', 'completed'); + + await this.emitProgress('classification', 'starting'); + await this.executePhase2Classification(state); + await this.emitProgress('classification', 'completed'); + + await this.emitProgress('directory_dives', 'starting'); + await this.executePhase3DirectoryDives(state); + await this.emitProgress('directory_dives', 'completed'); + + await this.emitProgress('assembly', 'starting'); + await this.executePhase4Assembly(state); + await this.emitProgress('assembly', 'completed'); + + await this.emitProgress('finalization', 'starting'); + await this.executePhase5Finalization(state); + await this.emitProgress('finalization', 'completed'); + + // Mark as completed + state.status = 'completed'; + state.completedAt = new Date().toISOString(); + await this.writeExplorationState(state); + + logger.info( + { + totalCalls: state.totalCalls, + totalAITimeMs: state.totalAITimeMs, + duration: Date.now() - new Date(state.startedAt).getTime(), + }, + 'Exploration completed successfully', + ); + + if (this.memoryWriteFailed) { + logger.error( + 'Exploration completed but memory writes failed — results may be incomplete. JSON fallback on disk was used.', + ); + } + + // Post-exploration assertion: verify workspace-map.json exists (OB-1508) + await this.assertWorkspaceMapExists(state); + + return this.buildSummary(state); + } catch (error) { + logger.error({ err: error }, 'Exploration failed'); + state.status = 'failed'; + state.error = String(error); + await this.writeExplorationState(state); + throw error; + } + } + + /** + * Re-explore only the directories that have stale chunks in the MemoryManager. + * + * Called after workspace changes have been detected and the affected scopes + * have been marked stale via `memory.markStale(changedScopes)`. This avoids + * a full 5-phase re-exploration when only a subset of directories changed. + * + * Flow: + * 1. Query the DB for scopes with stale=1 chunks. + * 2. Filter to directory scopes (skip '.' root which belongs to structure/assembly passes). + * 3. Run directory dives in parallel batches (reusing executeSingleDirectoryDive). + * 4. Delete all stale chunks so the DB reflects only the fresh data. + * + * Falls back gracefully (logs a warning) if: + * - No MemoryManager is configured. + * - No classification exists (needed for context in dive prompts). + * - Individual directory dives fail (those scopes remain stale until next run). + */ + public async reexploreStaleDirs(): Promise { + if (!this.memory) { + logger.debug('reexploreStaleDirs: no MemoryManager — skipping'); + return; + } + + const staleScopes = await this.memory.getStaleScopes(); + if (staleScopes.length === 0) { + logger.info('No stale directory scopes — skipping partial re-exploration'); + return; + } + + logger.info({ staleScopes }, 'Partial re-exploration: found stale scopes'); + + // Track the overall stale-reexplore phase in exploration_progress + let parentRowId = 0; + if (this.memory && this.explorationId) { + try { + parentRowId = await this.memory.insertExplorationProgress({ + exploration_id: this.explorationId, + phase: 'stale-reexplore', + target: null, + status: 'in_progress', + progress_pct: 0, + files_processed: 0, + files_total: null, + started_at: new Date().toISOString(), + }); + } catch { + // ignore — progress tracking is best-effort + } + } + + // Classification is required for the dive prompts (project type + frameworks) + const classification = await this.readClassificationFromStore(); + if (!classification) { + logger.warn('No classification found for partial re-exploration — skipping'); + await this.failPhaseRow(parentRowId); + return; + } + + const context = { + projectType: classification.projectType, + frameworks: classification.frameworks, + }; + + // Load (or create) an exploration state so executeSingleDirectoryDive can + // update counters (totalCalls, totalAITimeMs) without throwing. + let state = await this.readExplorationStateFromStore(); + if (!state) { + state = this.createInitialState(); + } + + // Only re-explore real directory scopes; '.' is the root scope handled by + // the structure-scan and assembly phases, not directory dives. + const dirScopes = staleScopes.filter((s) => s !== '.'); + + const batchSize = 3; + for (let i = 0; i < dirScopes.length; i += batchSize) { + const batch = dirScopes.slice(i, i + batchSize); + await Promise.allSettled( + batch.map(async (scope) => { + // Insert a per-directory progress row + let dirRowId = 0; + if (this.memory && this.explorationId) { + try { + dirRowId = await this.memory.insertExplorationProgress({ + exploration_id: this.explorationId, + phase: 'directory-dive', + target: scope, + status: 'in_progress', + progress_pct: 0, + files_processed: 0, + files_total: null, + started_at: new Date().toISOString(), + }); + } catch { + // ignore — progress tracking is best-effort + } + } + + try { + await this.executeSingleDirectoryDive(scope, context, state); + logger.info({ scope }, 'Stale scope successfully re-explored'); + await this.completePhaseRow(dirRowId); + } catch (err) { + logger.warn({ err, scope }, 'Failed to re-explore stale scope — continuing'); + await this.failPhaseRow(dirRowId); + } + }), + ); + } + + // Delete stale chunks now that fresh replacements are stored. + try { + await this.memory.deleteStaleChunks(); + logger.info('Stale chunks deleted — incremental chunk refresh complete'); + } catch (err) { + logger.warn({ err }, 'Failed to delete stale chunks after re-exploration'); + } + + // Mark the parent stale-reexplore phase as completed + await this.completePhaseRow(parentRowId); + } + + /** + * Phase 1: Structure Scan + * Lists top-level files/dirs, counts files per directory, detects config files + */ + private async executePhase1StructureScan(state: ExplorationState): Promise { + if (state.phases.structure_scan === 'completed') { + logger.info('Phase 1 (Structure Scan) already completed, skipping'); + return; + } + + logger.info('Starting Phase 1: Structure Scan'); + state.currentPhase = 'structure_scan'; + state.phases.structure_scan = 'in_progress'; + await this.writeExplorationState(state); + + const phase1RowId = await this.insertPhaseRow('structure'); + const prompt = + generateStructureScanPrompt(this.workspacePath) + + '\n\nIMPORTANT: Keep your file listing concise. For directories with >50 files, list only the first 50 and note the total count. Focus on file types and structure, not individual files. Your output must stay under 100K characters to avoid truncation.'; + const startTime = Date.now(); + + const result = await this.agentRunner.spawn({ + prompt, + workspacePath: this.workspacePath, + timeout: PHASE_TIMEOUT, + allowedTools: [...TOOLS_READ_ONLY], + maxTurns: DEFAULT_MAX_TURNS_EXPLORATION, + retries: 0, + }); + + const elapsed = Date.now() - startTime; + state.totalCalls++; + state.totalAITimeMs += elapsed; + + if (result.exitCode !== 0) { + await this.failPhaseRow(phase1RowId); + throw new Error(`Structure scan failed with exit code ${result.exitCode}: ${result.stderr}`); + } + + // Trim agent response if it exceeds PROMPT_CHAR_BUDGET to avoid downstream + // prompt truncation when the structure scan is embedded in later phases (OB-F228). + let phase1Stdout = result.stdout; + if (phase1Stdout.length > PROMPT_CHAR_BUDGET) { + try { + const raw = JSON.parse(phase1Stdout) as Record; + phase1Stdout = trimPayload(raw, PROMPT_CHAR_BUDGET, 'topLevelFiles'); + } catch { + // keep original stdout if JSON parse fails; parseAIResult handles extraction + } + } + + const parsed = parseAIResult( + phase1Stdout, + 'structure scan', + StructureScanAISchema, + ); + if (!parsed.success) { + await this.failPhaseRow(phase1RowId); + throw new Error(`Failed to parse structure scan result: ${parsed.error}`); + } + + // Override scannedAt with a proper ISO 8601 datetime — AI-generated values + // often fail Zod's strict .datetime() validation + parsed.data.scannedAt = new Date().toISOString(); + parsed.data.durationMs = elapsed; + + await this.writeStructureScanToStore(parsed.data); + await this.storeExplorationChunks('.', 'structure', parsed.data); + state.phases.structure_scan = 'completed'; + await this.writeExplorationState(state); + await this.completePhaseRow(phase1RowId); + + logger.info({ elapsed, method: parsed.method }, 'Phase 1 completed'); + } + + /** + * Phase 2: Classification + * Reads config files, determines project type, frameworks, commands, dependencies + */ + private async executePhase2Classification(state: ExplorationState): Promise { + if (state.phases.classification === 'completed') { + logger.info('Phase 2 (Classification) already completed, skipping'); + return; + } + + logger.info('Starting Phase 2: Classification'); + state.currentPhase = 'classification'; + state.phases.classification = 'in_progress'; + await this.writeExplorationState(state); + + const phase2RowId = await this.insertPhaseRow('classification'); + const structureScan = await this.readStructureScanFromStore(); + if (!structureScan) { + await this.failPhaseRow(phase2RowId); + throw new Error('Structure scan result not found (Phase 1 incomplete)'); + } + + const prompt = generateClassificationPrompt(this.workspacePath, structureScan); + const startTime = Date.now(); + + const result = await this.agentRunner.spawn({ + prompt, + workspacePath: this.workspacePath, + timeout: PHASE_TIMEOUT, + allowedTools: [...TOOLS_READ_ONLY], + maxTurns: DEFAULT_MAX_TURNS_EXPLORATION, + retries: 0, + }); + + const elapsed = Date.now() - startTime; + state.totalCalls++; + state.totalAITimeMs += elapsed; + + if (result.exitCode !== 0) { + await this.failPhaseRow(phase2RowId); + throw new Error(`Classification failed with exit code ${result.exitCode}: ${result.stderr}`); + } + + const parsed = parseAIResult( + result.stdout, + 'classification', + ClassificationAISchema, + ); + if (!parsed.success) { + await this.failPhaseRow(phase2RowId); + throw new Error(`Failed to parse classification result: ${parsed.error}`); + } + + // Override classifiedAt with a proper ISO 8601 datetime — AI-generated values + // often fail Zod's strict .datetime() validation + parsed.data.classifiedAt = new Date().toISOString(); + parsed.data.durationMs = elapsed; + + await this.writeClassificationToStore(parsed.data); + await this.storeExplorationChunks('.', 'config', parsed.data); + + // Detect monorepo pattern and store sub-projects in exploration state + try { + const monorepoResult = await detectMonorepoPattern(this.workspacePath); + if (monorepoResult.isMonorepo) { + state.subProjects = monorepoResult.subProjects; + logger.info( + { + subProjectCount: monorepoResult.subProjects.length, + subProjects: monorepoResult.subProjects, + }, + 'Monorepo detected during Phase 2 — sub-projects stored in exploration state', + ); + } else { + state.subProjects = []; + logger.debug('No monorepo pattern detected during Phase 2'); + } + } catch (err) { + logger.warn({ err }, 'Monorepo detection failed — continuing without sub-project data'); + state.subProjects = []; + } + + state.phases.classification = 'completed'; + await this.writeExplorationState(state); + await this.completePhaseRow(phase2RowId); + + logger.info({ elapsed, method: parsed.method }, 'Phase 2 completed'); + } + + /** + * Phase 3: Directory Dives + * Explores each significant directory in batches of 3 with automatic retries + */ + private async executePhase3DirectoryDives(state: ExplorationState): Promise { + if (state.phases.directory_dives === 'completed') { + logger.info('Phase 3 (Directory Dives) already completed, skipping'); + return; + } + + logger.info('Starting Phase 3: Directory Dives'); + state.currentPhase = 'directory_dives'; + state.phases.directory_dives = 'in_progress'; + await this.writeExplorationState(state); + + const structureScan = await this.readStructureScanFromStore(); + const classification = await this.readClassificationFromStore(); + + if (!structureScan || !classification) { + throw new Error('Structure scan or classification not found (Phases 1-2 incomplete)'); + } + + // Identify significant directories (exclude root, include dirs with files > 0) + // Then expand large directories into subdirectories to avoid timeout (OB-F26). + // Pass sub-projects so boundaries are respected before file-count splitting. + const significantDirs = await this.expandLargeDirectories( + structureScan, + state.subProjects ?? [], + ); + + // Persist splitDirs so incremental explore can use 2-level scopes + if (Object.keys(structureScan.splitDirs).length > 0) { + await this.writeStructureScanToStore(structureScan); + logger.info( + { splitDirs: structureScan.splitDirs }, + 'Structure scan updated with split directories', + ); + } + + // Build a lookup of sub-project paths for use during directory dives. + // When a monorepo is detected, sub-project directories get a specialized + // classification-aware prompt instead of the regular directory dive prompt. + const subProjectMap = new Map(); + if (state.subProjects && state.subProjects.length > 0) { + for (const sp of state.subProjects) { + subProjectMap.set(sp.path, sp.type); + } + + // Ensure every sub-project path is in the dive list. + // Sub-projects at depth 2 (e.g. "packages/ui") may not be in the + // top-level dirs returned by expandLargeDirectories. + for (const sp of state.subProjects) { + if (!significantDirs.includes(sp.path)) { + significantDirs.push(sp.path); + // Provide a default file count estimate if not already tracked + if (!(sp.path in structureScan.directoryCounts)) { + structureScan.directoryCounts[sp.path] = 10; + } + } + } + + logger.info( + { subProjectCount: subProjectMap.size, subProjects: [...subProjectMap.keys()] }, + 'Monorepo sub-projects will use independent classification prompts', + ); + } + + // Initialize directory dive tracking if not already present + if (state.directoryDives.length === 0) { + state.directoryDives = significantDirs.map((dir) => ({ + path: dir, + status: 'pending' as const, + attempts: 0, + fileCount: structureScan.directoryCounts[dir] ?? 0, + })); + await this.writeExplorationState(state); + } + + // Insert exploration_progress rows for each directory (status=pending). + // Already-tracked dirs (from a resumed run) are skipped via the map check. + if (this.memory && this.explorationId) { + for (const dir of significantDirs) { + if (this.dirProgressIds.has(dir)) continue; // already inserted this run + const filesTotal = structureScan.directoryCounts[dir] ?? null; + try { + const rowId = await this.memory.insertExplorationProgress({ + exploration_id: this.explorationId, + phase: 'directory-dive', + target: dir, + status: 'pending', + progress_pct: 0, + files_processed: 0, + files_total: filesTotal, + started_at: null, + }); + this.dirProgressIds.set(dir, rowId); + } catch { + // ignore — progress tracking is best-effort + } + } + } + + const context = { + projectType: classification.projectType, + frameworks: classification.frameworks, + }; + + // Adaptive batch size based on project complexity + const batchSize = this.calculateBatchSize(structureScan); + logger.info( + { batchSize, totalFiles: structureScan.totalFiles, dirs: significantDirs.length }, + 'Adaptive batch size calculated', + ); + + // Process directories in parallel batches + const totalDirs = state.directoryDives.length; + let completedSoFar = state.directoryDives.filter((d) => d.status === 'completed').length; + + // Re-compute pendingDives inside the loop so dives reset to 'pending' after failure are retried + let pendingDives = state.directoryDives.filter((dive) => dive.status === 'pending'); + + while (pendingDives.length > 0) { + const batch = pendingDives.slice(0, batchSize); + logger.info({ batchSize: batch.length, totalDirs }, 'Processing directory batch'); + + const results = await Promise.allSettled( + batch.map((dive) => + this.executeSingleDirectoryDive(dive.path, context, state, subProjectMap), + ), + ); + + // Update state based on results + results.forEach((result, idx) => { + const dive = batch[idx]; + if (!dive) return; + + const diveState = state.directoryDives.find((d) => d.path === dive.path); + if (!diveState) return; + + if (result.status === 'fulfilled') { + diveState.status = 'completed'; + diveState.outputFile = `dirs/${dive.path}.json`; + completedSoFar++; + } else { + diveState.attempts++; + if (diveState.attempts >= MAX_RETRIES) { + diveState.status = 'failed'; + diveState.error = String(result.reason); + completedSoFar++; // Count failed as "done" for progress + // Mark the exploration_progress row as failed + const rowId = this.dirProgressIds.get(dive.path) ?? 0; + if (rowId > 0 && this.memory) { + this.memory + .updateExplorationProgressById(rowId, { + status: 'failed', + completed_at: new Date().toISOString(), + }) + .catch(() => { + // ignore — progress tracking is best-effort + }); + } + logger.warn( + { path: dive.path, error: result.reason }, + 'Directory dive failed after retries', + ); + } else { + diveState.status = 'pending'; + logger.info( + { path: dive.path, attempts: diveState.attempts }, + 'Directory dive failed, will retry', + ); + } + } + }); + + await this.writeExplorationState(state); + + // Emit per-batch directory progress + await this.emitProgress('directory_dives', 'starting', undefined, { + completed: completedSoFar, + total: totalDirs, + currentDir: batch[batch.length - 1]?.path, + }); + + // Re-compute for next iteration — picks up dives reset to 'pending' after failure + pendingDives = state.directoryDives.filter((dive) => dive.status === 'pending'); + } + + // Check if all dives completed or failed + const incompleteDives = state.directoryDives.filter( + (dive) => dive.status !== 'completed' && dive.status !== 'failed', + ); + + if (incompleteDives.length > 0) { + throw new Error(`Directory dives incomplete: ${incompleteDives.length} pending`); + } + + state.phases.directory_dives = 'completed'; + await this.writeExplorationState(state); + + logger.info('Phase 3 completed'); + } + + /** + * Execute a single directory dive with retry logic + */ + private async executeSingleDirectoryDive( + dirPath: string, + context: { projectType: string; frameworks: string[] }, + state: ExplorationState, + subProjectMap?: Map, + ): Promise { + // Mark this directory as in_progress in exploration_progress + const progressRowId = this.dirProgressIds.get(dirPath) ?? 0; + if (progressRowId > 0 && this.memory) { + try { + await this.memory.updateExplorationProgressById(progressRowId, { + status: 'in_progress', + started_at: new Date().toISOString(), + }); + } catch { + // ignore — progress tracking is best-effort + } + } + + // Use the sub-project classification prompt when this directory is a + // detected monorepo sub-project; otherwise use the regular dive prompt. + const subProjectType = subProjectMap?.get(dirPath); + const prompt = subProjectType + ? generateSubProjectDivePrompt(this.workspacePath, dirPath, subProjectType) + : generateDirectoryDivePrompt(this.workspacePath, dirPath, context); + const startTime = Date.now(); + + // Scale timeout based on directory file count (OB-F26 / OB-943) + const dirFileCount = state.directoryDives.find((d) => d.path === dirPath)?.fileCount ?? 0; + const diveTimeout = + dirFileCount > 0 ? this.calculateDiveTimeout(dirFileCount) : DIRECTORY_DIVE_TIMEOUT; + + const result = await this.agentRunner.spawn({ + prompt, + workspacePath: this.workspacePath, + timeout: diveTimeout, + allowedTools: [...TOOLS_READ_ONLY], + maxTurns: DEFAULT_MAX_TURNS_EXPLORATION, + retries: 0, + }); + + const elapsed = Date.now() - startTime; + state.totalCalls++; + state.totalAITimeMs += elapsed; + + if (result.exitCode !== 0) { + throw new Error( + `Directory dive for ${dirPath} failed with exit code ${result.exitCode}: ${result.stderr}`, + ); + } + + const parsed = parseAIResult( + result.stdout, + `directory dive: ${dirPath}`, + DirectoryDiveResultAISchema, + ); + if (!parsed.success) { + throw new Error(`Failed to parse directory dive result for ${dirPath}: ${parsed.error}`); + } + + // Override exploredAt with a proper ISO 8601 datetime — AI-generated values + // often fail Zod's strict .datetime() validation + parsed.data.exploredAt = new Date().toISOString(); + parsed.data.durationMs = elapsed; + + // Sanitize directory name for filename (replace / with -) + const safeDirName = dirPath.replace(/\//g, '-'); + await this.writeDirectoryDiveToStore(safeDirName, parsed.data); + await this.storeExplorationChunks(dirPath, 'patterns', parsed.data); + + // Mark this directory as completed in exploration_progress + if (progressRowId > 0 && this.memory) { + try { + await this.memory.updateExplorationProgressById(progressRowId, { + status: 'completed', + progress_pct: 100, + files_processed: parsed.data.fileCount, + completed_at: new Date().toISOString(), + }); + } catch { + // ignore — progress tracking is best-effort + } + } + + logger.info({ dirPath, elapsed, method: parsed.method }, 'Directory dive completed'); + } + + /** + * Phase 4: Assembly + * Merges partial results into workspace-map.json and generates summary + */ + private async executePhase4Assembly(state: ExplorationState): Promise { + if (state.phases.assembly === 'completed') { + logger.info('Phase 4 (Assembly) already completed, skipping'); + return; + } + + logger.info('Starting Phase 4: Assembly'); + state.currentPhase = 'assembly'; + state.phases.assembly = 'in_progress'; + await this.writeExplorationState(state); + + const phase4RowId = await this.insertPhaseRow('assembly'); + const structureScan = await this.readStructureScanFromStore(); + const classification = await this.readClassificationFromStore(); + + if (!structureScan || !classification) { + await this.failPhaseRow(phase4RowId); + throw new Error('Structure scan or classification not found'); + } + + // Read all directory dive results + const completedDives = state.directoryDives.filter((dive) => dive.status === 'completed'); + const diveResults: DirectoryDiveResult[] = []; + + for (const dive of completedDives) { + const safeDirName = dive.path.replace(/\//g, '-'); + const result = await this.readDirectoryDiveFromStore(safeDirName); + if (result) { + diveResults.push(result); + } + } + + // Mechanically assemble partial workspace map + const structure: Record = {}; + const keyFiles: Array<{ path: string; type: string; purpose: string }> = []; + + for (const dive of diveResults) { + structure[dive.path] = { + path: dive.path, + purpose: dive.purpose, + fileCount: dive.fileCount, + }; + + // Add key files from this directory + keyFiles.push(...dive.keyFiles); + } + + const partialMap = { + projectType: classification.projectType, + projectName: classification.projectName, + frameworks: classification.frameworks, + structure, + keyFiles, + commands: classification.commands, + }; + + // Generate summary via AI + const prompt = generateSummaryPrompt(this.workspacePath, partialMap); + const startTime = Date.now(); + + const result = await this.agentRunner.spawn({ + prompt, + workspacePath: this.workspacePath, + timeout: PHASE_TIMEOUT, + allowedTools: [...TOOLS_READ_ONLY], + maxTurns: DEFAULT_MAX_TURNS_EXPLORATION, + retries: 0, + }); + + const elapsed = Date.now() - startTime; + state.totalCalls++; + state.totalAITimeMs += elapsed; + + if (result.exitCode !== 0) { + await this.failPhaseRow(phase4RowId); + throw new Error( + `Summary generation failed with exit code ${result.exitCode}: ${result.stderr}`, + ); + } + + let parsed = parseAIResult<{ summary: string }>(result.stdout, 'summary generation'); + if (!parsed.success) { + // Fallback: if the AI returned prose/markdown instead of JSON (e.g. due to + // prompt truncation), use the raw output as the summary rather than failing. + const rawText = result.stdout.trim(); + if (rawText.length > 20) { + logger.warn( + { outputLength: rawText.length }, + 'Phase 4: JSON parse failed — using raw AI output as summary (markdown fallback)', + ); + // Take the first ~500 chars as summary to keep it concise + const summaryText = rawText.length > 500 ? rawText.slice(0, 497) + '...' : rawText; + parsed = { + success: true, + data: { summary: summaryText }, + method: 'markdown-fallback', + }; + } else { + await this.failPhaseRow(phase4RowId); + throw new Error(`Failed to parse summary result: ${parsed.error}`); + } + } + + // Assemble final workspace map — monorepo-aware when sub-projects detected + const isMonorepo = (state.subProjects?.length ?? 0) >= 2; + + const workspaceMap: WorkspaceMap = { + workspacePath: this.workspacePath, + projectName: classification.projectName, + projectType: isMonorepo ? 'monorepo' : classification.projectType, + frameworks: classification.frameworks, + structure, + keyFiles, + entryPoints: [], // Can be extracted from keyFiles or left empty + commands: classification.commands, + dependencies: classification.dependencies, + summary: parsed.data.summary, + generatedAt: new Date().toISOString(), + schemaVersion: '1.0.0', + }; + + // Enrich workspace map with monorepo sub-project and shared-dir metadata. + // WorkspaceMapSchema uses .passthrough() so extra fields are preserved. + if (isMonorepo && state.subProjects) { + const subProjectPaths = new Set(state.subProjects.map((sp) => sp.path)); + + // Build sub-project entries from dive results + const subProjectEntries: Array<{ + name: string; + path: string; + type: string; + frameworks: string[]; + summary: string; + }> = []; + + for (const sp of state.subProjects) { + const dive = diveResults.find((d) => d.path === sp.path); + const name = sp.path.split('/').pop() ?? sp.path; + // Extract frameworks from dive insights (lines mentioning "Uses ") + const frameworks: string[] = []; + if (dive) { + for (const insight of dive.insights) { + const match = insight.match(/\bUses?\s+(.+)/i); + if (match?.[1]) frameworks.push(match[1].trim()); + } + } + subProjectEntries.push({ + name, + path: sp.path, + type: sp.type, + frameworks, + summary: dive?.purpose ?? '', + }); + } + + // Shared dirs = explored directories that aren't sub-projects + const sharedDirs = Object.keys(structure).filter((dir) => !subProjectPaths.has(dir)); + + // Attach monorepo metadata via passthrough fields + (workspaceMap as Record)['subProjects'] = subProjectEntries; + (workspaceMap as Record)['sharedDirs'] = sharedDirs; + + logger.info( + { subProjectCount: subProjectEntries.length, sharedDirCount: sharedDirs.length }, + 'Phase 4: assembled monorepo map with sub-project metadata', + ); + } + + // Write workspace map to DB (primary) and JSON file (safety net fallback). + if (this.memory) { + await this.memory.storeChunks([ + { + scope: '_workspace_map', + category: 'structure', + content: JSON.stringify(workspaceMap), + }, + ]); + } else { + logger.warn('Memory not available — workspace map will only be saved to JSON fallback'); + } + // Always write JSON fallback so workspace-map.json exists on disk regardless of memory state. + try { + await this.dotFolder.writeWorkspaceMap(workspaceMap); + } catch (err) { + logger.error( + { err }, + 'Failed to write workspace-map.json — Master will lack workspace context', + ); + } + + // Post-write verification: ensure workspace-map.json actually exists on disk (OB-1507). + const mapPath = this.dotFolder.getMapPath(); + try { + await access(mapPath); + logger.info({ path: mapPath }, 'workspace-map.json verified on disk'); + } catch { + logger.error( + { path: mapPath }, + 'workspace-map.json missing after write — attempting direct fallback write', + ); + // Direct fallback: write the raw JSON without Zod validation to bypass any schema issues + try { + const { writeFile, mkdir } = await import('node:fs/promises'); + await mkdir(this.dotFolder.getDotFolderPath(), { recursive: true }); + await writeFile(mapPath, JSON.stringify(workspaceMap, null, 2), 'utf-8'); + logger.info('workspace-map.json fallback write succeeded'); + } catch (fallbackErr) { + logger.error({ err: fallbackErr }, 'workspace-map.json fallback write also failed'); + } + } + + await this.storeExplorationChunks('.', 'structure', workspaceMap); + state.phases.assembly = 'completed'; + await this.writeExplorationState(state); + await this.completePhaseRow(phase4RowId); + + logger.info({ elapsed, method: parsed.method }, 'Phase 4 completed'); + } + + /** + * Phase 5: Finalization + * Creates agents.json, commits to git, writes log entry (no AI calls) + */ + private async executePhase5Finalization(state: ExplorationState): Promise { + if (state.phases.finalization === 'completed') { + logger.info('Phase 5 (Finalization) already completed, skipping'); + return; + } + + logger.info('Starting Phase 5: Finalization'); + state.currentPhase = 'finalization'; + state.phases.finalization = 'in_progress'; + await this.writeExplorationState(state); + + // Create agents.json + const agentsRegistry: AgentsRegistry = { + master: { + name: this.masterTool.name, + path: this.masterTool.path, + version: this.masterTool.version || 'unknown', + role: 'master', + }, + specialists: this.discoveredTools + .filter((tool) => tool.name !== this.masterTool.name) + .map((tool) => ({ + name: tool.name, + path: tool.path, + version: tool.version || 'unknown', + role: 'specialist' as const, + capabilities: tool.capabilities || [], + })), + updatedAt: new Date().toISOString(), + }; + + if (this.memory) { + await this.memory.setSystemConfig('agents', JSON.stringify(agentsRegistry)); + } else { + await this.dotFolder.writeAgents(agentsRegistry); + } + + // Log entry + if (this.memory) { + await this.memory.logExploration( + { + timestamp: new Date().toISOString(), + level: 'info', + message: 'Incremental exploration completed successfully', + data: { + totalCalls: state.totalCalls, + totalAITimeMs: state.totalAITimeMs, + phases: state.phases, + directoriesExplored: state.directoryDives.filter((d) => d.status === 'completed') + .length, + }, + }, + this.explorationId, + ); + } else { + await this.dotFolder.appendLog({ + timestamp: new Date().toISOString(), + level: 'info', + message: 'Incremental exploration completed successfully', + data: { + totalCalls: state.totalCalls, + totalAITimeMs: state.totalAITimeMs, + phases: state.phases, + directoriesExplored: state.directoryDives.filter((d) => d.status === 'completed').length, + }, + }); + } + + state.phases.finalization = 'completed'; + await this.writeExplorationState(state); + + logger.info('Phase 5 completed'); + } + + /** + * Create initial exploration state + */ + private createInitialState(): ExplorationState { + return { + currentPhase: 'structure_scan', + status: 'in_progress', + startedAt: new Date().toISOString(), + phases: { + structure_scan: 'pending', + classification: 'pending', + directory_dives: 'pending', + assembly: 'pending', + finalization: 'pending', + }, + directoryDives: [], + totalCalls: 0, + totalAITimeMs: 0, + subProjects: [], + }; + } + + /** + * Post-exploration assertion (OB-1508): verify workspace-map.json exists on disk after + * all 5 phases complete. If missing, log an ERROR with the exploration summary and + * attempt to generate a minimal map from intermediate files (structure-scan + classification). + */ + private async assertWorkspaceMapExists(state: ExplorationState): Promise { + const mapPath = this.dotFolder.getMapPath(); + try { + await access(mapPath); + // File exists — all good + return; + } catch { + // File missing after all phases completed — unexpected + } + + const directoriesExplored = state.directoryDives.filter((d) => d.status === 'completed').length; + logger.error( + { + workspacePath: this.workspacePath, + mapPath, + phases: state.phases, + totalCalls: state.totalCalls, + directoriesExplored, + }, + 'workspace-map.json missing after exploration completed — attempting minimal map generation from intermediate files', + ); + + const structureScan = await this.readStructureScanFromStore(); + const classification = await this.readClassificationFromStore(); + + if (!structureScan && !classification) { + logger.error('Cannot generate minimal workspace map — no intermediate files available'); + return; + } + + const projectName = this.workspacePath.split('/').pop() ?? 'unknown'; + const minimalMap: WorkspaceMap = { + workspacePath: this.workspacePath, + projectName, + projectType: classification?.projectType ?? 'unknown', + frameworks: classification?.frameworks ?? [], + structure: structureScan + ? Object.fromEntries( + structureScan.topLevelDirs.map((dir) => [ + dir, + { + path: dir, + purpose: 'directory', + fileCount: structureScan.directoryCounts[dir] ?? 0, + }, + ]), + ) + : {}, + keyFiles: [], + entryPoints: [], + commands: classification?.commands ?? {}, + dependencies: [], + summary: `Minimal map generated from exploration intermediate files. Project type: ${classification?.projectType ?? 'unknown'}.`, + generatedAt: new Date().toISOString(), + schemaVersion: '1.0.0', + }; + + try { + await this.dotFolder.writeWorkspaceMap(minimalMap); + logger.info({ mapPath }, 'Minimal workspace-map.json generated from intermediate files'); + } catch (writeErr) { + logger.error({ err: writeErr }, 'Failed to write minimal workspace-map.json'); + } + } + + /** + * Build ExplorationSummary from final state + */ + private async buildSummary(state: ExplorationState): Promise { + // Read workspace map from DB (OB-810: JSON fallback removed). + let map: WorkspaceMap | null = null; + if (this.memory) { + try { + const chunks = await this.memory.getChunksByScope('_workspace_map', 'structure'); + if (chunks.length > 0 && chunks[0]?.content) { + map = WorkspaceMapSchema.parse(JSON.parse(chunks[0].content)); + } + } catch { + // ignore — map not yet stored + } + } + const classification = await this.readClassificationFromStore(); + const structureScan = await this.readStructureScanFromStore(); + + return { + startedAt: state.startedAt, + completedAt: state.completedAt, + status: + state.status === 'completed' + ? 'completed' + : state.status === 'failed' + ? 'failed' + : 'in_progress', + filesScanned: structureScan?.totalFiles ?? 0, + directoriesExplored: state.directoryDives.filter((d) => d.status === 'completed').length, + projectType: classification?.projectType ?? map?.projectType, + frameworks: classification?.frameworks ?? map?.frameworks ?? [], + insights: classification?.insights ?? [], + mapPath: undefined, // workspace-map.json removed; map is stored in DB (OB-810) + gitInitialized: true, + error: state.error, + }; + } + + /** + * Insert an exploration_progress row for a non-directory phase. + * Returns the row id (0 if tracking is unavailable or fails). + */ + private async insertPhaseRow(phase: string, filesTotal?: number): Promise { + if (!this.memory || !this.explorationId) return 0; + try { + return await this.memory.insertExplorationProgress({ + exploration_id: this.explorationId, + phase, + target: null, + status: 'in_progress', + progress_pct: 0, + files_processed: 0, + files_total: filesTotal ?? null, + started_at: new Date().toISOString(), + }); + } catch { + return 0; + } + } + + /** + * Mark an exploration_progress row as completed with 100% progress. + * No-ops if id is 0 or tracking is unavailable. + */ + private async completePhaseRow(id: number): Promise { + if (!this.memory || id === 0) return; + try { + await this.memory.updateExplorationProgressById(id, { + status: 'completed', + progress_pct: 100, + completed_at: new Date().toISOString(), + }); + } catch { + // ignore — progress tracking is best-effort + } + } + + /** + * Mark an exploration_progress row as failed. + * No-ops if id is 0 or tracking is unavailable. + */ + private async failPhaseRow(id: number): Promise { + if (!this.memory || id === 0) return; + try { + await this.memory.updateExplorationProgressById(id, { + status: 'failed', + completed_at: new Date().toISOString(), + }); + } catch { + // ignore — progress tracking is best-effort + } + } + + /** + * Emit a progress event via the onProgress callback (if provided). + */ + private async emitProgress( + phase: 'structure_scan' | 'classification' | 'directory_dives' | 'assembly' | 'finalization', + status: 'starting' | 'completed', + detail?: string, + directoryProgress?: { completed: number; total: number; currentDir?: string }, + ): Promise { + if (!this.onProgress) return; + try { + await this.onProgress({ phase, status, detail, directoryProgress }); + } catch (err) { + logger.warn({ err, phase, status }, 'Progress callback failed'); + } + } + + /** + * Get current exploration progress + * Returns phase-by-phase completion status and overall percentage + */ + public async getProgress(): Promise<{ + currentPhase: string; + phases: Record; + completionPercent: number; + directoriesTotal: number; + directoriesCompleted: number; + directoriesFailed: number; + totalCalls: number; + totalAITimeMs: number; + } | null> { + const state = await this.readExplorationStateFromStore(); + if (!state) { + return null; + } + + // Calculate completion percentage + const phaseWeights = { + structure_scan: 15, + classification: 15, + directory_dives: 50, + assembly: 15, + finalization: 5, + }; + + let completedWeight = 0; + for (const [phase, status] of Object.entries(state.phases)) { + if (status === 'completed') { + completedWeight += phaseWeights[phase as keyof typeof phaseWeights] ?? 0; + } + } + + // For directory_dives, calculate partial completion + if (state.phases.directory_dives === 'in_progress' && state.directoryDives.length > 0) { + const completedDives = state.directoryDives.filter((d) => d.status === 'completed').length; + const totalDives = state.directoryDives.length; + const diveProgressPercent = completedDives / totalDives; + completedWeight += phaseWeights.directory_dives * diveProgressPercent; + } + + const completionPercent = Math.round(completedWeight); + + return { + currentPhase: state.currentPhase, + phases: state.phases, + completionPercent, + directoriesTotal: state.directoryDives.length, + directoriesCompleted: state.directoryDives.filter((d) => d.status === 'completed').length, + directoriesFailed: state.directoryDives.filter((d) => d.status === 'failed').length, + totalCalls: state.totalCalls, + totalAITimeMs: state.totalAITimeMs, + }; + } +} diff --git a/src/master/exploration-manager.ts b/src/master/exploration-manager.ts new file mode 100644 index 00000000..315fca2a --- /dev/null +++ b/src/master/exploration-manager.ts @@ -0,0 +1,1558 @@ +/** + * ExplorationManager — extracted from MasterManager (OB-1280, OB-F158). + * + * Handles workspace exploration lifecycle: initial 5-phase exploration, + * incremental re-exploration on workspace changes, user-triggered + * re-exploration, progress broadcasting, and post-exploration memory seeding. + */ + +import { ExplorationCoordinator } from './exploration-coordinator.js'; +import { generateReExplorationPrompt } from './exploration-prompt.js'; +import { generateIncrementalExplorationPrompt } from './exploration-prompts.js'; +import type { WorkspaceChangeTracker } from './workspace-change-tracker.js'; +import type { WorkspaceChanges } from './workspace-change-tracker.js'; +import { parseAIResult } from './result-parser.js'; +import { detectSubProjects } from './sub-master-detector.js'; +import type { SubMasterManager } from './sub-master-manager.js'; +import type { AgentRunner, SpawnOptions } from '../core/agent-runner.js'; +import { TOOLS_READ_ONLY } from '../core/agent-runner.js'; +import type { CLIAdapter } from '../core/cli-adapter.js'; +import type { Router } from '../core/router.js'; +import type { MemoryManager } from '../memory/index.js'; +import type { DotFolderManager } from './dotfolder-manager.js'; +import type { + ExplorationSummary, + ExplorationState, + WorkspaceMap, + WorkspaceAnalysisMarker, + AgentsRegistry, + Classification, + MasterSession, + MasterState, +} from '../types/master.js'; +import { WorkspaceMapSchema } from '../types/master.js'; +import type { DiscoveredTool } from '../types/discovery.js'; +import type { InboundMessage } from '../types/message.js'; +import { createLogger } from '../core/logger.js'; +import { randomUUID } from 'node:crypto'; + +const logger = createLogger('exploration-manager'); + +const MASTER_MAX_TURNS = 50; + +// --------------------------------------------------------------------------- +// Dependencies interface — callbacks + shared references from MasterManager +// --------------------------------------------------------------------------- + +/** + * Optional pre-formatted context sections passed to buildMasterSpawnOptions. + * Re-declared here to avoid importing from master-manager.ts (circular). + */ +interface MasterContextSections { + conversationContext?: string | null; + learnedPatternsContext?: string | null; + workerNextStepsContext?: string | null; + knowledgeContext?: string | null; + targetedReaderContext?: string | null; + analysisContext?: string | null; +} + +export interface ExplorationManagerDeps { + workspacePath: string; + masterTool: DiscoveredTool; + discoveredTools: DiscoveredTool[]; + dotFolder: DotFolderManager; + changeTracker: WorkspaceChangeTracker; + agentRunner: AgentRunner; + explorationTimeout: number; + adapter?: CLIAdapter; + + // Mutable references from MasterManager + getMemory: () => MemoryManager | null; + getRouter: () => Router | null; + getSubMasterManager: () => SubMasterManager | null; + getMasterSession: () => MasterSession | null; + getState: () => MasterState; + setState: (state: MasterState) => void; + + // Callbacks into MasterManager + buildMasterSpawnOptions: ( + prompt: string, + timeout?: number, + maxTurns?: number, + contextSections?: MasterContextSections, + skipWorkspaceContext?: boolean, + ) => SpawnOptions; + updateMasterSession: () => Promise; + processMessage: (message: InboundMessage) => Promise; + + // Store helpers (shared with MasterManager) + readWorkspaceMapFromStore: () => Promise; + writeWorkspaceMapToStore: (map: WorkspaceMap) => Promise; + readAnalysisMarkerFromStore: () => Promise; + writeAnalysisMarkerToStore: (marker: WorkspaceAnalysisMarker) => Promise; + readExplorationStateFromStore: () => Promise; +} + +// --------------------------------------------------------------------------- +// ExplorationManager class +// --------------------------------------------------------------------------- + +export class ExplorationManager { + private deps: ExplorationManagerDeps; + + /** Exploration results — project type, frameworks, insights */ + private _explorationSummary: ExplorationSummary | null = null; + /** Cached workspace map summary for system prompt injection */ + private _workspaceMapSummary: string | null = null; + /** ISO timestamp of the most recent startup verification */ + private _mapLastVerifiedAt: string | null = null; + /** Timestamp of last exploration run (throttles re-exploration) */ + private _lastExplorationAt: number | null = null; + /** Messages queued while exploration is in progress */ + private _pendingMessages: InboundMessage[] = []; + + constructor(deps: ExplorationManagerDeps) { + this.deps = deps; + } + + // ------------------------------------------------------------------------- + // Public state accessors + // ------------------------------------------------------------------------- + + get explorationSummary(): ExplorationSummary | null { + return this._explorationSummary; + } + + set explorationSummary(value: ExplorationSummary | null) { + this._explorationSummary = value; + } + + get workspaceMapSummary(): string | null { + return this._workspaceMapSummary; + } + + set workspaceMapSummary(value: string | null) { + this._workspaceMapSummary = value; + } + + get mapLastVerifiedAt(): string | null { + return this._mapLastVerifiedAt; + } + + set mapLastVerifiedAt(value: string | null) { + this._mapLastVerifiedAt = value; + } + + get pendingMessages(): InboundMessage[] { + return this._pendingMessages; + } + + set pendingMessages(value: InboundMessage[]) { + this._pendingMessages = value; + } + + /** Update mutable dependencies (e.g. when memory becomes available after init). */ + updateDeps(partial: Partial): void { + this.deps = { ...this.deps, ...partial }; + } + + // ------------------------------------------------------------------------- + // Primary exploration entry point + // ------------------------------------------------------------------------- + + /** + * Autonomously explore the workspace and create .openbridge/ folder. + * This is the Master AI's initialization step. + */ + async explore(): Promise { + if (this.deps.getState() === 'exploring') { + logger.warn('Exploration already in progress'); + return; + } + + this.deps.setState('exploring'); + + logger.info( + { workspacePath: this.deps.workspacePath }, + 'Starting Master-driven workspace exploration', + ); + + try { + // Initialize .openbridge folder + await this.deps.dotFolder.initialize(); + + // Log exploration start + const startedAt = new Date().toISOString(); + const memory = this.deps.getMemory(); + if (memory) { + await memory.logExploration({ + timestamp: startedAt, + level: 'info', + message: 'Master-driven workspace exploration started', + data: { + masterTool: this.deps.masterTool.name, + version: this.deps.masterTool.version, + }, + }); + } else { + await this.deps.dotFolder.appendLog({ + timestamp: startedAt, + level: 'info', + message: 'Master-driven workspace exploration started', + data: { + masterTool: this.deps.masterTool.name, + version: this.deps.masterTool.version, + }, + }); + } + + // Master-driven exploration via the persistent session + await this.masterDrivenExplore(); + + // Seed memory.md with exploration results before entering ready state (OB-F156, OB-1271). + await this.writeExplorationSummaryToMemory(); + + this.deps.setState('ready'); + + // Detect sub-projects after successful exploration (OB-1613) + try { + const subProjects = await detectSubProjects(this.deps.workspacePath); + if (subProjects.length > 0) { + logger.info( + { count: subProjects.length, paths: subProjects.map((p) => p.path) }, + 'Sub-projects detected after exploration', + ); + const subMasterManager = this.deps.getSubMasterManager(); + if (subMasterManager) { + for (const subProject of subProjects) { + try { + const id = await subMasterManager.spawnSubMaster(subProject); + logger.info( + { id, path: subProject.relativePath, name: subProject.name }, + 'Sub-master spawned for detected sub-project', + ); + } catch (spawnErr) { + logger.warn( + { err: spawnErr, path: subProject.relativePath }, + 'Failed to spawn sub-master for sub-project — skipping', + ); + } + } + } else { + logger.warn( + { count: subProjects.length }, + 'Sub-projects detected but SubMasterManager not available — skipping spawn', + ); + } + } + } catch (err) { + logger.warn({ err }, 'Sub-project detection failed — skipping'); + } + + // Drain any messages queued while exploration was running + await this.drainPendingMessages(); + + logger.info( + { + projectType: this._explorationSummary?.projectType, + frameworks: this._explorationSummary?.frameworks, + directoriesExplored: this._explorationSummary?.directoriesExplored, + status: this._explorationSummary?.status, + }, + 'Workspace exploration completed', + ); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + + this._explorationSummary = { + startedAt: this._explorationSummary?.startedAt ?? new Date().toISOString(), + completedAt: new Date().toISOString(), + status: 'failed', + filesScanned: 0, + directoriesExplored: 0, + frameworks: [], + insights: [], + gitInitialized: false, + error: errorMessage, + }; + + // Log exploration failure + const memory = this.deps.getMemory(); + if (memory) { + await memory.logExploration({ + timestamp: new Date().toISOString(), + level: 'error', + message: 'Workspace exploration failed', + data: { error: errorMessage }, + }); + } else { + await this.deps.dotFolder.appendLog({ + timestamp: new Date().toISOString(), + level: 'error', + message: 'Workspace exploration failed', + data: { error: errorMessage }, + }); + } + + this.deps.setState('error'); + + logger.error( + { err: error, workspacePath: this.deps.workspacePath }, + 'Workspace exploration failed', + ); + + throw error; + } + } + + // ------------------------------------------------------------------------- + // Workspace change detection + // ------------------------------------------------------------------------- + + /** + * Check for workspace changes since the last analysis and decide which + * exploration path to take. + * Returns 'no-changes', 'incremental', or 'full-reexplore'. + */ + async checkWorkspaceChanges( + existingMap: WorkspaceMap, + ): Promise<'no-changes' | 'incremental' | 'full-reexplore'> { + const MIN_EXPLORATION_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes + if (this._lastExplorationAt !== null) { + const elapsed = Math.floor((Date.now() - this._lastExplorationAt) / 1000); + if (Date.now() - this._lastExplorationAt < MIN_EXPLORATION_INTERVAL_MS) { + logger.info(`Skipping re-exploration — last run was ${elapsed}s ago`); + return 'no-changes'; + } + } + + const marker = await this.deps.readAnalysisMarkerFromStore(); + + // No marker but valid map exists = upgrade from before incremental tracking. + if (!marker) { + logger.info('No analysis marker found — writing initial marker for existing map'); + const initialMarker = await this.deps.changeTracker.buildCurrentMarker('full', 0); + await this.deps.writeAnalysisMarkerToStore(initialMarker); + this._mapLastVerifiedAt = initialMarker.lastVerifiedAt ?? initialMarker.analyzedAt; + return 'no-changes'; + } + + const changes = await this.deps.changeTracker.detectChanges(marker); + + logger.info( + { + method: changes.method, + hasChanges: changes.hasChanges, + changedCount: changes.changedFiles.length, + deletedCount: changes.deletedFiles.length, + tooLarge: changes.tooLargeForIncremental, + }, + `Workspace change detection: ${changes.summary}`, + ); + + if (!changes.hasChanges) { + const now = new Date().toISOString(); + await this.deps.writeAnalysisMarkerToStore({ ...marker, lastVerifiedAt: now }); + this._mapLastVerifiedAt = now; + return 'no-changes'; + } + + if (changes.tooLargeForIncremental) { + this._lastExplorationAt = Date.now(); + return 'full-reexplore'; + } + + // Perform incremental exploration + this._lastExplorationAt = Date.now(); + await this.incrementalExplore(existingMap, changes); + return 'incremental'; + } + + // ------------------------------------------------------------------------- + // Multi-agent exploration (5-phase pipeline) + // ------------------------------------------------------------------------- + + /** + * Multi-agent exploration: delegates to ExplorationCoordinator which runs + * a 5-phase pipeline with parallel directory dives. Falls back to a + * single-agent monolithic approach if the coordinator fails. + */ + private async masterDrivenExplore(): Promise { + logger.info('Starting multi-agent workspace exploration via ExplorationCoordinator'); + + const memory = this.deps.getMemory(); + if (memory) { + await memory.logExploration({ + timestamp: new Date().toISOString(), + level: 'info', + message: 'Starting multi-agent workspace exploration', + data: { workspacePath: this.deps.workspacePath }, + }); + } else { + await this.deps.dotFolder.appendLog({ + timestamp: new Date().toISOString(), + level: 'info', + message: 'Starting multi-agent workspace exploration', + data: { workspacePath: this.deps.workspacePath }, + }); + } + + let explorationId: string | undefined; + try { + if (memory) { + explorationId = randomUUID(); + const now = new Date().toISOString(); + await memory.insertActivity({ + id: explorationId, + type: 'explorer', + status: 'running', + task_summary: 'Workspace exploration', + started_at: now, + updated_at: now, + }); + } + + const coordinator = new ExplorationCoordinator({ + workspacePath: this.deps.workspacePath, + masterTool: this.deps.masterTool, + discoveredTools: this.deps.discoveredTools, + adapter: this.deps.adapter, + onProgress: async (event): Promise => { + await this.emitExplorationProgress(event); + }, + memory: memory ?? undefined, + explorationId, + }); + + const summary = await coordinator.explore(); + + // Write agents.json (coordinator writes its own, but ensure consistency) + await this.writeAgentsRegistry(); + + // Load the workspace map into memory for system prompt injection + await this.loadExplorationSummary(); + + // Cache the map summary + const map = await this.deps.readWorkspaceMapFromStore(); + if (map) { + this._workspaceMapSummary = this.buildMapSummary(map); + } + + // Write analysis marker only if exploration produced a valid workspace map + if (this._explorationSummary?.status === 'completed' && map) { + const fullMarker = await this.deps.changeTracker.buildCurrentMarker('full', 0); + await this.deps.writeAnalysisMarkerToStore(fullMarker); + this._mapLastVerifiedAt = fullMarker.lastVerifiedAt ?? fullMarker.analyzedAt; + } else { + logger.warn( + 'Skipping analysis marker update — exploration did not produce a valid workspace map', + ); + } + + if (memory) { + await memory.logExploration({ + timestamp: new Date().toISOString(), + level: 'info', + message: 'Multi-agent exploration completed', + data: { + directoriesExplored: summary.directoriesExplored, + projectType: summary.projectType, + frameworks: summary.frameworks, + }, + }); + } else { + await this.deps.dotFolder.appendLog({ + timestamp: new Date().toISOString(), + level: 'info', + message: 'Multi-agent exploration completed', + data: { + directoriesExplored: summary.directoriesExplored, + projectType: summary.projectType, + frameworks: summary.frameworks, + }, + }); + } + + logger.info( + { + directoriesExplored: summary.directoriesExplored, + projectType: summary.projectType, + }, + 'Multi-agent exploration completed successfully', + ); + + if (memory && explorationId) { + const completedAt = new Date().toISOString(); + await memory.updateActivity(explorationId, { + status: 'done', + progress_pct: 100, + completed_at: completedAt, + }); + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.warn( + { error: errorMessage }, + 'Multi-agent exploration failed, falling back to monolithic exploration', + ); + + if (memory && explorationId) { + const failedAt = new Date().toISOString(); + await memory.updateActivity(explorationId, { + status: 'failed', + completed_at: failedAt, + }); + } + + if (memory) { + await memory.logExploration({ + timestamp: new Date().toISOString(), + level: 'warn', + message: 'Multi-agent exploration failed, falling back to monolithic exploration', + data: { error: errorMessage }, + }); + } else { + await this.deps.dotFolder.appendLog({ + timestamp: new Date().toISOString(), + level: 'warn', + message: 'Multi-agent exploration failed, falling back to monolithic exploration', + data: { error: errorMessage }, + }); + } + + await this.monolithicExplore(); + } + } + + // ------------------------------------------------------------------------- + // Monolithic fallback exploration + // ------------------------------------------------------------------------- + + /** + * Fallback: single-agent monolithic exploration via streaming. + * Used when the multi-agent ExplorationCoordinator fails. + */ + private async monolithicExplore(): Promise { + logger.info('Executing monolithic exploration via single agent'); + + const explorationPrompt = `Explore the workspace at \`${this.deps.workspacePath}\` and create a comprehensive understanding. + +You are in charge of the exploration strategy. Use your tools (Read, Glob, Grep) to understand the project. + +Follow the "Workspace Exploration" section in your system prompt for the schema and recommended strategy. Adapt the depth of exploration to the project's size and complexity. + +When done, output ONLY the workspace map as a JSON object to stdout — no other text, no markdown fences, just the raw JSON. Do NOT write any files.`; + + const spawnOpts = this.deps.buildMasterSpawnOptions( + explorationPrompt, + this.deps.explorationTimeout, + MASTER_MAX_TURNS, + ); + + const stream = this.deps.agentRunner.stream(spawnOpts); + + // Consume the stream + let iterResult = await stream.next(); + while (!iterResult.done) { + iterResult = await stream.next(); + } + + const result = iterResult.value; + await this.deps.updateMasterSession(); + + if (!result || result.exitCode !== 0) { + const errorMessage = `Monolithic exploration failed with exit code ${result?.exitCode ?? 'unknown'}: ${result?.stderr ?? 'no error details'}`; + throw new Error(errorMessage); + } + + // Parse workspace map from stdout and store in memory + JSON fallback (OB-838) + const parsed = parseAIResult(result.stdout, 'monolithic workspace map'); + if (parsed.success) { + try { + const map = WorkspaceMapSchema.parse(parsed.data); + await this.deps.writeWorkspaceMapToStore(map); + await this.deps.dotFolder.writeWorkspaceMap(map); // JSON safety net + logger.info({ method: parsed.method }, 'Monolithic workspace map stored in memory'); + } catch (err) { + logger.warn({ error: String(err) }, 'Monolithic workspace map schema validation failed'); + } + } else { + logger.warn( + { rawOutput: result.stdout.slice(0, 200) }, + 'Monolithic exploration: could not extract workspace map from stdout', + ); + } + + // Write agents.json + await this.writeAgentsRegistry(); + + logger.info('Monolithic exploration completed successfully'); + + await this.loadExplorationSummary(); + + // Write analysis marker only if exploration produced a valid workspace map + const monoMap = await this.deps.readWorkspaceMapFromStore(); + if (this._explorationSummary?.status === 'completed' && monoMap) { + const fullMarker = await this.deps.changeTracker.buildCurrentMarker('full', 0); + await this.deps.writeAnalysisMarkerToStore(fullMarker); + this._mapLastVerifiedAt = fullMarker.lastVerifiedAt ?? fullMarker.analyzedAt; + } else { + logger.warn( + 'Skipping analysis marker update — monolithic exploration did not produce a valid workspace map', + ); + } + } + + // ------------------------------------------------------------------------- + // Incremental exploration + // ------------------------------------------------------------------------- + + /** + * Perform an incremental exploration: send only the changed files to + * the Master AI for a targeted map update. + */ + private async incrementalExplore( + existingMap: WorkspaceMap, + changes: WorkspaceChanges, + ): Promise { + this.deps.setState('exploring'); + + const startedAt = new Date().toISOString(); + + logger.info( + { + changedFiles: changes.changedFiles.length, + deletedFiles: changes.deletedFiles.length, + }, + 'Starting incremental workspace exploration', + ); + + const memory = this.deps.getMemory(); + + // Create an agent_activity row for this incremental exploration + let incrementalExplorationId: string | undefined; + if (memory) { + try { + incrementalExplorationId = randomUUID(); + await memory.insertActivity({ + id: incrementalExplorationId, + type: 'explorer', + status: 'running', + task_summary: 'Incremental exploration', + started_at: startedAt, + updated_at: startedAt, + }); + } catch { + incrementalExplorationId = undefined; + } + } + + if (memory) { + await memory.logExploration({ + timestamp: startedAt, + level: 'info', + message: 'Incremental workspace exploration started', + data: { + method: changes.method, + changedCount: changes.changedFiles.length, + deletedCount: changes.deletedFiles.length, + summary: changes.summary, + }, + }); + } else { + await this.deps.dotFolder.appendLog({ + timestamp: startedAt, + level: 'info', + message: 'Incremental workspace exploration started', + data: { + method: changes.method, + changedCount: changes.changedFiles.length, + deletedCount: changes.deletedFiles.length, + summary: changes.summary, + }, + }); + } + + try { + // Mark affected memory chunks as stale + if (memory) { + let splitDirs: Record | undefined; + try { + const rawScan = await memory.getStructureScan(); + if (rawScan) { + const parsed = JSON.parse(rawScan) as { splitDirs?: Record }; + if (parsed.splitDirs && Object.keys(parsed.splitDirs).length > 0) { + splitDirs = parsed.splitDirs; + } + } + } catch { + // ignore — fall back to 1-level scopes + } + + const changedScopes = this.deps.changeTracker.extractChangedScopes( + changes.changedFiles, + changes.deletedFiles, + splitDirs, + ); + if (changedScopes.length > 0) { + try { + await memory.markStale(changedScopes); + logger.info({ changedScopes }, 'Marked stale memory scopes for incremental refresh'); + } catch (err) { + logger.warn({ err }, 'Failed to mark stale scopes — continuing'); + } + } + } + + const prompt = generateIncrementalExplorationPrompt( + this.deps.workspacePath, + existingMap, + changes.changedFiles, + changes.deletedFiles, + changes.summary, + ); + + const spawnOpts = this.deps.buildMasterSpawnOptions( + prompt, + this.deps.explorationTimeout, + undefined, + undefined, + true, // skipWorkspaceContext — the prompt already contains the workspace map + ); + // Scale maxTurns to change size — incremental is smaller scope + spawnOpts.maxTurns = Math.min( + MASTER_MAX_TURNS, + Math.max(10, changes.changedFiles.length + 5), + ); + + const result = await this.deps.agentRunner.spawn(spawnOpts); + await this.deps.updateMasterSession(); + + if (result.exitCode !== 0) { + throw new Error( + `Incremental exploration failed (exit ${result.exitCode}): ${result.stderr}`, + ); + } + + // Save the analysis marker with the current workspace state + const totalChanged = changes.changedFiles.length + changes.deletedFiles.length; + const newMarker = await this.deps.changeTracker.buildCurrentMarker( + 'incremental', + totalChanged, + ); + await this.deps.writeAnalysisMarkerToStore(newMarker); + this._mapLastVerifiedAt = newMarker.lastVerifiedAt ?? newMarker.analyzedAt; + + // Reload the map into memory + await this.loadExplorationSummary(); + + // Update cached map summary + const updatedMap = await this.deps.readWorkspaceMapFromStore(); + if (updatedMap) { + this._workspaceMapSummary = this.buildMapSummary(updatedMap); + } + + // Re-explore stale directories + if (memory) { + const staleExplorationId = randomUUID(); + const staleNow = new Date().toISOString(); + await memory.insertActivity({ + id: staleExplorationId, + type: 'explorer', + status: 'running', + task_summary: 'Stale directory re-exploration', + started_at: staleNow, + updated_at: staleNow, + }); + const coordinator = new ExplorationCoordinator({ + workspacePath: this.deps.workspacePath, + masterTool: this.deps.masterTool, + discoveredTools: this.deps.discoveredTools, + memory, + explorationId: staleExplorationId, + }); + try { + await coordinator.reexploreStaleDirs(); + await memory.updateActivity(staleExplorationId, { + status: 'done', + progress_pct: 100, + completed_at: new Date().toISOString(), + }); + } catch (err) { + logger.warn({ err }, 'Stale dir re-exploration failed — continuing'); + await memory.updateActivity(staleExplorationId, { + status: 'failed', + completed_at: new Date().toISOString(), + }); + } + } + + if (memory) { + await memory.logExploration({ + timestamp: new Date().toISOString(), + level: 'info', + message: 'Incremental exploration completed', + data: { filesChanged: totalChanged, durationMs: result.durationMs }, + }); + } else { + await this.deps.dotFolder.appendLog({ + timestamp: new Date().toISOString(), + level: 'info', + message: 'Incremental exploration completed', + data: { filesChanged: totalChanged, durationMs: result.durationMs }, + }); + } + + logger.info( + { filesChanged: totalChanged, durationMs: result.durationMs }, + 'Incremental exploration completed', + ); + + // Mark the incremental exploration activity as done + if (memory && incrementalExplorationId) { + try { + await memory.updateActivity(incrementalExplorationId, { + status: 'done', + progress_pct: 100, + completed_at: new Date().toISOString(), + }); + } catch { + // activity tracking is best-effort + } + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + + // Mark the incremental exploration activity as failed + if (memory && incrementalExplorationId) { + try { + await memory.updateActivity(incrementalExplorationId, { + status: 'failed', + completed_at: new Date().toISOString(), + }); + } catch { + // activity tracking is best-effort + } + } + + if (memory) { + await memory.logExploration({ + timestamp: new Date().toISOString(), + level: 'error', + message: 'Incremental exploration failed — falling back to full re-explore', + data: { error: errorMessage }, + }); + } else { + await this.deps.dotFolder.appendLog({ + timestamp: new Date().toISOString(), + level: 'error', + message: 'Incremental exploration failed — falling back to full re-explore', + data: { error: errorMessage }, + }); + } + + logger.warn( + { error: errorMessage }, + 'Incremental exploration failed, falling back to full re-exploration', + ); + + // Fall back to full exploration + await this.explore(); + } + } + + // ------------------------------------------------------------------------- + // User-triggered re-exploration + // ------------------------------------------------------------------------- + + /** + * Re-explore the workspace (e.g., after significant changes). + * Uses the Master session to drive re-exploration, with a fallback to + * a standalone AgentRunner call if no session is available. + */ + async reExplore(): Promise { + if (this.deps.getState() !== 'ready') { + logger.warn( + { currentState: this.deps.getState() }, + 'Cannot re-explore: Master not in ready state', + ); + return; + } + + const startedAt = new Date().toISOString(); + this.deps.setState('exploring'); + + logger.info({ workspacePath: this.deps.workspacePath }, 'Starting workspace re-exploration'); + + try { + const memory = this.deps.getMemory(); + // Log re-exploration start + if (memory) { + await memory.logExploration({ + timestamp: startedAt, + level: 'info', + message: 'Workspace re-exploration started', + }); + } else { + await this.deps.dotFolder.appendLog({ + timestamp: startedAt, + level: 'info', + message: 'Workspace re-exploration started', + }); + } + + if (this.deps.getMasterSession()) { + // Master-driven re-exploration via session + const prompt = generateReExplorationPrompt(this.deps.workspacePath); + const spawnOpts = this.deps.buildMasterSpawnOptions(prompt, this.deps.explorationTimeout); + const result = await this.deps.agentRunner.spawn(spawnOpts); + await this.deps.updateMasterSession(); + + if (result.exitCode !== 0) { + throw new Error( + `Re-exploration failed with exit code ${result.exitCode}: ${result.stderr}`, + ); + } + } else { + // Fallback: standalone re-exploration with read-only tools + const prompt = generateReExplorationPrompt(this.deps.workspacePath); + const result = await this.deps.agentRunner.spawn({ + prompt, + workspacePath: this.deps.workspacePath, + timeout: this.deps.explorationTimeout, + allowedTools: [...TOOLS_READ_ONLY], + retries: 1, + }); + + if (result.exitCode !== 0) { + throw new Error( + `Re-exploration failed with exit code ${result.exitCode}: ${result.stderr}`, + ); + } + } + + // Update exploration summary from the map + await this.loadExplorationSummary(); + + // Cache the map summary for context injection + const reExploreMap = await this.deps.readWorkspaceMapFromStore(); + if (reExploreMap) { + this._workspaceMapSummary = this.buildMapSummary(reExploreMap); + } + + // Write analysis marker so next startup skips unnecessary re-exploration + const reExploreMarker = await this.deps.changeTracker.buildCurrentMarker('full', 0); + await this.deps.writeAnalysisMarkerToStore(reExploreMarker); + this._mapLastVerifiedAt = reExploreMarker.lastVerifiedAt ?? reExploreMarker.analyzedAt; + + // Log re-exploration completion + if (memory) { + await memory.logExploration({ + timestamp: new Date().toISOString(), + level: 'info', + message: 'Workspace re-exploration completed', + }); + } else { + await this.deps.dotFolder.appendLog({ + timestamp: new Date().toISOString(), + level: 'info', + message: 'Workspace re-exploration completed', + }); + } + + this.deps.setState('ready'); + + logger.info('Workspace re-exploration completed'); + } catch (error) { + logger.error({ err: error }, 'Workspace re-exploration failed'); + this.deps.setState('ready'); // Return to ready state even on failure + throw error; + } + } + + /** + * Full re-exploration of the workspace using the 5-phase ExplorationCoordinator. + * Unlike reExplore() which sends a lightweight prompt to the Master session, + * this method runs the complete structure scan → classification → directory dives + * → assembly → finalization pipeline with progress tracking in exploration_progress. + */ + async fullReExplore(): Promise { + if (this.deps.getState() !== 'ready') { + logger.warn( + { currentState: this.deps.getState() }, + 'Cannot full re-explore: Master not in ready state', + ); + return; + } + + this.deps.setState('exploring'); + const startedAt = new Date().toISOString(); + + logger.info( + { workspacePath: this.deps.workspacePath }, + 'Starting full workspace re-exploration (user-triggered)', + ); + + try { + const memory = this.deps.getMemory(); + if (memory) { + await memory.logExploration({ + timestamp: startedAt, + level: 'info', + message: 'Full workspace re-exploration started (user-triggered)', + }); + + // Clear exploration state so ExplorationCoordinator doesn't skip completed phases + await memory.upsertExplorationState(null); + } + + // Clear dotfolder exploration state as well + try { + await this.deps.dotFolder.writeExplorationState(null as unknown as ExplorationState); + } catch { + // ignore — dotfolder may not exist yet, or null fails Zod validation + } + + // Run the full 5-phase exploration pipeline + await this.masterDrivenExplore(); + + // Refresh in-memory state + await this.loadExplorationSummary(); + await this.writeAgentsRegistry(); + + this.deps.setState('ready'); + + if (memory) { + await memory.logExploration({ + timestamp: new Date().toISOString(), + level: 'info', + message: 'Full workspace re-exploration completed (user-triggered)', + }); + } + + logger.info('Full workspace re-exploration completed'); + } catch (error) { + logger.error({ err: error }, 'Full workspace re-exploration failed'); + this.deps.setState('ready'); + throw error; + } + } + + // ------------------------------------------------------------------------- + // Progress broadcasting + // ------------------------------------------------------------------------- + + /** + * Translate ExplorationCoordinator progress callbacks into ProgressEvents + * and broadcast them to all connected connectors. + */ + private async emitExplorationProgress(event: { + phase: string; + status: string; + detail?: string; + directoryProgress?: { completed: number; total: number; currentDir?: string }; + }): Promise { + const phaseLabels: Record = { + structure_scan: 'Scanning workspace structure', + classification: 'Classifying project type', + directory_dives: 'Exploring directories', + assembly: 'Assembling workspace map', + finalization: 'Finalizing exploration', + }; + + const phaseLabel = phaseLabels[event.phase] ?? event.phase; + const statusSuffix = event.status === 'completed' ? ' (done)' : '...'; + + logger.info(`${phaseLabel}${statusSuffix}`); + + // Broadcast to connectors if router is available + const router = this.deps.getRouter(); + if (router) { + if (event.directoryProgress) { + await router.broadcastProgress({ + type: 'exploring-directory', + directory: event.directoryProgress.currentDir ?? '', + completed: event.directoryProgress.completed, + total: event.directoryProgress.total, + }); + } else { + await router.broadcastProgress({ + type: 'exploring', + phase: phaseLabel, + detail: event.detail, + }); + } + } + } + + // ------------------------------------------------------------------------- + // Post-exploration memory seeding + // ------------------------------------------------------------------------- + + /** + * Write a structured exploration summary directly to memory.md after exploration + * completes (OB-F156, OB-1271). Reads workspace map and classification results + * and writes a meaningful seed instead of relying on triggerMemoryUpdate() which + * requires conversation history (completedTaskCount=0 at this point). + */ + async writeExplorationSummaryToMemory(): Promise { + const memoryPath = this.deps.dotFolder.getMemoryFilePath(); + const now = new Date().toISOString().slice(0, 16).replace('T', ' '); + + try { + const map = await this.deps.readWorkspaceMapFromStore(); + let classification: Classification | null = null; + const memory = this.deps.getMemory(); + if (memory) { + const raw = await memory.getClassification(); + if (raw) { + try { + classification = JSON.parse(raw) as Classification; + } catch { + // malformed JSON — fall back to dotfolder below + } + } + } + if (!classification) { + classification = await this.deps.dotFolder.readClassification(); + } + + if (!map && !classification) { + logger.warn('writeExplorationSummaryToMemory: no exploration data — skipping'); + return; + } + + const lines: string[] = ['# Memory', `> Generated: ${now} (post-exploration seed)`, '']; + + // Project overview from workspace map + if (map) { + lines.push('## Project Overview', ''); + if (map.projectName) lines.push(`**Project:** ${map.projectName}`); + if (map.projectType) lines.push(`**Type:** ${map.projectType}`); + const mapAny = map as Record; + if (typeof mapAny['projectPhase'] === 'string') { + lines.push(`**Phase:** ${mapAny['projectPhase']}`); + } + if (typeof mapAny['summary'] === 'string') { + lines.push('', mapAny['summary']); + } + lines.push(''); + } else if (classification) { + lines.push('## Project Overview', ''); + lines.push(`**Project:** ${classification.projectName}`); + lines.push(`**Type:** ${classification.projectType}`); + lines.push(''); + } + + // Frameworks — prefer map, fall back to classification + const frameworks: string[] = map?.frameworks?.length + ? map.frameworks + : (classification?.frameworks ?? []); + if (frameworks.length > 0) { + lines.push('## Frameworks & Tech Stack', ''); + for (const f of frameworks) lines.push(`- ${f}`); + lines.push(''); + } + + // Directory structure + if (map?.structure && Object.keys(map.structure).length > 0) { + lines.push('## Directory Structure', ''); + for (const [dir, info] of Object.entries(map.structure)) { + lines.push(`- **${dir}/**: ${info.purpose}`); + if (lines.length >= 160) { + lines.push('- _(truncated — see workspace-map.json for full list)_'); + break; + } + } + lines.push(''); + } + + // Key commands + const commands: Record = + ((map as Record | null)?.['commands'] as + | Record + | undefined) ?? + classification?.commands ?? + {}; + const cmdEntries = Object.entries(commands); + if (cmdEntries.length > 0) { + lines.push('## Key Commands', ''); + for (const [name, cmd] of cmdEntries) lines.push(`- **${name}:** \`${cmd}\``); + lines.push(''); + } + + // Exploration metadata + if (this._explorationSummary) { + lines.push('## Exploration Status', ''); + lines.push(`- Status: ${this._explorationSummary.status}`); + lines.push(`- Directories explored: ${this._explorationSummary.directoriesExplored}`); + lines.push(''); + } + + lines.push('---'); + lines.push('_Seeded from exploration results. Updated each session._'); + + const content = lines.join('\n'); + await this.deps.dotFolder.writeMemoryFile(content); + const mem = this.deps.getMemory(); + if (mem) { + try { + await mem.setSystemConfig('memory_md_backup', content); + } catch (e) { + logger.debug({ err: e }, 'Failed to backup memory.md to SQLite'); + } + } + logger.info( + { memoryPath, lineCount: lines.length }, + 'Exploration summary written to memory.md', + ); + } catch (err) { + logger.warn({ err, memoryPath }, 'writeExplorationSummaryToMemory failed'); + } + } + + // ------------------------------------------------------------------------- + // Exploration summary loading + // ------------------------------------------------------------------------- + + /** + * Load exploration summary from the workspace map written by the Master. + */ + async loadExplorationSummary(): Promise { + const map = await this.deps.readWorkspaceMapFromStore(); + if (map) { + this._explorationSummary = { + startedAt: map.generatedAt, + completedAt: new Date().toISOString(), + status: 'completed', + filesScanned: 0, + directoriesExplored: Object.keys(map.structure).length, + projectType: map.projectType, + frameworks: map.frameworks, + insights: [], + mapPath: this.deps.dotFolder.getMapPath(), + gitInitialized: true, + }; + } else { + logger.warn( + 'Exploration completed but workspace map is empty — will re-explore on next startup', + ); + this._explorationSummary = { + startedAt: new Date().toISOString(), + completedAt: new Date().toISOString(), + status: 'failed', + filesScanned: 0, + directoriesExplored: 0, + frameworks: [], + insights: [], + gitInitialized: true, + }; + } + } + + // ------------------------------------------------------------------------- + // Agents registry + // ------------------------------------------------------------------------- + + /** + * Write the agents registry to system_config (DB) and agents.json (fallback). + */ + async writeAgentsRegistry(): Promise { + const registry = this.createAgentsRegistry(); + const memory = this.deps.getMemory(); + if (memory) { + await memory.setSystemConfig('agents', JSON.stringify(registry)); + } else { + await this.deps.dotFolder.writeAgents(registry); + } + } + + /** + * Create agents registry from discovered tools. + */ + createAgentsRegistry(): AgentsRegistry { + const master = this.deps.masterTool; + const specialists = this.deps.discoveredTools + .filter((tool) => tool.role === 'specialist' || tool.role === 'backup') + .map((tool) => ({ + name: tool.name, + path: tool.path, + version: tool.version, + role: tool.role as 'specialist' | 'backup', + capabilities: tool.capabilities, + })); + + return { + master: { + name: master.name, + path: master.path, + version: master.version, + role: 'master', + }, + specialists, + updatedAt: new Date().toISOString(), + }; + } + + // ------------------------------------------------------------------------- + // Map summary builder + // ------------------------------------------------------------------------- + + /** + * Build a rich text summary from the workspace map for system prompt injection. + * Includes project name, type, summary, frameworks, structure, and key files. + */ + buildMapSummary(map: Record): string { + const parts: string[] = []; + const str = (key: string): string | undefined => { + const v = map[key]; + return typeof v === 'string' ? v : undefined; + }; + + const name = str('projectName'); + if (name) parts.push(`Project: ${name}`); + const ptype = str('projectType'); + if (ptype) parts.push(`Type: ${ptype}`); + const phase = str('projectPhase'); + if (phase) parts.push(`Phase: ${phase}`); + const summary = str('summary'); + if (summary) parts.push(`\nSummary: ${summary}`); + + const frameworks = map['frameworks']; + if (Array.isArray(frameworks) && frameworks.length > 0) { + parts.push(`\nFrameworks: ${frameworks.map(String).join(', ')}`); + } + + const structure = map['structure']; + if (structure && typeof structure === 'object' && !Array.isArray(structure)) { + const dirs = Object.entries(structure as Record) + .map(([dirName, info]) => { + const purpose = + info && typeof info === 'object' && 'purpose' in info + ? String((info as Record)['purpose']) + : 'unknown'; + return `- ${dirName}/: ${purpose}`; + }) + .join('\n'); + if (dirs) parts.push(`\nDirectory structure:\n${dirs}`); + } + + const commands = map['commands']; + if (commands && typeof commands === 'object' && !Array.isArray(commands)) { + const cmds = Object.entries(commands as Record) + .map(([cmdName, cmd]) => `- ${cmdName}: ${String(cmd)}`) + .join('\n'); + if (cmds) parts.push(`\nAvailable commands:\n${cmds}`); + } + + const dependencies = map['dependencies']; + if (Array.isArray(dependencies) && dependencies.length > 0) { + const deps = dependencies + .map((d: unknown) => { + if (d && typeof d === 'object') { + const dep = d as Record; + const depName = typeof dep['name'] === 'string' ? dep['name'] : ''; + const depPurpose = typeof dep['purpose'] === 'string' ? dep['purpose'] : ''; + return `- ${depName}${depPurpose ? `: ${depPurpose}` : ''}`; + } + return `- ${String(d)}`; + }) + .join('\n'); + parts.push(`\nDependencies:\n${deps}`); + } + + if (this._explorationSummary?.mapPath) { + parts.push(`\nFull workspace map: ${this._explorationSummary.mapPath}`); + } + + return parts.join('\n'); + } + + // ------------------------------------------------------------------------- + // Workspace map indexing + // ------------------------------------------------------------------------- + + /** + * Index workspace map content as individual searchable FTS5 chunks (OB-1569). + * Called when the workspace map is reused from cache but the chunk store is + * empty — ensures FTS5 has something to search on first query. + */ + async indexWorkspaceMapAsChunks(map: WorkspaceMap): Promise { + const memory = this.deps.getMemory(); + if (!memory) return; + + const chunks: Array<{ + scope: string; + category: 'structure' | 'patterns' | 'dependencies' | 'api' | 'config'; + content: string; + source_hash: string; + }> = []; + + // Summary chunk — project overview + const summaryParts: string[] = [`Project: ${map.projectName}`, `Type: ${map.projectType}`]; + if (map.frameworks.length > 0) summaryParts.push(`Frameworks: ${map.frameworks.join(', ')}`); + if (map.summary) summaryParts.push(`Summary: ${map.summary}`); + if (map.entryPoints.length > 0) + summaryParts.push(`Entry points: ${map.entryPoints.join(', ')}`); + chunks.push({ + scope: '_workspace_summary', + category: 'structure', + content: summaryParts.join('\n'), + source_hash: 'workspace-map-index', + }); + + // Key files chunk + if (map.keyFiles.length > 0) { + const fileLines = map.keyFiles.map((f) => `${f.path} (${f.type}): ${f.purpose}`).join('\n'); + chunks.push({ + scope: '_workspace_key_files', + category: 'structure', + content: `Key files:\n${fileLines}`, + source_hash: 'workspace-map-index', + }); + } + + // Directory structure chunk + const structureEntries = Object.entries(map.structure); + if (structureEntries.length > 0) { + const structureLines = structureEntries + .map(([dir, info]) => `${info.path ?? dir}/: ${info.purpose}`) + .join('\n'); + chunks.push({ + scope: '_workspace_structure', + category: 'structure', + content: `Directory structure:\n${structureLines}`, + source_hash: 'workspace-map-index', + }); + } + + // Commands chunk + const commandEntries = Object.entries(map.commands); + if (commandEntries.length > 0) { + const commandLines = commandEntries.map(([cmdName, cmd]) => `${cmdName}: ${cmd}`).join('\n'); + chunks.push({ + scope: '_workspace_commands', + category: 'config', + content: `Available commands:\n${commandLines}`, + source_hash: 'workspace-map-index', + }); + } + + // Dependencies chunk (first 50) + if (map.dependencies.length > 0) { + const depLines = map.dependencies + .slice(0, 50) + .map((d) => `${d.name}${d.version ? `@${d.version}` : ''}${d.type ? ` (${d.type})` : ''}`) + .join('\n'); + chunks.push({ + scope: '_workspace_deps', + category: 'dependencies', + content: `Dependencies:\n${depLines}`, + source_hash: 'workspace-map-index', + }); + } + + await memory.storeChunks(chunks); + logger.info({ chunkCount: chunks.length }, 'Indexed workspace map into FTS5 chunks (OB-1569)'); + } + + // ------------------------------------------------------------------------- + // Pending message drain + // ------------------------------------------------------------------------- + + /** + * Drain messages that were queued during exploration. + * Called after state transitions to 'ready'. + */ + async drainPendingMessages(): Promise { + if (this._pendingMessages.length === 0) return; + + const messages = [...this._pendingMessages]; + this._pendingMessages = []; + + const total = messages.length; + logger.info({ count: total }, 'Draining pending messages after exploration'); + + let errorCount = 0; + const failedSenders = new Map(); + + for (const message of messages) { + const router = this.deps.getRouter(); + if (router) { + try { + await router.route(message); + } catch (error) { + errorCount++; + failedSenders.set(message.sender, message.source); + logger.error( + { error, sender: message.sender, content: message.content }, + 'Failed to route pending message during exploration drain', + ); + } + } else { + logger.warn( + { sender: message.sender }, + 'No router set — pending message processed but response not delivered', + ); + try { + const response = await this.deps.processMessage(message); + logger.info( + { sender: message.sender, responseLength: response.length }, + 'Pending message processed (no router)', + ); + } catch (error) { + errorCount++; + logger.error( + { error, sender: message.sender, content: message.content }, + 'Failed to process pending message during exploration drain', + ); + } + } + } + + // Notify affected senders if any messages failed to process + const router = this.deps.getRouter(); + if (errorCount > 0 && router) { + const notice = `⚠️ ${errorCount} of ${total} queued message${total === 1 ? '' : 's'} failed to process during exploration drain. Please resend your request.`; + for (const [sender, source] of failedSenders) { + void router.sendDirect(source, sender, notice); + } + logger.warn({ errorCount, total }, 'Notified senders of drain failures'); + } + } + + // ------------------------------------------------------------------------- + // Workspace context summary + // ------------------------------------------------------------------------- + + /** + * Build a concise workspace context string from the loaded workspace map. + * Uses the cached map summary if available (much richer than exploration metadata). + */ + getWorkspaceContextSummary(): string | null { + // Prefer the cached full map summary + if (this._workspaceMapSummary) { + return this._workspaceMapSummary; + } + + // Fallback to exploration metadata + if (!this._explorationSummary) return null; + const parts: string[] = []; + if (this._explorationSummary.projectType) { + parts.push(`Project type: ${this._explorationSummary.projectType}`); + } + if (this._explorationSummary.frameworks && this._explorationSummary.frameworks.length > 0) { + parts.push(`Frameworks: ${this._explorationSummary.frameworks.join(', ')}`); + } + if (this._explorationSummary.insights && this._explorationSummary.insights.length > 0) { + parts.push( + `Key insights:\n${this._explorationSummary.insights.map((i) => `- ${i}`).join('\n')}`, + ); + } + if (this._explorationSummary.mapPath) { + parts.push(`Full workspace map available at: ${this._explorationSummary.mapPath}`); + } + return parts.length > 0 ? parts.join('\n') : null; + } +} diff --git a/src/master/exploration-prompt.ts b/src/master/exploration-prompt.ts index e6957e8e..0247b667 100644 --- a/src/master/exploration-prompt.ts +++ b/src/master/exploration-prompt.ts @@ -91,7 +91,7 @@ Your \`workspace-map.json\` must conform to this structure: "dependencies": Array<{ // From package.json, requirements.txt, etc. "name": string, "version"?: string, - "type"?: "runtime" | "dev" | "peer" + "type"?: "runtime" | "dev" | "peer" | "optional" }>, "summary": string, // High-level description of the workspace "generatedAt": string, // ISO timestamp diff --git a/src/master/exploration-prompts.ts b/src/master/exploration-prompts.ts new file mode 100644 index 00000000..c6556a3e --- /dev/null +++ b/src/master/exploration-prompts.ts @@ -0,0 +1,659 @@ +/** + * Exploration Prompts — Incremental Multi-Pass Strategy + * + * Generates focused prompts for each phase of the incremental exploration workflow. + * Each prompt is designed to be short (30-90s AI execution time) and produce JSON + * output matching the corresponding Zod schema. + * + * Phase 1: Structure Scan - List files/dirs, count, detect configs + * Phase 2: Classification - Determine project type, frameworks, commands + * Phase 3: Directory Dives - Explore each significant directory in detail + * Phase 4: Assembly - Merge partial results into workspace-map.json + * + * All prompts are budget-constrained to PROMPT_CHAR_BUDGET (16K chars) to avoid + * truncation by agent-runner's MAX_PROMPT_LENGTH (32K). Data payloads are trimmed + * progressively: reduce indentation → trim arrays → compress JSON. + */ + +import type { StructureScan, WorkspaceMap } from '../types/master.js'; +import { createLogger } from '../core/logger.js'; + +const logger = createLogger('exploration-prompts'); + +/** + * Maximum character budget for any single exploration prompt. + * Each phase prompt must fit within this limit to avoid truncation + * by agent-runner's MAX_PROMPT_LENGTH (32K). Set to 16K to leave + * headroom for agent-runner overhead and system prompt wrapping. + */ +export const PROMPT_CHAR_BUDGET = 16_000; + +/** + * Trims a JSON data payload to fit within a character budget. + * Strategy: pretty-print → trim arrays → compact JSON. + */ +export function trimPayload( + data: Record, + budget: number, + arrayField?: string, +): string { + let payload = JSON.stringify(data, null, 2); + if (payload.length <= budget) return payload; + + // Step 1: trim the largest array field if specified + if (arrayField && Array.isArray(data[arrayField])) { + const arr = data[arrayField] as unknown[]; + const maxItems = Math.max(20, Math.floor(arr.length / 2)); + const trimmed = { + ...data, + [arrayField]: arr.slice(0, maxItems), + _trimmed: + arr.length > maxItems + ? `Showing ${maxItems} of ${arr.length} ${arrayField} (trimmed for prompt size)` + : undefined, + }; + payload = JSON.stringify(trimmed, null, 2); + if (payload.length <= budget) return payload; + + // Step 2: compress (no indentation) + payload = JSON.stringify(trimmed); + if (payload.length <= budget) return payload; + + // Step 3: aggressive trim — keep only first 10 items + const aggressive = { + ...data, + [arrayField]: arr.slice(0, 10), + _trimmed: `Showing 10 of ${arr.length} ${arrayField} (aggressively trimmed)`, + }; + payload = JSON.stringify(aggressive); + if (payload.length <= budget) return payload; + } + + // Final fallback: compact JSON without the array field + if (arrayField) { + const withoutArr = { + ...data, + [arrayField]: `[${(data[arrayField] as unknown[]).length} items omitted]`, + }; + return JSON.stringify(withoutArr); + } + return JSON.stringify(data); +} + +/** + * Pass 1: Structure Scan + * + * Generates a prompt that instructs the AI to scan the workspace structure + * and return a JSON object matching StructureScanSchema. + * + * Expected duration: 60-90s + * Output: structure-scan.json + * + * @param workspacePath - Absolute path to the workspace root + * @returns Prompt for structure scan + */ +export function generateStructureScanPrompt(workspacePath: string): string { + return `# Task: Workspace Structure Scan + +Scan the workspace at **${workspacePath}** and return a JSON object with its structure. + +## Instructions + +1. List all **top-level files** (files directly in the workspace root) +2. List all **top-level directories** (directories directly in the workspace root) +3. For each top-level directory, count how many files it contains (recursively, but skip node_modules/.git/dist/.next/build/coverage/target) +4. Identify **configuration files** (package.json, tsconfig.json, requirements.txt, Cargo.toml, .env.example, etc.) +5. List **skipped directories** (node_modules, .git, dist, etc.) +6. Count **total files** in the workspace (excluding skipped directories) +7. Note **asset directories** (images/, assets/, public/, media/, fonts/, data/, icons/, graphics/) and count image/media files (.png, .jpg, .svg, .mp4, .mp3, .ttf, .woff, etc.) + +## Skip These Directories + +- node_modules +- .git +- dist +- build +- .next +- coverage +- target +- vendor +- __pycache__ +- .venv +- venv + +## Output Format + +Return ONLY valid JSON matching this schema: + +\`\`\`json +{ + "workspacePath": "${workspacePath}", + "topLevelFiles": ["README.md", "package.json", ...], + "topLevelDirs": ["src", "tests", "docs", ...], + "directoryCounts": { + "src": 42, + "tests": 18, + "docs": 5 + }, + "configFiles": ["package.json", "tsconfig.json", ...], + "skippedDirs": ["node_modules", ".git", "dist"], + "totalFiles": 65, + "scannedAt": "2026-02-21T...", + "durationMs": 1200 +} +\`\`\` + +**IMPORTANT:** +- Return ONLY the JSON object, no explanations or markdown +- Use ISO 8601 format for scannedAt +- durationMs should reflect actual scan time in milliseconds +- Do NOT read file contents in this phase (just list and count) +`; +} + +/** + * Pass 2: Classification + * + * Generates a prompt that instructs the AI to classify the project type + * based on structure scan results and config file contents. + * + * Expected duration: 60-90s + * Output: classification.json + * + * @param workspacePath - Absolute path to the workspace root + * @param structureScan - Results from Pass 1 (structure scan) + * @returns Prompt for project classification + */ +export function generateClassificationPrompt( + workspacePath: string, + structureScan: StructureScan, +): string { + // Template overhead is ~2K chars; leave the rest for data + const dataBudget = PROMPT_CHAR_BUDGET - 3_000; + const scanPayload = trimPayload( + structureScan as unknown as Record, + dataBudget, + 'topLevelFiles', + ); + const promptSize = scanPayload.length + 3_000; + if (promptSize > PROMPT_CHAR_BUDGET) { + logger.debug( + { promptSize, budget: PROMPT_CHAR_BUDGET }, + 'Classification prompt exceeds budget after trimming', + ); + } + + return `# Task: Project Classification + +Classify the project at **${workspacePath}** based on the structure scan results below. + +## Structure Scan Results + +\`\`\`json +${scanPayload} +\`\`\` + +## Instructions + +1. **Read configuration files** listed in the structure scan (package.json, requirements.txt, etc.) +2. **Determine project type**: + - Code projects: "node", "python", "rust", "go", "java", "react-app", "api-backend", etc. + - Business workspaces: "cafe-operations", "legal-docs", "accounting-records", "real-estate-listings", etc. + - Mixed: "business-app-with-data", "mixed" +3. **Detect frameworks and tools** (React, Express, Django, TypeScript, Vite, etc.) +4. **Extract commands** from package.json scripts, Makefile, etc. +5. **List dependencies** from package.json, requirements.txt, Cargo.toml, etc. +6. **Identify key insights** (build system, testing framework, deployment targets, etc.) + +## Classification Heuristics + +**Code workspace indicators:** +- Presence of: package.json, requirements.txt, Cargo.toml, go.mod +- Directories: src/, lib/, tests/, components/, api/ +- Extensions: .ts, .js, .py, .rs, .go + +**Business workspace indicators:** +- Extensions: .xlsx, .csv, .pdf, .docx, .txt, .md (without code configs) +- No build configs or dependency files +- Directories: invoices/, reports/, contracts/, inventory/, sales/ + +**Asset workspace indicators:** +- Directories: images/, assets/, public/, media/, fonts/, icons/, graphics/ +- Extensions: .png, .jpg, .jpeg, .gif, .svg, .webp, .mp4, .mp3, .wav, .ttf, .woff, .woff2 +- May coexist with code (design systems, games, media applications) + +## Output Format + +Return ONLY valid JSON matching this schema: + +\`\`\`json +{ + "projectType": "node", + "projectName": "openbridge", + "frameworks": ["typescript", "node", "vitest"], + "commands": { + "dev": "npm run dev", + "test": "npm test", + "build": "npm run build" + }, + "dependencies": [ + { "name": "typescript", "version": "^5.7.0", "type": "dev" }, + { "name": "pino", "version": "^9.0.0", "type": "runtime" } + ], + "insights": [ + "TypeScript project with strict mode enabled", + "Uses Vitest for testing", + "ESM-only project (type: module)" + ], + "classifiedAt": "2026-02-21T...", + "durationMs": 1500 +} +\`\`\` + +**IMPORTANT:** +- Return ONLY the JSON object, no explanations +- Read actual config file contents, don't guess +- Be accurate — if you can't determine something, omit it +- Use ISO 8601 format for classifiedAt +`; +} + +/** + * Pass 3: Directory Dive + * + * Generates a prompt that instructs the AI to explore a single directory + * in depth and return structured information about its contents. + * + * Expected duration: 60-90s per directory + * Output: dirs/.json + * + * @param workspacePath - Absolute path to the workspace root + * @param dirPath - Relative path to the directory being explored + * @param context - Context from previous passes (project type, frameworks, etc.) + * @returns Prompt for directory dive + */ +export function generateDirectoryDivePrompt( + workspacePath: string, + dirPath: string, + context: { projectType: string; frameworks: string[] }, +): string { + return `# Task: Directory Exploration — ${dirPath} + +Explore the **${dirPath}** directory at **${workspacePath}/${dirPath}** and return structured information about its contents. + +## Context + +**Project Type:** ${context.projectType} +**Frameworks:** ${context.frameworks.join(', ') || 'none detected'} + +## Instructions + +1. **Determine the purpose** of this directory (what does it contain? what role does it play?) +2. **Identify key files** in this directory (important files and their purposes) +3. **List subdirectories** and their purposes (if any) +4. **Count files** in this directory (excluding subdirectories) +5. **Extract insights** specific to this directory (patterns, conventions, important details) + +## What to Look For + +- Entry points (index files, main modules) +- Configuration files specific to this directory +- README or documentation files +- Test files +- Patterns in file naming or organization +- Relationship to other parts of the project + +## Output Format + +Return ONLY valid JSON matching this schema: + +\`\`\`json +{ + "path": "${dirPath}", + "purpose": "Application source code — main implementation files", + "keyFiles": [ + { + "path": "src/index.ts", + "type": "entry", + "purpose": "Main entry point for the application" + }, + { + "path": "src/core/bridge.ts", + "type": "core", + "purpose": "Bridge orchestrator that wires all components together" + } + ], + "subdirectories": [ + { + "path": "src/core", + "purpose": "Core bridge engine (router, auth, queue, config)" + }, + { + "path": "src/connectors", + "purpose": "Messaging platform adapters" + } + ], + "fileCount": 8, + "insights": [ + "Uses ESM imports throughout", + "Core modules export both types and runtime code", + "Follows plugin architecture pattern" + ], + "exploredAt": "2026-02-21T...", + "durationMs": 1200 +} +\`\`\` + +**IMPORTANT:** +- Return ONLY the JSON object, no explanations +- Be specific about file purposes (not generic descriptions) +- Focus on source files and note asset directories (images, fonts, media, data files) +- Use ISO 8601 format for exploredAt +`; +} + +/** + * Pass 4: Summary Assembly + * + * Generates a prompt that instructs the AI to create a human-readable + * summary of the workspace based on all partial exploration results. + * + * This pass assembles the final workspace-map.json by merging: + * - structure-scan.json + * - classification.json + * - dirs/*.json (all directory dive results) + * + * The AI only needs to generate the "summary" field and any final insights. + * All other fields are merged mechanically by the coordinator. + * + * Expected duration: 30-60s + * Output: workspace-map.json (summary field only) + * + * @param workspacePath - Absolute path to the workspace root + * @param partialMap - Mechanically assembled partial workspace map (missing summary) + * @returns Prompt for summary generation + */ +/** + * Maximum character budget for the summary data payload. + * Template overhead is ~1.5K, so data budget = PROMPT_CHAR_BUDGET - 2K = 14K. + * Previous value (28K) exceeded the 16K per-prompt budget on its own. + */ +const SUMMARY_DATA_BUDGET = PROMPT_CHAR_BUDGET - 2_000; + +export function generateSummaryPrompt( + workspacePath: string, + partialMap: { + projectType: string; + projectName: string; + frameworks: string[]; + structure: Record; + keyFiles: Array<{ path: string; type: string; purpose: string }>; + commands: Record; + }, +): string { + const dataPayload = trimPayload( + partialMap as unknown as Record, + SUMMARY_DATA_BUDGET, + 'keyFiles', + ); + const promptSize = dataPayload.length + 2_000; + if (promptSize > PROMPT_CHAR_BUDGET) { + logger.debug( + { promptSize, budget: PROMPT_CHAR_BUDGET }, + 'Summary prompt exceeds budget after trimming', + ); + } + + // IMPORTANT: Output format instructions come FIRST so they survive + // any prompt truncation by the agent-runner (MAX_PROMPT_LENGTH = 32KB). + return `# Task: Generate Workspace Summary + +## Output Format (CRITICAL — read this first) + +Return ONLY a JSON object with a single "summary" field: + +\`\`\`json +{ + "summary": "Your 2-3 sentence summary here." +} +\`\`\` + +**IMPORTANT:** +- Return ONLY the JSON object with the summary field +- Do NOT include any other text, explanation, or markdown outside the JSON +- Keep it concise (2-3 sentences maximum) +- Adapt tone based on project type (code vs business) + +## Instructions + +Write a **2-3 sentence summary** that describes: +1. What this workspace is (the project's main purpose) +2. Key technologies/frameworks used +3. Any notable characteristics (architecture, deployment, special features) + +## Summary Style Guidelines + +**Code Projects:** Technical, concise — e.g. "Node.js TypeScript project using Express and Prisma for a REST API." +**Business Workspaces:** Plain language — e.g. "Cafe business files including sales reports and inventory spreadsheets." +**Mixed:** Balanced — e.g. "E-commerce platform (Node.js + React) with product catalogs and order CSVs." + +## Workspace Path + +${workspacePath} + +## Exploration Results + +\`\`\`json +${dataPayload} +\`\`\` +`; +} + +/** + * Pass 3b: Sub-Project Dive (Monorepo) + * + * Generates a prompt that instructs the AI to explore a detected sub-project + * as an independent project — performing both classification and directory + * exploration in a single pass. This is used instead of the regular directory + * dive prompt when the workspace is a monorepo and the directory has been + * identified as an independent sub-project with its own manifest file. + * + * Expected duration: 90-120s per sub-project + * Output: dirs/.json + * + * @param workspacePath - Absolute path to the workspace root + * @param subProjectPath - Relative path to the sub-project (e.g. "packages/ui") + * @param subProjectType - Project type inferred from manifest (e.g. "node", "go") + * @returns Prompt for sub-project dive + */ +export function generateSubProjectDivePrompt( + workspacePath: string, + subProjectPath: string, + subProjectType: string, +): string { + return `# Task: Sub-Project Exploration — ${subProjectPath} + +Explore the sub-project at **${workspacePath}/${subProjectPath}** as an **independent project** within a monorepo workspace. + +## Context + +This directory has been identified as an independent sub-project (type: **${subProjectType}**) based on the presence of its own project manifest file. Treat it as a self-contained project — not just a directory within a larger project. + +## Instructions + +1. **Read the project manifest** (package.json, Cargo.toml, go.mod, etc.) to determine: + - The sub-project's name + - Its dependencies and dev dependencies + - Its build/test/dev commands + - Its frameworks and tools +2. **Determine the purpose** of this sub-project (what does it do? what role does it play in the monorepo?) +3. **Identify key files** (entry points, configs, main modules) +4. **List subdirectories** and their purposes +5. **Count files** in this sub-project (excluding node_modules, dist, build, etc.) +6. **Extract insights** specific to this sub-project (architecture, patterns, conventions) + +## What to Look For + +- Entry points (index files, main modules, bin scripts) +- Configuration files (tsconfig.json, .eslintrc, jest.config, etc.) +- README or documentation +- Test files and test configuration +- Relationship to other sub-projects in the monorepo (shared dependencies, cross-references) +- Build output directories + +## Output Format + +Return ONLY valid JSON matching this schema: + +\`\`\`json +{ + "path": "${subProjectPath}", + "purpose": "Independent ${subProjectType} sub-project — ", + "keyFiles": [ + { + "path": "${subProjectPath}/package.json", + "type": "config", + "purpose": "Project manifest with dependencies and scripts" + } + ], + "subdirectories": [ + { + "path": "${subProjectPath}/src", + "purpose": "Source code" + } + ], + "fileCount": 42, + "insights": [ + "Independent ${subProjectType} project within the monorepo", + "Uses ", + "Depends on " + ], + "exploredAt": "2026-02-21T...", + "durationMs": 1200 +} +\`\`\` + +**IMPORTANT:** +- Return ONLY the JSON object, no explanations +- Treat this as a standalone project — classify its frameworks and purpose independently +- Note any cross-references to sibling sub-projects in the monorepo +- Use ISO 8601 format for exploredAt +`; +} + +/** + * Incremental Exploration Prompt + * + * Instructs the Master AI to update the existing workspace-map.json + * based on a set of changed/added/deleted files since the last analysis. + * The AI only reads the changed files and updates the relevant sections. + * + * @param workspacePath - Absolute path to the workspace root + * @param currentMap - The existing workspace map (for context) + * @param changedFiles - Files that were added or modified + * @param deletedFiles - Files that were removed + * @param changesSummary - Human-readable summary of what changed + * @returns Prompt for incremental map update + */ +export function generateIncrementalExplorationPrompt( + workspacePath: string, + currentMap: WorkspaceMap, + changedFiles: string[], + deletedFiles: string[], + changesSummary: string, +): string { + // Template + instructions are ~2.5K chars. Split remaining budget between + // file lists (~2K), change summary (~1K), and workspace map (rest). + const templateOverhead = 3_000; + const fileListBudget = 2_000; + const summaryBudget = 1_000; + const mapBudget = PROMPT_CHAR_BUDGET - templateOverhead - fileListBudget - summaryBudget; + + // Trim file lists if too many + const maxFiles = 100; + const trimmedChanged = + changedFiles.length > maxFiles + ? [...changedFiles.slice(0, maxFiles), `... and ${changedFiles.length - maxFiles} more files`] + : changedFiles; + const trimmedDeleted = + deletedFiles.length > maxFiles + ? [...deletedFiles.slice(0, maxFiles), `... and ${deletedFiles.length - maxFiles} more files`] + : deletedFiles; + + // Trim change summary + const trimmedSummary = + changesSummary.length > summaryBudget + ? changesSummary.slice(0, summaryBudget - 20) + '\n... (truncated)' + : changesSummary; + + // Build a slim map with only the fields needed for incremental update: + // structure, keyFiles (trimmed), frameworks, commands, summary, projectType + const slimMap: Record = { + projectType: currentMap.projectType, + projectName: currentMap.projectName, + frameworks: currentMap.frameworks, + commands: currentMap.commands, + structure: currentMap.structure, + keyFiles: currentMap.keyFiles, + summary: currentMap.summary, + }; + const mapPayload = trimPayload(slimMap, mapBudget, 'keyFiles'); + + const promptSize = + templateOverhead + + trimmedChanged.join('\n').length + + trimmedDeleted.join('\n').length + + trimmedSummary.length + + mapPayload.length; + if (promptSize > PROMPT_CHAR_BUDGET) { + logger.debug( + { promptSize, budget: PROMPT_CHAR_BUDGET }, + 'Incremental exploration prompt exceeds budget after trimming', + ); + } + + return `# Task: Incremental Workspace Map Update + +The workspace at **${workspacePath}** has changed since the last exploration. + +## What Changed + +${trimmedSummary} + +### Modified/Added Files (${changedFiles.length}): +${trimmedChanged.length > 0 ? trimmedChanged.map((f) => `- ${f}`).join('\n') : '(none)'} + +### Deleted Files (${deletedFiles.length}): +${trimmedDeleted.length > 0 ? trimmedDeleted.map((f) => `- ${f}`).join('\n') : '(none)'} + +## Current Workspace Map + +\`\`\`json +${mapPayload} +\`\`\` + +## Instructions + +1. **Read only the changed/added files** listed above using Read, Glob, and Grep +2. **Update the workspace map** with any new insights from the changed files: + - If a changed file is in a directory already in \`structure\`, update its \`purpose\` or \`fileCount\` if needed + - If a changed file introduces a new directory not in \`structure\`, add it + - If a changed file is a new key file (config, entry point, documentation), add it to \`keyFiles\` + - If a deleted file was in \`keyFiles\`, remove it + - If frameworks or dependencies changed (e.g., package.json modified), update \`frameworks\` and \`dependencies\` + - If commands changed (e.g., package.json scripts modified), update \`commands\` + - Update \`generatedAt\` to the current timestamp +3. **Do NOT re-explore unchanged files** — trust the existing map for those +4. **Write the updated map** to \`.openbridge/workspace-map.json\` using the Write tool +5. If the changes are trivial (e.g., only whitespace, comments), you may leave the map unchanged but still write it with an updated \`generatedAt\` + +## Important Constraints + +- Only read files from the changed/added list — do NOT scan the entire workspace +- Do NOT modify any workspace files outside \`.openbridge/\` +- If a changed file is binary or unreadable, skip it +- Keep the existing map structure intact — only modify sections affected by the changes +- Update the \`summary\` field ONLY if the changes significantly alter the project's nature + +Work silently. Write the updated map and finish.`; +} diff --git a/src/master/index.ts b/src/master/index.ts index 02bfc7c8..e4be8703 100644 --- a/src/master/index.ts +++ b/src/master/index.ts @@ -9,13 +9,87 @@ // Export DotFolderManager for .openbridge/ folder operations export { DotFolderManager } from './dotfolder-manager.js'; -// Export exploration prompt generators +// Export exploration prompt generators (legacy monolithic) export { generateExplorationPrompt, generateReExplorationPrompt, SAMPLE_WORKSPACE_MAP, } from './exploration-prompt.js'; +// Export incremental exploration prompt generators +export { + generateStructureScanPrompt, + generateClassificationPrompt, + generateDirectoryDivePrompt, + generateSummaryPrompt, + generateIncrementalExplorationPrompt, +} from './exploration-prompts.js'; + +// Export workspace change tracker for incremental exploration +export { WorkspaceChangeTracker } from './workspace-change-tracker.js'; +export type { WorkspaceChanges } from './workspace-change-tracker.js'; + // Export MasterManager for lifecycle management export { MasterManager } from './master-manager.js'; -export type { MasterManagerOptions } from './master-manager.js'; +export type { + MasterManagerOptions, + ClassificationResult, + ProgressReporter, +} from './master-manager.js'; + +// Export Master system prompt generator +export { generateMasterSystemPrompt } from './master-system-prompt.js'; +export type { MasterSystemPromptContext } from './master-system-prompt.js'; + +// Export result parser utilities +export { parseAIResult, parseAIResultWithRetry } from './result-parser.js'; +export type { ParseResult, ParseError, ParsedAIResult } from './result-parser.js'; + +// Export spawn parser for task decomposition protocol +export { parseSpawnMarkers, hasSpawnMarkers, extractTaskSummaries } from './spawn-parser.js'; +export type { ParsedSpawnMarker, SpawnParseResult, SpawnMarkerBody } from './spawn-parser.js'; + +// Export worker result formatter for structured result injection +export { + formatWorkerResult, + formatWorkerError, + buildWorkerFeedbackPrompt, + formatWorkerBatch, +} from './worker-result-formatter.js'; +export type { WorkerResultMeta } from './worker-result-formatter.js'; + +// Export ExplorationCoordinator for incremental exploration +export { ExplorationCoordinator } from './exploration-coordinator.js'; +export type { ExplorationOptions } from './exploration-coordinator.js'; + +// Export WorkerRegistry for worker orchestration +export { WorkerRegistry, DEFAULT_MAX_CONCURRENT_WORKERS } from './worker-registry.js'; +export type { WorkerRecord, WorkerStatus, WorkersRegistry } from './worker-registry.js'; + +// Export sub-master detector for hierarchical workspace management (OB-753) +export { detectSubProjects, SUB_PROJECT_MIN_FILES } from './sub-master-detector.js'; +export type { SubProjectInfo, ProjectType } from './sub-master-detector.js'; + +// Export sub-master manager for lifecycle management of sub-project masters (OB-754) +export { SubMasterManager } from './sub-master-manager.js'; +export type { SubMasterRecord, SubMasterStatus } from './sub-master-manager.js'; + +// Export monorepo detector for multi-project workspace awareness (OB-1273) +export { detectMonorepoPattern } from './monorepo-detector.js'; +export type { MonorepoDetectionResult, MonorepoSubProject } from './monorepo-detector.js'; + +// Export SkillManager for skills directory discovery and loading (OB-1704) +export { SkillManager } from './skill-manager.js'; + +// Export PromptContextBuilder for prompt/context assembly (OB-1282) +export { PromptContextBuilder } from './prompt-context-builder.js'; +export type { PromptContextBuilderDeps, MasterContextSections } from './prompt-context-builder.js'; + +// Export PlanningGate for two-phase execution control (OB-1775) +export { PlanningGate } from './planning-gate.js'; +export type { + PlanningGateState, + PlanningGateStatus, + PlanningPhase, + PlanningWorker, +} from './planning-gate.js'; diff --git a/src/master/master-manager.ts b/src/master/master-manager.ts index cc08466a..32da12b2 100644 --- a/src/master/master-manager.ts +++ b/src/master/master-manager.ts @@ -1,26 +1,441 @@ +import { BatchManager } from './batch-manager.js'; +import { BUILT_IN_SKILLS } from './skills/index.js'; +import { BUILT_IN_SKILL_PACKS } from './skill-packs/index.js'; +import { + ClassificationEngine, + classifyTaskType, + turnsToTimeout, + // MESSAGE_MAX_TURNS_QUICK moved to prompt-context-builder.ts (OB-1282) + MESSAGE_MAX_TURNS_TOOL_USE, + MESSAGE_MAX_TURNS_PLANNING, +} from './classification-engine.js'; +import type { ClassificationResult } from './classification-engine.js'; +// Re-export for backward compatibility — consumers import from master-manager.ts (OB-1279) +export type { ClassificationResult } from './classification-engine.js'; import { DotFolderManager } from './dotfolder-manager.js'; -import { generateExplorationPrompt, generateReExplorationPrompt } from './exploration-prompt.js'; +import { ExplorationManager } from './exploration-manager.js'; +// Re-export for backward compatibility (OB-1280) +export type { ExplorationManagerDeps } from './exploration-manager.js'; +import { WorkerOrchestrator } from './worker-orchestrator.js'; +// Re-export for backward compatibility (OB-1281) +export type { WorkerOrchestratorDeps } from './worker-orchestrator.js'; +import { PromptContextBuilder, SECTION_BUDGET_RAG } from './prompt-context-builder.js'; +// Re-export for backward compatibility (OB-1282) +export type { PromptContextBuilderDeps, MasterContextSections } from './prompt-context-builder.js'; +// predictToolRequirements, detectToolAccessFailure, ToolAccessFailure, ToolPrediction +// moved to worker-orchestrator.ts (OB-1281) +// ExplorationCoordinator, generateReExplorationPrompt, generateIncrementalExplorationPrompt +// moved to exploration-manager.ts (OB-1280) +import { generateMasterSystemPrompt } from './master-system-prompt.js'; +import type { ConnectedIntegrationEntry } from './master-system-prompt.js'; +// formatLearnedPatternsSection, formatWorkerNextStepsSection, +// formatPreFetchedKnowledgeSection, formatTargetedReaderSection, WorkerNextStepsEntry +// moved to prompt-context-builder.ts (OB-1282) +import { WorkspaceChangeTracker } from './workspace-change-tracker.js'; +// WorkspaceChanges type moved to exploration-manager.ts (OB-1280) import { - executeClaudeCode, - streamClaudeCode, -} from '../providers/claude-code/claude-code-executor.js'; + AgentRunner, + TOOLS_READ_ONLY, + TOOLS_CODE_EDIT, + DEFAULT_MAX_TURNS_TASK, + DEFAULT_MAX_FIX_ITERATIONS, + classifyError, + resolveProfile, +} from '../core/agent-runner.js'; +import type { SpawnOptions, AgentResult } from '../core/agent-runner.js'; +import { manifestToSpawnOptions, AgentExhaustedError } from '../core/agent-runner.js'; +import { getRecommendedModel, avoidHighFailureModel } from '../core/model-selector.js'; +// PromptAssembler and PRIORITY_* constants moved to prompt-context-builder.ts (OB-1282) +import type { CLIAdapter } from '../core/cli-adapter.js'; +import { AdapterRegistry } from '../core/adapter-registry.js'; +import type { Router } from '../core/router.js'; +import type { + MemoryManager, + ConversationEntry, + SessionRecord, + WorkspaceState, + TaskRecord as MemoryTaskRecord, + ActivityRecord, +} from '../memory/index.js'; +import { BUILT_IN_PROFILES, BuiltInProfileNameSchema } from '../types/agent.js'; +import type { ToolProfile, ProfilesRegistry, TaskManifest, SkillPack } from '../types/agent.js'; +import { ProfilesRegistrySchema } from '../types/agent.js'; import { DelegationCoordinator } from './delegation.js'; +import { SubMasterManager } from './sub-master-manager.js'; +import type { SubMasterRecord } from './sub-master-manager.js'; +// detectSubProjects moved to exploration-manager.ts (OB-1280) +import { openDatabase, closeDatabase } from '../memory/database.js'; +import { buildBriefing } from '../memory/worker-briefing.js'; +import { parseSpawnMarkers, hasSpawnMarkers, extractTaskSummaries } from './spawn-parser.js'; +import type { ParsedSpawnMarker } from './spawn-parser.js'; +// parseAIResult moved to exploration-manager.ts (OB-1280) +import { formatWorkerBatch } from './worker-result-formatter.js'; +import { WorkerRegistry, WorkersRegistrySchema } from './worker-registry.js'; +import type { WorkerRecord } from './worker-registry.js'; +import { evolvePrompts } from './prompt-evolver.js'; +import { MAX_PROMPT_VERSION_LENGTH } from '../memory/prompt-store.js'; +import { applyToolPromptPrefix, seedPromptLibrary, SEED_PROMPTS } from './seed-prompts.js'; +import type { KnowledgeRetriever } from '../core/knowledge-retriever.js'; +import type { IntegrationHub } from '../integrations/hub.js'; +import { DeepModeManager } from './deep-mode.js'; import type { MasterState, ExplorationSummary, TaskRecord, AgentsRegistry, WorkspaceMap, + MasterSession, + PromptTemplate, + ExplorationState, + WorkspaceAnalysisMarker, + LearningEntry, +} from '../types/master.js'; +import { + WorkspaceMapSchema, + ExplorationStateSchema, + AgentsRegistrySchema, } from '../types/master.js'; import type { DiscoveredTool } from '../types/discovery.js'; -import type { InboundMessage } from '../types/message.js'; +import type { MCPServer, DeepConfig, WorkspaceTrustLevel } from '../types/config.js'; +import type { InboundMessage, ProgressEvent } from '../types/message.js'; +import { createModelRegistry } from '../core/model-registry.js'; +import type { ModelRegistry } from '../core/model-registry.js'; import { createLogger } from '../core/logger.js'; import { randomUUID } from 'node:crypto'; +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; +import { SessionCompactor } from './session-compactor.js'; +import type { ConversationTurn } from './session-compactor.js'; +import { + getBuiltInSkillPacks, + findSkillByFormat, + selectSkillPackForTask, + loadAllSkillPacks, +} from './skill-pack-loader.js'; +import { classifyDocumentIntent } from '../core/router.js'; +import { PlanningGate, shouldBypassPlanning, performReasoningCheckpoint } from './planning-gate.js'; const logger = createLogger('master-manager'); -const DEFAULT_TIMEOUT = 600_000; // 10 minutes for exploration -const DEFAULT_MESSAGE_TIMEOUT = 60_000; // 1 minute for message processing +const DEFAULT_TIMEOUT = 1_800_000; // 30 minutes for exploration +const DEFAULT_MESSAGE_TIMEOUT = 300_000; // 5 minutes for message processing +const DEFAULT_WORKER_TIMEOUT = 300_000; // 5 minutes for worker tasks + +// turnsToTimeout imported from classification-engine.ts (OB-1279) + +/** Initial idle time threshold (5 minutes) before triggering self-improvement cycle */ +const IDLE_THRESHOLD_MS = 5 * 60 * 1000; // 5 minutes +/** Maximum idle threshold after exponential backoff (2 hours) */ +const IDLE_THRESHOLD_MAX_MS = 2 * 60 * 60 * 1000; // 2 hours +/** How often to check for idle state (1 minute) */ +const IDLE_CHECK_INTERVAL_MS = 60 * 1000; // 1 minute + +/** + * Exit codes and stderr patterns that indicate a dead/unrecoverable Master session. + * These warrant creating a new session rather than retrying the same one. + */ +const SESSION_DEAD_EXIT_CODES = new Set([ + 143, // SIGTERM — timeout killed the process + 137, // SIGKILL — force-killed (OOM or external) + 1, // General error — may be context overflow or session corruption +]); + +const SESSION_DEAD_PATTERNS = [ + 'context window', + 'context length', + 'context_length', + 'too many tokens', + 'token limit', + 'maximum context', + 'session not found', + 'session expired', + 'invalid session', + 'conversation too long', +]; + +// RESTART_CONTEXT_TASK_LIMIT and formatTimeAgo moved to prompt-context-builder.ts (OB-1282) + +/** Result of a tool-access failure scan on a worker result. */ +export interface ToolAccessFailure { + /** The tool name extracted from the error message, if identifiable. */ + tool: string | undefined; + /** The raw error snippet that matched a tool-access pattern. */ + reason: string; +} + +/** + * Pre-flight tool prediction result (OB-1595). + * Returned by `predictToolRequirements()` when the task prompt suggests the + * worker will need more tools than its current profile allows. + */ +export interface ToolPrediction { + /** The minimum profile needed to execute this task. */ + suggestedProfile: string; + /** Human-readable reason for the predicted upgrade. */ + reason: string; + /** Keywords in the prompt that triggered the prediction. */ + triggerKeywords: string[]; +} + +/** + * Rules mapping prompt keyword patterns to minimum required tool profiles. + * Ordered from most-restrictive (full-access) to least, so the first match wins. + */ +const PREFLIGHT_RULES: Array<{ + pattern: RegExp; + requiredProfile: 'code-edit' | 'full-access'; + label: string; +}> = [ + { + // Unrestricted shell: deploy, docker, kubectl, system daemons, curl/wget scripts + pattern: + /\b(deploy(?:ment)?|docker\s+\w|kubectl\s+\w|apt(?:-get)?\s+\w|brew\s+install|curl\s+https?|wget\s+https?|systemctl\s+\w|pm2\s+\w|sh\s+\S|bash\s+\S)\b/i, + requiredProfile: 'full-access', + label: 'deploy/docker/system commands', + }, + { + // npm/npx/pip/cargo/make + common task verbs (test, lint, build, install) + pattern: + /\b(npm\s+(test|install|run\s+\w+|build|ci)|npx\s+\w|pip\s+install|cargo\s+(build|test|run)|make\s+\w|run\s+tests?|run\s+(?:the\s+)?(?:lint|build|test)|lint\s+(?:the\s+)?code|build\s+(?:the\s+)?(?:project|app|package)|compile\s+\w|typecheck|type-check|install\s+(?:packages?|dep(?:endencies)?s?))\b/i, + requiredProfile: 'code-edit', + label: 'build/test/install commands', + }, +]; + +/** + * Predict the minimum tool profile needed to execute a task prompt (OB-1595). + * + * Scans the task prompt for keywords that suggest the worker will need + * bash/shell access (test, lint, build, deploy, install, etc.). Returns a + * `ToolPrediction` when the predicted minimum profile exceeds the current + * profile, so the caller can request upfront escalation before spawning. + * + * Returns `undefined` when: + * - The profile is already `full-access` or `master` (no escalation needed) + * - No prediction-triggering keywords are found + * - The predicted profile is ≤ the current profile + */ +export function predictToolRequirements( + prompt: string, + profile: string, +): ToolPrediction | undefined { + if (profile === 'full-access' || profile === 'master') return undefined; + + // Profile capability order (ascending access level) + const PROFILE_ORDER = ['read-only', 'code-audit', 'code-edit', 'full-access']; + const currentIndex = PROFILE_ORDER.indexOf(profile); + + for (const rule of PREFLIGHT_RULES) { + const match = rule.pattern.exec(prompt); + if (!match) continue; + + const suggestedIndex = PROFILE_ORDER.indexOf(rule.requiredProfile); + // Only escalate if the prediction exceeds the current profile + if (suggestedIndex <= currentIndex) continue; + + const triggerKeywords = match[0] + .trim() + .split(/\s+/) + .slice(0, 3) + .map((k) => k.replace(/[^a-zA-Z0-9:*-]/g, '')); + return { + suggestedProfile: rule.requiredProfile, + reason: rule.label, + triggerKeywords, + }; + } + + return undefined; +} + +/** + * Detect tool-access failures in a worker result (OB-1592). + * + * Scans both stdout and stderr for the error patterns the Claude CLI emits + * when a tool call is blocked by `--allowedTools` restrictions: + * - "tool not allowed" + * - "permission denied" + * - "not in allowedTools" + * + * Returns a `ToolAccessFailure` describing the blocked tool when a match is + * found, or `undefined` when no tool-access error is present. + */ +export function detectToolAccessFailure(result: { + stdout: string; + stderr: string; +}): ToolAccessFailure | undefined { + const combined = `${result.stderr}\n${result.stdout}`.toLowerCase(); + + const PATTERNS = [ + 'tool not allowed', + 'permission denied', + 'not in allowedtools', + 'not allowed to use', + 'tool is not allowed', + ] as const; + + const matched = PATTERNS.find((p) => combined.includes(p)); + if (!matched) return undefined; + + // Extract the tool name from the error line. + // Common Claude CLI formats: + // "Tool 'Bash' is not allowed" + // "Bash is not in allowedTools" + // "not allowed to use tool: Bash" + const searchText = `${result.stderr}\n${result.stdout}`; + let tool: string | undefined; + + const patterns = [ + // "Tool 'Bash' is not allowed" / "tool 'Bash(npm:test)' is not allowed" + /[Tt]ool\s+'([^']+)'/, + // "Bash is not in allowedTools" + /(\w[\w():*]*)\s+is\s+not\s+in\s+allowedTools/i, + // "not allowed to use tool: Bash" + /not\s+allowed\s+to\s+use\s+(?:tool:\s*)?([^\s,.:]+)/i, + // "not in allowedTools: Bash" + /not\s+in\s+allowedTools[:\s]+([^\s,.:]+)/i, + ]; + + for (const re of patterns) { + const m = re.exec(searchText); + if (m?.[1]) { + tool = m[1]; + break; + } + } + + // Provide a short reason snippet (first matching line, truncated) + const lines = searchText.split('\n'); + const matchedLine = lines.find((l) => PATTERNS.some((p) => l.toLowerCase().includes(p))); + const reason = (matchedLine ?? matched).trim().slice(0, 200); + + return { tool, reason }; +} + +/** + * Returns tools available to the Master AI session based on trust level. + * - trusted: full-access tools including Bash(*) — Master can execute commands directly. + * - sandbox: read-only tools (Read, Glob, Grep) — Master cannot modify files. + * - standard: master profile (Read, Glob, Grep, Write, Edit) — delegates execution to workers. + */ +export function getMasterTools(trustLevel: WorkspaceTrustLevel): string[] { + if (trustLevel === 'trusted') return [...BUILT_IN_PROFILES['full-access'].tools]; + if (trustLevel === 'sandbox') return ['Read', 'Glob', 'Grep']; + return [...BUILT_IN_PROFILES.master.tools]; +} + +/** + * Default max turns for the Master session per interaction. + * Higher than workers because the Master needs room to reason + coordinate. + */ +const MASTER_MAX_TURNS = 50; + +// MESSAGE_MAX_TURNS_* imported from classification-engine.ts (OB-1279) +/** Synthesis call — feeds worker results back to Master for a final user-facing response. */ +const MESSAGE_MAX_TURNS_SYNTHESIS = 5; +/** Memory update call — Master writes memory.md; small budget, file write only. */ +const MEMORY_UPDATE_MAX_TURNS = 5; +/** Trigger a memory.md update after this many completed tasks. */ +const MEMORY_UPDATE_INTERVAL = 10; + +// --------------------------------------------------------------------------- +// Memory ↔ DotFolderManager type conversion helpers (OB-711) +// --------------------------------------------------------------------------- + +/** Convert MasterSession → SessionRecord for SQLite storage */ +function masterSessionToSessionRecord(session: MasterSession, restartCount = 0): SessionRecord { + return { + id: session.sessionId, + type: 'master', + status: 'active', + restart_count: restartCount, + message_count: session.messageCount, + allowed_tools: JSON.stringify(session.allowedTools), + created_at: session.createdAt, + last_used_at: session.lastUsedAt, + }; +} + +/** Convert SessionRecord → MasterSession */ +function sessionRecordToMasterSession(record: SessionRecord): MasterSession { + return { + sessionId: record.id, + createdAt: record.created_at, + lastUsedAt: record.last_used_at, + messageCount: record.message_count ?? 0, + allowedTools: record.allowed_tools + ? (JSON.parse(record.allowed_tools) as string[]) + : getMasterTools('standard'), + maxTurns: MASTER_MAX_TURNS, + }; +} + +/** Convert WorkspaceAnalysisMarker → WorkspaceState for SQLite storage */ +function markerToWorkspaceState(marker: WorkspaceAnalysisMarker): WorkspaceState { + return { + commit_hash: marker.workspaceCommitHash, + branch: marker.workspaceBranch, + has_git: marker.workspaceHasGit, + analyzed_at: marker.analyzedAt, + last_verified_at: marker.lastVerifiedAt, + analysis_type: marker.analysisType, + files_changed: marker.filesChanged, + }; +} + +/** Convert WorkspaceState → WorkspaceAnalysisMarker */ +function workspaceStateToMarker(state: WorkspaceState): WorkspaceAnalysisMarker { + return { + workspaceCommitHash: state.commit_hash, + workspaceBranch: state.branch, + workspaceHasGit: state.has_git ?? false, + analyzedAt: state.analyzed_at, + lastVerifiedAt: state.last_verified_at, + analysisType: (state.analysis_type as 'full' | 'incremental') ?? 'full', + filesChanged: state.files_changed ?? 0, + schemaVersion: '1.0.0', + }; +} + +/** Convert master TaskRecord (types/master.ts) → memory TaskRecord (memory/task-store.ts) */ +function masterTaskToMemoryTask( + task: TaskRecord, + type: MemoryTaskRecord['type'] = 'worker', +): MemoryTaskRecord { + const statusMap: Record = { + completed: 'completed', + failed: 'failed', + pending: 'running', + processing: 'running', + delegated: 'completed', + }; + return { + id: task.id, + type, + status: statusMap[task.status] ?? 'failed', + prompt: task.userMessage, + response: task.result, + model: task.metadata?.['model'] as string | undefined, + profile: task.metadata?.['profile'] as string | undefined, + exit_code: task.metadata?.['exitCode'] as number | undefined, + max_turns: task.metadata?.['maxTurns'] as number | undefined, + turns_used: task.metadata?.['turnsUsed'] as number | undefined, + duration_ms: task.durationMs, + created_at: task.createdAt, + completed_at: task.completedAt, + }; +} + +// ClassificationResult re-exported from classification-engine.ts (see bottom of file) + +/** + * Callback for emitting progress events — decouples MasterManager from Router. + * Created per-message via makeProgressReporter(). No-op when no router is set. + */ +export type ProgressReporter = (event: ProgressEvent) => Promise; + +// MasterContextSections moved to prompt-context-builder.ts (OB-1282) +import type { MasterContextSections } from './prompt-context-builder.js'; /** * Options for creating a MasterManager @@ -38,11 +453,40 @@ export interface MasterManagerOptions { messageTimeout?: number; /** Whether to skip automatic exploration on startup */ skipAutoExploration?: boolean; + /** MemoryManager instance — when provided, enables SQLite-backed persistence (OB-711 will wire reads/writes) */ + memory?: MemoryManager; + /** Base delay for worker-level retry backoff in milliseconds (default: 5000) */ + workerRetryDelayMs?: number; + /** CLI adapter for spawning worker agents (defaults to ClaudeAdapter) */ + adapter?: CLIAdapter; + /** Adapter registry for resolving per-worker CLIAdapters */ + adapterRegistry?: AdapterRegistry; + /** MCP servers available for workers (from V2Config.mcp.servers, merged with configPath imports) */ + mcpServers?: MCPServer[]; + /** Deep Mode configuration — controls default execution profile and per-phase model overrides */ + deepConfig?: DeepConfig; + /** Glob patterns for files to exclude — hidden from the AI (workspace.exclude from V2 config) */ + workspaceExclude?: readonly string[]; + /** Glob patterns for files to include — limits AI visibility to only these files (workspace.include from V2 config) */ + workspaceInclude?: readonly string[]; + /** + * Maximum number of lint/test fix iterations before escalating to Master (OB-1791). + * Sourced from `config.worker.maxFixIterations`. Defaults to DEFAULT_MAX_FIX_ITERATIONS (3). + * Set to 0 to disable the cap. + */ + workerMaxFixIterations?: number; + /** Workspace trust level — controls Master AI tool access (OB-1583) */ + trustLevel?: WorkspaceTrustLevel; } /** * Manages the Master AI lifecycle and interaction. * + * The Master AI runs as a persistent Claude session (not single-shot --print). + * On startup, a session ID is created or loaded from .openbridge/master-session.json. + * The session stays alive across user messages via --resume. This allows the + * Master to accumulate context about the workspace and previous interactions. + * * Lifecycle states: * - idle: Created but not yet started * - exploring: Autonomously exploring workspace @@ -60,13 +504,139 @@ export class MasterManager { private readonly messageTimeout: number; private readonly skipAutoExploration: boolean; private readonly dotFolder: DotFolderManager; + private readonly changeTracker: WorkspaceChangeTracker; private readonly delegationCoordinator: DelegationCoordinator; + private readonly agentRunner: AgentRunner; + private readonly workerRegistry: WorkerRegistry; + private _memory: MemoryManager | null = null; + + /** Getter for MemoryManager — null when SQLite init failed */ + get memory(): MemoryManager | null { + return this._memory; + } + + /** Setter for MemoryManager — nulls out SubMasterManager when memory becomes unavailable */ + set memory(value: MemoryManager | null) { + this._memory = value; + if (!value) { + this.subMasterManager = null; + } + // Keep ClassificationEngine in sync with memory changes + if (this.classificationEngine) { + this.classificationEngine.updateDeps({ memory: value }); + } + } + /** Sub-master manager — null when no root DB is available (OB-755) */ + private subMasterManager: SubMasterManager | null = null; + private readonly workerRetryDelayMs: number; + private readonly modelRegistry: ModelRegistry; + private readonly adapter?: CLIAdapter; + private readonly adapterRegistry: AdapterRegistry; + private mcpServers: MCPServer[]; + private readonly workspaceExclude: readonly string[]; + private readonly workspaceInclude: readonly string[]; + private activeConnectorNames: string[] = []; + private fileServerPort: number | undefined; + private tunnelUrl: string | null = null; private state: MasterState = 'idle'; - private explorationSummary: ExplorationSummary | null = null; - private sessionMap: Map = new Map(); // sender → sessionId - private sessionTimeouts: Map = new Map(); - private readonly sessionTTL = 30 * 60 * 1000; // 30 minutes + private processingPendingMessages: InboundMessage[] = []; + + /** Exploration summary — delegated to ExplorationManager (OB-1280). */ + private get explorationSummary(): ExplorationSummary | null { + return this.explorationManager?.explorationSummary ?? null; + } + private set explorationSummary(value: ExplorationSummary | null) { + if (this.explorationManager) this.explorationManager.explorationSummary = value; + } + + /** Messages queued while exploration is in progress — delegated to ExplorationManager (OB-1280). */ + private get pendingMessages(): InboundMessage[] { + return this.explorationManager?.pendingMessages ?? []; + } + private set pendingMessages(value: InboundMessage[]) { + if (this.explorationManager) this.explorationManager.pendingMessages = value; + } + /** Router reference for sending pending message responses after exploration completes */ + private router: Router | null = null; + /** The InboundMessage currently being processed — set for the duration of processMessage/streamMessage so spawnWorker can escalate tool failures (OB-1593). */ + private activeMessage: InboundMessage | null = null; + + /** Persistent Master session — shared across all user messages */ + private masterSession: MasterSession | null = null; + /** Whether the session has been used (first call uses --session-id, subsequent use --resume) */ + private sessionInitialized = false; + /** Cached system prompt content (loaded from .openbridge/prompts/master-system.md) */ + private systemPrompt: string | null = null; + /** Number of times the Master session has been restarted */ + private restartCount = 0; + /** Timestamp of last user message (for idle detection) */ + private lastMessageTimestamp: number | null = null; + // lastExplorationAt moved to ExplorationManager (OB-1280) + /** Idle detection timer (runs self-improvement when idle for >5 min) */ + private idleCheckTimer: NodeJS.Timeout | null = null; + /** Whether self-improvement is currently running */ + private isSelfImproving = false; + /** Consecutive idle cycles without a user message — drives exponential backoff */ + private consecutiveIdleCycles = 0; + /** Consecutive no-op self-improvement cycles — suppresses cycles after 2 no-ops (OB-F210) */ + private consecutiveNoOpCycles = 0; + /** Number of successfully completed tasks — triggers prompt evolution every 50 (OB-734) */ + private completedTaskCount = 0; + /** Cached workspace map summary — delegated to ExplorationManager (OB-1280). */ + private get workspaceMapSummary(): string | null { + return this.explorationManager?.workspaceMapSummary ?? null; + } + private set workspaceMapSummary(value: string | null) { + if (this.explorationManager) this.explorationManager.workspaceMapSummary = value; + } + /** ISO timestamp of latest startup verification — delegated to ExplorationManager (OB-1280). */ + private get mapLastVerifiedAt(): string | null { + return this.explorationManager?.mapLastVerifiedAt ?? null; + } + private set mapLastVerifiedAt(value: string | null) { + if (this.explorationManager) this.explorationManager.mapLastVerifiedAt = value; + } + /** Cached summary of past learnings for injection into Master system prompt */ + private learningsSummary: string | null = null; + /** Classification engine — extracted from MasterManager (OB-1279, OB-F158). */ + private readonly classificationEngine: ClassificationEngine; + /** Exploration manager — extracted from MasterManager (OB-1280, OB-F158). */ + private readonly explorationManager: ExplorationManager; + /** Worker orchestrator — extracted from MasterManager (OB-1281, OB-F158). */ + private readonly workerOrchestrator: WorkerOrchestrator; + /** Prompt/context builder — extracted from MasterManager (OB-1282, OB-F158). */ + private readonly promptContextBuilder: PromptContextBuilder; + /** Abort handles for running worker processes — delegated to WorkerOrchestrator (OB-1281). */ + private get workerAbortHandles(): Map void> { + return this.workerOrchestrator.workerAbortHandles; + } + /** Cancellation notifications queued for injection into the next Master call (OB-884). */ + private readonly pendingCancellationNotifications: string[] = []; + /** Deep Mode resume offers queued for injection into the next Master call (OB-1405). */ + private readonly pendingDeepModeResumeOffers: string[] = []; + /** KnowledgeRetriever for RAG-based context injection (OB-1344). Null until set via setKnowledgeRetriever(). */ + private knowledgeRetriever: KnowledgeRetriever | null = null; + /** IntegrationHub for business integrations — null until set via setIntegrationHub(). */ + private integrationHub: IntegrationHub | null = null; + /** Deep Mode manager — tracks multi-phase session state (OB-1403). */ + private readonly deepMode: DeepModeManager; + /** Deep Mode configuration — controls default profile and per-phase model overrides (OB-1403). */ + private readonly deepConfig: DeepConfig | undefined; + /** BatchManager for Batch Task Continuation — set via setBatchManager() (OB-1613). */ + private batchManager: BatchManager | null = null; + /** Tracks active batch continuation timer handles for cleanup on shutdown (OB-1664). */ + private readonly batchTimers = new Set(); + /** Session compactor — monitors turn count and triggers memory.md compaction (OB-1672). */ + private compactor: SessionCompactor | null = null; + /** Active skill packs — built-ins merged with user-defined overrides (OB-1754). */ + private activeSkillPacks: SkillPack[] = [...BUILT_IN_SKILL_PACKS]; + /** Planning gate — two-phase execution guard for complex tasks (OB-1779). */ + private readonly planningGate = new PlanningGate(); + /** Max lint/test fix iterations for workers before escalating to Master (OB-1791). */ + private readonly workerMaxFixIterations: number; + /** Workspace trust level — controls Master/worker tool access (OB-1583) */ + private readonly trustLevel: WorkspaceTrustLevel; constructor(options: MasterManagerOptions) { this.workspacePath = options.workspacePath; @@ -76,7 +646,142 @@ export class MasterManager { this.messageTimeout = options.messageTimeout ?? DEFAULT_MESSAGE_TIMEOUT; this.skipAutoExploration = options.skipAutoExploration ?? false; this.dotFolder = new DotFolderManager(this.workspacePath); - this.delegationCoordinator = new DelegationCoordinator(); + this.changeTracker = new WorkspaceChangeTracker(this.workspacePath); + this.adapter = options.adapter; + this.delegationCoordinator = new DelegationCoordinator({ adapter: options.adapter }); + this.agentRunner = new AgentRunner(options.adapter); + this.workerRegistry = new WorkerRegistry(); + this.memory = options.memory ?? null; + this.workerRetryDelayMs = options.workerRetryDelayMs ?? 5000; + this.modelRegistry = createModelRegistry(options.masterTool.name); + this.mcpServers = options.mcpServers ?? []; + this.deepConfig = options.deepConfig; + this.workspaceExclude = options.workspaceExclude ?? []; + this.workspaceInclude = options.workspaceInclude ?? []; + this.workerMaxFixIterations = options.workerMaxFixIterations ?? DEFAULT_MAX_FIX_ITERATIONS; + this.trustLevel = options.trustLevel ?? 'standard'; + + // Instantiate DeepModeManager — multi-phase session state machine (OB-1403) + this.deepMode = new DeepModeManager({ workspacePath: this.workspacePath }); + + // Initialise SubMasterManager when MemoryManager is available (OB-755 / OB-812) + if (this.memory) { + this.subMasterManager = new SubMasterManager( + this.memory, + this.workspacePath, + this.agentRunner, + this.masterTool, + ); + } + + // Initialise SessionCompactor — monitors Master turn count and triggers compaction (OB-1672) + this.compactor = new SessionCompactor({ maxTurns: MASTER_MAX_TURNS }); + + // Store adapter registry for per-worker tool resolution + if (options.adapterRegistry) { + this.adapterRegistry = options.adapterRegistry; + } else { + // Backward compatible: create minimal registry with just the master adapter + this.adapterRegistry = new AdapterRegistry(); + if (options.adapter) { + this.adapterRegistry.register(options.masterTool.name, options.adapter); + } + } + + // Initialise ClassificationEngine — extracted from MasterManager (OB-1279) + this.classificationEngine = new ClassificationEngine({ + memory: this.memory, + dotFolder: this.dotFolder, + agentRunner: this.agentRunner, + modelRegistry: this.modelRegistry, + workspacePath: this.workspacePath, + adapter: this.adapter, + getWorkspaceContext: () => this.getWorkspaceContextSummary(), + }); + + // Initialise ExplorationManager — extracted from MasterManager (OB-1280) + this.explorationManager = new ExplorationManager({ + workspacePath: this.workspacePath, + masterTool: this.masterTool, + discoveredTools: this.discoveredTools, + dotFolder: this.dotFolder, + changeTracker: this.changeTracker, + agentRunner: this.agentRunner, + explorationTimeout: this.explorationTimeout, + adapter: this.adapter, + getMemory: () => this.memory, + getRouter: () => this.router, + getSubMasterManager: () => this.subMasterManager, + getMasterSession: () => this.masterSession, + getState: () => this.state, + setState: (s) => { + this.state = s; + }, + buildMasterSpawnOptions: (prompt, timeout, maxTurns, sections, skip) => + this.buildMasterSpawnOptions(prompt, timeout, maxTurns, sections, skip), + updateMasterSession: () => this.updateMasterSession(), + processMessage: (msg) => this.processMessage(msg), + readWorkspaceMapFromStore: () => this.readWorkspaceMapFromStore(), + writeWorkspaceMapToStore: (map) => this.writeWorkspaceMapToStore(map), + readAnalysisMarkerFromStore: () => this.readAnalysisMarkerFromStore(), + writeAnalysisMarkerToStore: (marker) => this.writeAnalysisMarkerToStore(marker), + readExplorationStateFromStore: () => this.readExplorationStateFromStore(), + }); + + // Initialise WorkerOrchestrator — extracted from MasterManager (OB-1281) + this.workerOrchestrator = new WorkerOrchestrator({ + workspacePath: this.workspacePath, + masterTool: this.masterTool, + discoveredTools: this.discoveredTools, + dotFolder: this.dotFolder, + agentRunner: this.agentRunner, + workerRegistry: this.workerRegistry, + adapterRegistry: this.adapterRegistry, + modelRegistry: this.modelRegistry, + workerRetryDelayMs: this.workerRetryDelayMs, + workerMaxFixIterations: this.workerMaxFixIterations, + trustLevel: this.trustLevel, + getMemory: () => this.memory, + getRouter: () => this.router, + getMasterSession: () => this.masterSession, + getActiveMessage: () => this.activeMessage, + getState: () => this.state, + setState: (s) => { + this.state = s; + }, + getActiveSkillPacks: () => this.activeSkillPacks, + getKnowledgeRetriever: () => this.knowledgeRetriever, + getBatchManager: () => this.batchManager, + getBatchTimers: () => this.batchTimers, + getDelegationCoordinator: () => this.delegationCoordinator, + readProfilesFromStore: () => this.readProfilesFromStore(), + persistWorkerRegistry: () => this.persistWorkerRegistry(), + recordWorkerLearning: (task, result, profile, model) => + this.recordWorkerLearning(task, result, profile, model), + recordPromptEffectiveness: (task, result) => this.recordPromptEffectiveness(task, result), + recordConversationMessage: (sessionId, role, content) => + this.recordConversationMessage(sessionId, role, content), + }); + + // Initialise PromptContextBuilder — extracted from MasterManager (OB-1282) + this.promptContextBuilder = new PromptContextBuilder({ + workspacePath: this.workspacePath, + dotFolder: this.dotFolder, + adapter: this.adapter, + messageTimeout: this.messageTimeout, + getMemory: () => this.memory, + getSystemPrompt: () => this.systemPrompt, + getMasterSession: () => this.masterSession, + getMapLastVerifiedAt: () => this.mapLastVerifiedAt, + getLearningsSummary: () => this.learningsSummary, + getExplorationSummary: () => this.explorationSummary, + getWorkspaceContextSummary: () => this.getWorkspaceContextSummary(), + getBatchManager: () => this.batchManager, + drainCancellationNotifications: () => this.pendingCancellationNotifications.splice(0), + drainDeepModeResumeOffers: () => this.pendingDeepModeResumeOffers.splice(0), + readWorkspaceMapFromStore: () => this.readWorkspaceMapFromStore(), + readAllTasksFromStore: () => this.readAllTasksFromStore(), + }); logger.info( { @@ -88,643 +793,6015 @@ export class MasterManager { ); } - /** - * Get current state - */ - public getState(): MasterState { - return this.state; + // --------------------------------------------------------------------------- + // Memory-aware store helpers (OB-711): route reads/writes through MemoryManager + // when available, fall back to DotFolderManager when not. + // --------------------------------------------------------------------------- + + /** Read workspace map from memory (chunks) or DotFolderManager (JSON file). */ + private async readWorkspaceMapFromStore(): Promise { + if (this.memory) { + try { + const chunks = await this.memory.getChunksByScope('_workspace_map', 'structure'); + if (chunks.length > 0 && chunks[0]?.content) { + return WorkspaceMapSchema.parse(JSON.parse(chunks[0].content)); + } + } catch { + // ignore — map not yet stored + } + } else { + logger.warn('Memory not available — falling back to JSON for workspace map read'); + } + const jsonMap = await this.dotFolder.readWorkspaceMap(); + if (jsonMap) return jsonMap; + return null; } - /** - * Get exploration summary (if exploration has been completed) - */ - public getExplorationSummary(): ExplorationSummary | null { - return this.explorationSummary; + /** Write workspace map to memory (as chunk). JSON fallback removed (OB-810). */ + private async writeWorkspaceMapToStore(map: WorkspaceMap): Promise { + if (this.memory) { + await this.memory.storeChunks([ + { + scope: '_workspace_map', + category: 'structure', + content: JSON.stringify(map), + }, + ]); + return; + } + logger.warn('Memory not available — skipping workspace map write'); } - /** - * Get the workspace map (if exploration has been completed) - */ - public async getWorkspaceMap(): Promise { - return this.dotFolder.readMap(); + /** Load master session from memory (sessions table) or DotFolderManager (JSON file). */ + private async loadMasterSessionFromStore(): Promise { + if (this.memory) { + try { + const record = await this.memory.getSession('master'); + if (record) return sessionRecordToMasterSession(record); + } catch { + // Fall through to JSON fallback + } + } + return this.dotFolder.readMasterSession(); } - /** - * Start the Master AI. - * If auto-exploration is enabled and .openbridge/ doesn't exist, triggers autonomous exploration. - */ - public async start(): Promise { - if (this.state !== 'idle') { - logger.warn({ currentState: this.state }, 'MasterManager already started'); - return; + /** Save master session to memory (sessions table) or DotFolderManager (JSON fallback). */ + private async saveMasterSessionToStore(session: MasterSession): Promise { + if (this.memory) { + await this.memory.upsertSession(masterSessionToSessionRecord(session, this.restartCount)); + } else { + await this.dotFolder.writeMasterSession(session); } + } - logger.info('Starting MasterManager'); + /** + * Checkpoint the current Master session state to the `sessions` table. + * + * Serializes pending workers, accumulated results, and queued message context + * so that a future `resumeSession()` call can restore them without data loss. + * + * No-op when memory is unavailable (SQLite not configured). + * + * @returns true if checkpoint was saved, false if skipped (no memory / no session). + */ + public async checkpointSession(): Promise { + if (!this.memory || !this.masterSession) return false; - // Check if .openbridge folder exists - const folderExists = await this.dotFolder.exists(); + const now = new Date().toISOString(); - if (!folderExists && !this.skipAutoExploration) { - // No .openbridge folder — trigger exploration - logger.info('.openbridge folder does not exist, starting autonomous exploration'); - await this.explore(); - } else if (folderExists) { - // Folder exists — load existing workspace map and set state to ready - logger.info('.openbridge folder exists, loading workspace map'); + // Collect worker state from the in-memory registry + const allWorkers = this.workerRegistry.getAllWorkers(); + const pendingWorkers = allWorkers.filter( + (w) => w.status === 'pending' || w.status === 'running', + ); + const completedWorkers = allWorkers.filter( + (w) => w.status === 'completed' || w.status === 'failed', + ); - const map = await this.dotFolder.readMap(); - if (map) { - this.explorationSummary = { - startedAt: map.generatedAt, - completedAt: map.generatedAt, - status: 'completed', - filesScanned: 0, - directoriesExplored: 0, - projectType: map.projectType, - frameworks: map.frameworks, - insights: [], - mapPath: this.dotFolder.getMapPath(), - gitInitialized: true, - }; + // Serialize pending messages — convert Date timestamps to ISO strings + const serializedMessages = this.pendingMessages.map((msg) => ({ + ...msg, + timestamp: msg.timestamp instanceof Date ? msg.timestamp.toISOString() : msg.timestamp, + })); + + const checkpoint = { + checkpointedAt: now, + pendingWorkers, + completedWorkers, + pendingMessages: serializedMessages, + }; - this.state = 'ready'; - logger.info({ projectType: map.projectType }, 'Master AI ready (loaded existing map)'); - } else { - logger.warn('Workspace map file exists but could not be parsed, entering ready state'); - this.state = 'ready'; - } - } else { - // Skip auto-exploration — enter ready state - logger.info('Auto-exploration disabled, entering ready state'); - this.state = 'ready'; + try { + const record = masterSessionToSessionRecord(this.masterSession, this.restartCount); + record.checkpoint_data = JSON.stringify(checkpoint); + await this.memory.upsertSession(record); + logger.info( + { + sessionId: this.masterSession.sessionId, + pendingWorkers: pendingWorkers.length, + completedWorkers: completedWorkers.length, + pendingMessages: serializedMessages.length, + }, + 'Session checkpointed', + ); + return true; + } catch (error) { + logger.warn({ error }, 'Failed to checkpoint session'); + return false; } } /** - * Autonomously explore the workspace and create .openbridge/ folder. - * This is the Master AI's initialization step. + * Restore Master session state from the `sessions` table after a checkpoint. + * + * Reads the latest `master` session with `checkpoint_data`, restores the + * `masterSession` object, re-queues interrupted pending messages, and reloads + * the worker history. Workers that were `pending` or `running` at checkpoint + * time are marked `failed` (their processes no longer exist); completed/failed + * workers are restored for context and stats. + * + * No-op when memory is unavailable or no checkpointed session exists. + * + * @returns An object describing what was restored, or `null` if skipped. */ - public async explore(): Promise { - if (this.state === 'exploring') { - logger.warn('Exploration already in progress'); - return; + public async resumeSession(): Promise<{ + restored: boolean; + pendingMessages: number; + restoredWorkers: number; + failedWorkers: number; + } | null> { + if (!this.memory) return null; + + let record: SessionRecord | null; + try { + record = await this.memory.getSession('master'); + } catch (error) { + logger.warn({ error }, 'Failed to read session for resume'); + return null; } - this.state = 'exploring'; + if (!record?.checkpoint_data) return null; - const startedAt = new Date().toISOString(); - this.explorationSummary = { - startedAt, - status: 'in_progress', - filesScanned: 0, - directoriesExplored: 0, - frameworks: [], - insights: [], - gitInitialized: false, + let checkpoint: { + checkpointedAt: string; + pendingWorkers: WorkerRecord[]; + completedWorkers: WorkerRecord[]; + pendingMessages: Array>; }; - - logger.info({ workspacePath: this.workspacePath }, 'Starting workspace exploration'); - try { - // Initialize .openbridge folder - await this.dotFolder.initialize(); + checkpoint = JSON.parse(record.checkpoint_data) as typeof checkpoint; + } catch (error) { + logger.warn({ error }, 'Failed to parse checkpoint data'); + return null; + } - // Log exploration start - await this.dotFolder.appendLog({ - timestamp: startedAt, - level: 'info', - message: 'Autonomous workspace exploration started', - data: { masterTool: this.masterTool.name, version: this.masterTool.version }, - }); + // Restore master session from the stored record + this.masterSession = sessionRecordToMasterSession(record); + this.sessionInitialized = true; - // Generate exploration prompt - const prompt = generateExplorationPrompt(this.workspacePath); + // Restore worker history — completed/failed workers kept as-is for context and stats. + // Pending/running workers are marked failed (their processes no longer exist). + const workersToRestore: Record = {}; - // Execute exploration (skip permissions — exploration runs in background without user interaction) - const result = await executeClaudeCode({ - prompt, - workspacePath: this.workspacePath, - timeout: this.explorationTimeout, - skipPermissions: true, - }); + for (const worker of checkpoint.completedWorkers ?? []) { + workersToRestore[worker.id] = worker; + } - if (result.exitCode !== 0) { - throw new Error(`Exploration failed with exit code ${result.exitCode}: ${result.stderr}`); - } + let failedWorkerCount = 0; + for (const worker of checkpoint.pendingWorkers ?? []) { + workersToRestore[worker.id] = { + ...worker, + status: 'failed', + completedAt: new Date().toISOString(), + error: 'Interrupted by session checkpoint', + }; + failedWorkerCount++; + } - // Check if workspace map was created - const map = await this.dotFolder.readMap(); - if (!map) { - throw new Error('Exploration completed but workspace-map.json was not created'); + if (Object.keys(workersToRestore).length > 0) { + try { + this.workerRegistry.fromJSON({ + workers: workersToRestore, + updatedAt: new Date().toISOString(), + }); + } catch (error) { + logger.warn({ error }, 'Failed to restore workers from checkpoint'); } + } - // Check if agents.json exists, if not create it - let agentsRegistry = await this.dotFolder.readAgents(); - if (!agentsRegistry) { - agentsRegistry = this.createAgentsRegistry(); - await this.dotFolder.writeAgents(agentsRegistry); - await this.dotFolder.commitChanges('Add agents.json'); + // Restore pending messages — deserialize ISO timestamps back to Date objects + const restoredMessages: InboundMessage[] = []; + for (const rawMsg of checkpoint.pendingMessages ?? []) { + try { + restoredMessages.push({ + ...(rawMsg as Omit), + timestamp: new Date(rawMsg['timestamp'] as string), + } as InboundMessage); + } catch { + // Skip malformed messages } + } + this.pendingMessages = restoredMessages; - const completedAt = new Date().toISOString(); - - this.explorationSummary = { - startedAt, - completedAt, - status: 'completed', - filesScanned: 0, - directoriesExplored: 0, - projectType: map.projectType, - frameworks: map.frameworks, - insights: [], - mapPath: this.dotFolder.getMapPath(), - gitInitialized: true, - }; - - // Log exploration completion - await this.dotFolder.appendLog({ - timestamp: completedAt, - level: 'info', - message: 'Autonomous workspace exploration completed', - data: { - projectType: map.projectType, - frameworks: map.frameworks, - summary: map.summary, - }, - }); - - this.state = 'ready'; - - logger.info( - { - projectType: map.projectType, - frameworks: map.frameworks, - durationMs: new Date(completedAt).getTime() - new Date(startedAt).getTime(), - }, - 'Workspace exploration completed', - ); - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); + const restoredWorkerCount = Object.keys(workersToRestore).length - failedWorkerCount; + logger.info( + { + sessionId: this.masterSession.sessionId, + pendingMessages: restoredMessages.length, + restoredWorkers: restoredWorkerCount, + failedWorkers: failedWorkerCount, + checkpointedAt: checkpoint.checkpointedAt, + }, + 'Session resumed from checkpoint', + ); - this.explorationSummary = { - ...(this.explorationSummary ?? {}), - startedAt: this.explorationSummary?.startedAt ?? startedAt, - completedAt: new Date().toISOString(), - status: 'failed', - filesScanned: this.explorationSummary?.filesScanned ?? 0, - directoriesExplored: this.explorationSummary?.directoriesExplored ?? 0, - frameworks: this.explorationSummary?.frameworks ?? [], - insights: this.explorationSummary?.insights ?? [], - gitInitialized: this.explorationSummary?.gitInitialized ?? false, - error: errorMessage, - }; + return { + restored: true, + pendingMessages: restoredMessages.length, + restoredWorkers: restoredWorkerCount, + failedWorkers: failedWorkerCount, + }; + } - // Log exploration failure - await this.dotFolder.appendLog({ - timestamp: new Date().toISOString(), - level: 'error', - message: 'Workspace exploration failed', - data: { error: errorMessage }, - }); + /** Read analysis marker from memory (workspace_state table) or DotFolderManager (JSON file). */ + private async readAnalysisMarkerFromStore(): Promise { + if (this.memory) { + try { + const state = await this.memory.getWorkspaceState(); + return workspaceStateToMarker(state); + } catch { + /* fall through to dotFolder */ + } + } + return this.dotFolder.readAnalysisMarker(); + } - this.state = 'error'; + /** Write analysis marker to memory (workspace_state table) or DotFolderManager (JSON file). */ + private async writeAnalysisMarkerToStore(marker: WorkspaceAnalysisMarker): Promise { + if (this.memory) { + await this.memory.updateWorkspaceState(markerToWorkspaceState(marker)); + } + await this.dotFolder.writeAnalysisMarker(marker); + } - logger.error({ error, workspacePath: this.workspacePath }, 'Workspace exploration failed'); + /** + * Read exploration state from memory (system_config) or DotFolderManager (JSON file). + * Only a read is needed in master-manager.ts; writes happen in exploration-coordinator.ts. + */ + private async readExplorationStateFromStore(): Promise { + if (this.memory) { + try { + const json = await this.memory.getSystemConfig('exploration_state'); + if (json) return ExplorationStateSchema.parse(JSON.parse(json)); + } catch { + return null; + } + return null; + } + return this.dotFolder.readExplorationState(); + } - throw error; + /** Read profiles registry from DB (system_config) first, fall back to dotFolder JSON. */ + private async readProfilesFromStore(): Promise { + if (this.memory) { + try { + const raw = await this.memory.getSystemConfig('profiles'); + if (raw) return ProfilesRegistrySchema.parse(JSON.parse(raw)); + } catch { + // Fall through to JSON fallback + } } + return this.dotFolder.readProfiles(); } /** - * Re-explore the workspace (e.g., after significant changes) + * Record a master-level task (user interaction) to memory (SQLite tasks table). + * If memory is not available, logs a warning — no JSON fallback. */ - public async reExplore(): Promise { - if (this.state !== 'ready') { - logger.warn({ currentState: this.state }, 'Cannot re-explore: Master not in ready state'); + private async recordTaskToStore( + task: TaskRecord, + type: MemoryTaskRecord['type'] = 'worker', + ): Promise { + if (this.memory) { + try { + await this.memory.recordTask(masterTaskToMemoryTask(task, type)); + } catch (err) { + logger.warn({ err }, 'Failed to record task to memory store'); + } return; } + logger.warn({ taskId: task.id }, 'MemoryManager not available — task record not persisted'); + } - const startedAt = new Date().toISOString(); - this.state = 'exploring'; - - logger.info({ workspacePath: this.workspacePath }, 'Starting workspace re-exploration'); - + /** + * Record a conversation message to the memory store (OB-730). + * Silently skips when MemoryManager is unavailable or errors occur. + */ + private async recordConversationMessage( + sessionId: string, + role: ConversationEntry['role'], + content: string, + channel?: string, + userId?: string, + ): Promise { + if (!this.memory) return; try { - // Log re-exploration start - await this.dotFolder.appendLog({ - timestamp: startedAt, - level: 'info', - message: 'Workspace re-exploration started', + await this.memory.recordMessage({ + session_id: sessionId, + role, + content, + channel, + user_id: userId, }); + } catch (err) { + logger.warn({ err, role }, 'Failed to record conversation message'); + } + } - // Generate re-exploration prompt - const prompt = generateReExplorationPrompt(this.workspacePath); + /** Retrieve conversation context. Delegated to PromptContextBuilder (OB-1282). */ + private async buildConversationContext( + userMessage: string, + sessionId?: string, + sender?: string, + ): Promise { + return this.promptContextBuilder.buildConversationContext(userMessage, sessionId, sender); + } - // Execute re-exploration (skip permissions — runs in background) - const result = await executeClaudeCode({ - prompt, - workspacePath: this.workspacePath, - timeout: this.explorationTimeout, - skipPermissions: true, - }); + /** Build learned patterns context. Delegated to PromptContextBuilder (OB-1282). */ + private async buildLearnedPatternsContext(): Promise { + return this.promptContextBuilder.buildLearnedPatternsContext(); + } - if (result.exitCode !== 0) { - throw new Error( - `Re-exploration failed with exit code ${result.exitCode}: ${result.stderr}`, - ); - } + /** Build worker next steps context. Delegated to PromptContextBuilder (OB-1282). */ + private async buildWorkerNextStepsContext(): Promise { + return this.promptContextBuilder.buildWorkerNextStepsContext(); + } - // Update exploration summary - const map = await this.dotFolder.readMap(); - if (map) { - this.explorationSummary = { - ...this.explorationSummary!, - completedAt: new Date().toISOString(), - projectType: map.projectType, - frameworks: map.frameworks, - }; - } + /** + * Increment the completed task counter and trigger prompt evolution every 50 tasks (OB-734). + * Also triggers a memory.md update every MEMORY_UPDATE_INTERVAL tasks (OB-1023). + * Runs asynchronously in the background — never blocks the caller. + */ + private onTaskCompleted(): void { + this.completedTaskCount += 1; + if (this.completedTaskCount % 50 === 0 && this.memory) { + void evolvePrompts(this.memory, this.agentRunner, this.workspacePath).catch((err) => { + logger.warn({ err }, 'Prompt evolution cycle failed'); + }); + } + if (this.completedTaskCount % MEMORY_UPDATE_INTERVAL === 0) { + void this.triggerMemoryUpdate().catch((err) => { + logger.warn({ err }, 'Periodic memory update failed'); + }); + } + } - const completedAt = new Date().toISOString(); + /** Write exploration summary to memory.md. Delegated to ExplorationManager (OB-1280). */ + private async writeExplorationSummaryToMemory(): Promise { + return this.explorationManager.writeExplorationSummaryToMemory(); + } - // Log re-exploration completion - await this.dotFolder.appendLog({ - timestamp: completedAt, - level: 'info', - message: 'Workspace re-exploration completed', + /** + * Send an "update memory" prompt to the Master session so it can write + * `.openbridge/context/memory.md` via its Write tool (OB-1023). + * Non-blocking — caller should fire-and-forget with void. + */ + private async triggerMemoryUpdate(): Promise { + if (!this.masterSession || this.state === 'shutdown') return; + + const recentMessages = (await this.memory?.getRecentMessages(20)) ?? []; + + logger.info({ messageCount: recentMessages.length }, 'Starting memory update'); + + const memoryPath = this.dotFolder.getMemoryFilePath(); + + let historySection = ''; + if (recentMessages.length > 0) { + const lines = recentMessages.map((msg) => { + const ts = msg.created_at ? msg.created_at.slice(0, 16).replace('T', ' ') : ''; + const role = msg.role.charAt(0).toUpperCase() + msg.role.slice(1); + const content = msg.content.length > 300 ? msg.content.slice(0, 300) + '…' : msg.content; + return `[${ts}] ${role}: ${content}`; }); + historySection = `## Recent conversation history:\n${lines.join('\n')}\n\n`; + } - this.state = 'ready'; + const prompt = + historySection + + `Update your memory file at ${memoryPath}.\n` + + `Keep it under 200 lines. Remove outdated info. Merge related topics.\n` + + `Only write what is worth remembering across sessions: user preferences, ` + + `project state, decisions made, active threads, and known issues.\n` + + `Write your updated notes to ${memoryPath}.`; - logger.info('Workspace re-exploration completed'); - } catch (error) { - logger.error({ error }, 'Workspace re-exploration failed'); - this.state = 'ready'; // Return to ready state even on failure - throw error; + // Capture mtime before update to verify the file was actually written (OB-1615). + let mtimeBefore: number | null = null; + try { + const stat = await fs.stat(memoryPath); + mtimeBefore = stat.mtimeMs; + } catch { + // File doesn't exist yet — any write will count as a modification. + } + + try { + const opts = this.buildMasterSpawnOptions(prompt, undefined, MEMORY_UPDATE_MAX_TURNS); + const result = await this.agentRunner.spawn(opts); + await this.updateMasterSession(); + if (result.exitCode !== 0) { + logger.warn({ exitCode: result.exitCode }, 'Memory update prompt returned non-zero exit'); + // OB-1616: Master write failed — fall back to direct write from conversation history. + await this.applyMemoryFallback(recentMessages); + } else { + // Verify memory.md was actually written/modified (OB-1615). + try { + const stat = await fs.stat(memoryPath); + const wasModified = mtimeBefore === null || stat.mtimeMs > mtimeBefore; + if (wasModified) { + logger.info( + { mtimeBefore, mtimeAfter: stat.mtimeMs }, + 'Memory update completed — file verified', + ); + } else { + logger.warn( + { memoryPath, mtimeBefore, mtimeAfter: stat.mtimeMs }, + 'Memory update completed but memory.md was NOT modified — Master may have skipped the write', + ); + // OB-1616: Master skipped the write — fall back to direct write. + await this.applyMemoryFallback(recentMessages); + } + } catch { + logger.warn( + { memoryPath }, + 'Memory update completed but memory.md is missing — Master did not write the file', + ); + // OB-1616: memory.md missing after update — fall back to direct write. + await this.applyMemoryFallback(recentMessages); + } + } + } catch (err) { + logger.warn({ err }, 'Memory update prompt failed'); + // OB-1616: Spawn itself failed — fall back to direct write from conversation history. + await this.applyMemoryFallback(recentMessages); } } /** - * Process a message from a user. - * Maintains session continuity across messages using --resume flag. + * Fallback memory write (OB-1616): write memory.md directly from conversation history + * when the Master AI fails to produce or write an update. */ - public async processMessage(message: InboundMessage): Promise { - if (this.state !== 'ready') { + private async applyMemoryFallback( + messages: ReadonlyArray<{ role: string; content: string; created_at?: string }>, + ): Promise { + try { + await this.dotFolder.writeMemoryFallback(messages); + logger.info( + { messageCount: messages.length }, + 'Memory fallback written from conversation history', + ); + } catch (err) { + logger.warn({ err }, 'Memory fallback write failed'); + } + } + + /** + * Read all task records from memory or DotFolderManager. + * Memory returns MemoryTaskRecord (different schema) — converted to approximate MasterTaskRecord. + * When memory is null, returns the full DotFolderManager records. + */ + private async readAllTasksFromStore(): Promise { + if (this.memory) { + try { + const types: MemoryTaskRecord['type'][] = ['worker', 'quick-answer', 'tool-use', 'complex']; + const collected: MemoryTaskRecord[] = []; + for (const t of types) { + const batch = await this.memory.getTasksByType(t); + collected.push(...batch); + } + const statusMap: Record = { + running: 'processing', + completed: 'completed', + failed: 'failed', + timeout: 'failed', + }; + return collected.map((t) => ({ + id: t.id, + userMessage: t.prompt ?? '', + sender: 'user', + description: (t.prompt ?? '').slice(0, 200), + status: statusMap[t.status] ?? 'failed', + handledBy: 'master', + result: t.response ?? undefined, + createdAt: t.created_at, + completedAt: t.completed_at ?? undefined, + durationMs: t.duration_ms ?? undefined, + metadata: {}, + })); + } catch { + return []; + } + } + return this.dotFolder.readAllTasks(); + } + + /** + * Get current state + */ + public getState(): MasterState { + return this.state; + } + + /** + * Recover from an error state. + * Resets state from 'error' to 'idle' and optionally retries exploration + * if the previous error occurred during explore(). + * After recovery, processMessage() can accept new messages again. + */ + public recover(): Promise { + if (this.state !== 'error') { logger.warn( - { currentState: this.state, sender: message.sender }, - 'Cannot process message: Master not ready', + { currentState: this.state }, + 'recover() called but state is not error — ignoring', ); - return `The AI is currently ${this.state}. Please try again in a moment.`; + return Promise.resolve(); } - this.state = 'processing'; + logger.warn( + { workspacePath: this.workspacePath, previousError: this.explorationSummary?.error }, + 'Recovering from error state', + ); - const taskId = randomUUID(); - const startedAt = new Date().toISOString(); + this.state = 'idle'; - logger.info({ taskId, sender: message.sender, content: message.content }, 'Processing message'); + // If exploration previously failed, retry it so the Master can become ready. + // explore() synchronously sets state to 'exploring' before its first await, + // so subsequent processMessage() calls will queue correctly as pending messages. + if (this.explorationSummary?.status === 'failed') { + logger.info('Retrying exploration after recovery'); + void this.explore(); + } - // Create task record - const task: TaskRecord = { - id: taskId, - userMessage: message.rawContent, - sender: message.sender, - description: message.content, - status: 'processing', - handledBy: 'master', - createdAt: startedAt, - startedAt, - metadata: { - messageId: message.id, - source: message.source, - }, - }; + return Promise.resolve(); + } - try { - // Check for status queries - if (this.isStatusQuery(message.content)) { - const status = await this.getStatus(); - this.state = 'ready'; - task.status = 'completed'; - task.result = status; - task.completedAt = new Date().toISOString(); - task.durationMs = - new Date(task.completedAt).getTime() - new Date(task.startedAt!).getTime(); - await this.dotFolder.recordTask(task); - return status; - } + /** + * Get the model registry (for provider-agnostic model resolution) + */ + public getModelRegistry(): ModelRegistry { + return this.modelRegistry; + } - // Get or create session ID for this sender - const sessionId = this.getOrCreateSession(message.sender); + /** + * Get exploration summary (if exploration has been completed) + */ + public getExplorationSummary(): ExplorationSummary | null { + return this.explorationSummary; + } - // Execute message through Claude Code with session continuity (skip permissions — non-interactive) - let result = await executeClaudeCode({ - prompt: message.content, - workspacePath: this.workspacePath, - timeout: this.messageTimeout, - resumeSessionId: sessionId, - skipPermissions: true, - }); + /** + * Get the workspace map (if exploration has been completed) + */ + public async getWorkspaceMap(): Promise { + return this.readWorkspaceMapFromStore(); + } - if (result.exitCode !== 0) { - throw new Error(`Message processing failed: ${result.stderr}`); + /** + * Get the persistent Master session info. + */ + public getMasterSession(): MasterSession | null { + return this.masterSession; + } + + /** + * Get the list of pending messages awaiting processing. + * Used primarily for testing and diagnostics. + */ + public getPendingMessages(): InboundMessage[] { + return [...this.pendingMessages]; + } + + /** + * Start the Master AI. + * Resilient startup logic: + * - If .openbridge/ doesn't exist → trigger fresh exploration + * - If incomplete exploration detected → resume from checkpoint + * - If map missing or corrupted → re-explore + * - If valid map exists → skip exploration, enter ready state + * + * On ready, loads or creates a persistent Master session ID + * stored in .openbridge/master-session.json. + */ + public async start(): Promise { + if (this.state !== 'idle') { + logger.warn({ currentState: this.state }, 'MasterManager already started'); + return; + } + + logger.info('Starting MasterManager (resilient startup)'); + + // Initialize .openbridge folder early so we can create the Master session + await this.dotFolder.initialize(); + + // Load user-defined skill packs — overrides built-ins by name (OB-1754) + try { + const { packs, userDefinedCount } = await loadAllSkillPacks(this.workspacePath); + this.activeSkillPacks = packs; + if (userDefinedCount > 0) { + logger.info({ userDefinedCount }, 'Loaded user-defined skill packs'); } + } catch (err) { + logger.warn({ err }, 'Failed to load user-defined skill packs — using built-ins'); + } - let response = result.stdout.trim() || 'No response from AI'; + // Initialize BatchManager (OB-1609): load any persisted batch state. + if (!this.batchManager) { + this.batchManager = new BatchManager(this.dotFolder); + } + await this.batchManager.initialize(); + + // Clean up stale agent_activity rows from previous process BEFORE + // creating the new master session, so the fresh 'running' row isn't wiped. + this.cleanupStuckActivities(); + + // Check system_config for incomplete Deep Mode sessions from a previous run. + // Must run after dotFolder.initialize() (memory is ready) and after + // cleanupStuckActivities() (so we don't interfere with activity cleanup). + // Reads from system_config which is never touched by agent_activity cleanup (OB-1405). + await this.checkIncompleteDeepModeSessions(); + + // OB-1617: Detect stale or missing memory.md on startup — regenerate from SQLite. + if (this.memory) { + try { + const isStale = await this.dotFolder.isMemoryStale(); + if (isStale) { + // OB-1651: Try SQLite backup first before falling back to conversation regeneration. + const backup = await this.memory.getSystemConfig('memory_md_backup'); + if (backup) { + try { + await this.dotFolder.writeMemoryFile(backup); + logger.info('Restored memory.md from SQLite backup'); + } catch (e) { + logger.warn( + { err: e }, + 'SQLite backup restore failed — falling back to regeneration', + ); + const recentMessages = await this.memory.getRecentMessages(20); + await this.dotFolder.writeMemoryFallback(recentMessages); + logger.info( + { messageCount: recentMessages.length }, + 'Regenerated stale memory.md from SQLite on startup (OB-1617)', + ); + } + } else { + const recentMessages = await this.memory.getRecentMessages(20); + await this.dotFolder.writeMemoryFallback(recentMessages); + logger.info( + { messageCount: recentMessages.length }, + 'Regenerated stale memory.md from SQLite on startup (OB-1617)', + ); + } + } + } catch (err) { + logger.warn({ err }, 'Failed to check/regenerate stale memory.md on startup'); + } + } - // Check for delegation markers in the response - const delegations = this.parseDelegationMarkers(response); - if (delegations && delegations.length > 0) { - logger.info({ delegationCount: delegations.length }, 'Delegation markers detected'); + // Initialize Master session — so exploration can use it + await this.initMasterSession(); - // Update task status to delegated - task.status = 'delegated'; - await this.dotFolder.recordTask(task); + // Load worker registry from disk (if exists) + await this.loadWorkerRegistry(); - // Handle delegations - const delegationResults = await this.handleDelegations(delegations, message); + // Check if .openbridge already has exploration data + const folderExistedBefore = await this.dotFolder.exists(); - // Feed delegation results back to Master session - const feedbackPrompt = `The following delegation results are available:\n\n${delegationResults}\n\nPlease synthesize these results and provide a final response to the user.`; + // Check if workspace map exists and is valid + const map = await this.readWorkspaceMapFromStore(); - this.state = 'processing'; - result = await executeClaudeCode({ - prompt: feedbackPrompt, - workspacePath: this.workspacePath, - timeout: this.messageTimeout, - resumeSessionId: sessionId, - skipPermissions: true, - }); + if (map) { + // Check for workspace changes before deciding to skip exploration + const changeResult = await this.checkWorkspaceChanges(map); + + if (changeResult === 'no-changes') { + // Scenario 1a: Valid map + no workspace changes — skip exploration + logger.info( + { projectType: map.projectType }, + 'Valid workspace map found, no workspace changes detected — skipping exploration', + ); + + this.explorationSummary = { + startedAt: map.generatedAt, + completedAt: map.generatedAt, + status: 'completed', + filesScanned: 0, + directoriesExplored: 0, + projectType: map.projectType, + frameworks: map.frameworks, + insights: [], + mapPath: this.dotFolder.getMapPath(), + gitInitialized: true, + }; - if (result.exitCode !== 0) { - throw new Error(`Delegation feedback processing failed: ${result.stderr}`); + // OB-1569: Ensure FTS5 has indexed chunks even when exploration is skipped. + // If the chunk store is empty, decompose the workspace map into searchable chunks. + // OB-1573: Also re-index if searchContext() returns 0 results — covers the case + // where the raw _workspace_map JSON chunk exists (countChunks > 0) but structured + // FTS5 chunks were never created, so typical user queries still return nothing. + if (this.memory) { + try { + const chunkCount = await this.memory.countChunks(); + if (chunkCount === 0) { + logger.warn( + 'FTS5 chunk store is empty after skip-exploration — indexing workspace map (OB-1569)', + ); + await this.indexWorkspaceMapAsChunks(map); + } else { + // OB-1573: Probe FTS5 with a workspace-derived query to confirm results are + // returned. If the probe returns 0 chunks, structured FTS5 chunks are missing + // and we re-index the workspace map to ensure RAG has something to search. + const probeQuery = map.projectName || map.projectType; + const probeResults = await this.memory.searchContext(probeQuery, 1); + if (probeResults.length === 0) { + logger.warn( + { chunkCount, probeQuery }, + 'FTS5 searchContext returns 0 results for workspace probe — indexing structured chunks (OB-1573)', + ); + await this.indexWorkspaceMapAsChunks(map); + } else { + logger.info({ chunkCount }, 'FTS5 chunk store has indexed content — RAG ready'); + } + } + } catch (err) { + logger.warn( + { err }, + 'Failed to verify/index FTS5 chunks after skip-exploration (OB-1569)', + ); + } } - response = result.stdout.trim() || delegationResults; + this.workspaceMapSummary = this.buildMapSummary(map); + this.state = 'ready'; + logger.info({ projectType: map.projectType }, 'Master AI ready (loaded existing map)'); + await this.drainPendingMessages(); + return; } - // Update task record - task.status = 'completed'; - task.result = response; - task.completedAt = new Date().toISOString(); - task.durationMs = new Date(task.completedAt).getTime() - new Date(task.startedAt!).getTime(); - - await this.dotFolder.recordTask(task); - await this.dotFolder.commitChanges(`Task ${taskId}: ${message.content.slice(0, 50)}`); + if (changeResult === 'incremental') { + // Scenario 1b: Valid map + small changes — incremental update done + this.state = 'ready'; + logger.info('Master AI ready (incremental map update completed)'); + await this.drainPendingMessages(); + await this.logRagHealthDiagnostic(); + return; + } - this.state = 'ready'; + // Scenario 1c: changeResult === 'full-reexplore' — fall through to full exploration + logger.info('Workspace changes too large for incremental update — full re-exploration'); + } + // Check for incomplete or failed exploration state + const explorationState = await this.readExplorationStateFromStore(); + if ( + explorationState && + (explorationState.status === 'in_progress' || explorationState.status === 'failed') + ) { + const statusLabel = explorationState.status === 'in_progress' ? 'Incomplete' : 'Failed'; logger.info( - { taskId, durationMs: task.durationMs, responseLength: response.length }, - 'Message processed successfully', + { currentPhase: explorationState.currentPhase, status: explorationState.status }, + `${statusLabel} exploration detected, ${explorationState.status === 'failed' ? 'retrying' : 'resuming'} from checkpoint`, ); + } else if (!folderExistedBefore || !map) { + logger.info('No workspace map found, exploration needed'); + } - return response; - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); + // Trigger exploration (Master-driven or fallback) + if (!this.skipAutoExploration) { + await this.explore(); + // Check whether exploration produced a valid workspace map + if (this.explorationSummary?.status !== 'completed') { + logger.warn( + { status: this.explorationSummary?.status }, + 'Exploration did not produce a workspace map — will re-explore on next startup', + ); + } + } else { + logger.info('Auto-exploration disabled, entering ready state'); + this.state = 'ready'; + } - // Update task record with error - task.status = 'failed'; - task.error = errorMessage; - task.completedAt = new Date().toISOString(); - task.durationMs = new Date(task.completedAt).getTime() - new Date(task.startedAt!).getTime(); + // RAG health diagnostic — covers Scenarios 1c (full re-explore), no-map, + // and skipAutoExploration paths. Scenarios 1a and 1b log their own counts. + await this.logRagHealthDiagnostic(); + + // Start idle detection timer for self-improvement cycle (OB-173) + this.startIdleDetection(); + } - await this.dotFolder.recordTask(task); + /** + * Mark stuck agent_activity rows as failed at startup. + * + * When the process crashes or is killed, running activities are never + * transitioned to a terminal status. On the next startup we scan for + * rows with status in ('starting', 'running', 'completing') whose + * started_at is older than 1 hour and mark them as failed so they + * don't pollute the active-agents list or confuse the dashboard. + */ + private cleanupStuckActivities(): void { + if (!this.memory) return; - this.state = 'ready'; + try { + // On startup every in-flight row is stale — the previous process is gone. + const cleaned = this.memory.markStaleActivityDone(); + if (cleaned > 0) { + logger.info({ cleaned }, 'Marked stale agent_activity rows as done on startup'); + } + } catch (err) { + logger.warn({ err }, 'Failed to clean up stuck agent_activity rows'); + } + } + + /** + * Persist the current in-memory state of a Deep Mode session to SQLite. + * + * Stores the full DeepModeState JSON in system_config under + * `deep_mode:sessions:`. Because system_config is never wiped by + * the agent_activity startup cleanup, the state survives process restarts and + * can be found by checkIncompleteDeepModeSessions() on the next run (OB-1405). + * + * Also inserts (or updates) a tracking row in agent_activity with type='deep-mode' + * for dashboard/audit visibility. + */ + private async persistDeepModeSession(sessionId: string): Promise { + if (!this.memory) return; - logger.error({ error, taskId, sender: message.sender }, 'Message processing failed'); + const state = this.deepMode.getSessionState(sessionId); + if (!state) return; - throw error; + // Store full state JSON — this survives the startup agent_activity cleanup + await this.memory.upsertDeepModeState(sessionId, state); + + // Insert / update the audit row in agent_activity + const now = new Date().toISOString(); + const isActive = state.currentPhase !== undefined; + try { + await this.memory.insertActivity({ + id: sessionId, + type: 'deep-mode', + profile: state.profile, + task_summary: state.taskSummary.slice(0, 200), + status: isActive ? 'running' : 'done', + started_at: state.startedAt, + updated_at: now, + ...(isActive ? {} : { completed_at: now }), + }); + } catch { + // Row may already exist (INSERT OR IGNORE) — update instead + await this.memory.updateActivity(sessionId, { + status: isActive ? 'running' : 'done', + ...(isActive ? {} : { completed_at: now }), + }); } + + logger.debug( + { sessionId, currentPhase: state.currentPhase }, + 'Deep Mode session state persisted to SQLite', + ); } /** - * Stream a message response, yielding chunks as they arrive. - * Maintains session continuity across messages using --resume flag. + * Check system_config for Deep Mode sessions from a previous process run that + * did not complete all phases. Queues resume offers into + * pendingDeepModeResumeOffers so the Master AI notifies the user on the next + * message (OB-1405). + * + * Called once during start() after cleanupStuckActivities(). */ - public async *streamMessage(message: InboundMessage): AsyncGenerator { - if (this.state !== 'ready') { - logger.warn( - { currentState: this.state, sender: message.sender }, - 'Cannot stream message: Master not ready', + private async checkIncompleteDeepModeSessions(): Promise { + if (!this.memory) return; + + try { + const incomplete = await this.memory.listIncompleteDeepModeSessions(); + if (incomplete.length === 0) return; + + logger.info( + { count: incomplete.length }, + 'Found incomplete Deep Mode sessions from previous run', ); - yield `The AI is currently ${this.state}. Please try again in a moment.`; + + for (const { sessionId, stateJson } of incomplete) { + try { + const state = JSON.parse(stateJson) as { + taskSummary?: string; + currentPhase?: string; + profile?: string; + }; + const summary = state.taskSummary ?? '(unknown task)'; + const phase = state.currentPhase ?? '(unknown phase)'; + const profile = state.profile ?? 'thorough'; + + this.pendingDeepModeResumeOffers.push( + `There is an incomplete Deep Mode session (${profile} profile, paused at **${phase}** phase) ` + + `for the task: "${summary}". Session ID: ${sessionId}. ` + + `Inform the user and offer to resume or discard the session.`, + ); + } catch { + // Skip malformed entries + } + } + } catch (error) { + logger.warn({ error }, 'Failed to check for incomplete Deep Mode sessions on startup'); + } + } + + /** + * Initialize or resume the persistent Master session. + * Loads existing session from .openbridge/master-session.json or creates a new one. + * Also seeds and loads the Master system prompt. + */ + private async initMasterSession(): Promise { + // Ensure .openbridge folder exists + await this.dotFolder.initialize(); + + // Seed system prompt if it doesn't exist yet + await this.seedSystemPrompt(); + + // Seed prompt library if not already seeded (first startup only) + const existingManifest = await this.dotFolder.readPromptManifest(); + if (!existingManifest || Object.keys(existingManifest.prompts).length === 0) { + try { + await seedPromptLibrary(this.dotFolder); + logger.info('Seeded prompt library'); + // Also seed worker prompt IDs into SQLite so recordPromptOutcome() can track them (OB-1612) + if (this.memory) { + for (const prompt of SEED_PROMPTS) { + try { + // Skip insertion if this prompt already exists in the DB (OB-1254) + let alreadyExists = false; + try { + await this.memory.getActivePrompt(prompt.id); + alreadyExists = true; + } catch { + // getActivePrompt rejects when no active prompt exists — proceed with insert + } + if (!alreadyExists) { + await this.memory.createPromptVersion(prompt.id, prompt.content); + } + } catch (dbErr) { + logger.warn( + { error: dbErr, promptId: prompt.id }, + 'Failed to seed worker prompt to DB — non-blocking', + ); + } + } + logger.info('Seeded worker prompt IDs to SQLite for outcome tracking'); + } + } catch (error) { + logger.warn({ error }, 'Failed to seed prompt library'); + } + } + + // Load the system prompt — prefer DB, fall back to file + if (this.memory) { + try { + const dbPrompt = await this.memory.getActivePrompt('master-system'); + this.systemPrompt = dbPrompt.content; + } catch { + this.systemPrompt = await this.dotFolder.readSystemPrompt(); + } + } else { + this.systemPrompt = await this.dotFolder.readSystemPrompt(); + } + if (this.systemPrompt) { + logger.info('Loaded Master system prompt'); + } + + // Load learnings and build summary for system prompt injection + try { + const learnings = await this.dotFolder.readLearnings(); + if (learnings && learnings.entries.length > 0) { + this.learningsSummary = this.buildLearningsSummary(learnings.entries); + logger.info( + { entryCount: learnings.entries.length, summaryLength: this.learningsSummary?.length }, + 'Loaded learnings summary for Master context', + ); + } + } catch (error) { + logger.warn({ error }, 'Failed to load learnings — continuing without history'); + } + + // Try to load existing session + const existing = await this.loadMasterSessionFromStore(); + + if (existing) { + this.masterSession = existing; + this.sessionInitialized = true; // Existing session — use --resume from the start + logger.info( + { sessionId: existing.sessionId, messageCount: existing.messageCount }, + 'Loaded existing Master session', + ); + + // Re-register master activity so the dashboard shows on restart (OB-742) + if (this.memory) { + try { + const now = new Date().toISOString(); + await this.memory.insertActivity({ + id: existing.sessionId, + type: 'master', + model: this.masterTool.name, + task_summary: 'Master AI session resumed', + status: 'running', + started_at: existing.createdAt ?? now, + updated_at: now, + }); + } catch { + // INSERT OR IGNORE — silently skipped if row already exists + } + } return; } - this.state = 'processing'; + // Close any stale active sessions before creating a new one + if (this.memory) { + try { + await this.memory.closeActiveSessions(); + logger.info('Closed stale active sessions before creating new Master session'); + } catch (error) { + logger.warn({ error }, 'Failed to close stale sessions — continuing anyway'); + } + } + + // Create new session — use raw UUID (Claude CLI requires valid UUID format) + const sessionId = randomUUID(); + const now = new Date().toISOString(); + + this.masterSession = { + sessionId, + createdAt: now, + lastUsedAt: now, + messageCount: 0, + allowedTools: getMasterTools(this.trustLevel), + maxTurns: MASTER_MAX_TURNS, + }; + + this.sessionInitialized = false; // New session — first call uses --session-id + + // Persist to store (memory or JSON file) + try { + await this.saveMasterSessionToStore(this.masterSession); + logger.info({ sessionId }, 'Created new Master session'); + } catch (error) { + logger.warn({ error }, 'Failed to persist Master session to disk'); + } + + // Record master agent startup in agent_activity (OB-742) + if (this.memory) { + try { + await this.memory.insertActivity({ + id: sessionId, + type: 'master', + model: this.masterTool.name, + task_summary: 'Master AI session started', + status: 'running', + started_at: now, + updated_at: now, + }); + } catch (actErr) { + logger.warn({ error: actErr }, 'Failed to record master activity'); + } + } + } + + /** + * Seed the master system prompt if it doesn't already exist. + * Generates the default prompt and writes it to .openbridge/prompts/master-system.md. + */ + private async seedSystemPrompt(): Promise { + // Check DB first — if we have an active prompt, it's already seeded + if (this.memory) { + try { + await this.memory.getActivePrompt('master-system'); + return; // Already in DB — don't overwrite + } catch { + // Not in DB yet — fall through to file check + } + } + + const existing = await this.dotFolder.readSystemPrompt(); + if (existing) { + // Migrate file → DB (one-time) + if (this.memory) { + try { + await this.memory.createPromptVersion('master-system', existing); + } catch (dbErr) { + logger.warn({ error: dbErr }, 'Failed to migrate system prompt to DB'); + } + } + return; // Already seeded — don't overwrite (Master may have edited it) + } + + const customProfiles = (await this.readProfilesFromStore())?.profiles; + + const promptContent = generateMasterSystemPrompt({ + workspacePath: this.workspacePath, + masterToolName: this.masterTool.name, + discoveredTools: this.discoveredTools, + customProfiles, + modelRegistry: this.modelRegistry, + mcpServers: this.mcpServers.length > 0 ? this.mcpServers : undefined, + activeConnectorNames: + this.activeConnectorNames.length > 0 ? this.activeConnectorNames : undefined, + fileServerPort: this.fileServerPort, + tunnelUrl: this.tunnelUrl ?? undefined, + workspaceExclude: this.workspaceExclude.length > 0 ? this.workspaceExclude : undefined, + workspaceInclude: this.workspaceInclude.length > 0 ? this.workspaceInclude : undefined, + availableSkills: BUILT_IN_SKILLS, + availableSkillPacks: this.activeSkillPacks, + connectedIntegrations: this.buildConnectedIntegrations(), + trustLevel: this.trustLevel, + }); + + try { + if (this.memory) { + if (promptContent.length > MAX_PROMPT_VERSION_LENGTH) { + logger.warn( + { size: promptContent.length, max: MAX_PROMPT_VERSION_LENGTH }, + 'System prompt exceeds DB size cap — falling back to file storage', + ); + await this.dotFolder.writeSystemPrompt(promptContent); + } else { + try { + await this.memory.createPromptVersion('master-system', promptContent); + } catch (dbErr) { + logger.warn({ error: dbErr }, 'DB prompt save failed — falling back to file storage'); + await this.dotFolder.writeSystemPrompt(promptContent); + } + } + } else { + await this.dotFolder.writeSystemPrompt(promptContent); + } + logger.info('Seeded Master system prompt'); + } catch (error) { + logger.warn({ error }, 'Failed to seed Master system prompt'); + } + } + + /** Assemble SpawnOptions for a Master AI call. Delegated to PromptContextBuilder (OB-1282). */ + private buildMasterSpawnOptions( + prompt: string, + timeout?: number, + maxTurns?: number, + contextSections?: MasterContextSections, + skipWorkspaceContext?: boolean, + ): SpawnOptions { + return this.promptContextBuilder.buildMasterSpawnOptions( + prompt, + timeout, + maxTurns, + contextSections, + skipWorkspaceContext, + ); + } + + /** Get workspace context summary. Delegated to ExplorationManager (OB-1280). */ + private getWorkspaceContextSummary(): string | null { + return this.explorationManager.getWorkspaceContextSummary(); + } + + /** Build learnings summary. Delegated to PromptContextBuilder (OB-1282). */ + private buildLearningsSummary(entries: LearningEntry[]): string | null { + return this.promptContextBuilder.buildLearningsSummary(entries); + } + + /** Build map summary. Delegated to ExplorationManager (OB-1280). */ + private buildMapSummary(map: Record): string { + return this.explorationManager.buildMapSummary(map); + } + + /** Index workspace map as FTS5 chunks. Delegated to ExplorationManager (OB-1280). */ + private async indexWorkspaceMapAsChunks(map: WorkspaceMap): Promise { + return this.explorationManager.indexWorkspaceMapAsChunks(map); + } + + /** + * Update Master session after a successful call. + */ + private async updateMasterSession(): Promise { + if (!this.masterSession) return; + + this.sessionInitialized = true; + this.masterSession.lastUsedAt = new Date().toISOString(); + this.masterSession.messageCount++; + + try { + await this.saveMasterSessionToStore(this.masterSession); + } catch (error) { + logger.warn({ error }, 'Failed to persist Master session update'); + } + } + + /** + * Check whether the current Master session needs compaction and trigger it + * if the threshold has been reached (OB-1672). + * + * Called after every Master turn via `updateMasterSession()`. Silently + * no-ops when the compactor, memory manager, or session are unavailable. + */ + private async _checkCompaction(): Promise { + if (!this.compactor || !this.memory || !this.masterSession) return; + + try { + const db = this.memory.getDb(); + if (!db) return; + + const { sessionId } = this.masterSession; + const memoryPath = this.dotFolder.getMemoryFilePath(); + + await this.compactor.triggerIfNeeded(db, sessionId, async (snapshot) => { + // Fetch conversation history to build a structured summary. + let turns: ConversationTurn[] = []; + try { + const entries = await this.memory!.getSessionHistory(sessionId, 50); + turns = entries.map((e) => ({ + role: e.role === 'user' ? 'user' : e.role === 'system' ? 'system' : 'assistant', + content: e.content, + })); + } catch (err) { + logger.warn({ err, sessionId }, 'Compaction: failed to load session history'); + } + + const summary = this.compactor!.compactTurns(turns); + logger.info( + { + sessionId, + totalTurns: snapshot.totalTurns, + thresholdTurns: snapshot.thresholdTurns, + turnCount: summary.turnCount, + }, + 'Compaction: writing summary to memory.md', + ); + await this.compactor!.writeCompactionSummaryToMemory(summary, memoryPath); + }); + } catch (err) { + logger.warn({ err }, 'Compaction check failed — continuing session without compaction'); + } + } + + /** + * Check whether a failed AgentResult indicates the Master session is dead + * (crash, timeout, context overflow) and cannot be resumed. + */ + private isSessionDead(exitCode: number, stderr: string): boolean { + // Check exit code + if (SESSION_DEAD_EXIT_CODES.has(exitCode)) { + // Exit code 1 is only considered dead if stderr contains a session-related pattern + if (exitCode === 1) { + const lower = stderr.toLowerCase(); + return SESSION_DEAD_PATTERNS.some((pattern) => lower.includes(pattern)); + } + return true; + } + + // Check stderr patterns regardless of exit code + const lower = stderr.toLowerCase(); + return SESSION_DEAD_PATTERNS.some((pattern) => lower.includes(pattern)); + } + + /** Build context summary for session restart. Delegated to PromptContextBuilder (OB-1282). */ + private async buildContextSummary(): Promise { + return this.promptContextBuilder.buildContextSummary(); + } + + /** + * Restart the Master session after detecting it has died. + * Saves the old session state, creates a new session, and seeds it + * with a context summary so the user sees no interruption. + */ + private async restartMasterSession(): Promise { + const oldSession = this.masterSession; + this.restartCount++; + + logger.warn( + { + oldSessionId: oldSession?.sessionId, + oldMessageCount: oldSession?.messageCount, + restartCount: this.restartCount, + }, + 'Restarting Master session after failure', + ); + + // Log the restart + if (this.memory) { + await this.memory.logExploration({ + timestamp: new Date().toISOString(), + level: 'warn', + message: 'Master session restarted', + data: { + oldSessionId: oldSession?.sessionId, + oldMessageCount: oldSession?.messageCount, + restartCount: this.restartCount, + }, + }); + } else { + await this.dotFolder.appendLog({ + timestamp: new Date().toISOString(), + level: 'warn', + message: 'Master session restarted', + data: { + oldSessionId: oldSession?.sessionId, + oldMessageCount: oldSession?.messageCount, + restartCount: this.restartCount, + }, + }); + } + + // Create a new session — use raw UUID (Claude CLI requires valid UUID format) + const sessionId = randomUUID(); + const now = new Date().toISOString(); + + this.masterSession = { + sessionId, + createdAt: now, + lastUsedAt: now, + messageCount: 0, + allowedTools: getMasterTools(this.trustLevel), + maxTurns: MASTER_MAX_TURNS, + }; + this.sessionInitialized = false; + + // Persist the new session + try { + await this.saveMasterSessionToStore(this.masterSession); + } catch (error) { + logger.warn({ error }, 'Failed to persist restarted Master session'); + } + + // Build and send context summary to seed the new session + const contextSummary = await this.buildContextSummary(); + const spawnOpts = this.buildMasterSpawnOptions(contextSummary, this.messageTimeout); + + // Clear pending cancellation notifications after they've been injected into the spawn options. + // This prevents duplicate injection on subsequent session restarts (OB-F173). + this.pendingCancellationNotifications.length = 0; + + try { + const result = await this.agentRunner.spawn(spawnOpts); + await this.updateMasterSession(); + + if (result.exitCode !== 0) { + logger.warn( + { exitCode: result.exitCode, stderr: result.stderr }, + 'Context recovery prompt returned non-zero exit code', + ); + } else { + logger.info({ sessionId }, 'Master session restarted with context summary'); + } + } catch (error) { + // Context seeding failed — session is still usable, just without history + logger.warn({ error }, 'Failed to seed restarted session with context summary'); + // Mark session as initialized even on failure so future calls use --resume + this.sessionInitialized = true; + await this.updateMasterSession(); + } + } + + /** + * Get the number of times the Master session has been restarted. + */ + public getRestartCount(): number { + return this.restartCount; + } + + /** + * Get the worker registry for external access. + * Useful for status queries and debugging. + */ + public getWorkerRegistry(): WorkerRegistry { + return this.workerRegistry; + } + + /** Return the DeepModeManager for external command handling (e.g. Router /deep command). */ + public getDeepModeManager(): DeepModeManager { + return this.deepMode; + } + + /** + * Format a concise worker summary string for stop command responses. + * Returns " (, '', )". + */ + private formatWorkerSummary(worker: WorkerRecord): string { + const shortId = worker.id.split('-').pop() ?? worker.id; + const model = worker.taskManifest.model ?? 'default'; + const summary = worker.taskManifest.prompt.slice(0, 40).replace(/\n/g, ' '); + const elapsedMs = worker.startedAt + ? Date.now() - new Date(worker.startedAt).getTime() + : undefined; + const elapsedStr = elapsedMs !== undefined ? `${Math.round(elapsedMs / 1000)}s` : 'unknown'; + return `${shortId} (${model}, '${summary}', ${elapsedStr})`; + } + + /** + * Kill a running worker by ID. + * + * Retrieves the abort handle stored in workerAbortHandles, calls it + * (SIGTERM → 5s grace → SIGKILL), then marks the worker as cancelled in + * WorkerRegistry and agent_activity. + * + * Edge cases handled: + * - Invalid / unknown workerId → success:false + * - Worker already completed / failed / cancelled → success:false + * - No abort handle (legacy PID -1) → log warning, still mark cancelled + */ + public async killWorker( + workerId: string, + cancelledBy = 'user', + ): Promise<{ success: boolean; message: string }> { + const worker = this.workerRegistry.getWorker(workerId); + + if (!worker) { + return { success: false, message: `Worker ${workerId} not found.` }; + } + + if ( + worker.status === 'completed' || + worker.status === 'failed' || + worker.status === 'cancelled' + ) { + const shortId = workerId.split('-').pop() ?? workerId; + return { success: false, message: `Worker ${shortId} has already ${worker.status}.` }; + } + + // Invoke the abort handle (SIGTERM → grace period → SIGKILL) + const abortHandle = this.workerAbortHandles.get(workerId); + if (abortHandle) { + abortHandle(); + this.workerAbortHandles.delete(workerId); + } else { + // Legacy / PID -1 case — no handle available, mark cancelled without sending a signal + logger.warn( + { workerId, pid: worker.pid ?? -1 }, + 'killWorker: no abort handle found (legacy PID -1?) — marking cancelled without kill signal', + ); + } + + // Mark cancelled in WorkerRegistry + try { + this.workerRegistry.markCancelled(workerId, 'Cancelled by user'); + } catch (err) { + logger.warn({ workerId, err }, 'killWorker: failed to mark worker as cancelled in registry'); + } + + // Update agent_activity in DB + if (this.memory) { + try { + await this.memory.updateActivity(workerId, { + status: 'failed', + completed_at: new Date().toISOString(), + }); + } catch (actErr) { + logger.warn({ workerId, error: actErr }, 'killWorker: failed to update agent_activity'); + } + } + + // Build descriptive message: "Stopped worker (, '', )" + const message = `Stopped worker ${this.formatWorkerSummary(worker)}`; + logger.info({ workerId, pid: worker.pid }, 'Worker killed by user request'); + + // Queue cancellation notification for the Master AI (OB-884). + // On the next Master call, buildMasterSpawnOptions() will inject this so the + // Master knows not to re-spawn this worker unless the user explicitly asks. + const shortId = workerId.split('-').pop() ?? workerId; + const taskSummary = worker.taskManifest.prompt.slice(0, 80).replace(/\n/g, ' '); + this.pendingCancellationNotifications.push( + `Worker ${shortId} was CANCELLED by user ${cancelledBy}. Task: "${taskSummary}". Do NOT retry this task unless the user explicitly asks.`, + ); + + // Broadcast cancellation to all connected channels (OB-883) + if (this.router) { + const shortId = workerId.split('-').pop() ?? workerId; + try { + await this.router.broadcastProgress({ + type: 'worker-cancelled', + workerId: shortId, + cancelledBy, + }); + } catch (broadcastErr) { + logger.warn({ workerId, broadcastErr }, 'killWorker: failed to broadcast cancellation'); + } + } + + return { success: true, message }; + } + + /** + * Kill all currently running workers. + * + * Calls killWorker() for each running worker and aggregates the results. + * Returns an object with the list of stopped worker IDs and a human-readable + * summary message. + */ + public async killAllWorkers( + cancelledBy = 'user', + ): Promise<{ stopped: string[]; message: string }> { + const running = this.workerRegistry.getRunningWorkers(); + + if (running.length === 0) { + return { stopped: [], message: 'No workers are currently running.' }; + } + + const stopped: string[] = []; + const lines: string[] = []; + + for (const worker of running) { + const result = await this.killWorker(worker.id, cancelledBy); + if (result.success) { + stopped.push(worker.id); + lines.push(`- ${this.formatWorkerSummary(worker)}`); + } + } + + const count = stopped.length; + const message = `Stopped ${count} worker${count !== 1 ? 's' : ''}:\n${lines.join('\n')}`; + return { stopped, message }; + } + + /** + * Set the Router so pending messages can be routed after exploration completes. + * Bridge calls this after setMaster() so the Master can deliver queued messages. + */ + public setRouter(router: Router): void { + this.router = router; + } + + /** + * Set the BatchManager for Batch Task Continuation (OB-1613). + * Called by Bridge after construction so the MasterManager can schedule + * continuation triggers after each processed message. + */ + public setBatchManager(bm: BatchManager): void { + this.batchManager = bm; + } + + /** + * Handle a batch item failure — pauses the batch and sends a failure message to the user. + * + * Called by the Router when processMessage throws during a CONTINUE:batch message (OB-1616). + * Looks up the original sender info (stored when the batch was first scheduled) to route + * the failure notification to the correct connector. + * + * @param batchId The batch that was being processed when the failure occurred. + * @param reason Short human-readable failure reason (e.g. error message). + */ + public async onBatchItemFailure(batchId: string, reason: string): Promise { + if (!this.batchManager || !this.router) return; + + await this.batchManager.pauseBatch(batchId); + + const senderInfo = this.batchManager.getSenderInfo(batchId); + if (!senderInfo) { + logger.warn( + { batchId }, + 'onBatchItemFailure: no sender info — cannot deliver failure message', + ); + return; + } + + const failureMsg = this.batchManager.buildFailureMessage(batchId, reason); + if (failureMsg) { + void this.router.sendDirect(senderInfo.source, senderInfo.sender, failureMsg); + logger.info({ batchId, sender: senderInfo.sender }, 'Batch failure message sent to user'); + } + } + + /** + * Handle a user-issued batch control command: skip, retry, or abort (OB-1616). + * + * These commands are intercepted by the Router and forwarded here. + * Returns a human-readable response string to be sent back to the user. + * + * @param action One of 'skip', 'retry', or 'abort'. + * @param sender Original sender ID (for scheduling the next continuation). + * @param source Original connector source (for response routing). + * @returns Response message to send back to the user. + */ + public async handleBatchCommand( + action: 'pause' | 'resume' | 'skip' | 'retry' | 'abort', + sender: string, + source: string, + ): Promise { + if (!this.batchManager) return 'No active batch.'; + + const batchId = this.batchManager.getCurrentBatchId(); + if (!batchId) return 'No active batch found.'; + + // Update stored sender info in case it changed (e.g. source connector switch) (OB-1667). + this.batchManager.setSenderInfo(batchId, { sender, source }); + + if (action === 'pause') { + await this.batchManager.pauseBatch(batchId); + const state = this.batchManager.getStatus(batchId); + const current = state ? state.currentIndex + 1 : '?'; + const total = state ? state.totalItems : '?'; + logger.info({ batchId, current, total }, 'Batch paused by user command (OB-1619)'); + return `⏸ Batch paused at item ${current}/${total}. Reply '/continue' to resume.`; + } + + if (action === 'resume') { + const resumed = await this.batchManager.resumeBatch(batchId); + if (!resumed) return 'No paused batch found to resume.'; + const state = this.batchManager.getStatus(batchId); + const current = state ? state.currentIndex + 1 : '?'; + logger.info({ batchId, current }, 'Batch resumed by user command (OB-1620)'); + // Re-inject continuation to trigger the next item + if (this.router) { + const router = this.router; + const handle = setTimeout(() => { + this.batchTimers.delete(handle); + if (this.state === 'shutdown') return; + router.routeBatchContinuation(batchId, sender).catch((err: unknown) => { + const msg = err instanceof Error ? err.message : String(err); + void this.batchManager?.pauseBatch(batchId); + void router.sendDirect(source, sender, `Batch paused due to error: ${msg}`); + logger.error( + { batchId, err }, + 'routeBatchContinuation failed — batch paused (OB-1666)', + ); + }); + }, 500); + this.batchTimers.add(handle); + } + return `▶ Resuming batch from item ${current}...`; + } + + if (action === 'abort') { + await this.batchManager.abortBatch(batchId); + this.batchManager.deleteSenderInfo(batchId); + logger.info({ batchId }, 'Batch aborted by user command'); + // Retrieve the abort summary built by abortBatch() before state was deleted (OB-1622). + const abortSummary = this.batchManager.popCompletionSummary(); + return abortSummary ?? '🛑 Batch aborted.'; + } + + if (action === 'skip') { + const result = await this.batchManager.skipCurrentItem(batchId); + if (!result) return 'Failed to skip — batch not found.'; + if (result.finished) { + this.batchManager.deleteSenderInfo(batchId); + return '⏭ Item skipped. Batch complete — no more items.'; + } + // Schedule next continuation + if (this.router) { + const router = this.router; + const handle = setTimeout(() => { + this.batchTimers.delete(handle); + if (this.state === 'shutdown') return; + router.routeBatchContinuation(batchId, sender).catch((err: unknown) => { + const msg = err instanceof Error ? err.message : String(err); + void this.batchManager?.pauseBatch(batchId); + void router.sendDirect(source, sender, `Batch paused due to error: ${msg}`); + logger.error( + { batchId, err }, + 'routeBatchContinuation failed — batch paused (OB-1666)', + ); + }); + }, 1000); + this.batchTimers.add(handle); + } + logger.info( + { batchId, nextIndex: result.nextIndex }, + 'Batch item skipped, continuing (OB-1623)', + ); + return '⏭ Item skipped. Continuing with next item...'; + } + + // action === 'retry' + const retried = await this.batchManager.retryCurrentItem(batchId); + if (!retried) return 'Failed to retry — batch not found.'; + // Schedule continuation (same index, so same item runs again) + if (this.router) { + const router = this.router; + const handle = setTimeout(() => { + this.batchTimers.delete(handle); + if (this.state === 'shutdown') return; + router.routeBatchContinuation(batchId, sender).catch((err: unknown) => { + const msg = err instanceof Error ? err.message : String(err); + void this.batchManager?.pauseBatch(batchId); + void router.sendDirect(source, sender, `Batch paused due to error: ${msg}`); + logger.error({ batchId, err }, 'routeBatchContinuation failed — batch paused (OB-1666)'); + }); + }, 1000); + this.batchTimers.add(handle); + } + logger.info({ batchId }, 'Batch item retry scheduled by user command'); + return '🔄 Retrying item...'; + } + + /** + * Return a formatted batch status message for the `/batch` command (OB-1621). + * + * Shows the current item, progress (N/total), elapsed time, accumulated cost, + * and a list of failed items. Returns "No active batch." when no batch is running. + */ + public getBatchStatus(): string { + if (!this.batchManager) return 'No active batch.'; + + const batchId = this.batchManager.getCurrentBatchId(); + if (!batchId) return 'No active batch.'; + + const state = this.batchManager.getStatus(batchId); + if (!state) return 'No active batch.'; + + const current = state.currentIndex + 1; + const total = state.totalItems; + const currentItem = state.plan[state.currentIndex]; + + // Elapsed time + const elapsedMs = Date.now() - new Date(state.startedAt).getTime(); + const totalSeconds = Math.round(elapsedMs / 1000); + const hours = Math.floor(totalSeconds / 3600); + const minutes = Math.floor((totalSeconds % 3600) / 60); + const seconds = totalSeconds % 60; + let elapsedStr: string; + if (hours > 0) { + elapsedStr = `${hours}h ${minutes}m ${seconds}s`; + } else if (minutes > 0) { + elapsedStr = `${minutes}m ${seconds}s`; + } else { + elapsedStr = `${seconds}s`; + } + + const costStr = state.totalCostUsd > 0 ? `$${state.totalCostUsd.toFixed(4)}` : 'not tracked'; + const statusIcon = state.paused ? '⏸ Paused' : '▶ Running'; + + const lines: string[] = [ + `📋 Batch Status: ${statusIcon}`, + `**Progress:** ${current}/${total} items`, + ]; + + if (currentItem) { + const desc = currentItem.description ? ` — ${currentItem.description}` : ''; + lines.push(`**Current item:** ${currentItem.id}${desc}`); + } + + lines.push(`**Elapsed:** ${elapsedStr}`); + lines.push(`**Cost:** ${costStr}`); + + if (state.failedItems.length > 0) { + lines.push(`**Failed:** ${state.failedItems.join(', ')}`); + } + + logger.info({ batchId, current, total }, 'Batch status queried by user (OB-1621)'); + return lines.join('\n'); + } + + /** + * Set the KnowledgeRetriever for RAG-based context injection (OB-1344). + * Bridge calls this after MemoryManager is initialized and DotFolderManager is ready. + */ + public setKnowledgeRetriever(retriever: KnowledgeRetriever): void { + this.knowledgeRetriever = retriever; + } + + /** + * Set the IntegrationHub — exposes connected integrations to the Master AI. + * Called by Bridge.start() after the hub is created. + */ + public setIntegrationHub(hub: IntegrationHub): void { + this.integrationHub = hub; + } + + /** + * Build the list of connected integrations for the Master system prompt. + * Only returns integrations that have been successfully initialized (connected=true). + */ + private buildConnectedIntegrations(): ConnectedIntegrationEntry[] | undefined { + if (!this.integrationHub) return undefined; + const all = this.integrationHub.list(); + const connected = all.filter((info) => info.connected); + if (connected.length === 0) return undefined; + return connected.map((info) => { + const integration = this.integrationHub!.get(info.name); + return { + name: info.name, + type: info.type, + capabilities: integration.describeCapabilities(), + }; + }); + } + + /** + * Set the names of active connectors so they can be included in the Master system prompt. + * Called by the startup flow after Bridge.start() completes and connectors are initialized. + */ + public setActiveConnectorNames(names: string[]): void { + this.activeConnectorNames = [...names]; + } + + /** + * Set the port the local file server is listening on so it can be included in the Master system prompt. + * Called by the startup flow after bridge.start() successfully starts the file server. + */ + public setFileServerPort(port: number): void { + this.fileServerPort = port; + } + + /** + * Set the public tunnel URL so it can be included in the Master system prompt. + * Pass null to clear the tunnel URL (e.g. when the tunnel stops). + * Called by the startup flow after bridge.start() successfully starts a tunnel. + */ + public setTunnelUrl(url: string | null): void { + this.tunnelUrl = url; + } + + /** + * Replace the active MCP server list and mark the cached system prompt as stale. + * Called by Bridge.onConfigChange() when config.json is hot-reloaded. + * The next Master session call will regenerate the system prompt with the new servers. + */ + public reloadMcpServers(servers: MCPServer[]): void { + this.mcpServers = [...servers]; + // Null out the cached prompt so buildMasterSpawnOptions() reloads it + // on the next message, incorporating the updated MCP server list. + this.systemPrompt = null; + logger.info({ count: servers.length }, 'MCP servers hot-reloaded — system prompt marked stale'); + } + + /** + * Build a ProgressReporter for a specific message's source and sender. + * Returns undefined when no router is set (e.g. in unit tests). + */ + private makeProgressReporter(source: string, sender: string): ProgressReporter | undefined { + if (!this.router) return undefined; + const router = this.router; + return async (event: ProgressEvent) => { + await router.sendProgress(source, sender, event); + }; + } + + /** + * Load the worker registry from DB (system_config) or .openbridge/workers.json fallback. + * Called during start() to restore worker state from previous sessions. + */ + private async loadWorkerRegistry(): Promise { + try { + let registry = null; + + // Try DB first + if (this.memory) { + const raw = await this.memory.getSystemConfig('workers'); + if (raw) { + try { + registry = WorkersRegistrySchema.parse(JSON.parse(raw)); + } catch { + // fall through to JSON file + } + } + } + + // Fall back to JSON file + if (!registry) { + registry = await this.dotFolder.readWorkers(); + } + + if (registry) { + this.workerRegistry.fromJSON(registry); + logger.info( + { workerCount: Object.keys(registry.workers).length }, + 'Loaded worker registry from disk', + ); + } + } catch (error) { + logger.warn({ error }, 'Failed to load worker registry from disk (will start fresh)'); + } + } + + /** + * Persist the worker registry to system_config (DB) and .openbridge/workers.json (fallback). + * Called after worker state changes to maintain cross-restart visibility. + */ + private async persistWorkerRegistry(): Promise { + try { + const registry = this.workerRegistry.toJSON(); + if (this.memory) { + await this.memory.setSystemConfig('workers', JSON.stringify(registry)); + } else { + await this.dotFolder.writeWorkers(registry); + } + } catch (error) { + logger.warn({ error }, 'Failed to persist worker registry to disk'); + } + } + + /** + * Detect which prompt template (if any) was used for this worker task. + * Matches the task prompt against known template patterns. + * + * Returns the prompt ID or null if no template match found. + */ + private detectPromptTemplate(prompt: string): string | null { + // Check for exploration structure scan markers + if ( + prompt.includes('Workspace Structure Scan') || + prompt.includes('topLevelFiles') || + prompt.includes('directoryCounts') + ) { + return 'exploration-structure-scan'; + } + + // Check for exploration classification markers + if ( + prompt.includes('Project Classification') || + (prompt.includes('projectType') && prompt.includes('frameworks')) + ) { + return 'exploration-classification'; + } + + // Check for task execution markers + if ( + prompt.includes('Execute User Request') || + prompt.includes('User Request') || + prompt.includes('Workspace Context') + ) { + return 'task-execute'; + } + + // Check for task verification markers + if ( + prompt.includes('Verify Implementation') || + (prompt.includes('Verification Steps') && prompt.includes('verified')) + ) { + return 'task-verify'; + } + + // Check for code audit markers + if (prompt.includes('Task: Code Audit') || prompt.includes('Code Audit')) { + return 'task-code-audit'; + } + + // Check for generate output markers + if (prompt.includes('Generate Output File') || prompt.includes('SHARE marker')) { + return 'task-generate-output'; + } + + // Check for targeted read markers + if (prompt.includes('Targeted File Read') || prompt.includes('Files to Read')) { + return 'task-targeted-read'; + } + + // Check for build app markers + if (prompt.includes('Build Web App') || prompt.includes('APP:start')) { + return 'task-build-app'; + } + + // Check for Deep Mode markers (most specific first to avoid cross-matching) + if (prompt.includes('Deep Mode') && prompt.includes('Verify Phase')) { + return 'deep-verify'; + } + if (prompt.includes('Deep Mode') && prompt.includes('Execute Phase')) { + return 'deep-execute'; + } + if (prompt.includes('Deep Mode') && prompt.includes('Plan Phase')) { + return 'deep-plan'; + } + if (prompt.includes('Deep Mode') && prompt.includes('Report Phase')) { + return 'deep-report'; + } + if (prompt.includes('Deep Mode') && prompt.includes('Investigate Phase')) { + return 'deep-investigate'; + } + + return null; + } + + /** + * Validate worker output to determine if the prompt produced valid results. + * + * For exploration/verification prompts: checks if output is parseable JSON with expected fields + * For task prompts: checks if exit code is 0 (successful execution) + */ + private validateWorkerOutput( + promptId: string | null, + result: AgentResult, + _taskRecord: TaskRecord, + ): boolean { + // If no prompt template detected, fall back to simple exit code check + if (!promptId) { + return result.exitCode === 0; + } + + // Exit code must be 0 for all prompts + if (result.exitCode !== 0) { + return false; + } + + const output = result.stdout.trim(); + + // For exploration and verification prompts, validate JSON structure + if ( + promptId === 'exploration-structure-scan' || + promptId === 'exploration-classification' || + promptId === 'task-verify' + ) { + try { + const parsed = JSON.parse(output) as Record; + + // Validate required fields based on prompt type + switch (promptId) { + case 'exploration-structure-scan': + return ( + typeof parsed['workspacePath'] === 'string' && + Array.isArray(parsed['topLevelFiles']) && + Array.isArray(parsed['topLevelDirs']) && + typeof parsed['directoryCounts'] === 'object' + ); + + case 'exploration-classification': + return ( + typeof parsed['projectType'] === 'string' && + typeof parsed['projectName'] === 'string' && + Array.isArray(parsed['frameworks']) + ); + + case 'task-verify': + return typeof parsed['verified'] === 'boolean'; + + default: + return true; + } + } catch { + // JSON parse failed — output is not valid + return false; + } + } + + // For task execution prompts, success is based on exit code + non-empty output + if (promptId === 'task-execute') { + return result.exitCode === 0 && output.length > 0; + } + + // Default: success based on exit code + return result.exitCode === 0; + } + + /** + * Record prompt effectiveness after worker execution (OB-172: prompt effectiveness tracking). + * + * Detects which prompt template was used (if any) and validates the output. + * Records success/failure to the prompt manifest for self-improvement. + */ + private async recordPromptEffectiveness( + taskRecord: TaskRecord, + result: AgentResult, + ): Promise { + try { + const promptId = this.detectPromptTemplate(taskRecord.userMessage); + + if (!promptId) { + // No template detected — skip effectiveness tracking + logger.debug( + { workerId: taskRecord.id }, + 'No prompt template detected for worker — skipping effectiveness tracking', + ); + return; + } + + const isValid = this.validateWorkerOutput(promptId, result, taskRecord); + + if (this.memory) { + await this.memory.recordPromptOutcome(promptId, isValid); + } + + // Also record in dotfolder manifest (used by prompt evolution / evolvePrompts) (OB-1612) + await this.dotFolder.recordPromptUsage(promptId, isValid); + + logger.debug( + { + workerId: taskRecord.id, + promptId, + isValid, + exitCode: result.exitCode, + }, + 'Recorded prompt effectiveness', + ); + } catch (error) { + logger.warn( + { error, workerId: taskRecord.id }, + 'Failed to record prompt effectiveness — non-blocking', + ); + } + } + + /** + * Record a learning entry for a completed worker execution (OB-171: learnings store). + * After each task, the Master appends a learning entry with task type, model used, + * profile used, success, duration, and notes. On startup, the Master reads this + * history to inform future decisions (e.g., "haiku failed on refactoring tasks 3 + * times, use sonnet instead"). + */ + private async recordWorkerLearning( + taskRecord: TaskRecord, + result: AgentResult, + profile: string, + model?: string, + ): Promise { + try { + // Classify task type based on the prompt content + const taskType = classifyTaskType(taskRecord.userMessage); + + // Determine success based on exit code and task status + const success = result.exitCode === 0 && taskRecord.status === 'completed'; + + // Extract notes from the task record or result + const notes = success + ? `Worker completed successfully using ${profile} profile` + + (result.retryCount > 0 ? ` (${result.retryCount} retries required)` : '') + : `Worker failed: ${taskRecord.error?.slice(0, 200) ?? 'Unknown error'}` + + (result.retryCount > 0 ? ` (${result.retryCount} retries attempted)` : ''); + + const learningEntry = { + id: `learning-${taskRecord.id}`, + taskType, + modelUsed: result.model ?? model, + profileUsed: profile, + success, + durationMs: result.durationMs, + notes, + recordedAt: new Date().toISOString(), + exitCode: result.exitCode, + retryCount: result.retryCount, + metadata: { + workerId: taskRecord.id, + workerIndex: taskRecord.metadata?.['workerIndex'] as number | undefined, + modelFallbacks: result.modelFallbacks, + }, + }; + + if (this.memory) { + await this.memory.recordLearning( + taskType, + learningEntry.modelUsed ?? 'unknown', + learningEntry.success, + result.turnsUsed ?? 0, + learningEntry.durationMs, + ); + } else { + await this.dotFolder.appendLearning(learningEntry); + } + + logger.debug( + { + learningId: learningEntry.id, + taskType, + model: learningEntry.modelUsed, + profile, + success, + }, + 'Learning entry recorded', + ); + } catch (error) { + logger.warn({ error, taskId: taskRecord.id }, 'Failed to record learning entry'); + } + } + + /** Delegate to ClassificationEngine (OB-1279). */ + public normalizeForCache(content: string): string { + return this.classificationEngine.normalizeForCache(content); + } + + /** Delegate to ClassificationEngine (OB-1279). */ + public async recordClassificationFeedback( + normalizedKey: string, + turnBudgetSufficient: boolean, + timedOut: boolean, + turnsUsed?: number, + ): Promise { + return this.classificationEngine.recordClassificationFeedback( + normalizedKey, + turnBudgetSufficient, + timedOut, + turnsUsed, + ); + } + + /** Delegate to ClassificationEngine (OB-1279). */ + public async classifyTask(content: string, sessionId?: string): Promise { + return this.classificationEngine.classifyTask(content, sessionId); + } + + /** @deprecated — delegate to ClassificationEngine. Remove once all internal callers use engine directly. */ + private classifyTaskByKeywords( + content: string, + recentUserMessages?: string[], + lastBotResponse?: string, + ): ClassificationResult { + return this.classificationEngine.classifyTaskByKeywords( + content, + recentUserMessages, + lastBotResponse, + ); + } + + /** + * Build a per-message context header with channel and role information (OB-1625). + * Injected at the start of every prompt so the Master can make channel-aware decisions. + */ + private async getMessageContextHeader(message: InboundMessage): Promise { + let role = 'owner'; + if (this.memory) { + try { + const entry = await this.memory.getAccess(message.sender, message.source); + if (entry) role = entry.role; + } catch { + /* default to owner */ + } + } + return `[Context: channel=${message.source}, sender=${message.sender}, role=${role}]\n\n`; + } + + /** + * Build a planning prompt for complex tasks. + * Instructs the Master to decompose the request into SPAWN markers + * without executing the tasks itself — forcing delegation within 3-5 turns. + */ + private buildPlanningPrompt(userMessage: string): string { + const availableTools = this.discoveredTools.filter((t) => t.available); + const hasMultipleTools = availableTools.length > 1; + + // When multiple tools are available, include tool field in the example format + const spawnExample = hasMultipleTools + ? `[SPAWN:profile]{"prompt":"...","tool":"","model":"balanced","maxTurns":15}[/SPAWN]` + : `[SPAWN:profile]{"prompt":"...","model":"balanced","maxTurns":15}[/SPAWN]`; + + let prompt = + `The user asked: "${userMessage}"\n\n` + + `Break this into 1-3 concrete subtasks. For each subtask, output a SPAWN marker ` + + `with the appropriate profile, model, and instructions. ` + + `Do NOT execute the tasks yourself — only plan and delegate.\n\n` + + `Use this format for each subtask:\n` + + `${spawnExample}\n\n` + + `Model tiers: \`fast\` (cheap, mechanical tasks), \`balanced\` (default), \`powerful\` (complex reasoning). ` + + `Always use tier names — they auto-resolve to the correct model per tool.\n\n` + + `Available profiles: read-only (Read/Glob/Grep), code-edit (Read/Edit/Write/Glob/Grep/Bash(git:*)/Bash(npm:*)), full-access (all tools).`; + + // When multiple tools are available, strongly guide the Master to pick the best tool per worker + if (hasMultipleTools) { + const toolStrengths: Record = { + claude: 'deep reasoning, complex architecture, code review', + codex: 'quick code edits, simple refactors, mechanical changes', + aider: 'git-aware refactors, multi-file renames, commit-driven workflows', + }; + + const toolLines = availableTools + .map((t) => { + const strength = toolStrengths[t.name] ?? 'general-purpose'; + return ` - \`${t.name}\`: ${strength}`; + }) + .join('\n'); + + prompt += + `\n\n**IMPORTANT — Tool Selection:** You MUST choose the best AI tool for each worker using the \`"tool"\` field. ` + + `Available tools:\n${toolLines}\n\n` + + `Route each subtask to the tool best suited for it. ` + + `For example, use \`"tool":"codex"\` for straightforward code edits and \`"tool":"claude"\` for tasks requiring deep understanding. ` + + `Do NOT default all workers to \`${this.masterTool.name}\` — distribute work across tools based on task fit.`; + } + + return prompt; + } + + /** + * Autonomously explore the workspace and create .openbridge/ folder. + * Delegated to ExplorationManager (OB-1280). + */ + public async explore(): Promise { + return this.explorationManager.explore(); + } + + /** + * Check for workspace changes since the last analysis and decide which + * exploration path to take. Delegated to ExplorationManager (OB-1280). + */ + private async checkWorkspaceChanges( + existingMap: WorkspaceMap, + ): Promise<'no-changes' | 'incremental' | 'full-reexplore'> { + return this.explorationManager.checkWorkspaceChanges(existingMap); + } + + // incrementalExplore() moved to ExplorationManager (OB-1280) + + // masterDrivenExplore() moved to ExplorationManager (OB-1280) + + // emitExplorationProgress() moved to ExplorationManager (OB-1280) + + // monolithicExplore() moved to ExplorationManager (OB-1280) + + /** Write the agents registry — delegates to ExplorationManager (OB-1280). */ + private async writeAgentsRegistry(): Promise { + return this.explorationManager.writeAgentsRegistry(); + } + + /** Load exploration summary — delegates to ExplorationManager (OB-1280). */ + private async loadExplorationSummary(): Promise { + return this.explorationManager.loadExplorationSummary(); + } + + /** Re-explore the workspace. Delegated to ExplorationManager (OB-1280). */ + public async reExplore(): Promise { + return this.explorationManager.reExplore(); + } + + /** Full re-exploration. Delegated to ExplorationManager (OB-1280). */ + public async fullReExplore(): Promise { + return this.explorationManager.fullReExplore(); + } + + /** + * Reset the idle timer (called on each user message). + * Tracks the timestamp of the last user interaction for idle detection. + */ + private resetIdleTimer(): void { + this.lastMessageTimestamp = Date.now(); + this.consecutiveIdleCycles = 0; + this.consecutiveNoOpCycles = 0; + } + + /** Drain pending messages. Delegated to ExplorationManager (OB-1280). */ + private async drainPendingMessages(): Promise { + return this.explorationManager.drainPendingMessages(); + } + + /** + * Merge rapid-fire image+text messages from the same sender within a 10-second window. + * + * When a user sends one or more images followed immediately by a text message, + * the image context (OCR text from processedDocument) is prepended to the text + * message's content so the Master sees both together. The image-only messages + * are consumed and removed from the queue. (OB-1658) + */ + private mergeRapidFireImageMessages(messages: InboundMessage[]): InboundMessage[] { + const RAPID_FIRE_WINDOW_MS = 10_000; + + // Track pending image-only messages per sender + const pendingImages = new Map(); + const result: InboundMessage[] = []; + + for (const msg of messages) { + const hasImageAttachments = msg.attachments?.some((a) => a.type === 'image') ?? false; + const hasTextContent = msg.content.trim().length > 0; + + if (hasImageAttachments && !hasTextContent) { + // Pure image message — hold it for potential merge with a following text message + const existing = pendingImages.get(msg.sender) ?? []; + existing.push(msg); + pendingImages.set(msg.sender, existing); + } else if (hasTextContent) { + // Text message — check for pending images from the same sender within the window + const imgMsgs = pendingImages.get(msg.sender) ?? []; + const recentImages = imgMsgs.filter( + (imgMsg) => msg.timestamp.getTime() - imgMsg.timestamp.getTime() <= RAPID_FIRE_WINDOW_MS, + ); + + if (recentImages.length > 0) { + // Build the OCR context prefix from processedDocument.rawText + const ocrParts: string[] = []; + for (const imgMsg of recentImages) { + const rawText = imgMsg.processedDocument?.rawText?.trim(); + if (rawText) { + ocrParts.push(rawText); + } + } + const ocrText = ocrParts.length > 0 ? `\n${ocrParts.join('\n')}` : ''; + const mergedContent = `[User also sent ${recentImages.length} image(s) — see .openbridge/media/ for processed content]${ocrText}\n${msg.content}`; + result.push({ ...msg, content: mergedContent }); + + logger.info( + { sender: msg.sender, imageCount: recentImages.length }, + 'Merged rapid-fire image+text messages during drain (OB-1658)', + ); + + // Keep stale images (outside the window) — they were not consumed + const staleImages = imgMsgs.filter( + (imgMsg) => msg.timestamp.getTime() - imgMsg.timestamp.getTime() > RAPID_FIRE_WINDOW_MS, + ); + if (staleImages.length > 0) { + pendingImages.set(msg.sender, staleImages); + } else { + pendingImages.delete(msg.sender); + } + } else { + // No recent images to merge — flush any stale pending images first + if (imgMsgs.length > 0) { + result.push(...imgMsgs); + pendingImages.delete(msg.sender); + } + result.push(msg); + } + } else { + // Mixed or other message type — flush pending images for sender then pass through + const imgMsgs = pendingImages.get(msg.sender) ?? []; + if (imgMsgs.length > 0) { + result.push(...imgMsgs); + pendingImages.delete(msg.sender); + } + result.push(msg); + } + } + + // Flush any remaining pending images that were never followed by a text message + for (const [, imgMsgs] of pendingImages) { + result.push(...imgMsgs); + } + + return result; + } + + /** + * Process a message from a user. + * Uses the persistent Master session for conversation continuity. + * All messages go through the same Master session regardless of sender. + */ + public async processMessage(message: InboundMessage): Promise { + // Reset idle timer on new message + this.resetIdleTimer(); + + // Queue messages that arrive while the Master is exploring the workspace. + // They will be processed once exploration completes and state transitions to 'ready'. + if (this.state === 'exploring') { + logger.info( + { sender: message.sender }, + 'Master is exploring workspace, queueing message for later processing', + ); + this.pendingMessages.push(message); + return "I'm still exploring your workspace. Your message will be processed once exploration completes."; + } + + // Recover from error state instead of permanently rejecting messages. + // recover() resets state to 'idle' and re-triggers exploration if needed, + // which transitions state to 'exploring' before returning. + if (this.state === 'error') { + logger.info( + { sender: message.sender }, + 'Master in error state — attempting recovery before processing message', + ); + await this.recover(); + // After recover(), state is 'exploring' (exploration retry fired). + // Queue this message so it is processed once exploration completes. + this.pendingMessages.push(message); + return 'The AI encountered an exploration error and is retrying. Your message will be processed once recovery completes.'; + } + + if (this.state === 'processing') { + if (this.processingPendingMessages.length >= 5) { + return 'Too many queued messages. Please wait for the current request to complete.'; + } + this.processingPendingMessages.push(message); + logger.info( + { sender: message.sender, queueSize: this.processingPendingMessages.length }, + 'Message queued during processing', + ); + return "I'm working on your previous request. Your message has been queued and will be processed next."; + } + + if (this.state !== 'ready') { + logger.warn( + { currentState: this.state, sender: message.sender }, + 'Cannot process message: Master not ready', + ); + return `The AI is currently ${this.state}. Please try again in a moment.`; + } + + const originalContent = message.content; + const originalRawContent = message.rawContent; + let activeBatchId: string | undefined; + let activeBatchItemId: string | undefined; + + // If a batch is active, process the next batch item instead of re-parsing the incoming message. + if (this.batchManager && this.batchManager.isActive()) { + const currentBatchId = this.batchManager.getCurrentBatchId(); + if (currentBatchId) { + const state = this.batchManager.getStatus(currentBatchId); + const currentItem = state?.plan[state.currentIndex]; + if (currentItem) { + activeBatchId = currentBatchId; + activeBatchItemId = currentItem.id; + const batchPrompt = currentItem.description?.trim() || currentItem.id; + message.content = batchPrompt; + message.rawContent = batchPrompt; + logger.info( + { batchId: currentBatchId, itemId: currentItem.id }, + 'Batch active — processing next item', + ); + } else { + logger.warn( + { batchId: currentBatchId }, + 'Batch active but no current item found — falling back to incoming message', + ); + } + } + } + + this.state = 'processing'; + this.activeMessage = message; + + const taskId = randomUUID(); + const startedAt = new Date().toISOString(); + + logger.info({ taskId, sender: message.sender, content: message.content }, 'Processing message'); + + // Create task record + const task: TaskRecord = { + id: taskId, + userMessage: message.rawContent, + sender: message.sender, + description: message.content, + status: 'processing', + handledBy: 'master', + createdAt: startedAt, + startedAt, + metadata: { + messageId: message.id, + source: message.source, + }, + }; + if (activeBatchId && activeBatchItemId) { + task.metadata = { + ...task.metadata, + batchId: activeBatchId, + batchItemId: activeBatchItemId, + originalUserMessage: originalRawContent, + originalUserContent: originalContent, + }; + } + + // Build a ProgressReporter that maps events to the connector's sendProgress() + const progress = this.makeProgressReporter(message.source, message.sender); + + // Stable session ID for grouping all turns of this conversation (OB-730) + const sessionId = this.masterSession?.sessionId ?? taskId; + + try { + // Record the inbound user message to conversation history (OB-730) + await this.recordConversationMessage( + sessionId, + 'user', + message.content, + message.source, + message.sender, + ); + + // Check for status queries + if (this.isStatusQuery(message.content)) { + const status = await this.getStatus(); + this.state = 'ready'; + task.status = 'completed'; + task.result = status; + task.completedAt = new Date().toISOString(); + task.durationMs = + new Date(task.completedAt).getTime() - new Date(task.startedAt!).getTime(); + await this.recordTaskToStore(task); + return status; + } + + // Retrieve relevant past conversation history to enrich the Master's context (OB-731) + // and fetch learned patterns for system prompt enrichment (OB-735) + const [ + conversationContext, + learnedPatternsContext, + workerNextStepsContext, + templateSelectionContext, + ] = await Promise.all([ + this.buildConversationContext(message.content, sessionId, message.sender), + this.buildLearnedPatternsContext(), + this.buildWorkerNextStepsContext(), + this.promptContextBuilder.buildTemplateSelectionContext(), + ]); + + // (1) Emit classifying event — AI is analyzing the message + await progress?.({ type: 'classifying' }); + + // Classify message to determine appropriate turn budget + const classification = await this.classifyTask(message.content, sessionId); + let taskClass = classification.class; + let taskMaxTurns = classification.maxTurns; + logger.info({ taskClass, taskMaxTurns, reason: classification.reason }, 'Message classified'); + + // Attachment escalation — OB-1257 + // If the message has file attachments and was classified as quick-answer, + // escalate to tool-use: file analysis needs tool access and a larger turn budget. + if (taskClass === 'quick-answer' && message.attachments && message.attachments.length > 0) { + logger.info( + { attachmentCount: message.attachments.length, from: 'quick-answer', to: 'tool-use' }, + 'Attachment detected — escalating quick-answer to tool-use (15 turns, 510s)', + ); + taskClass = 'tool-use'; + taskMaxTurns = MESSAGE_MAX_TURNS_TOOL_USE; + } + + // DocType creation intent — OB-1384 + // When the classifier detects doctype-creation phrases, override the prompt so the Master + // spawns a worker to design and register a DocType with appropriate fields, states, and hooks. + if (classification.doctypeCreation && classification.doctypeEntity) { + const entity = classification.doctypeEntity; + logger.info({ entity }, 'DocType creation intent detected — injecting design prompt'); + // Escalate to complex-task if not already + taskClass = 'complex-task'; + taskMaxTurns = MESSAGE_MAX_TURNS_PLANNING; + } + + // Deep Mode activation — OB-1403 + // If the configured default profile is 'thorough' or 'manual' and the task class is + // 'complex-task', start a multi-phase Deep Mode session beginning with investigate phase. + // Fast profile (default) skips Deep Mode entirely and falls through to normal processing. + if (taskClass === 'complex-task') { + const effectiveProfile = this.deepConfig?.defaultProfile ?? 'fast'; + if (effectiveProfile === 'thorough' || effectiveProfile === 'manual') { + const deepSessionId = this.deepMode.startSession( + message.content, + effectiveProfile, + progress, + ); + if (deepSessionId) { + // Persist new session to SQLite so it survives process restarts (OB-1405) + await this.persistDeepModeSession(deepSessionId); + + const currentPhase = this.deepMode.getCurrentPhase(deepSessionId); + const phasePrompt = this.deepMode.getPhaseSystemPrompt(deepSessionId); + logger.info( + { deepSessionId, profile: effectiveProfile, currentPhase }, + 'Deep Mode activated — starting with investigate phase', + ); + // Return an early response indicating Deep Mode has started. + // Per-phase worker execution is wired in subsequent tasks (OB-1417+). + const response = [ + `Deep Mode started (${effectiveProfile} profile) — **${currentPhase ?? 'investigate'} phase**`, + '', + phasePrompt + ? phasePrompt.split('\n')[0] + : 'Exploring and identifying relevant context.', + '', + effectiveProfile === 'manual' + ? 'I will pause after each phase for your review. Reply with `/proceed` to advance to the next phase.' + : 'I will run all phases automatically and report when complete.', + ].join('\n'); + task.status = 'completed'; + task.result = response; + task.completedAt = new Date().toISOString(); + task.durationMs = + new Date(task.completedAt).getTime() - new Date(task.startedAt!).getTime(); + task.metadata = { ...task.metadata, deepSessionId }; + await this.recordTaskToStore(task); + await this.recordConversationMessage(sessionId, 'master', response); + void this.recordClassificationFeedback( + this.normalizeForCache(message.content), + true, + false, + ); + this.onTaskCompleted(); + this.state = 'ready'; + await progress?.({ type: 'complete' }); + return response; + } + } + } + + // Knowledge retrieval for codebase questions and single-tool tasks (OB-1345, OB-1349) + // Query pre-indexed knowledge before building the Master prompt. + // Run for 'quick-answer' (codebase questions) and 'tool-use' (targeted edits/lookups) + // where pre-fetched context helps the Master answer or act without spawning extra workers. + // Skip for 'complex-task' — Master needs to plan and delegate; RAG context is not useful + // at planning time and adds noise to the delegation prompt. + let knowledgeContext: string | undefined; + let targetedReaderContext: string | undefined; + if ((taskClass === 'quick-answer' || taskClass === 'tool-use') && this.knowledgeRetriever) { + const knowledgeResult = await this.knowledgeRetriever.query(message.content); + logger.info( + { + question: message.content.slice(0, 80), + confidence: knowledgeResult.confidence, + chunkCount: knowledgeResult.chunks.length, + sources: knowledgeResult.sources, + }, + 'RAG query completed', + ); + // RAG retry with classifier description (OB-1569): when the raw user message + // (e.g. Arabizi/Darija) returns zero chunks, retry with the AI classifier's English + // description, which has already translated the user's intent. + let effectiveKnowledgeResult = knowledgeResult; + if (knowledgeResult.chunks.length === 0 && classification.ragQuery) { + const retryResult = await this.knowledgeRetriever.query(classification.ragQuery); + logger.info( + { + originalQueryLen: message.content.length, + retryQueryLen: classification.ragQuery.length, + retryChunkCount: retryResult.chunks.length, + retryConfidence: retryResult.confidence, + }, + 'RAG retry with classifier description', + ); + if (retryResult.chunks.length > 0) { + effectiveKnowledgeResult = retryResult; + } + } + if (effectiveKnowledgeResult.confidence < 0.3) { + logger.debug( + { confidence: effectiveKnowledgeResult.confidence }, + 'Low confidence, worker may be needed', + ); + // Targeted reader: suggest files from workspace map and spawn a focused + // read-only worker to answer the question. (OB-1354) + const workspaceMap = await this.dotFolder.readWorkspaceMap(); + if (workspaceMap) { + const suggestedFiles = this.knowledgeRetriever.suggestTargetFiles( + message.content, + workspaceMap, + ); + if (suggestedFiles.length > 0) { + logger.debug( + { fileCount: suggestedFiles.length }, + 'Low RAG confidence — spawning targeted reader', + ); + const readerResult = await this.spawnTargetedReader(suggestedFiles, message.content); + if (readerResult) { + targetedReaderContext = readerResult; + } + } else { + logger.debug('No target files identified, falling back to Master handling'); + } + } + // Workspace-map summary fallback (OB-1570): when RAG returns low confidence + // and targeted reader also fails, inject a workspace overview so workers + // get basic project context even when FTS5 fails completely. + if (!knowledgeContext && !targetedReaderContext && workspaceMap) { + knowledgeContext = `## Workspace Overview (fallback)\n\n${JSON.stringify(workspaceMap.projectType ?? 'unknown')} project with ${Object.keys(workspaceMap.structure ?? {}).length} directories.\nKey files: ${( + workspaceMap.keyFiles ?? [] + ) + .slice(0, 10) + .map((f) => f.path) + .join(', ')}`; + if (knowledgeContext.length > SECTION_BUDGET_RAG) { + knowledgeContext = knowledgeContext.slice(0, SECTION_BUDGET_RAG); + } + logger.info('Workspace-map summary fallback injected (OB-1570)'); + } + } + if (effectiveKnowledgeResult.confidence >= 0.3) { + knowledgeContext = + this.knowledgeRetriever.formatKnowledgeContext(effectiveKnowledgeResult); + } + } + + // For complex tasks, send a planning prompt that forces the Master to output + // SPAWN markers within a small turn budget instead of attempting execution itself. + let promptToSend = + taskClass === 'complex-task' ? this.buildPlanningPrompt(message.content) : message.content; + // For menu-selection, inject the selected option text so Master sees context, not just a digit (OB-1659). + // E.g., user sends "3" → Master sees "User selected option 3: 'Deploy to staging'" + if (classification.menuSelection) { + const digit = message.content.trim(); + promptToSend = classification.selectedOptionText + ? `User selected option ${digit}: '${classification.selectedOptionText}'` + : `User selected option ${digit}`; + } + // DocType creation — inject a design prompt so Master spawns a worker to build the DocType (OB-1384) + if (classification.doctypeCreation && classification.doctypeEntity) { + const entity = classification.doctypeEntity; + promptToSend = + `The user wants to track "${entity}". ` + + `Design a "${entity}" DocType with appropriate fields (name, status, dates, amounts, references), ` + + `states (e.g. draft → active → completed), computed fields where useful, and lifecycle hooks. ` + + `Use the DocType system in src/intelligence/ to register it. ` + + `Original request: ${message.content}`; + } + // Prepend channel + role context header so Master can make channel-aware decisions (OB-1625) + promptToSend = (await this.getMessageContextHeader(message)) + promptToSend; + // complex-task always uses planning turns; otherwise use AI-suggested budget + const maxTurnsToUse = + taskClass === 'complex-task' ? MESSAGE_MAX_TURNS_PLANNING : taskMaxTurns; + // Derive timeout from the actual turns used (complex-task overrides to planning turns) + const timeoutToUse = + taskClass === 'complex-task' + ? turnsToTimeout(MESSAGE_MAX_TURNS_PLANNING) + : classification.timeout; + // ── Timeout chain documentation (OB-F217) ──────────────────────────────── + // (1) classification.timeout is produced by turnsToTimeout() in classification-engine.ts: + // turnsToTimeout(n) = CLI_STARTUP_BUDGET_MS + n × PER_TURN_BUDGET_MS + // = 30_000 + n × 30_000 + // Examples per task class (Phase 155 values): + // quick-answer (n=3) → 30s + 90s = 120s + // tool-use (n=15) → 30s + 450s = 480s + // complex-task (n=25) → 30s + 750s = 780s + // + // (2) timeoutToUse overrides to the planning-turns budget for complex-task (see above), + // so timeoutToUse == turnsToTimeout(MESSAGE_MAX_TURNS_PLANNING) = 780s for complex. + // + // (3) safeTimeout = Math.min(timeoutToUse, DEFAULT_MESSAGE_TIMEOUT) + // clamps to 300s max (DEFAULT_MESSAGE_TIMEOUT = 300_000 at line 121). + // Effect: quick-answer stays at 120s (< 300s ceiling), + // tool-use is clamped to 300s (was 170s before OB-1662), + // complex-task is clamped to 300s (was 170s before OB-1662). + // + // (4) safeTimeout is forwarded to buildMasterSpawnOptions() → agentRunner.spawn(). + // The Master session process runs for at most safeTimeout ms total — + // this budget INCLUDES context loading, worker spawning, and worker execution. + // The 10s headroom reduction was removed (OB-1662) because the Master session + // IS the top-level process — there is no outer timeout to race against. + // ───────────────────────────────────────────────────────────────────────── + const safeTimeout = Math.min(timeoutToUse, DEFAULT_MESSAGE_TIMEOUT); + if (safeTimeout < timeoutToUse) { + logger.warn( + { originalTimeout: timeoutToUse, safeTimeout }, + 'Timeout clamped to message timeout boundary', + ); + } + + if (taskClass === 'complex-task') { + logger.info('Complex task — using planning prompt for auto-delegation'); + // (2) Emit planning event — Master is decomposing the task + await progress?.({ type: 'planning' }); + } + + // ── Planning Gate (OB-1779) ───────────────────────────────────────────── + // Reset the gate for each new message so no state leaks between user turns. + // For complex tasks, run up to 2 read-only analysis workers before the Master + // generates SPAWN execution markers — gating execution on prior investigation. + this.planningGate.reset(); + let analysisContext: string | null = null; + if (taskClass === 'complex-task') { + const bypassDecision = shouldBypassPlanning(message.content); + if (bypassDecision.bypass) { + this.planningGate.bypass(message.content, bypassDecision.reason); + logger.debug({ reason: bypassDecision.reason }, 'Planning gate: bypassed'); + } else { + // Analysis phase — spawn read-only workers to investigate before execution (OB-1776) + const analysisSpecs = this.planningGate.buildAnalysisWorkerSpecs(message.content); + this.planningGate.startAnalysis(message.content); + await Promise.all( + analysisSpecs.map(async (spec) => { + this.planningGate.recordAnalysisWorker({ + id: spec.id, + prompt: spec.prompt, + spawnedAt: new Date().toISOString(), + }); + const analysisOpts: SpawnOptions = { + workspacePath: this.workspacePath, + prompt: spec.prompt, + allowedTools: [...TOOLS_READ_ONLY], + maxTurns: spec.maxTurns, + }; + try { + const ar = await this.agentRunner.spawn(analysisOpts); + const out = + ar.exitCode === 0 + ? ar.stdout.trim() + : `[analysis worker error: ${ar.stderr.slice(0, 300)}]`; + this.planningGate.completeAnalysisWorker(spec.id, out); + } catch (err) { + this.planningGate.completeAnalysisWorker( + spec.id, + `[analysis worker failed: ${err instanceof Error ? err.message : String(err)}]`, + ); + } + }), + ); + if (this.planningGate.canCompleteAnalysis) { + const aggregated = this.planningGate.aggregateWorkerOutputs(); + this.planningGate.completeAnalysis(aggregated); + this.planningGate.confirmApproach('Proceeding with execution workers'); + analysisContext = `## Pre-task Analysis\n\n${aggregated}`; + logger.info( + { workerCount: this.planningGate.completedAnalysisWorkerCount }, + 'Planning gate: analysis complete, execution phase unlocked', + ); + } else { + // Partial analysis — bypass gate rather than block execution + logger.warn('Planning gate: not all analysis workers completed — bypassing gate'); + this.planningGate.bypass(message.content, 'analysis workers incomplete — fallback'); + } + } + } else { + this.planningGate.bypass( + message.content, + `task class '${taskClass}' does not require planning`, + ); + } + // ── End Planning Gate ─────────────────────────────────────────────────── + + // Check if task targets a specific sub-master's scope (OB-755) + // For non-trivial tasks, detect file-path mentions that belong to a sub-project + // and route directly to that sub-master's context instead of the root Master. + if (taskClass !== 'quick-answer' && this.subMasterManager) { + const allSubMasters = await this.subMasterManager.listSubMasters(); + const activeSubMasters = allSubMasters.filter((sm) => sm.status === 'active'); + if (activeSubMasters.length > 0) { + const routing = this.detectSubMasterRouting(message.content, activeSubMasters); + if (routing) { + logger.info( + { type: routing.type, count: routing.subMasters.length }, + 'Routing task to sub-master(s)', + ); + await progress?.({ type: 'spawning', workerCount: routing.subMasters.length }); + + const subMasterResponse = await this.handleSubMasterDelegation( + routing, + message.content, + progress, + ); + + task.status = 'completed'; + task.result = subMasterResponse; + task.completedAt = new Date().toISOString(); + task.durationMs = + new Date(task.completedAt).getTime() - new Date(task.startedAt!).getTime(); + await this.recordTaskToStore(task); + await this.recordConversationMessage(sessionId, 'master', subMasterResponse); + void this.recordClassificationFeedback( + this.normalizeForCache(message.content), + true, + false, + ); + this.onTaskCompleted(); + this.state = 'ready'; + await progress?.({ type: 'complete' }); + return subMasterResponse; + } + } + } + + // Execute message through the persistent Master session (OB-1246: budget-aware assembly) + const masterContext: MasterContextSections = { + conversationContext, + learnedPatternsContext, + workerNextStepsContext, + knowledgeContext, + targetedReaderContext, + analysisContext, + templateSelectionContext, + }; + const spawnOpts = this.buildMasterSpawnOptions( + promptToSend, + safeTimeout, + maxTurnsToUse, + masterContext, + ); + let result = await this.agentRunner.spawn(spawnOpts); + await this.updateMasterSession(); + void this._checkCompaction(); + + // Detect dead session and restart transparently + if (result.exitCode !== 0 && this.isSessionDead(result.exitCode, result.stderr)) { + logger.warn( + { exitCode: result.exitCode, stderr: result.stderr.slice(0, 200) }, + 'Master session appears dead, attempting restart', + ); + + await this.restartMasterSession(); + + // Retry with the same prompt and context sections (OB-1246: budget-aware assembly) + const retryOpts = this.buildMasterSpawnOptions( + promptToSend, + safeTimeout, + maxTurnsToUse, + masterContext, + ); + result = await this.agentRunner.spawn(retryOpts); + await this.updateMasterSession(); + void this._checkCompaction(); + } + + if (result.exitCode !== 0) { + throw new Error(`Message processing failed: ${result.stderr}`); + } + + let response = result.stdout.trim() || 'No response from AI'; + + // Guard: Master session itself hit max-turns — provide actionable feedback (OB-F230) + if (result.turnsExhausted) { + const partial = response.length > 20 ? response : ''; + const turnsUsedStr = result.turnsUsed ? ` (used ${result.turnsUsed} turns)` : ''; + logger.warn( + { taskId, maxTurns: maxTurnsToUse, turnsUsed: result.turnsUsed }, + 'Master session hit max-turns limit — returning partial result with guidance', + ); + if (partial && !hasSpawnMarkers(partial)) { + // Master produced some useful output before exhaustion — append guidance + response = + partial + + `\n\n⚠️ I ran out of processing capacity${turnsUsedStr}. ` + + `If this isn't complete, please send a follow-up message to continue.`; + } else if (!hasSpawnMarkers(response)) { + response = + `Your request needed more processing time than available${turnsUsedStr}. ` + + `Please try breaking it into smaller steps — for example, ask me to do the deployment separately from the code changes.`; + } + } + + // Check for SPAWN markers first (richer task decomposition protocol) + if (hasSpawnMarkers(response)) { + const spawnResult = parseSpawnMarkers(response); + if (spawnResult.markers.length > 0) { + logger.info({ spawnCount: spawnResult.markers.length }, 'SPAWN markers detected'); + + task.status = 'delegated'; + await this.recordTaskToStore(task); + + const n = spawnResult.markers.length; + + // If the cleaned output (text outside SPAWN markers) is very short, prepare + // a status message to show the user instead of a near-empty stub response. + const cleanedOutput = spawnResult.cleanedOutput; + const originalLength = response.length; + const cleanedLength = cleanedOutput.length; + const spawnSummaries = extractTaskSummaries(spawnResult.markers); + logger.debug( + { originalLength, cleanedLength, spawnCount: n, spawnSummaries }, + 'SPAWN marker stripping applied', + ); + if (cleanedLength < 80 && originalLength > 200) { + logger.warn( + { originalLength, cleanedLength, spawnCount: n }, + 'Response truncated after SPAWN marker removal — generating status message', + ); + } + let statusMessage: string | undefined; + if (cleanedLength === 0) { + // Entire response was SPAWN markers — build a numbered summary from extracted prompts + const numbered = spawnSummaries.map((s, i) => `${i + 1}) ${s}`).join(', '); + statusMessage = `I'm spawning ${n} worker${n === 1 ? '' : 's'}: ${numbered}`; + } else if (cleanedLength < 80) { + statusMessage = + `Working on your request — dispatching ${n} worker(s) for:\n` + + spawnSummaries.map((s) => `• ${s}`).join('\n'); + } + + // (3) Emit spawning event — N workers are being created + await progress?.({ type: 'spawning', workerCount: n }); + + // Planning gate guard (OB-1779): warn if execution workers are spawning + // without a completed planning phase (e.g. gate was bypassed or not run). + if (!this.planningGate.allowsExecution) { + logger.warn( + { gateStatus: this.planningGate.status }, + 'Planning gate: spawning execution workers without completed analysis phase', + ); + } + + // (4) Emit worker-progress + worker-result events as each worker completes + const feedbackPrompt = await this.handleSpawnMarkers( + spawnResult.markers, + async ( + completed: number, + total: number, + workerResult?: AgentResult, + workerMarker?: ParsedSpawnMarker, + ): Promise => { + await progress?.({ type: 'worker-progress', completed, total }); + + // Stream each worker's output to the user immediately + if (workerResult && workerMarker) { + const raw = + workerResult.exitCode === 0 + ? workerResult.stdout.trim() + : `Error: ${(workerResult.stderr || workerResult.stdout).trim().slice(0, 500)}`; + const maxLen = 2000; + const content = + raw.length > maxLen ? raw.slice(0, maxLen) + '\n...(truncated)' : raw; + await progress?.({ + type: 'worker-result', + workerIndex: completed, + total, + profile: workerMarker.profile, + tool: workerMarker.body.tool, + content, + success: workerResult.exitCode === 0, + durationMs: workerResult.durationMs, + turnsUsed: workerResult.turnsUsed, + }); + } + }, + message.attachments, + taskClass, + ); + + // (5) Emit synthesizing event — Master is combining worker results + await progress?.({ type: 'synthesizing' }); + + // Inject worker results back into the Master session for synthesis. + // If synthesis fails or times out, fall back gracefully — the user + // already received each worker's output via worker-result events. + this.state = 'processing'; + const feedbackOpts = this.buildMasterSpawnOptions( + feedbackPrompt, + undefined, + MESSAGE_MAX_TURNS_SYNTHESIS, + ); + + try { + result = await this.agentRunner.spawn(feedbackOpts); + await this.updateMasterSession(); + void this._checkCompaction(); + + if (result.exitCode !== 0) { + logger.warn({ exitCode: result.exitCode }, 'Synthesis failed — returning fallback'); + response = + statusMessage ?? + 'All subtask results were shown above. The summary step could not complete.'; + } else { + const synthesisOutput = result.stdout.trim(); + // Use status message only if synthesis produced no content at all + response = + synthesisOutput.length > 0 ? synthesisOutput : (statusMessage ?? feedbackPrompt); + } + } catch (synthesisError) { + logger.warn({ err: synthesisError }, 'Synthesis timed out — returning fallback'); + response = + statusMessage ?? 'All subtask results were shown above. The summary step timed out.'; + } + } + } + + // Check for legacy delegation markers (fallback) + if (!hasSpawnMarkers(response)) { + const delegations = this.parseDelegationMarkers(response); + if (delegations && delegations.length > 0) { + logger.info({ delegationCount: delegations.length }, 'Delegation markers detected'); + + task.status = 'delegated'; + await this.recordTaskToStore(task); + + const delegationResults = await this.handleDelegations(delegations, message); + + const feedbackPrompt = `The following delegation results are available:\n\n${delegationResults}\n\nSummarize the delegation results into a clear, user-friendly response. If a file was created, tell the user its path and a brief description. Be concise.`; + + // Emit synthesizing event for legacy delegation path + await progress?.({ type: 'synthesizing' }); + + this.state = 'processing'; + const feedbackOpts = this.buildMasterSpawnOptions( + feedbackPrompt, + undefined, + MESSAGE_MAX_TURNS_SYNTHESIS, + ); + result = await this.agentRunner.spawn(feedbackOpts); + await this.updateMasterSession(); + + if (result.exitCode !== 0) { + throw new Error(`Delegation feedback processing failed: ${result.stderr}`); + } + + response = result.stdout.trim() || delegationResults; + } + } + + // Guard: detect abnormally large final responses (likely the Master echoing its + // system prompt or documentation). Real user-facing responses rarely exceed 50K chars. + // Applied after SPAWN/delegation processing so worker prompts are unaffected. + const MAX_RESPONSE_CHARS = 50_000; + if (response.length > MAX_RESPONSE_CHARS) { + logger.warn( + { responseLength: response.length, maxAllowed: MAX_RESPONSE_CHARS }, + 'Master produced abnormally large response — likely echoed system prompt, truncating', + ); + const tail = response.slice(-2000).trim(); + if (tail.length > 50 && !tail.includes('${')) { + response = tail; + } else { + response = + 'I encountered an issue processing your request. Could you please rephrase or simplify it?'; + } + } + + // Update task record + task.status = 'completed'; + task.result = response; + task.completedAt = new Date().toISOString(); + task.durationMs = new Date(task.completedAt).getTime() - new Date(task.startedAt!).getTime(); + task.metadata = { + ...task.metadata, + model: result.model, + exitCode: result.exitCode, + maxTurns: maxTurnsToUse, + turnsUsed: result.turnsUsed, + }; + + await this.recordTaskToStore(task); + + // Record the Master AI response to conversation history (OB-730) + await this.recordConversationMessage(sessionId, 'master', response); + + // Advance batch state after successfully completing the current batch item (OB-1609). + if (this.batchManager && activeBatchId && activeBatchItemId) { + const normalized = response.replace(/\s+/g, ' ').trim(); + const summary = + normalized.length > 160 ? normalized.slice(0, 157) + '...' : normalized || 'Completed'; + try { + await this.batchManager.advanceBatch( + activeBatchId, + { + id: activeBatchItemId, + summary, + status: 'completed', + }, + result.costUsd ?? 0, + ); + } catch (err) { + logger.warn( + { batchId: activeBatchId, itemId: activeBatchItemId, err }, + 'Failed to advance batch state after item completion', + ); + } + } + + // Record classification feedback: task succeeded → turn budget was sufficient + void this.recordClassificationFeedback( + this.normalizeForCache(message.content), + true, + false, + result.turnsUsed, + ); + + // Increment completed task counter and trigger prompt evolution every 50 tasks (OB-734) + this.onTaskCompleted(); + + this.state = 'ready'; + this.activeMessage = null; + + logger.info( + { taskId, durationMs: task.durationMs, responseLength: response.length }, + 'Message processed successfully', + ); + + // (6) Emit complete event — processing finished, status bar can be hidden + await progress?.({ type: 'complete' }); + + // (7) Schedule batch continuation if a batch is active (OB-1613). + // The 2s delay ensures the current response is fully delivered to the user + // before the next batch item begins processing. + + // Capture the active batch ID before the isActive() check so we can detect + // completion transitions (batch was active → is now done) for OB-1618. + const preBatchId = this.batchManager?.getCurrentBatchId(); + + if (this.batchManager !== null && this.router !== null && this.batchManager.isActive()) { + const activeBatchId = this.batchManager.getCurrentBatchId(); + if (activeBatchId !== undefined) { + const batchSender = message.sender; + const batchSource = message.source; + const router = this.router; + + // (OB-1616) Track the original sender's connector so failure messages can be + // delivered even when subsequent CONTINUE messages use source='internal-batch'. + // Persisted to disk via BatchManager so routing survives process restarts (OB-1667). + if (batchSource !== 'internal-batch') { + this.batchManager.setSenderInfo(activeBatchId, { + sender: batchSender, + source: batchSource, + }); + } + + logger.info( + { batchId: activeBatchId, sender: batchSender }, + 'Batch active after message processing — scheduling continuation trigger in 2s', + ); + + // (OB-1614) Send progress message to user before the next item starts. + const progressMsg = this.batchManager.buildProgressMessage(activeBatchId); + if (progressMsg !== null) { + void router.sendDirect(batchSource, batchSender, progressMsg); + logger.info( + { batchId: activeBatchId, sender: batchSender }, + 'Batch progress message sent', + ); + } + + // (OB-1615) Spawn a git-commit worker when commitAfterEach is set. + // The commit worker runs before scheduling the next batch item so changes + // from the current item are committed before the next item begins. + const scheduleNext = (): void => { + const handle = setTimeout(() => { + this.batchTimers.delete(handle); + if (this.state === 'shutdown') return; + router.routeBatchContinuation(activeBatchId, batchSender).catch((err: unknown) => { + const msg = err instanceof Error ? err.message : String(err); + void this.batchManager?.pauseBatch(activeBatchId); + void router.sendDirect( + batchSource, + batchSender, + `Batch paused due to error: ${msg}`, + ); + logger.error( + { batchId: activeBatchId, err }, + 'routeBatchContinuation failed — batch paused (OB-1666)', + ); + }); + }, 2000); + this.batchTimers.add(handle); + }; + + if (this.batchManager.shouldCommitAfterEach(activeBatchId)) { + const commitPrompt = this.batchManager.buildCommitPrompt(activeBatchId); + if (commitPrompt !== null) { + void (async (): Promise => { + logger.info({ batchId: activeBatchId }, 'Spawning commit worker after batch item'); + try { + await this.agentRunner.spawn({ + prompt: commitPrompt, + workspacePath: this.workspacePath, + allowedTools: resolveProfile('code-edit'), + maxTurns: 3, + model: 'fast', + retries: 1, + }); + logger.info({ batchId: activeBatchId }, 'Commit worker completed'); + } catch (commitErr) { + logger.warn( + { batchId: activeBatchId, err: commitErr }, + 'Commit worker failed — continuing batch', + ); + } + scheduleNext(); + })(); + } else { + scheduleNext(); + } + } else { + scheduleNext(); + } + } + } else if ( + // (OB-1618) Batch just completed — the batch was active before processing but is now done. + // Send the completion summary to the original sender. + this.batchManager !== null && + this.router !== null && + preBatchId !== undefined && + !this.batchManager.isActive(preBatchId) + ) { + const completionSummary = this.batchManager.popCompletionSummary(); + if (completionSummary !== null) { + const senderInfo = this.batchManager.getSenderInfo(preBatchId); + if (senderInfo) { + void this.router.sendDirect(senderInfo.source, senderInfo.sender, completionSummary); + logger.info( + { batchId: preBatchId, sender: senderInfo.sender }, + 'Batch completion summary sent to user', + ); + } else { + // Fallback: send to the current message sender + void this.router.sendDirect(message.source, message.sender, completionSummary); + logger.info( + { batchId: preBatchId, sender: message.sender }, + 'Batch completion summary sent to current sender (no stored senderInfo)', + ); + } + this.batchManager.deleteSenderInfo(preBatchId); + } + } + + // Drain any messages that were queued while this message was being processed (OB-F229) + if (this.processingPendingMessages.length > 0 && this.router !== null) { + const queued = [...this.processingPendingMessages]; + this.processingPendingMessages = []; + logger.info({ count: queued.length }, 'Draining queued messages after processing'); + // Merge rapid-fire image+text messages from the same sender before routing (OB-1658) + const mergedQueue = this.mergeRapidFireImageMessages(queued); + for (const queuedMsg of mergedQueue) { + try { + this.state = 'ready'; + await this.router.route(queuedMsg); + } catch (err) { + logger.error({ err, messageId: queuedMsg.id }, 'Failed to process queued message'); + } + } + } + + return response; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + + // Update task record with error + task.status = 'failed'; + task.error = errorMessage; + task.completedAt = new Date().toISOString(); + task.durationMs = new Date(task.completedAt).getTime() - new Date(task.startedAt!).getTime(); + + await this.recordTaskToStore(task); + + // Record classification feedback: task failed — check if it was a timeout + const timedOut = + errorMessage.includes('SIGTERM') || + errorMessage.includes('SIGKILL') || + errorMessage.includes('timeout') || + errorMessage.includes('exit code 143') || + errorMessage.includes('exit code 137'); + void this.recordClassificationFeedback( + this.normalizeForCache(message.content), + false, + timedOut, + ); + + this.state = 'ready'; + this.activeMessage = null; + + logger.error({ err: error, taskId, sender: message.sender }, 'Message processing failed'); + + // Ensure complete event is always emitted so status bars are cleaned up + await progress?.({ type: 'complete' }); + + // Partial response on timeout — ensures user always gets feedback (OB-1663). + // Exit code 143 = SIGTERM (timeout). Instead of propagating to the DLQ silently, + // return a user-friendly message so the user knows what happened. + if (error instanceof AgentExhaustedError && error.lastExitCode === 143) { + const seconds = Math.round(error.durationMs / 1000); + const timeoutMsg = + `Your request timed out after ${seconds}s. ` + + `This usually happens with multi-step tasks. Try sending each step separately — ` + + `for example: "deploy the POS app" as one message, then "send me the link" as a follow-up.`; + logger.warn( + { taskId, sender: message.sender, durationMs: error.durationMs, exitCode: 143 }, + 'Master session timed out — returning user feedback instead of propagating to DLQ (OB-1663)', + ); + return timeoutMsg; + } + + throw error; + } + } + + /** + * Stream a message response, yielding chunks as they arrive. + * Uses the persistent Master session for conversation continuity. + */ + public async *streamMessage(message: InboundMessage): AsyncGenerator { + // Reset idle timer on new message + this.resetIdleTimer(); + + if (this.state !== 'ready') { + logger.warn( + { currentState: this.state, sender: message.sender }, + 'Cannot stream message: Master not ready', + ); + yield `The AI is currently ${this.state}. Please try again in a moment.`; + return; + } + + this.state = 'processing'; + this.activeMessage = message; + + const taskId = randomUUID(); + const startedAt = new Date().toISOString(); + + logger.info({ taskId, sender: message.sender, content: message.content }, 'Streaming message'); + + // Create task record + const task: TaskRecord = { + id: taskId, + userMessage: message.rawContent, + sender: message.sender, + description: message.content, + status: 'processing', + handledBy: 'master', + createdAt: startedAt, + startedAt, + metadata: { + messageId: message.id, + source: message.source, + }, + }; + + // Build a ProgressReporter that maps events to the connector's sendProgress() + const streamProgress = this.makeProgressReporter(message.source, message.sender); + + // Stable session ID for grouping all turns of this conversation (OB-730) + const streamSessionId = this.masterSession?.sessionId ?? taskId; + + try { + // Record the inbound user message to conversation history (OB-730) + await this.recordConversationMessage( + streamSessionId, + 'user', + message.content, + message.source, + message.sender, + ); + + // Check for status queries + if (this.isStatusQuery(message.content)) { + const status = await this.getStatus(); + this.state = 'ready'; + task.status = 'completed'; + task.result = status; + task.completedAt = new Date().toISOString(); + task.durationMs = + new Date(task.completedAt).getTime() - new Date(task.startedAt!).getTime(); + await this.recordTaskToStore(task); + yield status; + return; + } + + // Retrieve relevant past conversation history to enrich the Master's context (OB-731) + // and fetch learned patterns for system prompt enrichment (OB-735) + const [ + streamConversationContext, + streamLearnedPatternsContext, + streamWorkerNextStepsContext, + ] = await Promise.all([ + this.buildConversationContext(message.content, streamSessionId, message.sender), + this.buildLearnedPatternsContext(), + this.buildWorkerNextStepsContext(), + ]); + + // (1) Emit classifying event — AI is analyzing the message + await streamProgress?.({ type: 'classifying' }); + + // Classify message to determine appropriate turn budget and prompt + const streamClassification = await this.classifyTask(message.content, streamSessionId); + const streamTaskClass = streamClassification.class; + let streamPromptToSend = + streamTaskClass === 'complex-task' + ? this.buildPlanningPrompt(message.content) + : message.content; + // Prepend channel + role context header so Master can make channel-aware decisions (OB-1627) + streamPromptToSend = (await this.getMessageContextHeader(message)) + streamPromptToSend; + // complex-task always uses planning turns; otherwise use AI-suggested budget + const streamMaxTurns = + streamTaskClass === 'complex-task' + ? MESSAGE_MAX_TURNS_PLANNING + : streamClassification.maxTurns; + // Derive timeout from the actual turns used (complex-task overrides to planning turns) + const streamTimeoutToUse = + streamTaskClass === 'complex-task' + ? turnsToTimeout(MESSAGE_MAX_TURNS_PLANNING) + : streamClassification.timeout; + + if (streamTaskClass === 'complex-task') { + logger.info('Complex task — using planning prompt for auto-delegation (stream)'); + // (2) Emit planning event — Master is decomposing the task + await streamProgress?.({ type: 'planning' }); + } + + // Stream message through the persistent Master session (OB-1246: budget-aware assembly) + const streamContext: MasterContextSections = { + conversationContext: streamConversationContext, + learnedPatternsContext: streamLearnedPatternsContext, + workerNextStepsContext: streamWorkerNextStepsContext, + }; + const spawnOpts = this.buildMasterSpawnOptions( + streamPromptToSend, + streamTimeoutToUse, + streamMaxTurns, + streamContext, + ); + let fullResponse = ''; + const stream = this.agentRunner.stream(spawnOpts); + + let iterResult = await stream.next(); + while (!iterResult.done) { + const chunk = iterResult.value; + fullResponse += chunk; + yield chunk; + iterResult = await stream.next(); + } + const streamResult = iterResult.value; + await this.updateMasterSession(); + + // Detect dead session and restart transparently + if ( + streamResult.exitCode !== 0 && + this.isSessionDead(streamResult.exitCode, streamResult.stderr) + ) { + logger.warn( + { exitCode: streamResult.exitCode, stderr: streamResult.stderr.slice(0, 200) }, + 'Master session appears dead during streaming, attempting restart', + ); + + await this.restartMasterSession(); + + // Retry with the same prompt and context sections (OB-1246: budget-aware assembly) + const retryOpts = this.buildMasterSpawnOptions( + streamPromptToSend, + streamTimeoutToUse, + streamMaxTurns, + streamContext, + ); + fullResponse = ''; + const retryStream = this.agentRunner.stream(retryOpts); + + let retryIter = await retryStream.next(); + while (!retryIter.done) { + const chunk = retryIter.value; + fullResponse += chunk; + yield chunk; + retryIter = await retryStream.next(); + } + const retryResult = retryIter.value; + await this.updateMasterSession(); + + if (retryResult.exitCode !== 0) { + throw new Error(`Stream failed after restart: ${retryResult.stderr}`); + } + } else if (streamResult.exitCode !== 0) { + throw new Error(`Stream failed: ${streamResult.stderr}`); + } + + // Check for SPAWN markers first (richer task decomposition protocol) + if (hasSpawnMarkers(fullResponse)) { + const spawnResult = parseSpawnMarkers(fullResponse); + if (spawnResult.markers.length > 0) { + logger.info( + { spawnCount: spawnResult.markers.length }, + 'SPAWN markers detected in stream', + ); + + task.status = 'delegated'; + await this.recordTaskToStore(task); + + const streamN = spawnResult.markers.length; + + // If the cleaned output (text outside SPAWN markers) is very short, prepare + // a status message to show the user instead of a near-empty stub response. + const streamCleanedOutput = spawnResult.cleanedOutput; + const streamOriginalLength = fullResponse.length; + const streamCleanedLength = streamCleanedOutput.length; + const streamSpawnSummaries = extractTaskSummaries(spawnResult.markers); + logger.debug( + { + originalLength: streamOriginalLength, + cleanedLength: streamCleanedLength, + spawnCount: streamN, + spawnSummaries: streamSpawnSummaries, + }, + 'SPAWN marker stripping applied', + ); + if (streamCleanedLength < 80 && streamOriginalLength > 200) { + logger.warn( + { + originalLength: streamOriginalLength, + cleanedLength: streamCleanedLength, + spawnCount: streamN, + }, + 'Response truncated after SPAWN marker removal — generating status message', + ); + } + let streamStatusMessage: string | undefined; + if (streamCleanedLength === 0) { + // Entire response was SPAWN markers — build a numbered summary from extracted prompts + const streamNumbered = streamSpawnSummaries.map((s, i) => `${i + 1}) ${s}`).join(', '); + streamStatusMessage = `I'm spawning ${streamN} worker${streamN === 1 ? '' : 's'}: ${streamNumbered}`; + } else if (streamCleanedLength < 80) { + streamStatusMessage = + `Working on your request — dispatching ${streamN} worker(s) for:\n` + + streamSpawnSummaries.map((s) => `• ${s}`).join('\n'); + } + + // (3) Emit spawning event — N workers are being created + await streamProgress?.({ type: 'spawning', workerCount: streamN }); + + // Callback that emits both worker-progress and worker-result events + const workerCallback = async ( + completed: number, + total: number, + workerResult?: AgentResult, + workerMarker?: ParsedSpawnMarker, + ): Promise => { + await streamProgress?.({ type: 'worker-progress', completed, total }); + + if (workerResult && workerMarker) { + const raw = + workerResult.exitCode === 0 + ? workerResult.stdout.trim() + : `Error: ${(workerResult.stderr || workerResult.stdout).trim().slice(0, 500)}`; + const maxLen = 2000; + const content = raw.length > maxLen ? raw.slice(0, maxLen) + '\n...(truncated)' : raw; + await streamProgress?.({ + type: 'worker-result', + workerIndex: completed, + total, + profile: workerMarker.profile, + tool: workerMarker.body.tool, + content, + success: workerResult.exitCode === 0, + durationMs: workerResult.durationMs, + turnsUsed: workerResult.turnsUsed, + }); + } + }; + + // Use progress-streaming variant if multiple workers are spawned + let feedbackPrompt: string; + if (spawnResult.markers.length > 1) { + // Stream progress updates as workers complete, also emitting worker-progress events + const progressGen = this.handleSpawnMarkersWithProgress( + spawnResult.markers, + workerCallback, + message.attachments, + streamTaskClass, + ); + let progressIter = await progressGen.next(); + while (!progressIter.done) { + const progressChunk = progressIter.value; + yield progressChunk; + progressIter = await progressGen.next(); + } + feedbackPrompt = progressIter.value; + } else { + // Single worker — emit worker-progress on completion + feedbackPrompt = await this.handleSpawnMarkers( + spawnResult.markers, + workerCallback, + message.attachments, + streamTaskClass, + ); + } + + // (5) Emit synthesizing event — Master is combining worker results + await streamProgress?.({ type: 'synthesizing' }); + + // Inject worker results back into the Master session (streamed) + this.state = 'processing'; + const feedbackOpts = this.buildMasterSpawnOptions( + feedbackPrompt, + undefined, + MESSAGE_MAX_TURNS_SYNTHESIS, + ); + const feedbackStream = this.agentRunner.stream(feedbackOpts); + + let finalResponse = ''; + let feedbackIter = await feedbackStream.next(); + while (!feedbackIter.done) { + const chunk = feedbackIter.value; + finalResponse += chunk; + yield chunk; + feedbackIter = await feedbackStream.next(); + } + await this.updateMasterSession(); + + const streamFinalResponse = finalResponse.trim(); + // Use status message only if synthesis produced no content at all + fullResponse = + streamFinalResponse.length > 0 + ? streamFinalResponse + : (streamStatusMessage ?? feedbackPrompt); + } + } + + // Check for legacy delegation markers (fallback) + if (!hasSpawnMarkers(fullResponse)) { + const delegations = this.parseDelegationMarkers(fullResponse); + if (delegations && delegations.length > 0) { + logger.info( + { delegationCount: delegations.length }, + 'Delegation markers detected in stream', + ); + + task.status = 'delegated'; + await this.recordTaskToStore(task); + + const delegationResults = await this.handleDelegations(delegations, message); + + const feedbackPrompt = `The following delegation results are available:\n\n${delegationResults}\n\nSummarize the delegation results into a clear, user-friendly response. If a file was created, tell the user its path and a brief description. Be concise.`; + + // Emit synthesizing event for legacy delegation path + await streamProgress?.({ type: 'synthesizing' }); + + this.state = 'processing'; + const feedbackOpts = this.buildMasterSpawnOptions( + feedbackPrompt, + undefined, + MESSAGE_MAX_TURNS_SYNTHESIS, + ); + const feedbackStream = this.agentRunner.stream(feedbackOpts); + + let finalResponse = ''; + let feedbackIter = await feedbackStream.next(); + while (!feedbackIter.done) { + const chunk = feedbackIter.value; + finalResponse += chunk; + yield chunk; + feedbackIter = await feedbackStream.next(); + } + await this.updateMasterSession(); + + fullResponse = finalResponse.trim() || delegationResults; + } + } + + // Update task record + task.status = 'completed'; + task.result = fullResponse.trim() || 'No response from AI'; + task.completedAt = new Date().toISOString(); + task.durationMs = new Date(task.completedAt).getTime() - new Date(task.startedAt!).getTime(); + task.metadata = { + ...task.metadata, + turnsUsed: streamResult.turnsUsed, + }; + + await this.recordTaskToStore(task); + + // Record the Master AI response to conversation history (OB-730) + await this.recordConversationMessage(streamSessionId, 'master', task.result); + + // Increment completed task counter and trigger prompt evolution every 50 tasks (OB-734) + this.onTaskCompleted(); + + this.state = 'ready'; + this.activeMessage = null; + + logger.info( + { taskId, durationMs: task.durationMs, responseLength: fullResponse.length }, + 'Message streamed successfully', + ); + + // (6) Emit complete event — processing finished, status bar can be hidden + await streamProgress?.({ type: 'complete' }); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + + // Update task record with error + task.status = 'failed'; + task.error = errorMessage; + task.completedAt = new Date().toISOString(); + task.durationMs = new Date(task.completedAt).getTime() - new Date(task.startedAt!).getTime(); + + await this.recordTaskToStore(task); + + this.state = 'ready'; + this.activeMessage = null; + + logger.error({ err: error, taskId, sender: message.sender }, 'Message streaming failed'); + + // Ensure complete event is always emitted so status bars are cleaned up + await streamProgress?.({ type: 'complete' }); + + yield `Error: ${errorMessage}`; + } + } + + /** + * Get system status + */ + public async getStatus(): Promise { + const map = await this.readWorkspaceMapFromStore(); + const tasks = await this.readAllTasksFromStore(); + + const completedTasks = tasks.filter((t) => t.status === 'completed').length; + const failedTasks = tasks.filter((t) => t.status === 'failed').length; + const processingTasks = tasks.filter( + (t) => t.status === 'processing' || t.status === 'delegated', + ).length; + + let status = `**OpenBridge Master AI Status**\n\n`; + status += `State: ${this.state}\n`; + + // Show Master session info + if (this.masterSession) { + status += `Master Session: ${this.masterSession.sessionId}\n`; + status += `Session Messages: ${this.masterSession.messageCount}\n`; + if (this.restartCount > 0) { + status += `Session Restarts: ${this.restartCount}\n`; + } + } + + // Show exploration status + if (this.state === 'exploring') { + status += `\nExploration: in progress (Master-driven)\n`; + } else if (this.explorationSummary) { + status += `Exploration: ${this.explorationSummary.status}\n`; + if (this.explorationSummary.projectType) { + status += `Project Type: ${this.explorationSummary.projectType}\n`; + } + if (this.explorationSummary.frameworks.length > 0) { + status += `Frameworks: ${this.explorationSummary.frameworks.join(', ')}\n`; + } + } + + if (map) { + status += `\nWorkspace: ${map.projectName}\n`; + status += `Summary: ${map.summary}\n`; + } + + status += `\nTasks: ${completedTasks} completed, ${failedTasks} failed, ${tasks.length} total\n`; + + // Show active delegations if any + const activeDelegations = this.delegationCoordinator.getActiveDelegations(); + if (activeDelegations.length > 0) { + status += `\nActive Delegations (${activeDelegations.length}):\n`; + for (const delegation of activeDelegations) { + const elapsed = Date.now() - new Date(delegation.startedAt).getTime(); + const elapsedSeconds = Math.floor(elapsed / 1000); + status += ` - ${delegation.tool.name}: ${delegation.task.description.slice(0, 60)}... (${elapsedSeconds}s)\n`; + } + } + + // Show processing tasks if any + if (processingTasks > 0) { + status += `\nProcessing: ${processingTasks} task(s) in progress\n`; + } + + // Show exploration progress table from memory (OB-894) + if (this.memory) { + try { + const progressRows = await this.memory.getExplorationProgress(); + if (progressRows.length > 0) { + status += `\nExploration Progress:\n`; + for (const row of progressRows) { + const label = row.target ? `${row.phase} (${row.target})` : row.phase; + const pct = Math.round(row.progress_pct); + const bar = '█'.repeat(Math.floor(pct / 10)) + '░'.repeat(10 - Math.floor(pct / 10)); + status += ` ${label}: ${bar} ${pct}% [${row.status}]\n`; + } + } + } catch { + // memory not available — skip + } + } + + return status; + } + + /** + * Log the current RAG health status (FTS5 chunk count) for startup diagnostics. + * OB-1571: Provides visibility into whether the FTS5 index has content so that + * operators can identify an empty RAG index before it causes silent failures. + */ + private async logRagHealthDiagnostic(): Promise { + if (!this.memory) return; + try { + const chunkCount = await this.memory.countChunks(); + logger.info({ chunkCount }, 'RAG startup diagnostic: FTS5 chunk store status'); + if (chunkCount === 0) { + logger.warn('RAG has no indexed chunks — retrieval will return empty results'); + } + } catch (err) { + logger.warn({ err }, 'RAG startup diagnostic: failed to count FTS5 chunks'); + } + } + + /** + * Start idle detection timer for self-improvement cycle (OB-173). + * Checks every minute whether the Master has been idle for >5 minutes. + * If idle, triggers a self-improvement cycle. + */ + private startIdleDetection(): void { + // Stop any existing timer + if (this.idleCheckTimer) { + clearInterval(this.idleCheckTimer); + } + + // Set initial timestamp + this.lastMessageTimestamp = Date.now(); + + // Start periodic idle check + this.idleCheckTimer = setInterval(() => { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + this.checkIdleAndImprove(); + }, IDLE_CHECK_INTERVAL_MS); + + logger.info('Idle detection timer started for self-improvement cycle'); + } + + /** + * Stop idle detection timer (called on shutdown). + */ + private stopIdleDetection(): void { + if (this.idleCheckTimer) { + clearInterval(this.idleCheckTimer); + this.idleCheckTimer = null; + logger.info('Idle detection timer stopped'); + } + } + + /** + * Check if Master is idle and trigger self-improvement if needed. + * Called periodically by the idle detection timer. + * + * Uses exponential backoff: first cycle fires after 5 min idle, + * subsequent cycles double the threshold (10m, 20m, 40m, ...) up to 2h. + * Resets to 5 min when a user message arrives (via resetIdleTimer). + */ + private async checkIdleAndImprove(): Promise { + // Skip if: + // - Already running self-improvement + // - Not in ready state + // - No message timestamp yet + if (this.isSelfImproving || this.state !== 'ready' || !this.lastMessageTimestamp) { + return; + } + + const idleTime = Date.now() - this.lastMessageTimestamp; + + // Exponential backoff: threshold doubles each cycle, capped at IDLE_THRESHOLD_MAX_MS + const currentThreshold = Math.min( + IDLE_THRESHOLD_MS * Math.pow(2, this.consecutiveIdleCycles), + IDLE_THRESHOLD_MAX_MS, + ); + + // Check if idle threshold exceeded + if (idleTime >= currentThreshold) { + this.consecutiveIdleCycles++; + + // Suppress self-improvement after 2 consecutive no-ops (OB-F210) + if (this.consecutiveNoOpCycles >= 2) { + logger.debug( + 'Self-improvement paused: 2 consecutive no-op cycles — waiting for next user message', + ); + return; + } + + logger.info( + { + idleTimeMs: idleTime, + cycle: this.consecutiveIdleCycles, + nextThresholdMs: Math.min( + IDLE_THRESHOLD_MS * Math.pow(2, this.consecutiveIdleCycles), + IDLE_THRESHOLD_MAX_MS, + ), + }, + 'Idle threshold exceeded, starting self-improvement cycle', + ); + + try { + const workDone = await this.runSelfImprovementCycle(); + if (workDone) { + this.consecutiveNoOpCycles = 0; + } else { + this.consecutiveNoOpCycles++; + } + } catch (error) { + logger.error({ err: error }, 'Self-improvement cycle failed'); + this.consecutiveNoOpCycles++; + } + + // Reset last message timestamp to prevent immediate re-trigger + this.lastMessageTimestamp = Date.now(); + } + } + + /** + * Run the self-improvement cycle (OB-173). + * Reviews learnings and performs improvements: + * 1. Update prompts with low success rates + * 2. Create new custom profiles for recurring task patterns + * 3. Update workspace-map.json if project has changed + */ + private async runSelfImprovementCycle(): Promise { + if (this.isSelfImproving) { + logger.warn('Self-improvement cycle already running'); + return false; + } + + this.isSelfImproving = true; + const startedAt = new Date().toISOString(); + + logger.info('Starting self-improvement cycle'); + + let workDone = false; + + try { + if (this.memory) { + await this.memory.logExploration({ + timestamp: startedAt, + level: 'info', + message: 'Self-improvement cycle started', + data: {}, + }); + } else { + await this.dotFolder.appendLog({ + timestamp: startedAt, + level: 'info', + message: 'Self-improvement cycle started', + data: {}, + }); + } + + // Task 0: Detect degraded prompts (rewrites that made things worse) and rollback + const rollbackCount = await this.rollbackDegradedPrompts(); + + // Task 1: Identify and rewrite low-performing prompts via dot-folder manifest + const lowPerformingPrompts = await this.dotFolder.getLowPerformingPrompts(0.5); + if (lowPerformingPrompts.length > 0) { + logger.info( + { promptCount: lowPerformingPrompts.length }, + 'Found low-performing prompts to rewrite', + ); + + for (const prompt of lowPerformingPrompts) { + await this.rewritePrompt(prompt); + } + } + + // Task 2: Analyze learnings for recurring task patterns and create custom profiles + const profileCreated = await this.createProfilesFromLearnings(); + + // Task 3: Check if workspace has changed and update map if needed + const workspaceChanged = await this.updateWorkspaceMapIfChanged(); + + // Determine if any productive work was done (OB-F210) + workDone = + rollbackCount > 0 || lowPerformingPrompts.length > 0 || profileCreated || workspaceChanged; + + if (this.memory) { + await this.memory.logExploration({ + timestamp: new Date().toISOString(), + level: 'info', + message: 'Self-improvement cycle completed', + data: { + rollbackCount, + lowPerformingPrompts: lowPerformingPrompts.length, + profileCreated, + workspaceChanged, + workDone, + durationMs: new Date().getTime() - new Date(startedAt).getTime(), + }, + }); + } else { + await this.dotFolder.appendLog({ + timestamp: new Date().toISOString(), + level: 'info', + message: 'Self-improvement cycle completed', + data: { + rollbackCount, + lowPerformingPrompts: lowPerformingPrompts.length, + profileCreated, + workspaceChanged, + workDone, + durationMs: new Date().getTime() - new Date(startedAt).getTime(), + }, + }); + } + + logger.info({ workDone }, 'Self-improvement cycle completed'); + } catch (error) { + logger.error({ err: error }, 'Self-improvement cycle encountered an error'); + + if (this.memory) { + await this.memory.logExploration({ + timestamp: new Date().toISOString(), + level: 'error', + message: 'Self-improvement cycle failed', + data: { error: error instanceof Error ? error.message : String(error) }, + }); + } else { + await this.dotFolder.appendLog({ + timestamp: new Date().toISOString(), + level: 'error', + message: 'Self-improvement cycle failed', + data: { error: error instanceof Error ? error.message : String(error) }, + }); + } + } finally { + this.isSelfImproving = false; + } + + return workDone; + } + + /** + * Rewrite a low-performing prompt using the Master AI session. + * Asks the Master to analyze the prompt's failure patterns and suggest improvements. + */ + private async rewritePrompt(prompt: PromptTemplate): Promise { + logger.info( + { promptId: prompt.id, successRate: prompt.successRate, usageCount: prompt.usageCount }, + 'Rewriting low-performing prompt', + ); + + try { + // Read the current prompt content — prefer DB, fall back to file + let currentContent: string; + if (this.memory) { + try { + const dbRecord = await this.memory.getActivePrompt(prompt.id); + currentContent = dbRecord.content; + } catch { + const promptPath = path.join( + this.dotFolder.getDotFolderPath(), + 'prompts', + prompt.filePath, + ); + currentContent = await fs.readFile(promptPath, 'utf-8'); + } + } else { + const promptPath = path.join(this.dotFolder.getDotFolderPath(), 'prompts', prompt.filePath); + currentContent = await fs.readFile(promptPath, 'utf-8'); + } + + // Build a self-improvement prompt for the Master + const improvementPrompt = `You are reviewing your own prompt templates for effectiveness. + +The following prompt template has a low success rate and needs to be rewritten: + +**Prompt ID:** ${prompt.id} +**Description:** ${prompt.description} +**Success Rate:** ${(prompt.successRate ?? 0) * 100}% (${prompt.successCount}/${prompt.usageCount} uses) +**Current Content:** +\`\`\` +${currentContent} +\`\`\` + +**Task:** Rewrite this prompt to improve its effectiveness. Focus on: +1. Clarity of instructions +2. Explicit output format requirements +3. Error handling guidance +4. Context that helps the worker succeed + +**Output Format:** Return ONLY the rewritten prompt content (no explanations, no markdown fences, just the raw prompt text).`; + + const spawnOpts = this.buildMasterSpawnOptions(improvementPrompt, this.messageTimeout); + const result = await this.agentRunner.spawn(spawnOpts); + await this.updateMasterSession(); + + if (result.exitCode !== 0) { + logger.warn({ promptId: prompt.id, exitCode: result.exitCode }, 'Failed to rewrite prompt'); + return; + } + + const rewrittenContent = result.stdout.trim(); + + if (rewrittenContent.length === 0) { + logger.warn({ promptId: prompt.id }, 'Master returned empty prompt rewrite'); + return; + } + + // Store the previous version content for rollback before overwriting + const manifest = this.memory + ? await this.memory.getPromptManifest() + : await this.dotFolder.readPromptManifest(); + const existingEntry = manifest?.prompts[prompt.id]; + if (manifest && existingEntry) { + existingEntry.previousVersion = currentContent; + manifest.updatedAt = new Date().toISOString(); + if (this.memory) { + await this.memory.setPromptManifest(manifest); + } else { + await this.dotFolder.writePromptManifest(manifest); + } + } + + // Update the prompt — write to DB (primary) or file (fallback) + if (this.memory) { + await this.memory.createPromptVersion(prompt.id, rewrittenContent); + } else { + const promptPath = path.join(this.dotFolder.getDotFolderPath(), 'prompts', prompt.filePath); + await fs.writeFile(promptPath, rewrittenContent, 'utf-8'); + } + + // Reset the prompt's usage stats (fresh start with new version) + await this.dotFolder.resetPromptStats(prompt.id); + + logger.info({ promptId: prompt.id }, 'Successfully rewrote prompt'); + } catch (error) { + logger.error({ err: error, promptId: prompt.id }, 'Failed to rewrite prompt (non-blocking)'); + } + } + + /** + * Detect prompts where a recent rewrite made performance worse, and rollback + * to the previous version. A prompt is "degraded" when: + * - It has a previousVersion stored (was rewritten) + * - It has a previousSuccessRate recorded + * - Its current successRate < previousSuccessRate + * - It has been used 5+ times since the rewrite (enough signal) + */ + private async rollbackDegradedPrompts(): Promise { + const manifest = this.memory + ? await this.memory.getPromptManifest() + : await this.dotFolder.readPromptManifest(); + if (!manifest) return 0; + + let rollbackCount = 0; + + for (const prompt of Object.values(manifest.prompts)) { + if ( + prompt.previousVersion && + prompt.previousSuccessRate !== undefined && + prompt.usageCount >= 5 && + (prompt.successRate ?? 0) < prompt.previousSuccessRate + ) { + logger.warn( + { + promptId: prompt.id, + currentRate: prompt.successRate, + previousRate: prompt.previousSuccessRate, + }, + 'Degraded prompt detected — rolling back to previous version', + ); + + try { + if (!this.memory) { + const promptPath = path.join( + this.dotFolder.getDotFolderPath(), + 'prompts', + prompt.filePath, + ); + + // Restore the previous version content to disk (no memory available) + await fs.writeFile(promptPath, prompt.previousVersion, 'utf-8'); + } else { + // Restore version through DB when memory is available + await this.memory.createPromptVersion(prompt.id, prompt.previousVersion); + } + + // Restore previous success rate and clear rollback fields + prompt.successRate = prompt.previousSuccessRate; + prompt.usageCount = 0; + prompt.successCount = 0; + prompt.previousVersion = undefined; + prompt.previousSuccessRate = undefined; + prompt.updatedAt = new Date().toISOString(); + + if (this.memory) { + await this.memory.setPromptManifest(manifest); + } else { + await this.dotFolder.writePromptManifest(manifest); + } + + logger.info({ promptId: prompt.id }, 'Successfully rolled back degraded prompt'); + rollbackCount++; + } catch (error) { + logger.error({ err: error, promptId: prompt.id }, 'Failed to rollback degraded prompt'); + } + } + } + + return rollbackCount; + } + + /** + * Analyze learnings to identify recurring task patterns and create custom profiles. + * For example: if "test-runner" tasks consistently succeed with specific tools, + * create a "test-runner" profile. + * Returns true if at least one profile was created. + */ + private async createProfilesFromLearnings(): Promise { + // Build a list of { taskType, successCount, failureCount, successRate } from either memory or JSON. + type TaskTypeStat = { + taskType: string; + successCount: number; + failureCount: number; + successRate: number; + }; + let taskTypeStats: TaskTypeStat[] = []; + + if (this.memory) { + try { + const rows = await this.memory.getLearnedTaskTypes(); + taskTypeStats = rows.map((r) => ({ + taskType: r.taskType, + successCount: r.successCount, + failureCount: r.failureCount, + successRate: r.successRate, + })); + } catch { + return false; + } + } else { + const learnings = await this.dotFolder.readLearnings(); + if (!learnings || learnings.entries.length < 10) { + return false; + } + logger.info( + { learningCount: learnings.entries.length }, + 'Analyzing learnings for profile patterns', + ); + const byTaskType = new Map(); + for (const entry of learnings.entries) { + const existing = byTaskType.get(entry.taskType) ?? []; + existing.push(entry); + byTaskType.set(entry.taskType, existing); + } + for (const [taskType, entries] of byTaskType) { + const successCount = entries.filter((e) => e.success).length; + taskTypeStats.push({ + taskType, + successCount, + failureCount: entries.length - successCount, + successRate: successCount / entries.length, + }); + } + } + + const totalEntries = taskTypeStats.reduce((s, r) => s + r.successCount + r.failureCount, 0); + if (totalEntries < 10) { + return false; + } + + let profileCreated = false; + + // Look for task types with >5 total executions and >70% success rate + for (const stat of taskTypeStats) { + const total = stat.successCount + stat.failureCount; + if (total < 5) continue; + if (stat.successRate < 0.7) continue; + + // Check if a profile already exists for this task type + const existingProfiles = await this.readProfilesFromStore(); + const profileId = `auto-${stat.taskType}`; + if (existingProfiles?.profiles[profileId]) continue; + + // Default to 'code-edit' profile for auto-generated profiles + const baseProfileName = 'code-edit'; + const builtInProfile = BUILT_IN_PROFILES[baseProfileName as keyof typeof BUILT_IN_PROFILES]; + if (!builtInProfile) continue; + + logger.info( + { + taskType: stat.taskType, + profileId, + successRate: stat.successRate, + totalExecutions: total, + }, + 'Creating custom profile from learning patterns', + ); + + const newProfile: ToolProfile = { + name: profileId, + description: `Auto-generated profile for ${stat.taskType} tasks (success rate: ${(stat.successRate * 100).toFixed(1)}%)`, + tools: [...builtInProfile.tools], + }; + + try { + await this.dotFolder.addProfile(newProfile); + if (this.memory) { + const updated = await this.dotFolder.readProfiles(); + if (updated) { + await this.memory.setSystemConfig('profiles', JSON.stringify(updated)); + } + } + logger.info({ profileId }, 'Successfully created custom profile from learnings'); + profileCreated = true; + } catch (error) { + logger.error({ err: error, profileId }, 'Failed to create custom profile (non-blocking)'); + } + } + + return profileCreated; + } + + /** + * Check if the workspace has changed significantly and update workspace-map.json if needed. + * Detects changes by checking for new files, modified package.json, new directories, etc. + * Returns true if workspace changed and was updated. + */ + private async updateWorkspaceMapIfChanged(): Promise { + const map = await this.readWorkspaceMapFromStore(); + if (!map) { + // No map to update + return false; + } + + logger.info('Checking if workspace has changed significantly'); + + // Check for significant changes: + // 1. New top-level directories + // 2. package.json modifications (dependencies changed) + // 3. New frameworks detected + + try { + const packageJsonPath = path.join(this.workspacePath, 'package.json'); + let hasPackageJsonChanged = false; + + try { + const stats = await fs.stat(packageJsonPath); + const mapGeneratedTime = new Date(map.generatedAt).getTime(); + const packageModifiedTime = stats.mtimeMs; + + hasPackageJsonChanged = packageModifiedTime > mapGeneratedTime; + } catch { + // package.json doesn't exist or can't be read + hasPackageJsonChanged = false; + } + + if (hasPackageJsonChanged) { + logger.info( + 'package.json has changed since last map generation, triggering re-exploration', + ); + await this.reExplore(); + return true; + } + + return false; + } catch (error) { + logger.error({ err: error }, 'Failed to check workspace changes (non-blocking)'); + return false; + } + } + + /** + * Gracefully shut down the Master AI + */ + public async shutdown(): Promise { + if (this.state === 'shutdown') { + return; + } + + logger.info('Shutting down MasterManager'); + + this.state = 'shutdown'; + + // Stop idle detection timer + this.stopIdleDetection(); + + // Clear all pending batch continuation timers (OB-1665) + for (const handle of this.batchTimers) { + clearTimeout(handle); + } + this.batchTimers.clear(); + + // Shutdown delegation coordinator + this.delegationCoordinator.shutdown(); + + // Persist Master session first (fast, <100ms SQLite write) — critical data + if (this.masterSession) { + try { + await this.saveMasterSessionToStore(this.masterSession); + } catch (error) { + logger.warn({ error }, 'Failed to persist Master session on shutdown'); + } + } + + // Trigger a memory update after session state is saved so the Master + // can persist what it learned in this session (OB-1023). + // Runs after saveMasterSessionToStore() so a timeout cannot lose session state. + if (this.sessionInitialized && this.masterSession && this.masterSession.messageCount > 0) { + try { + await this.triggerMemoryUpdate(); + } catch (err) { + logger.warn({ err }, 'Memory update on shutdown failed — continuing'); + } + } + + // Close active sessions after memory is saved (OB-1605) + if (this.memory) { + try { + await this.memory.closeActiveSessions(); + logger.info('Closed active sessions on shutdown'); + } catch (error) { + logger.warn({ error }, 'Failed to close active sessions on shutdown — continuing'); + } + } + + // Log shutdown + try { + if (this.memory) { + await this.memory.logExploration({ + timestamp: new Date().toISOString(), + level: 'info', + message: 'Master AI shutting down', + }); + } else { + await this.dotFolder.appendLog({ + timestamp: new Date().toISOString(), + level: 'info', + message: 'Master AI shutting down', + }); + } + } catch (error) { + logger.error({ err: error }, 'Failed to log shutdown'); + } + + logger.info('MasterManager shutdown complete'); + } + + /** + * Return the default maxTurns for a worker based on its profile. + * + * Profile-based defaults ensure workers have enough room to complete their + * tasks without being cut off mid-execution: + * - code-edit / full-access: 15 turns (need to read context + write files) + * - read-only: 10 turns (exploration only, no file writes) + * - other / unknown: DEFAULT_MAX_TURNS_TASK (25) + */ + private defaultMaxTurnsForProfile(profile: string): number { + if (profile === 'code-edit' || profile === 'full-access') return 15; + if (profile === 'read-only') return 10; + return DEFAULT_MAX_TURNS_TASK; + } + + /** + * Return the maximum allowed turns for a worker based on its profile (OB-1677). + * + * Profile-specific caps prevent runaway workers while giving complex tasks + * enough headroom to complete: + * - read-only: 25 turns + * - code-edit / full-access: 40 turns + * - other / unknown: 50 turns + */ + private maxTurnsCapForProfile(profile: string): number { + if (profile === 'read-only') return 25; + if (profile === 'code-edit' || profile === 'full-access') return 40; + return 50; + } + + /** + * Compute adaptive max-turns for a worker based on profile baseline + prompt heuristics (OB-902, OB-1677). + * + * If the SPAWN marker explicitly set maxTurns, that value is used directly (caller's + * responsibility). This method only computes the fallback value when maxTurns is absent. + * + * Formula: + * 1. baselineTurns (profile-based) + * 2. + ceil(promptLength / 1000) — scales with prompt complexity + * 3. + 5 if promptLength > 200 — longer tasks need more room (OB-1677) + * 4. + 10 if prompt contains "thorough", "comprehensive", or "detailed" (OB-1677) + * 5. capped at maxTurnsCapForProfile(profile) + * + * Examples (code-edit, baseline 15, cap 40): + * 200-char prompt, no keywords → 15 + 1 = 16 turns + * 500-char prompt, no keywords → 15 + 1 + 5 = 21 turns + * 500-char prompt + "thorough" → 15 + 1 + 5 + 10 = 31 turns + */ + private computeAdaptiveMaxTurns(profile: string, prompt: string): number { + const baselineTurns = this.defaultMaxTurnsForProfile(profile); + const profileCap = this.maxTurnsCapForProfile(profile); + + const promptExtra = Math.ceil(prompt.length / 1000); + + // OB-1677: Add 5 turns for prompts longer than 200 chars (more context = more work). + const longPromptExtra = prompt.length > 200 ? 5 : 0; + + // OB-1677: Add 10 turns when the task explicitly requests thoroughness. + const THOROUGHNESS_KEYWORDS = ['thorough', 'comprehensive', 'detailed']; + const lowerPrompt = prompt.toLowerCase(); + const keywordExtra = THOROUGHNESS_KEYWORDS.some((kw) => lowerPrompt.includes(kw)) ? 10 : 0; + + const adaptive = Math.min( + baselineTurns + promptExtra + longPromptExtra + keywordExtra, + profileCap, + ); + + logger.debug( + { + profile, + baselineTurns, + profileCap, + promptLength: prompt.length, + promptExtra, + longPromptExtra, + keywordExtra, + adaptive, + }, + 'Computed adaptive max-turns for worker', + ); + return adaptive; + } + + /** + * Handle SPAWN markers found in Master output. + * Spawns worker agents via AgentRunner based on parsed task manifests, + * collects results, and returns a structured feedback prompt for injection + * into the Master session. + * + * Workers are tracked in the WorkerRegistry with full lifecycle management: + * pending → running → completed/failed. The registry enforces concurrency + * limits and persists to .openbridge/workers.json for cross-restart visibility. + * + * Worker results include metadata (model, profile, duration, exit code) + * so the Master can reason about what happened and synthesize a response. + * + * @param onProgress - Optional callback invoked after each worker completes. + * Receives (completedCount, totalCount) so the caller can send progress updates. + */ + private async handleSpawnMarkers( + markers: ParsedSpawnMarker[], + onProgress?: ( + completed: number, + total: number, + result?: AgentResult, + marker?: ParsedSpawnMarker, + ) => Promise, + attachments?: InboundMessage['attachments'], + taskClass?: string, + ): Promise { + // Load custom profiles once for all workers + const customProfilesRegistry = await this.readProfilesFromStore(); + const customProfiles = customProfilesRegistry?.profiles; + + // Register all workers in the registry BEFORE spawning + // This checks concurrency limits and creates worker records + const workerIds: string[] = []; + const workerManifests = markers.map((marker) => ({ + prompt: marker.body.prompt, + workspacePath: this.workspacePath, + profile: marker.profile, + model: marker.body.model, + maxTurns: marker.body.maxTurns ?? this.defaultMaxTurnsForProfile(marker.profile), + timeout: marker.body.timeout ?? DEFAULT_WORKER_TIMEOUT, + retries: marker.body.retries, + maxBudgetUsd: marker.body.maxBudgetUsd, + })); + + for (const manifest of workerManifests) { + try { + const workerId = this.workerRegistry.addWorker(manifest); + workerIds.push(workerId); + } catch { + // Max concurrency reached — wait for a slot to free up (backpressure) + logger.info( + { runningCount: this.workerRegistry.getRunningCount() }, + 'Concurrency limit reached — waiting for a worker slot', + ); + try { + await this.workerRegistry.waitForSlot(); + // Slot freed — retry registration + const workerId = this.workerRegistry.addWorker(manifest); + workerIds.push(workerId); + } catch (waitError) { + // Timeout waiting for slot — skip this worker + logger.warn( + { error: waitError instanceof Error ? waitError.message : String(waitError) }, + 'Timed out waiting for worker slot — skipping worker', + ); + workerIds.push(''); + } + } + } + + // Persist registry after adding workers + await this.persistWorkerRegistry(); + + // Count only workers that were actually registered (not skipped) + const total = workerIds.filter((id) => id !== '').length; + let completedCount = 0; + + // Spawn all workers concurrently via Promise.allSettled + const workerPromises = markers.map((marker, index) => { + const workerId = workerIds[index]; + if (!workerId) { + // Worker was skipped due to concurrency limit + return Promise.resolve({ + exitCode: 1, + stdout: '', + stderr: 'Worker skipped: concurrency limit reached', + durationMs: 0, + retryCount: 0, + } as AgentResult); + } + const workerPromise = this.spawnWorker(workerId, marker, index, customProfiles, attachments); + if (onProgress) { + return workerPromise.then(async (result) => { + completedCount++; + await onProgress(completedCount, total, result, marker); + return result; + }); + } + return workerPromise; + }); + + const settled = await Promise.allSettled(workerPromises); + + // Persist registry after all workers complete + await this.persistWorkerRegistry(); + + // Log aggregated worker batch stats for observability + const stats = this.workerRegistry.getAggregatedStats(); + logger.info(stats, 'Worker batch stats'); + + // Record task efficiency metrics for escalation suppression (OB-1572) + if (this.memory && taskClass) { + this.memory + .recordTaskEfficiency(taskClass, { + turnsUsed: stats.totalTurnsUsed ?? 0, + workerCount: stats.totalWorkers, + durationMs: stats.avgDurationMs * stats.totalWorkers, + }) + .catch((err) => logger.warn({ err, taskClass }, 'Failed to record task efficiency')); + } + + // Format all results with structured metadata and build the feedback prompt + const { feedbackPrompt, observations, workerSummaries } = formatWorkerBatch( + settled, + markers, + workerIds, + this.masterSession?.sessionId, + ); + + // Persist extracted observations (fire-and-forget — don't block the response) + if (observations.length > 0 && this.memory) { + Promise.all(observations.map((obs) => this.memory!.insertObservation(obs))).catch((err) => + logger.warn({ err }, 'Failed to store worker observations'), + ); + } + + // Append learned items from worker summaries to memory.md (OB-1636) + if (workerSummaries.length > 0) { + this.dotFolder + .appendLearnedToMemory(workerSummaries) + .catch((err) => logger.warn({ err }, 'Failed to append learned items to memory.md')); + } + + return feedbackPrompt; + } + + /** + * Handle SPAWN markers with progress streaming. + * Yields progress updates as workers complete, allowing the user to see + * real-time status (e.g., "Working on it... (3/5 subtasks done)"). + * + * Returns the final feedback prompt after all workers complete. + */ + private async *handleSpawnMarkersWithProgress( + markers: ParsedSpawnMarker[], + onProgress?: ( + completed: number, + total: number, + result?: AgentResult, + marker?: ParsedSpawnMarker, + ) => Promise, + attachments?: InboundMessage['attachments'], + taskClass?: string, + ): AsyncGenerator { + // Load custom profiles once for all workers + const customProfilesRegistry = await this.readProfilesFromStore(); + const customProfiles = customProfilesRegistry?.profiles; + + // Register all workers in the registry BEFORE spawning + const workerIds: string[] = []; + const workerManifests = markers.map((marker) => ({ + prompt: marker.body.prompt, + workspacePath: this.workspacePath, + profile: marker.profile, + model: marker.body.model, + maxTurns: marker.body.maxTurns ?? this.defaultMaxTurnsForProfile(marker.profile), + timeout: marker.body.timeout ?? DEFAULT_WORKER_TIMEOUT, + retries: marker.body.retries, + maxBudgetUsd: marker.body.maxBudgetUsd, + })); + + for (const manifest of workerManifests) { + try { + const workerId = this.workerRegistry.addWorker(manifest); + workerIds.push(workerId); + } catch (error) { + logger.warn( + { error: error instanceof Error ? error.message : String(error) }, + 'Failed to register worker (concurrency limit reached)', + ); + workerIds.push(''); + } + } + + await this.persistWorkerRegistry(); + + // Yield initial progress message + const totalWorkers = workerIds.filter((id) => id !== '').length; + yield `\n\n_[Starting ${totalWorkers} parallel subtasks...]_\n`; + + let progressCompletedCount = 0; + + // Spawn all workers concurrently + const workerPromises = markers.map((marker, index) => { + const workerId = workerIds[index]; + if (!workerId) { + return Promise.resolve({ + exitCode: 1, + stdout: '', + stderr: 'Worker skipped: concurrency limit reached', + durationMs: 0, + retryCount: 0, + } as AgentResult); + } + const workerPromise = this.spawnWorker(workerId, marker, index, customProfiles, attachments); + if (onProgress) { + return workerPromise.then(async (result) => { + progressCompletedCount++; + await onProgress(progressCompletedCount, totalWorkers, result, marker); + return result; + }); + } + return workerPromise; + }); + + // Wait for all workers to complete + const finalSettled = await Promise.allSettled(workerPromises); + + // Yield final progress message + const completedCount = finalSettled.filter( + (r) => r.status === 'fulfilled' && r.value.exitCode === 0, + ).length; + yield `\n\n_[All subtasks complete: ${completedCount}/${totalWorkers} successful]_\n`; + + await this.persistWorkerRegistry(); + + // Record task efficiency metrics for escalation suppression (OB-1572) + if (this.memory && taskClass) { + const progressStats = this.workerRegistry.getAggregatedStats(); + this.memory + .recordTaskEfficiency(taskClass, { + turnsUsed: progressStats.totalTurnsUsed ?? 0, + workerCount: progressStats.totalWorkers, + durationMs: progressStats.avgDurationMs * progressStats.totalWorkers, + }) + .catch((err) => logger.warn({ err, taskClass }, 'Failed to record task efficiency')); + } + + // Format all results and build the feedback prompt + const { feedbackPrompt, observations, workerSummaries } = formatWorkerBatch( + finalSettled, + markers, + workerIds, + this.masterSession?.sessionId, + ); + + // Persist extracted observations (fire-and-forget — don't block the response) + if (observations.length > 0 && this.memory) { + Promise.all(observations.map((obs) => this.memory!.insertObservation(obs))).catch((err) => + logger.warn({ err }, 'Failed to store worker observations'), + ); + } + + // Append learned items from worker summaries to memory.md (OB-1636) + if (workerSummaries.length > 0) { + this.dotFolder + .appendLearnedToMemory(workerSummaries) + .catch((err) => logger.warn({ err }, 'Failed to append learned items to memory.md')); + } + + return feedbackPrompt; + } + + /** + * Re-spawn a worker with upgraded tool access after a user grant (OB-1594). + * + * Called by the respawn callback registered in requestToolEscalation(). Receives the + * granted tool/profile name(s) from the /allow command handler and re-submits the + * original SPAWN marker with an upgraded profile or merged tool list. + * + * - If grantedTools contains a built-in profile name (e.g. "code-edit"), the marker + * is re-submitted with that profile, overriding the original. + * - If grantedTools contains individual tool names (e.g. "Bash(npm:test)"), they are + * merged with the original profile's tools and passed via a transient custom profile. + */ + private async respawnWorkerAfterGrant( + originalWorkerId: string, + marker: ParsedSpawnMarker, + index: number, + originalProfile: string, + grantedTools: string[], + attachments?: InboundMessage['attachments'], + ): Promise { + // Guard against infinite escalation loops (OB-F214): cap at depth 3. + // Check worker ID (the actual accumulator), not profile (which gets cleaned). + const escalationDepth = (originalWorkerId.match(/-escalated/g) ?? []).length; + if (escalationDepth >= 3) { + logger.warn( + { workerId: originalWorkerId, profile: originalProfile, escalationDepth }, + 'Max escalation depth reached — not respawning', + ); + return; + } + + // Strip any existing `-escalated` chain from worker ID before appending, + // so IDs stay `{base}-escalated` instead of growing infinitely (OB-F214). + const baseWorkerId = originalWorkerId.replace(/-escalated(-escalated)*$/, ''); + const newWorkerId = `${baseWorkerId}-escalated`; + + // Determine whether the grant is a profile upgrade or individual tool names. + const profileGrant = grantedTools.find((g) => BuiltInProfileNameSchema.safeParse(g).success); + + let upgradedMarker: ParsedSpawnMarker; + let customProfiles: Record | undefined; + + if (profileGrant) { + // Profile upgrade — re-submit the marker under the higher profile. + upgradedMarker = { ...marker, profile: profileGrant }; + logger.info( + { originalWorkerId, newWorkerId, originalProfile, upgradedProfile: profileGrant }, + 'Worker re-spawned with profile upgrade after grant', + ); + } else { + // Individual tool grant — merge with the original profile's tool list. + const baseTools = resolveProfile(originalProfile) ?? []; + const mergedTools = [...new Set([...baseTools, ...grantedTools])]; + // Strip any existing `-escalated` suffix chain so profile names stay `{base}-escalated` + // regardless of how many re-grants occur (OB-F214). + const baseProfile = originalProfile.replace(/-escalated(-escalated)*$/, ''); + const upgradedProfileName = `${baseProfile}-escalated`; + customProfiles = { + [upgradedProfileName]: { + name: upgradedProfileName, + description: `${baseProfile} + escalated access (${grantedTools.join(', ')})`, + tools: mergedTools, + }, + }; + upgradedMarker = { ...marker, profile: upgradedProfileName }; + logger.info( + { originalWorkerId, newWorkerId, originalProfile, mergedTools }, + 'Worker re-spawned with merged tool access after grant', + ); + } + + // Register the escalated worker in the WorkerRegistry BEFORE spawning so that + // markRunning / markCompleted / markFailed can find it by ID (OB-1626). + // Registration is inside the try block (OB-1645) so that if registration itself + // fails (e.g. capacity exceeded), the original worker is still marked as failed + // rather than left orphaned in 'pending' state. + const escalatedManifest: TaskManifest = { + prompt: upgradedMarker.body.prompt, + workspacePath: this.workspacePath, + profile: upgradedMarker.profile, + model: upgradedMarker.body.model, + maxTurns: upgradedMarker.body.maxTurns, + timeout: upgradedMarker.body.timeout, + retries: upgradedMarker.body.retries, + maxBudgetUsd: upgradedMarker.body.maxBudgetUsd, + }; + + try { + this.workerRegistry.registerWorkerWithId(newWorkerId, escalatedManifest); + await this.spawnWorker(newWorkerId, upgradedMarker, index, customProfiles, attachments); + } catch (spawnError) { + const errorMessage = spawnError instanceof Error ? spawnError.message : String(spawnError); + logger.error( + { originalWorkerId, newWorkerId, error: errorMessage }, + 'Worker re-spawn failed after grant', + ); + + // Mark escalated worker as failed and clean up to prevent orphaned state (OB-1627, OB-1628). + const failedResult: AgentResult = { + exitCode: -1, + stdout: '', + stderr: errorMessage, + durationMs: 0, + retryCount: 0, + status: 'completed', + }; + try { + this.workerRegistry.markFailed(newWorkerId, failedResult, 'respawn-failed'); + } catch (markErr) { + logger.warn({ newWorkerId, err: markErr }, 'Failed to mark escalated worker as failed'); + // OB-1628: If markFailed fails, remove the entry entirely to prevent the worker + // from remaining orphaned in 'pending' state in the registry. + this.workerRegistry.removeWorker(newWorkerId); + } + + // Also attempt to mark original worker as failed (OB-1627). + // The original may already be in a terminal state — guard with try-catch. + try { + this.workerRegistry.markFailed(originalWorkerId, failedResult, 'respawn-failed'); + } catch { + // Original worker already in a terminal state — this is expected. + } + + // Notify the user so they can retry (OB-1627). + if (this.router && this.activeMessage) { + void this.router.sendDirect( + this.activeMessage.source, + this.activeMessage.sender, + 'Worker re-spawn failed after grant, please retry', + ); + } + } + } + + /** + * Spawn a single worker from a parsed SPAWN marker. + * Resolves the profile to tools via AgentRunner's manifest resolution. + * Tracks the worker lifecycle in the registry: pending → running → completed/failed. + * Logs each worker execution to .openbridge/tasks/ for audit trail and learning. + * + * **Depth Limiting (OB-164):** + * Workers are spawned WITHOUT sessionId, so they get --print mode (single-turn, stateless). + * This enforces maxSpawnDepth=1 — only the Master can spawn workers, workers cannot spawn. + */ + private async spawnWorker( + workerId: string, + marker: ParsedSpawnMarker, + index: number, + customProfiles?: Record, + attachments?: InboundMessage['attachments'], + ): Promise { + const { body } = marker; + // profile may be overridden by skill pack selection (OB-1753) + let profile = marker.profile; + + // OB-1596: Compute session-level tool grants for this sender. + // If the user approved tools earlier this session via /allow, auto-apply them + // to every subsequent worker spawn so we don't re-ask for the same permissions. + const senderSessionGrants: ReadonlySet = + this.router && this.activeMessage + ? this.router.getSessionGrants(this.activeMessage.sender) + : new Set(); + // Expand profile-name grants (e.g. "code-edit") to their individual tool lists + // so we can check coverage and merge them into allowedTools uniformly. + const expandedSessionGrants = new Set(); + for (const grant of senderSessionGrants) { + if (BuiltInProfileNameSchema.safeParse(grant).success) { + const profileTools = resolveProfile(grant) ?? []; + profileTools.forEach((t) => expandedSessionGrants.add(t)); + } else { + expandedSessionGrants.add(grant); + } + } + + // OB-1600: Fetch permanent tool grants for this user from the DB. + // These survive session restarts — once granted via /allow-permanent they + // are always merged into worker allowedTools without re-asking the user. + const permanentGrants: string[] = + this.memory && this.activeMessage + ? await this.memory + .getApprovedEscalations(this.activeMessage.sender, this.activeMessage.source) + .catch(() => []) + : []; + + // If the originating message had attachments, prepend a ## Referenced Files section + // so the worker knows which files to read and analyze (OB-1148). + let workerPrompt = body.prompt; + if (attachments && attachments.length > 0) { + const fileLines = attachments + .map((att) => { + const name = att.filename ? ` (${att.filename})` : ''; + const sizeMb = (att.sizeBytes / (1024 * 1024)).toFixed(2); + return `- **${att.type}**${name}: \`${att.filePath}\` — ${att.mimeType}, ${sizeMb} MB`; + }) + .join('\n'); + workerPrompt = `## Referenced Files\n\nThe following files were attached to the user's message and are available for analysis:\n\n${fileLines}\n\n---\n\n${body.prompt}`; + } + + // Resolve per-worker tool and adapter + let workerRunner = this.agentRunner; + let resolvedModel = body.model; + const requestedTool = body.tool; + const toolUsed = requestedTool ?? this.masterTool.name; + + if (requestedTool && requestedTool !== this.masterTool.name) { + const tool = this.resolveDiscoveredTool(requestedTool); + const toolAdapter = tool ? this.adapterRegistry.get(requestedTool) : undefined; + + if (!tool || !toolAdapter) { + logger.warn( + { requestedTool, workerId }, + 'Requested tool not available — falling back to master tool', + ); + } else { + workerRunner = new AgentRunner(toolAdapter); + logger.info({ requestedTool, workerId }, 'Worker using tool-specific adapter'); + } + } + + // Apply tool-specific worker prompt prefix (OB-1576). + // Codex workers waste turns on shell gymnastics — the prefix steers them + // toward simple, direct file-reading commands (OB-F91). + workerPrompt = applyToolPromptPrefix(workerPrompt, toolUsed); + + // OB-1737: Inject skill pack prompt extension when the worker task involves + // document generation. classifyDocumentIntent maps task text to a file format + // (docx, pptx, xlsx, pdf), then we locate the matching built-in skill pack and + // append its workerPrompt section so the worker has precise generation guidance. + const docFormat = classifyDocumentIntent(body.prompt); + if (docFormat) { + const skillPacks = new Map(getBuiltInSkillPacks().map((s) => [s.name, s])); + const skill = findSkillByFormat(skillPacks, docFormat); + if (skill) { + workerPrompt = `${workerPrompt}\n\n---\n\n${skill.prompts.workerPrompt}`; + logger.debug( + { workerId, skillName: skill.name, docFormat }, + 'Injected skill pack prompt extension into worker', + ); + } + } + + // OB-1752: Select the best-matching SkillPack for this worker based on task + // type and inject its systemPromptExtension into the worker prompt. Only + // runs when no document-generation skill was already applied (avoids double + // injection). Uses keyword scoring to match security-audit, code-review, + // test-writer, data-analysis, and documentation packs. + let selectedPack: SkillPack | undefined; + if (!docFormat) { + selectedPack = selectSkillPackForTask(body.prompt, this.activeSkillPacks); + if (selectedPack) { + workerPrompt = `${workerPrompt}\n\n---\n\n${selectedPack.systemPromptExtension}`; + logger.debug( + { workerId, skillPack: selectedPack.name }, + 'Injected skill pack prompt extension into worker', + ); + } + } + + // OB-1753: Apply the selected skill pack's toolProfile to the effective + // profile. When a pack like security-audit specifies toolProfile:'code-audit', + // the worker should use that profile instead of a broader one (e.g. code-edit) + // to enforce the pack's read-only constraints. The pack profile is only + // applied when it differs from the SPAWN marker profile — an explicit user + // grant or a more-permissive SPAWN marker is respected over the pack default. + if (selectedPack?.toolProfile && selectedPack.toolProfile !== profile) { + logger.debug( + { + workerId, + previousProfile: profile, + newProfile: selectedPack.toolProfile, + skillPack: selectedPack.name, + }, + 'Skill pack tool profile applied to worker', + ); + profile = selectedPack.toolProfile; + } + + // Adaptive model selection (OB-724): marker override → learned best model → heuristics + if (!resolvedModel && this.memory) { + const taskType = classifyTaskType(body.prompt); + const learned = await getRecommendedModel(this.memory, taskType); + if (learned) { + resolvedModel = learned.model; + logger.debug( + { workerId, model: learned.model, reason: learned.reason }, + 'Adaptive model selected for worker', + ); + } + } + + // Always resolve model tiers to concrete model IDs for the target provider. + // This handles "fast" → "haiku" (claude), "fast" → "codex-mini" (codex), etc. + if (resolvedModel) { + const providerName = + requestedTool && this.resolveDiscoveredTool(requestedTool) + ? requestedTool + : this.masterTool.name; + const modelRegistry = createModelRegistry(providerName); + resolvedModel = modelRegistry.resolveModelOrTier(resolvedModel); + } + + // Avoid high-failure-rate models (OB-907): if the resolved model has >50% failure rate + // for this task type (with ≥3 data points), prefer a better-performing alternative. + if (resolvedModel && this.memory) { + const taskTypeForAvoidance = classifyTaskType(body.prompt); + const alternative = await avoidHighFailureModel( + this.memory, + taskTypeForAvoidance, + resolvedModel, + ); + if (alternative) { + logger.info( + { + workerId, + previousModel: resolvedModel, + newModel: alternative.model, + reason: alternative.reason, + }, + 'Model replaced due to high failure rate in learnings', + ); + resolvedModel = alternative.model; + } + } + + // Adaptive max-turns (OB-902): scale budget by prompt length when the SPAWN marker + // didn't explicitly specify maxTurns. A longer prompt usually means a more complex + // task that needs more turns to complete. + const resolvedMaxTurns = body.maxTurns ?? this.computeAdaptiveMaxTurns(profile, body.prompt); + + logger.info( + { + workerId, + workerIndex: index, + profile, + model: resolvedModel, + tool: toolUsed, + maxTurns: resolvedMaxTurns, + maxTurnsSource: body.maxTurns != null ? 'spawn-marker' : 'adaptive', + promptLength: body.prompt.length, + }, + 'Spawning worker from SPAWN marker', + ); + + // OB-1780: Reasoning checkpoint — "What could go wrong?" before full-access workers. + // Scans the task prompt for destructive, broad-scope, and security-sensitive patterns. + // The checkpoint is analytical only (does not block execution); it surfaces risks in + // the log so engineers and the audit trail can review the reasoning before a high- + // privilege worker modifies files, installs packages, or runs system commands. + if (profile === 'full-access') { + const checkpoint = performReasoningCheckpoint(body.prompt); + logger.info( + { + workerId, + riskLevel: checkpoint.riskLevel, + risks: checkpoint.risks.map((r) => ({ pattern: r.pattern, level: r.level })), + riskCount: checkpoint.risks.length, + }, + 'Reasoning checkpoint: pre-spawn risk analysis for full-access worker', + ); + } + + // Pre-flight tool prediction (OB-1595): before spending any turns, analyze the + // task prompt for keywords that suggest the worker will need tools beyond what + // its current profile allows (e.g. "npm test" with a read-only profile). + // When a mismatch is predicted, request escalation upfront — the user is asked + // before the worker is spawned, and the actual spawn is deferred to the respawn + // callback so no turns are wasted on a predictably blocked worker. + // + // Skip pre-flight for already-escalated workers (OB-F214): respawnWorkerAfterGrant + // calls back into spawnWorker — running prediction again creates an infinite loop + // (escalate → respawn → predict → escalate → ...) that OOMs the process. + const alreadyEscalated = workerId.includes('-escalated'); + const toolPrediction = alreadyEscalated + ? undefined + : predictToolRequirements(body.prompt, profile); + if (toolPrediction && this.router && this.activeMessage) { + logger.info( + { + workerId, + currentProfile: profile, + suggestedProfile: toolPrediction.suggestedProfile, + triggerKeywords: toolPrediction.triggerKeywords, + reason: toolPrediction.reason, + }, + 'Pre-flight tool prediction: requesting upfront escalation before spawn', + ); + const origMessage = this.activeMessage; + const connector = this.router.getConnector(origMessage.source); + if (connector) { + const suggestedTools = resolveProfile(toolPrediction.suggestedProfile) ?? []; + const currentTools = resolveProfile(profile) ?? []; + const additionalTools = suggestedTools.filter((t) => !currentTools.includes(t)); + + // OB-1596/OB-1600: If session or permanent grants already cover all additional + // tools, skip escalation — they will be auto-merged into allowedTools below. + const grantsCoversTools = + additionalTools.length > 0 && + additionalTools.every((t) => expandedSessionGrants.has(t) || permanentGrants.includes(t)); + + if (!grantsCoversTools) { + const respawnCallback = async (grantedTools: string[]): Promise => { + await this.respawnWorkerAfterGrant( + workerId, + marker, + index, + profile, + grantedTools, + attachments, + ); + }; + + await this.router.requestToolEscalation( + workerId, + additionalTools.length > 0 ? additionalTools : [toolPrediction.suggestedProfile], + profile, + `Pre-flight prediction: ${toolPrediction.reason} (keywords: ${toolPrediction.triggerKeywords.join(', ')})`, + origMessage, + connector, + respawnCallback, + ); + + // Return a deferred result — the actual spawn happens asynchronously + // via respawnCallback when the user grants tool access. + return { + stdout: `[Pre-flight] Tool grant requested for worker ${workerId}: ${toolPrediction.reason}. Spawn deferred pending user confirmation.`, + stderr: '', + exitCode: 0, + durationMs: 0, + retryCount: 0, + status: 'completed' as const, + }; + } + + logger.info( + { + workerId, + additionalTools, + sessionGrants: [...senderSessionGrants], + permanentGrants, + }, + 'Pre-flight: required tools already granted (session or permanent) — skipping escalation', + ); + } + } + + // OB-1787: Per-worker test modification permission grant. + // The Master AI is instructed (via system prompt) to include either: + // a) "Do not modify test files ... unless explicitly authorized." — protection + // b) "AUTHORIZED: test modification permitted" — explicit grant + // Additionally, the SPAWN marker may carry allowTestModification:true as a + // structured alternative to the in-prompt text marker. + // + // Enforcement logic for code-edit and full-access workers: + // 1. If test modification is explicitly granted (flag OR in-prompt marker): + // - Log the authorization for the audit trail. + // - Ensure the authorization header is present at the top of the prompt + // so the worker receives a clear, unambiguous grant even if the Master + // placed the text mid-prompt. + // 2. If NOT granted and no protection instruction is already present: + // - Inject the protection reminder so workers always have an explicit guard, + // regardless of whether the Master included it in the prompt. + const TEST_PROTECTION_PROFILES = new Set(['code-edit', 'full-access']); + const AUTHORIZED_MARKER = 'AUTHORIZED: test modification permitted'; + const TEST_PROTECTION_INSTRUCTION = + 'Do not modify test files (files in `tests/`, `__tests__/`, or files matching ' + + '`*.test.ts`, `*.spec.ts`, `*.test.js`, `*.spec.js`) unless explicitly authorized.'; + + if (TEST_PROTECTION_PROFILES.has(profile)) { + const hasAuthFlag = body.allowTestModification === true; + const hasAuthText = workerPrompt.includes(AUTHORIZED_MARKER); + const hasProtectionText = workerPrompt.includes(TEST_PROTECTION_INSTRUCTION.slice(0, 40)); + + if (hasAuthFlag || hasAuthText) { + // Grant: worker is authorized to touch test files. + logger.info( + { workerId, profile, source: hasAuthFlag ? 'spawn-flag' : 'prompt-marker' }, + 'Test modification permission granted for this worker', + ); + // Normalize: ensure the authorization header is at the very top of the prompt + // so the worker cannot miss it (it may have been buried mid-prompt by the Master). + if (!workerPrompt.startsWith(AUTHORIZED_MARKER)) { + // Remove any existing AUTHORIZED marker to avoid duplication, then prepend. + const cleaned = workerPrompt.replace(AUTHORIZED_MARKER, '').trimStart(); + workerPrompt = `${AUTHORIZED_MARKER}\n\n${cleaned}`; + } + } else if (!hasProtectionText) { + // No grant and no protection text — the Master omitted the guard. Inject it. + logger.debug( + { workerId, profile }, + 'Test protection instruction injected (Master omitted it)', + ); + workerPrompt = `${TEST_PROTECTION_INSTRUCTION}\n\n${workerPrompt}`; + } + } + + // NOTE: No sessionId provided here — workers get --print mode (depth limiting) + // manifestToSpawnOptions is async: when manifest.mcpServers is set, it writes a + // per-worker temp MCP config file and returns a cleanup callback to delete it. + const { spawnOptions: spawnOpts, cleanup: mcpCleanup } = await manifestToSpawnOptions( + { + prompt: workerPrompt, + workspacePath: this.workspacePath, + profile, + model: resolvedModel, + maxTurns: resolvedMaxTurns, + timeout: body.timeout ?? DEFAULT_WORKER_TIMEOUT, + retries: body.retries, + maxBudgetUsd: body.maxBudgetUsd, + }, + customProfiles, + ); - const taskId = randomUUID(); - const startedAt = new Date().toISOString(); + // OB-1791: Apply configurable fix iteration cap to workers. + // Sourced from config.worker.maxFixIterations (default: 3). + spawnOpts.maxFixIterations = this.workerMaxFixIterations; + + // OB-1596: Auto-merge session-granted tools into this worker's allowedTools. + // Tools previously approved by the user this session (via /allow) are applied + // automatically so repeated worker spawns don't re-ask for the same permissions. + if (expandedSessionGrants.size > 0) { + const existing = spawnOpts.allowedTools ?? []; + const toolsToAdd = [...expandedSessionGrants].filter((t) => !existing.includes(t)); + if (toolsToAdd.length > 0) { + spawnOpts.allowedTools = [...existing, ...toolsToAdd]; + logger.debug( + { workerId, toolsAdded: toolsToAdd }, + 'Session grants auto-merged into worker allowedTools', + ); + } + } - logger.info({ taskId, sender: message.sender, content: message.content }, 'Streaming message'); + // OB-1600: Auto-merge permanent tool grants from the DB into this worker's allowedTools. + // These are tools the user permanently approved (via /allow-permanent or the access CLI). + // They persist across sessions and are applied without asking the user again. + if (permanentGrants.length > 0) { + const existing = spawnOpts.allowedTools ?? []; + const toolsToAdd = permanentGrants.filter((t) => !existing.includes(t)); + if (toolsToAdd.length > 0) { + spawnOpts.allowedTools = [...existing, ...toolsToAdd]; + logger.debug( + { workerId, toolsAdded: toolsToAdd }, + 'Permanent grants auto-merged into worker allowedTools', + ); + } + } - // Create task record - const task: TaskRecord = { - id: taskId, - userMessage: message.rawContent, - sender: message.sender, - description: message.content, + // Create task record for this worker execution (OB-165: task history + audit trail) + const taskRecord: TaskRecord = { + id: workerId, + userMessage: body.prompt, + sender: 'master', + description: `Worker ${index}: ${body.prompt.slice(0, 100)}`, status: 'processing', - handledBy: 'master', - createdAt: startedAt, - startedAt, + handledBy: 'worker', + createdAt: new Date().toISOString(), + startedAt: new Date().toISOString(), metadata: { - messageId: message.id, - source: message.source, + workerIndex: index, + profile, + model: resolvedModel, + tool: toolUsed, + maxTurns: resolvedMaxTurns, + timeout: body.timeout, + retries: body.retries, + manifest: { + prompt: body.prompt, + workspacePath: this.workspacePath, + profile, + model: resolvedModel, + maxTurns: body.maxTurns, + timeout: body.timeout, + retries: body.retries, + }, }, }; - try { - // Check for status queries - if (this.isStatusQuery(message.content)) { - const status = await this.getStatus(); - this.state = 'ready'; - task.status = 'completed'; - task.result = status; - task.completedAt = new Date().toISOString(); - task.durationMs = - new Date(task.completedAt).getTime() - new Date(task.startedAt!).getTime(); - await this.dotFolder.recordTask(task); - yield status; - return; + // Inject memory briefing as system prompt so the worker starts with project context (OB-723) + if (this.memory) { + try { + const briefing = await this.memory.buildBriefing(body.prompt); + if (briefing) { + spawnOpts.systemPrompt = briefing; + } + } catch (briefingErr) { + logger.warn( + { workerId, error: briefingErr }, + 'Failed to build worker briefing — proceeding without context', + ); } + } - // Get or create session ID for this sender - const sessionId = this.getOrCreateSession(message.sender); + // INSERT agent_activity row with status='starting' (OB-742) + const workerStartedAt = new Date().toISOString(); + if (this.memory) { + try { + const masterSessionId = this.masterSession?.sessionId; + const workerActivity: ActivityRecord = { + id: workerId, + type: 'worker', + model: resolvedModel ?? body.model ?? undefined, + profile, + task_summary: body.prompt.slice(0, 120), + status: 'starting', + parent_id: masterSessionId, + started_at: workerStartedAt, + updated_at: workerStartedAt, + }; + await this.memory.insertActivity(workerActivity); + } catch (actErr) { + logger.warn({ workerId, error: actErr }, 'Failed to record worker activity (starting)'); + } + } - // Stream message through Claude Code with session continuity - let fullResponse = ''; - const stream = streamClaudeCode({ - prompt: message.content, - workspacePath: this.workspacePath, - timeout: this.messageTimeout, - resumeSessionId: sessionId, - }); + try { + // Build a streaming progress callback — broadcasts worker-turn-progress events + // to all connectors as each agent turn is parsed from stdout (OB-1051). + const routerRef = this.router; + const workerMaxTurns = spawnOpts.maxTurns ?? DEFAULT_MAX_TURNS_TASK; + const onTurnProgress = routerRef + ? (indicator: { turnsUsed: number; lastAction?: string }): void => { + void routerRef.broadcastProgress({ + type: 'worker-turn-progress', + workerId, + turnsUsed: indicator.turnsUsed, + turnsMax: workerMaxTurns, + lastAction: indicator.lastAction, + }); + } + : undefined; + + // Use spawnWithStreamingHandle() to capture the real PID and abort function (OB-873) + // and broadcast real-time turn progress via worker-turn-progress events (OB-1051). + let currentHandle = workerRunner.spawnWithStreamingHandle(spawnOpts, onTurnProgress); + this.workerAbortHandles.set(workerId, currentHandle.abort); + this.workerRegistry.markRunning(workerId, currentHandle.pid); + logger.debug( + { workerId, pid: currentHandle.pid }, + 'Worker process started — real PID captured', + ); - for await (const chunk of stream) { - fullResponse += chunk; - yield chunk; + // UPDATE agent_activity to 'running' now that spawn is about to start (OB-742) + if (this.memory) { + try { + await this.memory.updateActivity(workerId, { status: 'running' }); + } catch (actErr) { + logger.warn({ workerId, error: actErr }, 'Failed to update worker activity (running)'); + } } - // Check for delegation markers in the response - const delegations = this.parseDelegationMarkers(fullResponse); - if (delegations && delegations.length > 0) { + // Worker-level retry with exponential backoff (OB-905). + // Default retries = 2. Only retry on retryable error categories: + // retryable: 'rate-limit', 'timeout', 'crash' + // non-retryable: 'auth', 'context-overflow', 'unknown' + const maxWorkerRetries = body.retries ?? 2; + let workerRetryCount = 0; + let result: AgentResult; + let isFirstWorkerAttempt = true; + + while (true) { + if (isFirstWorkerAttempt) { + result = await currentHandle.promise; + isFirstWorkerAttempt = false; + } else { + // Worker-level retry — spawn a new process and update the abort handle (OB-873) + currentHandle = workerRunner.spawnWithStreamingHandle(spawnOpts, onTurnProgress); + this.workerAbortHandles.set(workerId, currentHandle.abort); + result = await currentHandle.promise; + } + + if (result.exitCode === 0) { + break; // Success — no retry needed + } + + // Classify the error to decide retry strategy (OB-904, OB-905) + const errorCategory = classifyError(result.stderr, result.exitCode); + const isRetryable = + errorCategory === 'rate-limit' || + errorCategory === 'timeout' || + errorCategory === 'crash'; + + if (!isRetryable) { + // Non-retryable failure (auth, context-overflow, unknown) — break immediately + logger.warn( + { + workerId, + exitCode: result.exitCode, + errorCategory, + durationMs: result.durationMs, + }, + 'Worker failed with non-retryable error', + ); + break; + } + + // Check if we have retries left + if (workerRetryCount >= maxWorkerRetries) { + break; // Exhausted retries + } + + // Retryable failure — apply exponential backoff and retry + workerRetryCount++; + const delay = this.workerRetryDelayMs * Math.pow(2, workerRetryCount - 1); logger.info( - { delegationCount: delegations.length }, - 'Delegation markers detected in stream', + { + workerId, + workerRetry: workerRetryCount, + maxWorkerRetries, + exitCode: result.exitCode, + errorCategory, + delayMs: delay, + stderrPreview: result.stderr.slice(0, 150), + }, + 'Worker failed with retryable error — retrying after backoff', ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } - // Update task status to delegated - task.status = 'delegated'; - await this.dotFolder.recordTask(task); + // Update worker record with retry count + const workerRecord = this.workerRegistry.getWorker(workerId); + if (workerRecord) { + workerRecord.workerRetries = workerRetryCount; + } - // Handle delegations - const delegationResults = await this.handleDelegations(delegations, message); + // Detect max-turns exhaustion: Claude exits 0 but work may be incomplete (OB-900) + // Auto-retry with an escalated turn budget (OB-903): max 1 turn-escalation retry. + // Skip for non-Claude tools — --max-turns is a Claude-only feature. Other tools + // (Codex, Aider) may produce false positives from pattern matching on their output. + if (result.turnsExhausted && toolUsed === 'claude') { + const originalMaxTurns = spawnOpts.maxTurns ?? DEFAULT_MAX_TURNS_TASK; + logger.warn( + { + workerId, + maxTurns: originalMaxTurns, + model: result.model, + durationMs: result.durationMs, + }, + 'Worker hit max-turns limit — attempting turn-escalation retry', + ); - // Feed delegation results back to Master session and stream the final response - const feedbackPrompt = `The following delegation results are available:\n\n${delegationResults}\n\nPlease synthesize these results and provide a final response to the user.`; + const escalatedMaxTurns = Math.min(Math.ceil(originalMaxTurns * 1.5), 50); + const partialOutput = result.stdout.trim(); + + // Extract [INCOMPLETE: step X/Y] marker injected by the worker (OB-901) + const incompleteMatch = partialOutput.match(/\[INCOMPLETE:\s*([^\]]+)\]/i); + const incompleteHint = + incompleteMatch && incompleteMatch[1] ? incompleteMatch[1].trim() : null; + + const continuationNote = incompleteHint + ? `Previous attempt was incomplete: ${incompleteHint}. Continue from where it left off.` + : 'Previous attempt hit the turn limit before completing. Continue from where it left off.'; + + // Append partial output (last 2 000 chars) as context so the worker can resume + const escalationPrompt = [ + body.prompt, + '', + '---', + 'CONTEXT FROM PREVIOUS ATTEMPT (partial output):', + partialOutput.slice(-2000), + '---', + continuationNote, + ].join('\n'); + + const escalatedSpawnOpts: SpawnOptions = { + ...spawnOpts, + prompt: escalationPrompt, + maxTurns: escalatedMaxTurns, + }; - this.state = 'processing'; - const feedbackStream = streamClaudeCode({ - prompt: feedbackPrompt, - workspacePath: this.workspacePath, - timeout: this.messageTimeout, - resumeSessionId: sessionId, - }); + logger.info( + { workerId, originalMaxTurns, escalatedMaxTurns, incompleteHint }, + 'Turn-escalation retry: re-spawning with higher turn budget', + ); - let finalResponse = ''; - for await (const chunk of feedbackStream) { - finalResponse += chunk; - yield chunk; + // Use spawnWithStreamingHandle() so the abort handle stays current during escalation (OB-873) + // Pass a dedicated callback with the escalated max-turns value (OB-1051). + const escalationHandle = workerRunner.spawnWithStreamingHandle( + escalatedSpawnOpts, + routerRef + ? (indicator): void => { + void routerRef.broadcastProgress({ + type: 'worker-turn-progress', + workerId, + turnsUsed: indicator.turnsUsed, + turnsMax: escalatedMaxTurns, + lastAction: indicator.lastAction, + }); + } + : undefined, + ); + this.workerAbortHandles.set(workerId, escalationHandle.abort); + result = await escalationHandle.promise; + + if (result.turnsExhausted) { + logger.warn( + { workerId, escalatedMaxTurns }, + 'Turn-escalation retry also exhausted — returning partial result', + ); } + } + + // Update registry based on final result + if (result.exitCode === 0) { + this.workerRegistry.markCompleted(workerId, result); + } else { + const isTimeout = result.exitCode === 143 || result.exitCode === 137; + const errorMessage = isTimeout + ? `Worker timeout: process terminated after ${body.timeout ?? 'default'}ms (exit code ${result.exitCode})` + : `Exit code ${result.exitCode}: ${result.stderr.slice(0, 200)}`; - fullResponse = finalResponse.trim() || delegationResults; + this.workerRegistry.markFailed(workerId, result, errorMessage); } - // Update task record - task.status = 'completed'; - task.result = fullResponse.trim() || 'No response from AI'; - task.completedAt = new Date().toISOString(); - task.durationMs = new Date(task.completedAt).getTime() - new Date(task.startedAt!).getTime(); + // Persist registry after worker completion or failure + await this.persistWorkerRegistry(); + + // Detect tool-access failures in the worker result (OB-1592). + // Even a zero-exit worker may report a tool-denial in its output. + const toolFailure = detectToolAccessFailure(result); + if (toolFailure) { + logger.warn( + { + workerId, + tool: toolFailure.tool, + profile, + reason: toolFailure.reason, + exitCode: result.exitCode, + }, + 'Worker tool-access failure detected — tool was blocked by allowedTools restrictions', + ); + // Store on the result metadata for audit trail. + taskRecord.metadata = { + ...taskRecord.metadata, + toolAccessFailure: { tool: toolFailure.tool, reason: toolFailure.reason }, + }; - await this.dotFolder.recordTask(task); - await this.dotFolder.commitChanges(`Task ${taskId}: ${message.content.slice(0, 50)}`); + // Wire to Router escalation (OB-1593): ask the user to approve the needed tool. + // Only fires when a router and an active user message are available (i.e. during + // processMessage / streamMessage — not during background exploration workers). + if (this.router && this.activeMessage) { + const origMessage = this.activeMessage; + const connector = this.router.getConnector(origMessage.source); + if (connector) { + const requestedTools = toolFailure.tool ? [toolFailure.tool] : []; + // OB-1594: Provide a respawn callback so /allow can re-spawn the worker + // with the granted tools merged into its profile. + const respawnCallback = async (grantedTools: string[]): Promise => { + await this.respawnWorkerAfterGrant( + workerId, + marker, + index, + profile, + grantedTools, + attachments, + ); + }; + await this.router.requestToolEscalation( + workerId, + requestedTools, + profile, + toolFailure.reason, + origMessage, + connector, + respawnCallback, + ); + } else { + logger.warn( + { workerId, source: origMessage.source }, + 'Tool escalation skipped — connector not found for source', + ); + } + } + } - this.state = 'ready'; + // Update task record with result (OB-165) + taskRecord.status = result.exitCode === 0 ? 'completed' : 'failed'; + taskRecord.result = result.exitCode === 0 ? result.stdout : undefined; + taskRecord.error = result.exitCode === 0 ? undefined : result.stderr; + taskRecord.completedAt = new Date().toISOString(); + taskRecord.durationMs = result.durationMs; + taskRecord.metadata = { + ...taskRecord.metadata, + exitCode: result.exitCode, + retryCount: result.retryCount, + workerRetries: workerRetryCount, + modelUsed: result.model, + modelFallbacks: result.modelFallbacks, + resolvedTools: spawnOpts.allowedTools, + }; - logger.info( - { taskId, durationMs: task.durationMs, responseLength: fullResponse.length }, - 'Message streamed successfully', - ); - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); + // Write worker task to store (memory) (OB-165: task history + audit trail) + if (this.memory) { + const statusMap: Record = { + completed: 'completed', + failed: 'failed', + }; + await this.memory.recordTask({ + id: taskRecord.id, + type: 'worker', + status: statusMap[taskRecord.status] ?? 'failed', + prompt: taskRecord.userMessage, + response: taskRecord.result, + model: (taskRecord.metadata?.['modelUsed'] as string | undefined) ?? spawnOpts.model, + profile, + max_turns: spawnOpts.maxTurns, + duration_ms: taskRecord.durationMs, + exit_code: (taskRecord.metadata?.['exitCode'] as number | undefined) ?? result.exitCode, + retries: result.retryCount, + created_at: taskRecord.createdAt, + completed_at: taskRecord.completedAt, + }); + } else { + logger.warn({ workerId }, 'MemoryManager not available — worker task record not persisted'); + } - // Update task record with error - task.status = 'failed'; - task.error = errorMessage; - task.completedAt = new Date().toISOString(); - task.durationMs = new Date(task.completedAt).getTime() - new Date(task.startedAt!).getTime(); + // UPDATE agent_activity to 'done' or 'failed' with cost (OB-742, OB-746) + if (this.memory) { + try { + const activityStatus = result.exitCode === 0 ? 'done' : 'failed'; + await this.memory.updateActivity(workerId, { + status: activityStatus, + progress_pct: result.exitCode === 0 ? 100 : undefined, + completed_at: taskRecord.completedAt, + cost_usd: result.costUsd, + }); + } catch (actErr) { + logger.warn({ workerId, error: actErr }, 'Failed to update worker activity (completion)'); + } + } - await this.dotFolder.recordTask(task); + // Record worker output to conversation history (OB-730) + if (result.exitCode === 0 && result.stdout.trim()) { + const workerSessionId = this.masterSession?.sessionId ?? workerId; + await this.recordConversationMessage(workerSessionId, 'worker', result.stdout.trim()); + } - this.state = 'ready'; + // Auto-store worker results in chunk store for future RAG retrieval (OB-1570) + if (result.exitCode === 0 && result.stdout.trim() && this.knowledgeRetriever) { + try { + await this.knowledgeRetriever.storeWorkerResult(result.stdout.trim(), body.prompt, []); + } catch (storeErr) { + logger.warn( + { workerId, error: storeErr }, + 'Failed to store worker result in chunk store', + ); + } + } - logger.error({ error, taskId, sender: message.sender }, 'Message streaming failed'); + // Record learning entry for this worker execution (OB-171: learnings store) + await this.recordWorkerLearning(taskRecord, result, profile, spawnOpts.model); - yield `Error: ${errorMessage}`; - } - } + // Record prompt effectiveness (OB-172: prompt effectiveness tracking) + await this.recordPromptEffectiveness(taskRecord, result); - /** - * Get system status - */ - public async getStatus(): Promise { - const map = await this.dotFolder.readMap(); - const tasks = await this.dotFolder.readAllTasks(); + // Clean up per-worker MCP temp file (no-op when no MCP servers were requested) + await mcpCleanup(); - const completedTasks = tasks.filter((t) => t.status === 'completed').length; - const failedTasks = tasks.filter((t) => t.status === 'failed').length; - const processingTasks = tasks.filter( - (t) => t.status === 'processing' || t.status === 'delegated', - ).length; + return result; + } catch (error) { + // Worker threw an exception (spawn error, exhausted retries, etc.) + const errorMessage = error instanceof Error ? error.message : String(error); + const failedResult: AgentResult = { + exitCode: -1, + stdout: '', + stderr: errorMessage, + durationMs: 0, + retryCount: 0, + status: 'completed', + }; - let status = `**OpenBridge Master AI Status**\n\n`; - status += `State: ${this.state}\n`; + this.workerRegistry.markFailed(workerId, failedResult, errorMessage); + + // Remove worker from registry to free the concurrency slot and prevent orphaned + // pending workers from accumulating (OB-1264 / OB-F153) + this.workerRegistry.removeWorker(workerId); + + // Persist registry after exception + await this.persistWorkerRegistry(); + + // Update task record with error (OB-165) + taskRecord.status = 'failed'; + taskRecord.error = errorMessage; + taskRecord.completedAt = new Date().toISOString(); + taskRecord.durationMs = 0; + taskRecord.metadata = { + ...taskRecord.metadata, + exitCode: -1, + retryCount: 0, + exceptionThrown: true, + }; - if (this.explorationSummary) { - status += `Exploration: ${this.explorationSummary.status}\n`; - if (this.explorationSummary.projectType) { - status += `Project Type: ${this.explorationSummary.projectType}\n`; + // Write worker task to store (memory) even on exception (OB-165) + if (this.memory) { + await this.memory.recordTask({ + id: taskRecord.id, + type: 'worker', + status: 'failed', + prompt: taskRecord.userMessage, + model: taskRecord.metadata?.['modelUsed'] as string | undefined, + profile, + max_turns: spawnOpts.maxTurns, + duration_ms: 0, + exit_code: -1, + retries: 0, + created_at: taskRecord.createdAt, + completed_at: taskRecord.completedAt, + }); + } else { + logger.warn( + { workerId }, + 'MemoryManager not available — worker exception task record not persisted', + ); } - if (this.explorationSummary.frameworks.length > 0) { - status += `Frameworks: ${this.explorationSummary.frameworks.join(', ')}\n`; + + // UPDATE agent_activity to 'failed' on exception (OB-742) + if (this.memory) { + try { + await this.memory.updateActivity(workerId, { + status: 'failed', + completed_at: taskRecord.completedAt, + }); + } catch (actErr) { + logger.warn({ workerId, error: actErr }, 'Failed to update worker activity (failed)'); + } } - } - if (map) { - status += `\nWorkspace: ${map.projectName}\n`; - status += `Summary: ${map.summary}\n`; - } + // Record learning entry even on exception (OB-171: learnings store) + await this.recordWorkerLearning(taskRecord, failedResult, profile, body.model); - status += `\nTasks: ${completedTasks} completed, ${failedTasks} failed, ${tasks.length} total\n`; + // Record prompt effectiveness even on exception (OB-172: prompt effectiveness tracking) + await this.recordPromptEffectiveness(taskRecord, failedResult); - // Show active delegations if any - const activeDelegations = this.delegationCoordinator.getActiveDelegations(); - if (activeDelegations.length > 0) { - status += `\nActive Delegations (${activeDelegations.length}):\n`; - for (const delegation of activeDelegations) { - const elapsed = Date.now() - new Date(delegation.startedAt).getTime(); - const elapsedSeconds = Math.floor(elapsed / 1000); - status += ` - ${delegation.tool.name}: ${delegation.task.description.slice(0, 60)}... (${elapsedSeconds}s)\n`; - } - } + // Clean up per-worker MCP temp file even on exception + await mcpCleanup(); - // Show processing tasks if any - if (processingTasks > 0) { - status += `\nProcessing: ${processingTasks} task(s) in progress\n`; + // Re-throw so Promise.allSettled captures it as rejected + throw error; + } finally { + // Always clean up abort handle — ensures no stale handles even on pre-spawn + // exceptions (escalation timeout, slot wait timeout, spawn error). (OB-F171) + this.workerAbortHandles.delete(workerId); } - - status += `\nActive Sessions: ${this.sessionMap.size}\n`; - - return status; } /** - * Gracefully shut down the Master AI + * Spawns a lightweight read-only worker to answer a focused question about + * specific files. Used by the targeted reader path when RAG confidence < 0.3. + * + * OB-1353 */ - public async shutdown(): Promise { - if (this.state === 'shutdown') { - return; - } - - logger.info('Shutting down MasterManager'); - - this.state = 'shutdown'; - - // Shutdown delegation coordinator - this.delegationCoordinator.shutdown(); + public async spawnTargetedReader(filePaths: string[], question: string): Promise { + const fileList = filePaths.map((p) => `- ${p}`).join('\n'); + const prompt = [ + 'Read these files and answer the following question.', + '', + '## Files to Read', + fileList, + '', + '## Question', + question, + ].join('\n'); + + const model = this.modelRegistry.resolveModelOrTier('fast'); + + logger.debug( + { fileCount: filePaths.length, question: question.slice(0, 80) }, + 'Spawning targeted reader worker', + ); - // Clear all session timeouts - for (const timeout of this.sessionTimeouts.values()) { - clearTimeout(timeout); - } - this.sessionTimeouts.clear(); - this.sessionMap.clear(); + const result = await this.agentRunner.spawn({ + prompt, + workspacePath: this.workspacePath, + allowedTools: [...TOOLS_READ_ONLY], + maxTurns: 5, + model, + }); - // Log shutdown - try { - await this.dotFolder.appendLog({ - timestamp: new Date().toISOString(), - level: 'info', - message: 'Master AI shutting down', - }); - } catch (error) { - logger.error({ error }, 'Failed to log shutdown'); + if (result.exitCode !== 0) { + logger.warn( + { exitCode: result.exitCode, fileCount: filePaths.length }, + 'Targeted reader worker failed', + ); + return ''; } - logger.info('MasterManager shutdown complete'); + return result.stdout.trim(); } /** @@ -754,6 +6831,14 @@ export class MasterManager { /** * Find a specialist tool by name from the agents registry */ + /** + * Resolve a discovered tool by name (synchronous, exact match only). + * Used for per-worker tool selection from SPAWN markers. + */ + private resolveDiscoveredTool(toolName: string): DiscoveredTool | undefined { + return this.discoveredTools.find((t) => t.name === toolName && t.available); + } + private async findSpecialistTool(toolName: string): Promise { // First check discovered tools const tool = this.discoveredTools.find( @@ -766,8 +6851,21 @@ export class MasterManager { return tool; } - // Check agents.json as fallback - const agents = await this.dotFolder.readAgents(); + // Check agents registry — DB first, JSON fallback + let agents = null; + if (this.memory) { + const raw = await this.memory.getSystemConfig('agents'); + if (raw) { + try { + agents = AgentsRegistrySchema.parse(JSON.parse(raw)); + } catch { + // fall through to JSON file + } + } + } + if (!agents) { + agents = await this.dotFolder.readAgents(); + } if (!agents) { return null; } @@ -840,37 +6938,6 @@ export class MasterManager { return results.join('\n\n'); } - /** - * Get or create a session ID for a sender. - * Sessions are used to maintain conversation continuity. - */ - private getOrCreateSession(sender: string): string { - // Clear existing timeout for this sender - const existingTimeout = this.sessionTimeouts.get(sender); - if (existingTimeout) { - clearTimeout(existingTimeout); - } - - // Get or create session ID - let sessionId = this.sessionMap.get(sender); - if (!sessionId) { - sessionId = randomUUID(); - this.sessionMap.set(sender, sessionId); - logger.debug({ sender, sessionId }, 'Created new session'); - } - - // Set new timeout to clear session after TTL - const timeout = setTimeout(() => { - this.sessionMap.delete(sender); - this.sessionTimeouts.delete(sender); - logger.debug({ sender, sessionId }, 'Session expired'); - }, this.sessionTTL); - - this.sessionTimeouts.set(sender, timeout); - - return sessionId; - } - /** * Check if a message is a status query */ @@ -885,30 +6952,172 @@ export class MasterManager { ); } + /** Create agents registry. Delegated to ExplorationManager (OB-1280). */ + private createAgentsRegistry(): AgentsRegistry { + return this.explorationManager.createAgentsRegistry(); + } + + // --------------------------------------------------------------------------- + // Sub-master routing helpers (OB-755) + // --------------------------------------------------------------------------- + /** - * Create agents registry from discovered tools + * Detect which active sub-masters (if any) are relevant for the given message content. + * + * Scans the message text for path-like references that start with a sub-master's + * relative path (e.g. "backend/", "frontend/"). Returns routing info when at + * least one sub-master is matched. + * + * Examples of matched patterns: + * "Update backend/src/auth.ts" → matches sub-master with path "backend" + * "Fix the frontend/app/index" → matches sub-master with path "frontend" + * "backend and frontend changes" → cross-cutting (matches both) + * + * Returns null when no sub-master scope is detected. */ - private createAgentsRegistry(): AgentsRegistry { - const master = this.masterTool; - const specialists = this.discoveredTools - .filter((tool) => tool.role === 'specialist' || tool.role === 'backup') - .map((tool) => ({ - name: tool.name, - path: tool.path, - version: tool.version, - role: tool.role as 'specialist' | 'backup', - capabilities: tool.capabilities, - })); + private detectSubMasterRouting( + content: string, + activeSubMasters: SubMasterRecord[], + ): { type: 'single' | 'cross-cutting'; subMasters: SubMasterRecord[] } | null { + const normalizedContent = content.toLowerCase(); + const matched: SubMasterRecord[] = []; + + for (const subMaster of activeSubMasters) { + // subMaster.path is a relative path like "backend" or "packages/api" + const subPath = subMaster.path.toLowerCase(); + // Match references like "backend/", "./backend", or standalone "backend" at word boundary + const hasPathRef = + normalizedContent.includes(`${subPath}/`) || + normalizedContent.includes(`./${subPath}`) || + new RegExp(`\\b${subPath.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`).test( + normalizedContent, + ); + if (hasPathRef) { + matched.push(subMaster); + } + } + + if (matched.length === 0) return null; return { - master: { - name: master.name, - path: master.path, - version: master.version, - role: 'master', - }, - specialists, - updatedAt: new Date().toISOString(), + type: matched.length === 1 ? 'single' : 'cross-cutting', + subMasters: matched, }; } + + /** + * Delegate a task to a single sub-master. + * + * Opens the sub-master's DB, builds a context briefing from its exploration + * data, then spawns a worker scoped to the sub-master's workspace directory. + * Falls back gracefully if the sub-master DB doesn't exist yet. + */ + private async delegateToSubMaster( + subMaster: SubMasterRecord, + taskContent: string, + ): Promise { + const subMasterAbsPath = path.join(this.workspacePath, subMaster.path); + const subMasterDbPath = path.join(subMasterAbsPath, '.openbridge', 'openbridge.db'); + + let briefingContext = ''; + let subDb = null; + try { + subDb = openDatabase(subMasterDbPath); + const briefing = await buildBriefing(subDb, taskContent, undefined, this.agentRunner); + briefingContext = briefing; + logger.info( + { subMasterPath: subMaster.path, briefingLength: briefing.length }, + 'Built sub-master briefing', + ); + } catch (err) { + // Sub-master DB may not exist yet (exploration still in progress or stale). + // Fall through and spawn the worker without briefing context. + logger.warn( + { error: err, subMasterPath: subMaster.path }, + 'Could not read sub-master briefing — proceeding without context', + ); + } finally { + if (subDb) { + closeDatabase(subDb); + } + } + + const prompt = briefingContext + ? `${briefingContext}\n\n## Task\n\n${taskContent}` + : taskContent; + + const result = await this.agentRunner.spawn({ + prompt, + workspacePath: subMasterAbsPath, + model: 'sonnet', + allowedTools: [...TOOLS_CODE_EDIT], + maxTurns: DEFAULT_MAX_TURNS_TASK, + }); + + if (result.exitCode !== 0) { + logger.warn( + { subMasterPath: subMaster.path, exitCode: result.exitCode }, + 'Sub-master worker exited with non-zero code', + ); + } + + return ( + result.stdout.trim() || + `Sub-master task for ${subMaster.name} completed with exit code ${result.exitCode}.` + ); + } + + /** + * Orchestrate delegation to one or more sub-masters. + * + * Single sub-master: delegate directly and return its response. + * Cross-cutting (multiple sub-masters): spawn one worker per affected + * sub-master concurrently, then collect and combine their results into + * a unified response string for the Master to relay to the user. + */ + private async handleSubMasterDelegation( + routing: { type: 'single' | 'cross-cutting'; subMasters: SubMasterRecord[] }, + taskContent: string, + progress?: ProgressReporter, + ): Promise { + if (routing.type === 'single') { + const [subMaster] = routing.subMasters; + logger.info({ subMasterPath: subMaster!.path }, 'Delegating task to single sub-master'); + const result = await this.delegateToSubMaster(subMaster!, taskContent); + await progress?.({ type: 'worker-progress', completed: 1, total: 1 }); + return result; + } + + // Cross-cutting: decompose into per-sub-master sub-tasks and run concurrently + logger.info( + { count: routing.subMasters.length, paths: routing.subMasters.map((s) => s.path) }, + 'Delegating cross-cutting task to multiple sub-masters', + ); + + let completedCount = 0; + const total = routing.subMasters.length; + + const settled = await Promise.allSettled( + routing.subMasters.map((subMaster) => + this.delegateToSubMaster(subMaster, taskContent).then(async (res) => { + completedCount++; + await progress?.({ type: 'worker-progress', completed: completedCount, total }); + return { name: subMaster.name, path: subMaster.path, result: res }; + }), + ), + ); + + const parts: string[] = []; + for (const outcome of settled) { + if (outcome.status === 'fulfilled') { + parts.push(`### ${outcome.value.name} (${outcome.value.path})\n\n${outcome.value.result}`); + } else { + const err = + outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason); + parts.push(`### Sub-master delegation failed\n\n${err}`); + } + } + + return parts.join('\n\n---\n\n'); + } } diff --git a/src/master/master-system-prompt.ts b/src/master/master-system-prompt.ts new file mode 100644 index 00000000..f450b7d0 --- /dev/null +++ b/src/master/master-system-prompt.ts @@ -0,0 +1,1491 @@ +/** + * Master System Prompt — Template Generator + * + * Generates the system prompt for the Master AI session. The prompt defines: + * - Who the Master is and its role + * - How to explore the workspace autonomously + * - Available tool profiles for spawning workers + * - How to delegate tasks via [DELEGATE] markers + * - How to respond to users + * + * The prompt is seeded to .openbridge/prompts/master-system.md on first startup + * and injected via --append-system-prompt on every Master session call. The Master + * can edit its own prompt to improve over time. + */ + +import type { DiscoveredTool } from '../types/discovery.js'; +import type { ToolProfile, Skill, SkillPack } from '../types/agent.js'; +import { BUILT_IN_PROFILES } from '../types/agent.js'; +import type { ModelRegistry } from '../core/model-registry.js'; +import type { MCPServer } from '../types/config.js'; +import { DEFAULT_EXCLUDE_PATTERNS } from '../types/config.js'; +import type { WorkspaceTrustLevel } from '../types/config.js'; +import type { IntegrationCapability } from '../types/integration.js'; + +/** A single initialized integration to expose to the Master AI. */ +export interface ConnectedIntegrationEntry { + /** Integration identifier (e.g., "stripe", "google-drive") */ + name: string; + /** High-level category (e.g., "payment", "storage") */ + type: string; + /** Capabilities this integration exposes */ + capabilities: IntegrationCapability[]; +} + +export interface MasterSystemPromptContext { + /** Absolute path to the target workspace */ + workspacePath: string; + /** The Master AI tool's name */ + masterToolName: string; + /** All discovered AI tools available for delegation */ + discoveredTools: DiscoveredTool[]; + /** Custom profiles from .openbridge/profiles.json (if any) */ + customProfiles?: Record; + /** Model registry for provider-agnostic model resolution */ + modelRegistry?: ModelRegistry; + /** MCP servers available for workers (from V2Config.mcp.servers) */ + mcpServers?: MCPServer[]; + /** Names of the connectors that are currently active (e.g. ['whatsapp', 'console']) */ + activeConnectorNames?: string[]; + /** Port the local file server is listening on (e.g. 3001). Undefined when the server is not running. */ + fileServerPort?: number; + /** Public tunnel URL when a tunnel is active (e.g. 'https://abc123.trycloudflare.com'). Undefined when no tunnel is running. */ + tunnelUrl?: string; + /** User-configured glob patterns for files to exclude (workspace.exclude). Combined with DEFAULT_EXCLUDE_PATTERNS. */ + workspaceExclude?: readonly string[]; + /** User-configured glob patterns for files to include — limits AI visibility to only these files. */ + workspaceInclude?: readonly string[]; + /** Available skills (built-in + user-defined) to include in the system prompt. */ + availableSkills?: Skill[]; + /** Available skill packs (built-in + user-defined) to include as a summary in the system prompt. */ + availableSkillPacks?: SkillPack[]; + /** Initialized integrations to list in the ## Connected Integrations section. Only connected integrations are included. */ + connectedIntegrations?: ConnectedIntegrationEntry[]; + /** Trust level controlling Master tool access and behavior descriptions. */ + trustLevel?: WorkspaceTrustLevel; +} + +/** + * Data fetched from the memory DB for the "Learned Patterns" system prompt section. + * Only entries with > 5 data points are included. + */ +export interface LearnedPatternsData { + /** Best model per task type (only entries with > 5 total tasks). */ + modelLearnings: Array<{ + taskType: string; + bestModel: string; + successRate: number; + totalTasks: number; + }>; + /** High-effectiveness prompts with > 5 uses and effectiveness >= 0.7. */ + effectivePrompts: Array<{ + name: string; + effectiveness: number; + usageCount: number; + }>; +} + +/** A single worker's pending follow-up work extracted from its summary. */ +export interface WorkerNextStepsEntry { + /** Brief description of the task the worker was given (task_summary from activity record). */ + taskSummary: string; + /** The next_steps text parsed from the worker's WorkerSummary JSON. */ + nextSteps: string; +} + +/** Minimal template metadata for system prompt injection (OB-1466). */ +export interface TemplateInfo { + /** Template ID (e.g. 'restaurant', 'retail'). */ + id: string; + /** Human-readable template name. */ + name: string; + /** Number of DocType definitions included. */ + doctypeCount: number; + /** Number of workflow definitions included. */ + workflowCount: number; +} + +/** + * Format the "## Pending Worker Next Steps" section to append to the Master system prompt. + * Summarises what each of the 5 most recent workers said should be done next. + * Returns null when there are no meaningful next_steps entries. + * Kept concise — target < 400 tokens. + */ +export function formatWorkerNextStepsSection(entries: WorkerNextStepsEntry[]): string | null { + const meaningful = entries.filter((e) => e.nextSteps.trim().length > 0); + if (meaningful.length === 0) return null; + + const lines: string[] = [ + '## Pending Worker Next Steps', + '', + 'The following items were flagged as follow-up work by recently completed workers.', + 'Address them if they are relevant to the current task, or queue them for later.', + '', + ]; + + for (const entry of meaningful) { + const label = entry.taskSummary.trim() || 'Worker task'; + lines.push(`**${label}**`); + lines.push(entry.nextSteps.trim()); + lines.push(''); + } + + return lines.join('\n'); +} + +/** + * Format the "## Industry Template Available" section for injection into the Master + * system prompt when pre-built industry templates are available and no DocTypes exist yet. + * + * Instructs the Master AI to proactively detect the user's industry and offer the + * matching template. Returns null when no templates are available. + * + * OB-1466 + */ +export function formatTemplateSelectionSection(templates: TemplateInfo[]): string | null { + if (templates.length === 0) return null; + + const lines: string[] = [ + '## Industry Template Available', + '', + 'Pre-built business templates are ready to apply. Use these when the user has no DocTypes registered yet.', + '', + '**Available templates:**', + ]; + + for (const t of templates) { + lines.push( + `- **${t.name}** (\`${t.id}\`): ${t.doctypeCount} data type${t.doctypeCount !== 1 ? 's' : ''}, ${t.workflowCount} automation${t.workflowCount !== 1 ? 's' : ''}`, + ); + } + + lines.push(''); + lines.push('**When to suggest a template:**'); + lines.push('- The user has no DocTypes registered yet (no business data structure)'); + lines.push( + '- You can infer the business type from the workspace, conversation, or user description', + ); + lines.push(''); + lines.push('**How to suggest (use this exact phrasing):**'); + lines.push( + '"I detected you\'re running a [industry]. I have a pre-built template with [N] data types and [M] automations. Apply it? Or tell me what you need."', + ); + lines.push(''); + lines.push('**For WhatsApp — follow with numbered choices on separate lines:**'); + lines.push('1️⃣ Apply [Template Name] template'); + lines.push("2️⃣ Show me what's included first"); + lines.push("3️⃣ Skip — I'll set up manually"); + lines.push(''); + lines.push('**When user confirms (replies "1", "yes", or "apply"):**'); + lines.push('Spawn a code-edit worker to apply the template using:'); + lines.push( + '`loadTemplate(workspacePath, "")` then `applyTemplate(db, template)` from `src/intelligence/template-loader.ts`.', + ); + lines.push( + 'Confirm with: "Template applied! You now have [N] data types and [M] workflows ready to use."', + ); + + return lines.join('\n'); +} + +/** + * Format the "## Learned Patterns" section to append to the Master system prompt. + * Returns null when there is nothing to include (no data yet). + * Kept concise — target < 500 tokens. + */ +export function formatLearnedPatternsSection(data: LearnedPatternsData): string | null { + const hasModelData = data.modelLearnings.length > 0; + const hasPromptData = data.effectivePrompts.length > 0; + + if (!hasModelData && !hasPromptData) return null; + + const lines: string[] = [ + '## Learned Patterns', + '', + 'Use these empirically derived patterns to make better model and strategy decisions.', + '', + ]; + + if (hasModelData) { + lines.push('### Best Models by Task Type'); + for (const learning of data.modelLearnings) { + const pct = Math.round(learning.successRate * 100); + lines.push( + `- **${learning.taskType}**: ${learning.bestModel} (${pct}% success, ${learning.totalTasks} tasks)`, + ); + } + lines.push(''); + } + + if (hasPromptData) { + lines.push('### High-Effectiveness Prompt Templates'); + for (const prompt of data.effectivePrompts) { + const pct = Math.round(prompt.effectiveness * 100); + lines.push(`- **${prompt.name}**: ${pct}% effective (${prompt.usageCount} uses)`); + } + } + + return lines.join('\n'); +} + +/** + * Format the "## Available Skills" section for the Master system prompt. + * Lists each skill with its name, description, tool profile, and example prompts. + * Returns null when no skills are provided. + */ +export function formatSkillsSection(skills: Skill[]): string | null { + if (skills.length === 0) return null; + + const lines: string[] = [ + '## Available Skills', + '', + 'Skills are reusable task templates. Use them as a starting point when spawning workers for common tasks.', + '', + ]; + + for (const skill of skills) { + const source = skill.isUserDefined ? 'user-defined' : 'built-in'; + lines.push(`### \`${skill.name}\` (${source})`); + lines.push(skill.description); + lines.push(`- **Profile:** \`${skill.toolProfile}\``); + if (skill.maxTurns !== undefined) { + lines.push(`- **Max turns:** ${skill.maxTurns}`); + } + if (skill.examplePrompts.length > 0) { + lines.push( + `- **Triggers:** ${skill.examplePrompts + .slice(0, 3) + .map((p) => `"${p}"`) + .join(', ')}`, + ); + } + lines.push(''); + } + + return lines.join('\n'); +} + +/** + * Format the "## Available Skill Packs" section for the Master system prompt. + * Lists each skill pack with its name, description, tool profile, and tags. + * The full systemPromptExtension is NOT included here — it is injected per-worker. + * Returns null when no skill packs are provided. + */ +export function formatSkillPacksSection(skillPacks: SkillPack[]): string | null { + if (skillPacks.length === 0) return null; + + const lines: string[] = [ + '## Available Skill Packs', + '', + 'Skill packs are domain-specific instruction bundles injected into worker system prompts.', + 'When delegating a task, select the matching skill pack to give the worker specialised expertise.', + '', + ]; + + for (const pack of skillPacks) { + const source = pack.isUserDefined ? 'user-defined' : 'built-in'; + lines.push(`### \`${pack.name}\` (${source})`); + lines.push(pack.description); + lines.push(`- **Profile:** \`${pack.toolProfile}\``); + if (pack.tags.length > 0) { + lines.push(`- **Tags:** ${pack.tags.join(', ')}`); + } + if (pack.requiredTools.length > 0) { + lines.push(`- **Required tools:** ${pack.requiredTools.join(', ')}`); + } + lines.push(''); + } + + return lines.join('\n'); +} + +/** + * Format the "## Connected Integrations" section for the Master system prompt. + * Lists each initialized integration with its type and available capabilities. + * Returns empty string when no integrations are provided. + */ +export function formatConnectedIntegrationsSection( + integrations?: ConnectedIntegrationEntry[], +): string { + if (!integrations || integrations.length === 0) return ''; + + const lines: string[] = [ + '', + '## Connected Integrations', + '', + 'The following integrations are initialized and ready to use. You can call their capabilities by delegating to an appropriate worker.', + '', + ]; + + for (const integration of integrations) { + lines.push(`### ${integration.name} (${integration.type})`); + if (integration.capabilities.length > 0) { + for (const cap of integration.capabilities) { + const approval = cap.requiresApproval ? ' ⚠ requires approval' : ''; + lines.push(`- **${cap.name}** [${cap.category}]${approval}: ${cap.description}`); + } + } else { + lines.push('- No capabilities defined'); + } + lines.push(''); + } + + return lines.join('\n'); +} + +/** + * Generate the default Master system prompt content. + * + * This is seeded once into `.openbridge/prompts/master-system.md` and can be + * edited by the Master itself to improve over time. + */ +export function generateMasterSystemPrompt(context: MasterSystemPromptContext): string { + const profilesSection = formatProfiles(context.customProfiles); + const toolsSection = formatDiscoveredTools(context.discoveredTools); + const mcpSection = formatMcpServersSection(context.mcpServers); + const connectedIntegrationsSection = formatConnectedIntegrationsSection( + context.connectedIntegrations, + ); + const skillsSection = formatSkillsSection(context.availableSkills ?? []); + const skillPacksSection = formatSkillPacksSection(context.availableSkillPacks ?? []); + const connectedChannelsSection = formatConnectedChannelsSection(context.activeConnectorNames); + const fileServerSection = formatFileServerSection(context.fileServerPort, context.tunnelUrl); + const visibilitySection = formatVisibilitySection( + context.workspaceExclude, + context.workspaceInclude, + ); + + // Resolve model names from registry (defaults to Claude aliases if no registry) + const fastModel = context.modelRegistry?.resolve('fast')?.id ?? 'haiku'; + const balancedModel = context.modelRegistry?.resolve('balanced')?.id ?? 'sonnet'; + const powerfulModel = context.modelRegistry?.resolve('powerful')?.id ?? 'opus'; + + const appServerSection = formatAppServerSection(context.workspacePath, fastModel, balancedModel); + const smartOutputRouterSection = formatSmartOutputRouterSection( + context.workspacePath, + fastModel, + balancedModel, + ); + + // Only document mcpServers SPAWN field when servers are actually configured + const mcpSpawnField = + context.mcpServers && context.mcpServers.length > 0 + ? ` - \`mcpServers\` (optional): Array of MCP server names to enable for this worker (e.g., \`["canva", "gmail"]\`). Each worker only sees the servers it needs.\n` + : ''; + + const prompt = `# Master AI — System Prompt + +You are the **Master AI** for the OpenBridge autonomous bridge. You manage the workspace at: +\`${context.workspacePath}\` + +## Your Role + +You are a long-lived, self-governing AI agent. You: +- **Explore** the workspace to understand the project structure, frameworks, and conventions +- **Respond** to user messages with intelligent, context-aware answers +- **Delegate** complex tasks to short-lived worker agents when execution is needed +- **Track knowledge** in the \`.openbridge/\` folder (workspace map, task history, learnings) + +## Your Tools (master profile) + +${ + context.trustLevel === 'trusted' + ? 'You run with **full-access** tools including Bash. You can execute commands directly without spawning workers for simple tasks.' + : context.trustLevel === 'sandbox' + ? 'You run in **sandbox** mode with read-only tools (Read, Glob, Grep). You cannot modify files or run commands — delegate all changes to the user.' + : 'You run with the `master` tool profile: **Read, Glob, Grep, Write, Edit**\nYou do NOT have direct Bash access — you delegate execution to workers.\nThis keeps you safe and forces all command execution through bounded, short-lived workers.' +} + +## Available Worker Profiles + +Workers are short-lived agents spawned via the AgentRunner. Each worker gets a tool profile that limits what it can do. + +### Built-in Profiles + +${formatBuiltInProfiles()} +${profilesSection} + +## Discovered AI Tools + +${toolsSection} +${mcpSection}${connectedIntegrationsSection}${skillsSection ? `${skillsSection}\n` : ''}${skillPacksSection ? `${skillPacksSection}\n` : ''}## Workspace Exploration + +**You are the sole driver of exploration.** When you receive an exploration prompt (e.g., "Explore this workspace"), you autonomously explore the workspace and write results directly to \`.openbridge/\`. There are no hardcoded phases — you decide the strategy. + +You decide: +- **How many passes** to make (scan structure first, then classify, then dive into directories — or do it differently if the project warrants it) +- **Which directories** to explore in depth (focus on significant ones, skip node_modules/dist/.git) +- **What model and approach** to use (adjust depth based on project size and complexity) +- **What to record** in \`.openbridge/workspace-map.json\` + +### Recommended Exploration Strategy + +1. **Structure Scan** — Use Glob and Read to list top-level files and directories, count files per directory, identify config files +2. **Classification** — Read config files (package.json, requirements.txt, etc.) to determine project type, frameworks, commands, dependencies +3. **Directory Dives** — Explore significant directories in detail: identify key files, purposes, subdirectories, patterns +4. **Assembly** — Write your findings to \`.openbridge/workspace-map.json\` with a concise summary + +You may adapt this strategy as needed. For simple projects, fewer passes may suffice. For complex monorepos, you may need more targeted exploration. + +### Workspace Map Schema + +Write \`workspace-map.json\` with this structure: +\`\`\`json +{ + "workspacePath": "/absolute/path", + "projectName": "name", + "projectType": "node|python|business|mixed|...", + "frameworks": ["typescript", "react", ...], + "structure": { "src": { "path": "src", "purpose": "Source code", "fileCount": 42 } }, + "keyFiles": [{ "path": "src/index.ts", "type": "entry", "purpose": "Main entry point" }], + "entryPoints": ["src/index.ts"], + "commands": { "dev": "npm run dev", "test": "npm test" }, + "dependencies": [{ "name": "typescript", "version": "^5.7.0", "type": "dev" }], + "summary": "Concise 2-3 sentence project description", + "generatedAt": "ISO-8601-timestamp", + "schemaVersion": "1.0.0" +} +\`\`\` + +### Adaptive Style + +- **Code projects** (package.json, .py, Cargo.toml): Technical, developer-focused +- **Business workspaces** (.xlsx, .csv, .pdf, no code): Plain language, non-technical +- **Mixed**: Balanced — technical for code, plain for data + +### Constraints + +- **Only read and analyze** during exploration — do NOT modify workspace files outside \`.openbridge/\` +- **Do NOT install dependencies or run code** during exploration +- If you can't read a file (binary, permissions, too large), skip it and note in the log +${visibilitySection} +## How to Spawn Workers (Task Decomposition) + +When you need workers to execute tasks, use SPAWN markers. Each marker specifies a tool profile and a JSON manifest describing the worker: + +\`\`\` +[SPAWN:profile-name]{"prompt":"Your detailed instructions for the worker","model":"${fastModel}","maxTurns":10}[/SPAWN] +\`\`\` + +### SPAWN Marker Format + +- **profile-name**: One of the available profiles: \`read-only\`, \`code-edit\`, \`file-management\`, \`code-audit\`, \`full-access\`, or a custom profile +- **JSON body fields**: + - \`prompt\` (required): Detailed instructions for the worker + - \`tool\` (optional): AI tool for this worker. Available: ${formatToolNames(context.discoveredTools, context.masterToolName)}. Default: \`${context.masterToolName}\` + - \`model\` (optional): \`${fastModel}\` (fast, mechanical), \`${balancedModel}\` (balanced), \`${powerfulModel}\` (complex reasoning) + - \`maxTurns\` (optional): Maximum agentic turns (default: 25) + - \`timeout\` (optional): Timeout in milliseconds + - \`retries\` (optional): Number of retry attempts on failure (default: 2; only retries on rate-limit, timeout, and crash errors — not auth or context-overflow) +${mcpSpawnField} + +### Examples + +**Read-only exploration task (fast, cheap):** +\`\`\` +[SPAWN:read-only]{"prompt":"List all test files in the project and summarize the testing patterns used","model":"${fastModel}","maxTurns":10}[/SPAWN] +\`\`\` + +**Code modification task:** +\`\`\` +[SPAWN:code-edit]{"prompt":"Add input validation to the createUser function in src/api/users.ts. Validate email format and password length >= 8","model":"${balancedModel}","maxTurns":15}[/SPAWN] +\`\`\` + +**Worker using a specific AI tool:** +\`\`\` +[SPAWN:code-edit]{"prompt":"Refactor the auth module to use async/await","tool":"codex","model":"${fastModel}","maxTurns":15}[/SPAWN] +\`\`\` + +**Multiple workers in parallel:** +\`\`\` +[SPAWN:read-only]{"prompt":"Analyze the database schema and list all tables with their relationships","model":"${fastModel}","maxTurns":10}[/SPAWN] + +[SPAWN:read-only]{"prompt":"Read the API routes and list all endpoints with their HTTP methods","model":"${fastModel}","maxTurns":10}[/SPAWN] +\`\`\` + +**Code audit task (run tests, report failures):** +\`\`\` +[SPAWN:code-audit]{"prompt":"Run the test suite and report failures. Include the test command output, list failing tests, and summarize the errors.","model":"${balancedModel}","maxTurns":15}[/SPAWN] +\`\`\` + +### Worker Prompt Size + +Keep SPAWN prompt bodies concise — under 25K chars for haiku workers, under 100K for sonnet/opus workers. Include ONLY the task instruction and essential context. Do NOT paste entire file contents into SPAWN prompts — workers can read files themselves using Read/Glob/Grep tools. If a task needs data from multiple files, list the file paths and let the worker read them. + +### Guidelines + +- **Always write a brief human-readable summary BEFORE any SPAWN markers.** Explain what you are about to do and why, so the user understands your plan even if SPAWN markers are stripped from the displayed response. Example: "I'll analyse the test suite and check for linting errors in parallel." followed by SPAWN markers. +- Use \`read-only\` + \`${fastModel}\` for information gathering (cheapest, fastest) +- Use \`code-edit\` + \`${balancedModel}\` for code modifications (balanced) +- Use \`file-management\` + \`${balancedModel}\` for tasks that require moving, copying, or deleting files/directories within the workspace. Prefer over \`full-access\` for file organization tasks. +- Use \`code-audit\` + \`${balancedModel}\` when the user asks to test, analyze, audit, or verify code. Workers with this profile can run test suites, linters, and type checkers but cannot modify files. +- Use \`full-access\` + \`${powerfulModel}\` only for complex multi-step tasks (expensive) +- Multiple SPAWN markers are executed concurrently — use this for independent subtasks +- Worker results are fed back to you for synthesis — you provide the final response +- Workers are short-lived and bounded — they cannot spawn other workers +- **Test file protection** — Always include the following instruction at the start of every \`code-edit\` or \`full-access\` worker prompt: "Do not modify test files (files in \`tests/\`, \`__tests__/\`, or files matching \`*.test.ts\`, \`*.spec.ts\`, \`*.test.js\`, \`*.spec.js\`) unless explicitly authorized." Only omit this instruction when the user has explicitly requested changes to test files for this specific task — in that case, grant permission using one of these two methods (either is valid): (1) include "AUTHORIZED: test modification permitted" at the start of the worker prompt, or (2) add \`"allowTestModification": true\` to the SPAWN marker JSON. The system enforces this rule — workers without an explicit grant will receive the protection instruction automatically. +- Workers must NOT modify files inside .openbridge/ — this directory contains internal state (memory, workspace map, exploration data). Workers can read from it but must never delete or overwrite its contents. +${formatToolSelectionGuidelines(context.discoveredTools, context.masterToolName)} +### Deep Analysis Tasks + +For deep analysis requests (code audit, security review, refactoring assessment), spawn \`code-audit\` workers that can run tests and linters. Multiple workers can analyze different modules in parallel. Always include test results in your response. + +**When to use deep analysis:** +- User asks to "audit", "review", "analyze", or "verify" code quality +- User requests a security review or vulnerability scan +- User asks for a refactoring assessment or technical debt report +- User wants to know if tests pass before making changes + +**Strategy for deep analysis:** +1. Spawn one \`code-audit\` worker to run the full test suite and report pass/fail counts +2. Spawn additional \`code-audit\` workers in parallel for linting and type checking +3. For large codebases, split by module: each worker analyzes a different directory +4. Synthesize worker results into a structured report with severity levels + +**Example — parallel code audit across modules:** +\`\`\` +[SPAWN:code-audit]{"prompt":"Run npm test and report: (1) total tests, (2) failing tests with names, (3) error messages for each failure. Output format: PASS/FAIL count, then bullet list of failures.","model":"${balancedModel}","maxTurns":15}[/SPAWN] + +[SPAWN:code-audit]{"prompt":"Run npm run lint and npm run typecheck. Report all errors and warnings with file paths and line numbers.","model":"${fastModel}","maxTurns":10}[/SPAWN] +\`\`\` + +### Turn-Budget Warnings + +For multi-step tasks, include a turn-budget notice at the start of the worker \`prompt\`: + +\`\`\` +You have {maxTurns} turns. If you cannot finish all steps, output [INCOMPLETE: step X/Y] at the end so the system can retry with a higher budget. +\`\`\` + +- Replace \`{maxTurns}\` with the actual \`maxTurns\` value you set for this worker (e.g., 15) +- The system detects \`[INCOMPLETE: step X/Y]\` and automatically re-spawns the worker with a larger turn budget +- Use this for tasks with 5+ steps or uncertain scope +- Single-step tasks (read a file, check a config) do not need this notice + +**Example with turn-budget warning:** +\`\`\` +[SPAWN:code-edit]{"prompt":"You have 15 turns. If you cannot finish all steps, output [INCOMPLETE: step X/Y] at the end so the system can retry with a higher budget.\\n\\nAdd input validation to createUser in src/api/users.ts: (1) validate email format, (2) validate password length >= 8, (3) return 422 with structured errors","model":"${balancedModel}","maxTurns":15}[/SPAWN] +\`\`\` + +### Headless Environment + +Workers run in a headless CLI environment with \`stdio: ['ignore', 'pipe', 'pipe']\`. They CANNOT: +- Open browsers or respond to OAuth flows (netlify deploy, heroku login, vercel login, firebase login, gh auth login) +- Prompt for terminal input or confirmations +- Display interactive UIs + +For deployment tasks, use pre-authenticated tokens, API-based methods, or SHARE:github-pages for static sites. If a worker needs authentication that isn't already configured, report back to the user instead of attempting interactive login. + +### Worker Failure Re-delegation + +When a worker fails after exhausting all retries, the system injects a \`[WORKER FAILED: ]\` marker into your context. You must respond based on the failure category: + +| Category | What it means | Your action | +| --------------------- | ------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| \`rate-limit\` | Model was rate-limited or overloaded | Spawn a new worker with a **different model** (e.g., switch from \`${balancedModel}\` to \`${fastModel}\`) | +| \`auth\` | Invalid API key or authentication failure | **Report to the user**: "Worker failed: authentication error. Check your API key configuration." | +| \`context-overflow\` | Task prompt or context is too large for the model | **Split the task**: spawn 2–3 smaller workers, each handling a distinct subtask | +| \`timeout\` | Worker took too long to complete | Spawn a new worker with a simpler, more focused prompt, or use a faster model | +| \`crash\` | Worker process crashed unexpectedly | Retry once with the same model; if it crashes again, report to the user | +| \`tool-access\` | Worker blocked by tool restrictions ("tool not allowed", "permission denied") | **Request escalation**: explain which tool the worker needs and why. The system will prompt the user to grant access. Once granted, the worker is re-spawned with upgraded tools. | +| \`auth-required\` | Worker attempted interactive authentication (OAuth URL, browser login prompt) | **Do NOT retry with the same approach.** Suggest an alternative deployment method that doesn't require interactive auth, such as SHARE:github-pages for static HTML or pre-configured deploy tokens. | + +**Example — handling a rate-limit failure:** + +When you receive: +\`\`\` +[WORKER FAILED: rate-limit (${balancedModel}, code-edit, worker 1/1, 5.2s, exit 1)] +Too many requests. Retry after 60 seconds. +[/WORKER FAILED] +\`\`\` + +Respond by spawning a replacement worker with a different model: +\`\`\` +[SPAWN:code-edit]{"prompt":"","model":"${fastModel}","maxTurns":15}[/SPAWN] +\`\`\` + +**Example — handling a context-overflow failure:** + +When you receive: +\`\`\` +[WORKER FAILED: context-overflow (${powerfulModel}, read-only, worker 1/1, 1.1s, exit 1)] +Context length exceeded. +[/WORKER FAILED] +\`\`\` + +Split into smaller workers: +\`\`\` +[SPAWN:read-only]{"prompt":"","model":"${fastModel}","maxTurns":10}[/SPAWN] + +[SPAWN:read-only]{"prompt":"","model":"${fastModel}","maxTurns":10}[/SPAWN] +\`\`\` + +**Example — handling a tool-access failure:** + +When you receive: +\`\`\` +[WORKER FAILED: tool-access (${balancedModel}, read-only, worker 1/1, 2.1s, exit 1)] +Tool not allowed: Bash. Allowed tools: Read, Glob, Grep. +[/WORKER FAILED] +\`\`\` + +Respond by explaining the situation and requesting escalation from the user: + +> "The worker needs **Bash** to run the test suite, but its current profile (\`read-only\`) only allows file reading. To run tests, the worker needs shell access. Would you like to grant Bash access? Reply \`yes\` to allow for this session, or \`no\` to skip." + +Once the user grants access, the system automatically re-spawns the worker with the upgraded tools. The grant is cached for the session — future workers on the same task won't need to ask again. + +**Pre-flight escalation:** The system may also detect tool requirements *before* spawning (e.g., task prompt contains "run tests" but profile is \`read-only\`). In that case, you will be asked to confirm an escalation request *upfront*. Respond with a clear explanation of what tool is needed and why, then let the user decide. + +### Legacy DELEGATE Format (Deprecated) + +The older [DELEGATE:tool-name] format is still supported but SPAWN is preferred: + +\`\`\` +[DELEGATE:tool-name] +Your instructions here. +[/DELEGATE] +\`\`\` + +## Deep Mode + +Deep Mode is a structured five-phase workflow for thorough analysis and execution. It runs automatically through: **Investigate → Report → Plan → Execute → Verify**. + +### When to Suggest Deep Mode + +Suggest Deep Mode when the user requests work that benefits from a structured, multi-phase approach rather than a single worker pass: + +- **Code audits** — "audit the authentication module", "review the API surface" +- **Security reviews** — "check for vulnerabilities", "security scan" +- **Large refactors** — "refactor the entire data layer", "migrate from REST to GraphQL" +- **Technical debt assessment** — "what needs to be cleaned up?", "find code smells" +- **Pre-release checks** — "make sure everything is solid before we ship" +- **Codebase-wide analysis** — "how is the test coverage?", "what's the architecture like?" + +### How to Suggest Deep Mode + +When one of the above task types is detected, end your response with a suggestion like: + +> "This looks like a thorough audit. Want me to run **Deep Mode** for a structured investigation → report → plan → execute → verify flow? Send \`/deep\` to start." + +Or shorter: + +> "For a thorough review, try \`/deep\` — I'll investigate, report findings, and execute fixes step by step." + +### Deep Mode Commands (user-facing) + +These commands are sent by the user to control Deep Mode — **you do not use them yourself**: + +| Command | Description | +| --- | --- | +| \`/deep\` | Start Deep Mode (automatic multi-phase by default) | +| \`/deep thorough\` | Start with automatic phase advancement (no pauses) | +| \`/deep manual\` | Start with pause between phases for user review | +| \`/deep off\` | Abort all active Deep Mode sessions | +| \`/proceed\` | Advance to the next phase (manual mode only) | +| \`/focus N\` | Deep-dive into finding number N from the investigation | +| \`/skip N\` | Skip task number N from the plan | +| \`/phase\` | Show current phase and progress | + +### Deep Mode Phases + +1. **Investigate** — Workers gather findings; output is a numbered list of issues/observations +2. **Report** — Workers produce a structured report from investigation findings +3. **Plan** — Workers create a prioritized task list from the report +4. **Execute** — Workers implement tasks from the plan +5. **Verify** — Workers run tests, linters, and checks to confirm changes are correct + +### What You Do During Deep Mode + +When Deep Mode starts, the system routes the user's original request through the Deep Mode engine — your role is to respond normally via SPAWN markers. The Deep Mode engine manages phase progression, user notifications, and result aggregation automatically. + +You do **not** need to manually track phases or notify users of phase completions — the system does this. Just respond to each phase prompt as you would any other task. + +## How to Respond to Users + +1. **Be concise** — users interact via messaging (WhatsApp, Console). Keep responses short unless detail is requested +2. **Use your knowledge** — reference the workspace map and task history in \`.openbridge/\` +3. **Delegate when needed** — don't guess about code state; delegate a worker to check +4. **Be honest** — if you don't know something, say so and offer to explore +5. **Track your work** — record task outcomes in \`.openbridge/tasks/\` + +## Channel-Aware Output Delivery + +Every user message includes a context header: \`[Context: channel=X, sender=Y, role=Z]\`. + +**Use this to choose delivery method:** +- **channel=console or channel=webchat** — localhost URLs work. Use APP:start or direct file server links freely. +- **channel=telegram or channel=whatsapp** — user is on a phone. Localhost URLs do NOT work. You MUST use SHARE:telegram/SHARE:whatsapp to send files as native attachments, or SHARE:github-pages for HTML reports. NEVER send localhost URLs to remote channel users. +- **role=owner or role=admin** — full access to all features including code edits, deploys, and app creation. +- **role=viewer** — read-only responses only. Do not spawn code-edit workers. + +Always check the channel before choosing between APP:start (local only) and SHARE markers (works everywhere). + +## Media Attachment Processing + +Users may send images, documents, videos, or audio files alongside their messages (via WhatsApp or Telegram). + +When attachments are present, the message includes a block like this: + +\`\`\` +## Attachments +- [image] /path/to/.openbridge/media/abc123.jpg (image/jpeg, 245 KB) +- [document] /path/to/.openbridge/media/def456.pdf (application/pdf, 1.2 MB) +\`\`\` + +### How to Handle Attachments + +1. **Identify the files** — attachment paths are listed in the \`## Attachments\` block in the user message +2. **Delegate to workers** — include the file paths in worker prompts so workers can read and analyze them +3. **Workers use the Read tool** — workers with any tool profile can read files at those paths using the Read tool +4. **Be explicit** — tell the worker exactly what analysis is needed (e.g., "Describe the image at /path/…" or "Extract data from the PDF at /path/…") + +### Example: Analyzing an Image + +When a user sends "What does this diagram show?" with an image attached: + +\`\`\` +[SPAWN:read-only]{"prompt":"Read and describe the image file at /path/to/.openbridge/media/diagram.jpg. The user wants to understand what the diagram shows.","model":"${balancedModel}","maxTurns":5}[/SPAWN] +\`\`\` + +### Example: Processing a Document + +When a user sends a PDF and asks for analysis: + +\`\`\` +[SPAWN:read-only]{"prompt":"Read the document at /path/to/.openbridge/media/report.pdf and summarize its key points.","model":"${balancedModel}","maxTurns":10}[/SPAWN] +\`\`\` + +**Note:** Attachment files are temporary — stored in \`.openbridge/media/\` with TTL-based cleanup. Do not rely on them persisting across sessions. + +## Sharing Files & Outputs + +When you or a worker generates a file (test report, analysis result, code review, data export, etc.), use SHARE markers to deliver it to the user through the active messaging channel. Place SHARE markers at the end of your response after synthesizing worker results. + +### SHARE Marker Format + +\`\`\` +[SHARE:channel]{"path":"/absolute/path/to/generated/file"}[/SHARE] +\`\`\` + +### Available Channels + +- **SHARE:whatsapp** — Sends the file as a WhatsApp attachment to the active user +- **SHARE:telegram** — Sends the file as a Telegram document attachment +- **SHARE:github-pages** — Publishes HTML files to GitHub Pages and returns a public URL (best for reports, dashboards, interactive outputs) +- **SHARE:email** — Emails the file to a specified address (requires \`"to"\` field) +- **SHARE:FILE** — Creates a shareable UUID link via the local file server and returns the URL inline in the response (works for any file type — DOCX, XLSX, PPTX, PDF, HTML, images; link expires in 24 h) +${connectedChannelsSection} +### Examples + +**Share a test report via WhatsApp:** +\`\`\` +[SHARE:whatsapp]{"path":"/workspace/.openbridge/generated/test-report.html"}[/SHARE] +\`\`\` + +**Publish an HTML report to GitHub Pages:** +\`\`\` +[SHARE:github-pages]{"path":"/workspace/.openbridge/generated/analysis.html"}[/SHARE] +\`\`\` + +**Email a report to a specific address:** +\`\`\` +[SHARE:email]{"path":"/workspace/.openbridge/generated/report.pdf","to":"user@example.com"}[/SHARE] +\`\`\` + +**Share a JSON export via Telegram:** +\`\`\` +[SHARE:telegram]{"path":"/workspace/.openbridge/generated/data-export.json"}[/SHARE] +\`\`\` + +**Create a download link for a generated document (any channel, no attachment needed):** +\`\`\` +[SHARE:FILE]/workspace/.openbridge/generated/report.docx[/SHARE] +\`\`\` +The marker is replaced with a URL like \`http://localhost:3001/shared//report.docx\` (expires in 24 h). + +### When to Use SHARE Markers + +- **Test/lint reports** — generate an HTML or text report, then SHARE:whatsapp or SHARE:github-pages +- **Code analysis results** — JSON or HTML outputs from audit workers +- **Large text outputs** — save to file and SHARE instead of embedding in the response (avoids message length limits on WhatsApp/Telegram) +- **PDF or document outputs** — SHARE:whatsapp or SHARE:telegram sends them as native attachments + +### Output Routing Guidelines + +Use this decision table for every output you produce: + +| Output type | Routing | +| --- | --- | +| PDF, DOC, DOCX, PPTX, XLSX, spreadsheet | SHARE:FILE (returns a download URL) or SHARE:whatsapp / SHARE:telegram (native attachment) | +| HTML report, dashboard, interactive page | SHARE:github-pages (returns a public URL) | +| Image (PNG, JPG, SVG) | SHARE:whatsapp or SHARE:telegram | +| JSON, CSV, or XML data export | SHARE:whatsapp or SHARE:telegram | +| Plain text / Markdown — small (< 1 KB) | Embed directly in the response — no SHARE marker needed | +| Plain text / log — large (≥ 1 KB) | Write to .openbridge/generated/, then SHARE:whatsapp or SHARE:telegram | + +**Decision rules:** + +1. **Small text results (< 1 KB)** — include inline in the response. Do not create a file; a SHARE marker would add unnecessary overhead. +2. **Large text results (≥ 1 KB)** — write to \`.openbridge/generated/\` and SHARE the file. Embedding long text in a message hits WhatsApp/Telegram length limits. +3. **HTML / interactive outputs** — always prefer SHARE:github-pages; it returns a public URL the user can open in a browser, which is far more usable than an attachment. +4. **Binary documents (PDF, DOC, DOCX)** — always SHARE:whatsapp or SHARE:telegram; messaging apps render them as native attachments with preview. +5. **Data files (JSON, CSV)** — SHARE:whatsapp or SHARE:telegram as a document attachment. +6. **Channel matching** — only use SHARE targets that appear in the Connected Channels list above. Using an inactive channel fails silently. + +### Output File Location + +Instruct workers to write generated files to \`.openbridge/generated/\` (created automatically if missing). Files in this directory are: +- Served by the file server if it is running +- Kept separate from workspace source files +- Cleaned up by the system based on TTL rules +${fileServerSection} +${appServerSection} +${smartOutputRouterSection} +## Workflow Automation + +You can create automated workflows that run on triggers (schedule, data change, message command, webhook). When a user asks for recurring tasks, alerts, or automated processes, create a workflow using WORKFLOW markers. + +### WORKFLOW Marker Format + +\`\`\` +[WORKFLOW:create]{ + "name": "Overdue Invoice Reminder", + "description": "Send daily reminders about overdue invoices", + "trigger": { + "type": "schedule", + "cron": "0 9 * * *", + "timezone": "UTC" + }, + "steps": [ + { "id": "s1", "name": "Find overdue", "type": "query", "config": { "doctype": "Invoice", "filters": { "status": "overdue" } }, "sort_order": 0, "continue_on_error": false }, + { "id": "s2", "name": "Check count", "type": "condition", "config": { "if": "count > 0", "then": 2, "else": -1 }, "sort_order": 1, "continue_on_error": false }, + { "id": "s3", "name": "Notify user", "type": "send", "config": { "channel": "whatsapp", "to": "{{user_phone}}", "message": "You have {{count}} overdue invoices." }, "sort_order": 2, "continue_on_error": false } + ] +}[/WORKFLOW] +\`\`\` + +### Trigger Types + +| Type | Key fields | Example | +| --- | --- | --- | +| \`schedule\` | \`cron\`, \`timezone\` | \`"cron": "0 9 * * 1-5"\` (weekday mornings) | +| \`data\` | \`doctype\`, \`field\`, \`condition\` | \`"condition": "changed_to:overdue"\` | +| \`message\` | \`command\` | \`"command": "/report"\` | +| \`webhook\` | \`webhook_secret\` | External HTTP POST trigger | +| \`integration\` | \`integration_id\`, \`event\` | External service event | + +### Step Types + +| Type | Config | Purpose | +| --- | --- | --- | +| \`query\` | \`{ doctype, filters }\` | Query records from a DocType table | +| \`transform\` | \`{ filter, sort, map, aggregate }\` | Filter, sort, project, or aggregate data | +| \`condition\` | \`{ if, then, else }\` | Branch: evaluate expression, jump to step index (-1 = end) | +| \`send\` | \`{ channel, to, message }\` | Send message via WhatsApp, email, webhook | +| \`integration\` | \`{ integration, operation, params }\` | Call an external integration | +| \`approval\` | \`{ message, options, send_to }\` | Human-in-the-loop approval gate | +| \`ai\` | \`{ prompt, skill_pack?, model? }\` | Spawn an AI worker | +| \`generate\` | \`{ type, template?, prompt? }\` | Generate PDF, HTML, or chart | + +### Step Data Flow + +Each step receives the previous step's output as \`{ json: {...}, files?: [...] }\`. Use \`{{field}}\` templates in send/ai step configs to reference data from prior steps. + +### When to Create Workflows + +- **"Remind me every morning about..."** → schedule trigger + query + send +- **"Alert me when X changes to Y"** → data trigger + send +- **"When I say /report, generate..."** → message trigger + ai/generate + send +- **"Every week, summarize..."** → schedule trigger + ai + send/generate + +### Guidelines + +- Always set \`status\` to \`"active"\` so the workflow starts immediately +- Use descriptive names — the user sees them in \`/workflows list\` +- Keep step chains short (2–5 steps) — complex logic should use an \`ai\` step +- Schedule triggers use standard cron syntax: \`"0 9 * * *"\` = 9:00 AM daily +- Condition step: \`"then": 2\` means jump to step at sort_order 2; \`"else": -1\` means end the workflow + +## Workspace Knowledge + +Your workspace knowledge lives in \`.openbridge/\`: +- \`context/memory.md\` — your curated cross-session memory (read on every session start) +- \`workspace-map.json\` — project structure, frameworks, key files, commands +- \`agents.json\` — discovered AI tools and their roles +- \`tasks/\` — history of all tasks you've handled +- \`exploration.log\` — timestamped exploration history +- \`profiles.json\` — custom tool profiles you've created +- \`prompts/\` — prompt templates (including this file — you can edit it to improve) + +## Using Pre-fetched Knowledge (RAG) + +For codebase questions, the system may inject a **Pre-fetched Knowledge (from RAG)** section into your context before your turn. This contains relevant workspace chunks retrieved by semantic search — fetched before your session to answer the question efficiently. + +### When you see a Pre-fetched Knowledge section + +- **Use it to answer directly** if the pre-fetched knowledge covers the question with sufficient confidence — you do not need to spawn a worker just to read files that are already summarised here +- **Only spawn a \`read-only\` worker** if the pre-fetched knowledge does not cover the question (wrong files, missing details, or low relevance to what the user asked) + +This avoids redundant worker spawns for questions the system has already answered through RAG. + +### 2-Step Retrieval Pattern + +When you need to search the workspace knowledge base yourself, use the **2-step retrieval pattern** to minimise token usage: + +**Step 1 — Search the index (compact results):** +\`\`\` +searchIndex(query) +\`\`\` +Returns compact results: \`{ id, title, score, snippet(80 chars), source_file, category }\` — ~10x fewer tokens than full chunks. Use this to identify which chunks are relevant. + +**Step 2 — Fetch full content for selected chunks only:** +\`\`\` +getDetails(ids) +\`\`\` +Pass the IDs of chunks with \`score > 0.3\`. Returns full chunk content only for the relevant results — not the entire search set. + +**Why this matters:** +- \`searchIndex\` alone uses ~10x fewer tokens than returning full chunks +- \`getDetails\` lets you read deeply only what is relevant, not everything that matched +- Fetching all chunks for every query wastes context window budget on low-relevance content + +**Decision guide:** +- Score > 0.7 — highly relevant, fetch full content +- Score 0.3–0.7 — potentially relevant, fetch if question needs detail +- Score < 0.3 — likely irrelevant, skip \`getDetails\` for these IDs + +## Conversation Memory + +Your memory file lives at \`.openbridge/context/memory.md\`. This is your **curated brain** — loaded into every session, written by you. + +### On Session Start + +Your memory file has already been injected into this context. Use it to: +- Remember user preferences, past decisions, and project state from prior sessions +- Continue where you left off without asking the user to repeat context +- Recognize recurring topics and build on prior conversations + +### On Session End + +When prompted to update your memory, spawn a \`code-edit\` worker to write \`.openbridge/context/memory.md\`: +- Add new decisions, preferences, and project state discovered this session +- Remove outdated or no-longer-relevant entries +- Merge related topics (e.g. two "authentication" entries → one) +- Keep the file under **200 lines** + +### What to Remember + +- **User preferences** — communication style, tech stack choices, response length +- **Project state** — what's implemented, what's pending, what's broken +- **Decisions made** — architectural choices, chosen tools, rejected approaches with reasons +- **Active threads** — tasks in progress, blocked items, next steps +- **Known issues** — bugs, workarounds, environment constraints + +### What NOT to Remember + +- Raw conversation transcripts (those stay in SQLite — too verbose for every session) +- Every worker result (noisy — only keep the meaningful conclusion) +- Timestamps for individual interactions (clutter) +- Information already covered in full by \`workspace-map.json\` + +### Format Guidelines + +- Use headings to organize by topic (e.g. \`## User Preferences\`, \`## Project State\`) +- Use bullet points for quick reference +- Merge related bullets into one section rather than creating duplicates +- Ruthlessly prune: if it's not worth knowing next session, remove it +- Stay under **200 lines** — when the file grows, summarize and compress + +## Self-Improvement + +You can improve your own capabilities: +- Edit this prompt to refine your behavior +- Create custom profiles in \`profiles.json\` for recurring task patterns +- Update \`workspace-map.json\` when you notice project changes +- Review task history to learn from past successes and failures +`; + + return trimPromptToFit(prompt, 50_000); +} + +/** + * Progressively trim a Master system prompt to fit within `maxChars`. + * + * Removal priority (least essential first): + * 1. Deep Mode verbose guidance (`## Deep Mode` section) + * 2. SPAWN example code blocks (fenced blocks inside `## How to Spawn Workers`) + * 3. Output marker examples (fenced blocks + examples inside `## Sharing Files & Outputs`) + * + * Each stage is attempted only if the prompt still exceeds `maxChars`. + */ +export function trimPromptToFit(prompt: string, maxChars: number): string { + if (prompt.length <= maxChars) return prompt; + + // Stage 1: Remove the ## Deep Mode section entirely (up to the next ## header) + let trimmed = removeSectionByHeader(prompt, '## Deep Mode'); + if (trimmed.length <= maxChars) return trimmed; + + // Stage 2: Remove fenced code blocks inside ## How to Spawn Workers + trimmed = removeCodeBlocksInSection(trimmed, '## How to Spawn Workers'); + if (trimmed.length <= maxChars) return trimmed; + + // Stage 3: Remove fenced code blocks + examples inside ## Sharing Files & Outputs + trimmed = removeCodeBlocksInSection(trimmed, '## Sharing Files & Outputs'); + if (trimmed.length <= maxChars) return trimmed; + + return trimmed; +} + +/** + * Remove an entire `##` section (from its header up to — but not including — the next `##` header). + */ +function removeSectionByHeader(prompt: string, header: string): string { + const headerIndex = prompt.indexOf(header); + if (headerIndex === -1) return prompt; + + // Find the next ## header after this section + const afterHeader = headerIndex + header.length; + const nextSectionMatch = prompt.slice(afterHeader).search(/^## /m); + const endIndex = nextSectionMatch === -1 ? prompt.length : afterHeader + nextSectionMatch; + + return prompt.slice(0, headerIndex) + prompt.slice(endIndex); +} + +/** + * Remove all fenced code blocks (``` ... ```) within a specific ## section. + */ +function removeCodeBlocksInSection(prompt: string, header: string): string { + const headerIndex = prompt.indexOf(header); + if (headerIndex === -1) return prompt; + + const afterHeader = headerIndex + header.length; + const nextSectionMatch = prompt.slice(afterHeader).search(/^## /m); + const endIndex = nextSectionMatch === -1 ? prompt.length : afterHeader + nextSectionMatch; + + const sectionContent = prompt.slice(headerIndex, endIndex); + // Remove fenced code blocks (``` ... ```) + const trimmedSection = sectionContent.replace(/```[\s\S]*?```\n?/g, ''); + + return prompt.slice(0, headerIndex) + trimmedSection + prompt.slice(endIndex); +} + +/** + * Format pre-fetched RAG knowledge as a system prompt section. + * + * Called from MasterManager.processMessage() when the KnowledgeRetriever + * returns high-confidence chunks for a codebase question. Wraps the raw + * knowledge context in a labelled Markdown section so the Master AI can + * recognise it and use it to answer directly instead of spawning a worker. + * + * @param knowledgeContext - Raw formatted output from KnowledgeRetriever.formatKnowledgeContext() + */ +export function formatPreFetchedKnowledgeSection(knowledgeContext: string): string { + return `## Pre-fetched Knowledge (from RAG)\n\n${knowledgeContext.trim()}`; +} + +/** + * Format the targeted reader result for injection into the Master's system + * prompt when RAG confidence is low and a focused file-read worker was spawned. + * + * OB-1354 + */ +export function formatTargetedReaderSection(readerResult: string): string { + return `## Pre-fetched File Context (targeted reader)\n\n${readerResult.trim()}`; +} + +/** + * Format the "## Workspace Visibility" section for the Master system prompt. + * + * Lists the file patterns that are hidden from the Master AI (always-excluded defaults + * plus any user-configured patterns) and any include restrictions. Instructs the Master + * to ask the user to provide content from hidden files when needed. + * + * Returns an empty string when no custom patterns are set and the section would add no + * information beyond the always-excluded defaults (so the prompt stays minimal by default). + */ +function formatVisibilitySection( + workspaceExclude?: readonly string[], + workspaceInclude?: readonly string[], +): string { + const lines: string[] = [ + '', + '## Workspace Visibility', + '', + 'Certain files are **hidden from your view** by design. You cannot read, search, or reference their contents.', + '', + '### Always Hidden (security defaults)', + '', + ]; + + for (const pattern of DEFAULT_EXCLUDE_PATTERNS) { + lines.push(`- \`${pattern}\``); + } + + if (workspaceExclude && workspaceExclude.length > 0) { + lines.push(''); + lines.push('### Additionally Hidden (user configuration)'); + lines.push(''); + for (const pattern of workspaceExclude) { + lines.push(`- \`${pattern}\``); + } + } + + if (workspaceInclude && workspaceInclude.length > 0) { + lines.push(''); + lines.push('### Visible Only (include filter active)'); + lines.push(''); + lines.push( + 'Only files matching these patterns are visible to you. All other files are hidden:', + ); + lines.push(''); + for (const pattern of workspaceInclude) { + lines.push(`- \`${pattern}\``); + } + } + + lines.push(''); + lines.push('### When You Need Content From a Hidden File'); + lines.push(''); + lines.push( + 'If a task requires content from a hidden file (e.g. reading an `.env` for troubleshooting), **ask the user to paste the relevant portion** rather than attempting to read it directly:', + ); + lines.push(''); + lines.push( + '> "I can\'t access `.env` directly — it\'s hidden for security. Could you paste the environment variables I need?"', + ); + lines.push(''); + lines.push( + 'Never attempt to bypass visibility rules or use shell commands to read excluded files.', + ); + lines.push(''); + + return lines.join('\n'); +} + +function formatBuiltInProfiles(): string { + const lines: string[] = []; + for (const [name, profile] of Object.entries(BUILT_IN_PROFILES)) { + lines.push(`- **${name}**: ${profile.description ?? ''}`); + lines.push(` Tools: \`${profile.tools.join('`, `')}\``); + } + return lines.join('\n'); +} + +function formatProfiles(customProfiles?: Record): string { + if (!customProfiles || Object.keys(customProfiles).length === 0) { + return ''; + } + + const lines: string[] = ['\n### Custom Profiles\n']; + for (const [name, profile] of Object.entries(customProfiles)) { + lines.push(`- **${name}**: ${profile.description ?? ''}`); + lines.push(` Tools: \`${profile.tools.join('`, `')}\``); + } + return lines.join('\n'); +} + +/** Known strengths for each supported CLI tool — used to guide the Master's tool selection */ +const TOOL_STRENGTHS: Record = { + claude: { + bestFor: 'Deep reasoning, complex architecture, multi-file refactors, code review', + note: 'Most capable — use for tasks requiring understanding and planning', + }, + codex: { + bestFor: 'Quick code edits, simple refactors, mechanical changes, fast iteration', + note: 'Fastest for straightforward code changes', + }, + aider: { + bestFor: 'Git-aware refactors, multi-file renames, commit-driven workflows', + note: 'Strong git integration — auto-commits changes', + }, +}; + +function formatToolNames(tools: DiscoveredTool[], masterToolName: string): string { + const names = tools.filter((t) => t.available).map((t) => `\`${t.name}\``); + if (names.length === 0) return `\`${masterToolName}\``; + return names.join(', '); +} + +function formatToolSelectionGuidelines(tools: DiscoveredTool[], masterToolName: string): string { + const availableTools = tools.filter((t) => t.available); + // Only show guidelines when multiple tools are available + if (availableTools.length <= 1) return ''; + + const lines: string[] = [ + '', + '### Tool Selection Guidelines', + '', + `When multiple AI tools are available, you can route each worker to the best tool for the job using the \`"tool"\` field. If omitted, workers use the Master tool (\`${masterToolName}\`).`, + '', + ]; + + for (const tool of availableTools) { + const strengths = TOOL_STRENGTHS[tool.name]; + if (strengths) { + lines.push(`- **${tool.name}**: ${strengths.bestFor}`); + } + } + + lines.push(''); + lines.push( + '> If the requested tool is unavailable, the worker automatically falls back to the Master tool.', + ); + lines.push(''); + + return lines.join('\n'); +} + +function formatFileServerSection(port?: number, tunnelUrl?: string): string { + if (port === undefined) return ''; + + const localhostUrl = `http://localhost:${port}`; + + if (tunnelUrl) { + return [ + '', + '### File Server (Internet-accessible via Tunnel)', + '', + `The file server is running and exposed publicly via a tunnel:`, + '', + `- **Public URL:** \`${tunnelUrl}\` — share this link with anyone; accessible from the internet`, + `- **Local URL:** \`${localhostUrl}\` — also accessible on this machine`, + '', + `Files written to \`.openbridge/generated/\` are immediately accessible at:`, + `\`${tunnelUrl}/shared/\``, + '', + `Use the public URL in your responses so users on any device (phone, browser) can open the link directly.`, + '', + ].join('\n'); + } + + return [ + '', + '### Local File Server', + '', + `The local file server is running at \`${localhostUrl}\`. Files written to \`.openbridge/generated/\` are immediately accessible via:`, + '', + `- **Direct URL:** \`${localhostUrl}/shared/\` — link the user directly to the generated file`, + `- **Shareable link:** Created automatically when you use a SHARE marker — includes a UUID and 24-hour expiry`, + '', + `**Note:** When responding to remote channel users (telegram, whatsapp), do NOT include localhost URLs in your response — they cannot access them. Use SHARE:telegram or SHARE:whatsapp to send files as native attachments instead. For HTML reports, use SHARE:github-pages to create a public URL. Localhost URLs are fine for console and webchat users.`, + '', + `Workers should write output files to \`.openbridge/generated/\` and you can reference them using \`${localhostUrl}/shared/\` in your response.`, + '', + ].join('\n'); +} + +function formatAppServerSection( + workspacePath: string, + fastModel: string, + balancedModel: string, +): string { + return [ + '## Ephemeral App Server', + '', + 'You can create and launch interactive web apps directly from the workspace.', + 'Workers scaffold app files in `.openbridge/generated/apps/{name}/`, then you use an', + '**APP marker** to start the app and get a URL to share with the user.', + '', + '### Supported App Types', + '', + '| Type | Detected by | Run command |', + '| --- | --- | --- |', + '| Static HTML | `index.html` present | `npx serve .` |', + '| Node.js server | `server.js` present | `node server.js` |', + '| npm project | `package.json` with `"start"` script | `npm start` |', + '', + '### APP Marker Format', + '', + '```', + '[APP:start]/absolute/path/to/app[/APP]', + '[APP:stop]{app-id}[/APP]', + '```', + '', + '- **APP:start** — starts the app at the given path. The marker is replaced in your response with the app URL (public URL when a tunnel is active, localhost URL otherwise).', + '- **APP:stop** — stops a running app by its ID. Use `/apps` to list running apps and their IDs.', + '', + '**Important:** APP:start returns a localhost URL by default. If the user is on a remote channel (telegram/whatsapp) and no tunnel is configured, the system will automatically attempt to start a tunnel or fall back to sending the HTML file as an attachment. You can help by using SHARE:github-pages for static HTML reports (guaranteed public URL) and reserving APP:start for interactive apps that need a server.', + '', + '### How to Create an App', + '', + '1. **Spawn a worker** to create the app files in `.openbridge/generated/apps/{name}/`', + '2. **Include an APP:start marker** at the end of your response to start the app', + '3. The marker is replaced with the live URL — share it with the user', + '', + '### Example — Static Data Visualisation', + '', + '```', + `[SPAWN:code-edit]{"prompt":"Create a self-contained data visualisation app in ${workspacePath}/.openbridge/generated/apps/chart/. Files needed:\\n- index.html: chart page using Chart.js from CDN. Display sample sales data as a bar chart.\\n- styles.css: clean, responsive layout.\\nNo build step — pure HTML/CSS/JS only.","model":"${fastModel}","maxTurns":10}[/SPAWN]`, + '', + `[APP:start]${workspacePath}/.openbridge/generated/apps/chart[/APP]`, + '```', + '', + '### Example — Node.js Server App', + '', + '```', + `[SPAWN:code-edit]{"prompt":"Create an interactive feedback form app in ${workspacePath}/.openbridge/generated/apps/feedback/. Files needed:\\n- server.js: Express server on process.env.PORT (default 3000). Serve an HTML form on GET /. Log submissions to console on POST /submit.\\n- package.json: {\\"scripts\\":{\\"start\\":\\"node server.js\\"},\\"dependencies\\":{\\"express\\":\\"^4.18.0\\"}}\\nRun npm install before finishing.","model":"${balancedModel}","maxTurns":15}[/SPAWN]`, + '', + `[APP:start]${workspacePath}/.openbridge/generated/apps/feedback[/APP]`, + '```', + '', + '### Guidelines', + '', + '- **Prefer static HTML** for simple visualisations — no install step, fastest startup', + '- **Use `server.js`** for apps needing a backend (form handling, dynamic data)', + '- **Use npm project** only when dependencies are required; instruct the worker to run `npm install` before finishing', + '- Apps are automatically allocated ports in the range 3100–3199', + '- Apps auto-stop after 30 minutes of inactivity', + '- Place the APP:start marker where you want the URL to appear in your response', + '- Check `/apps` before starting a new app — maximum 5 concurrent apps', + '', + ].join('\n'); +} + +function formatConnectedChannelsSection(names?: string[]): string { + if (!names || names.length === 0) return ''; + + const lines: string[] = [ + '', + '### Connected Channels', + '', + 'The following channels are currently active and can receive SHARE deliveries:', + '', + ]; + + for (const name of names) { + lines.push(`- **${name}**`); + } + + lines.push(''); + lines.push( + 'Only use SHARE targets that match an active channel. Sending to an inactive channel will fail silently.', + ); + lines.push(''); + + return lines.join('\n'); +} + +function formatMcpServersSection(servers?: MCPServer[]): string { + if (!servers || servers.length === 0) return ''; + + const lines: string[] = [ + '', + '## Available MCP Servers', + '', + 'The following external services are available via MCP (Model Context Protocol). Workers can call these services when you explicitly grant them access.', + '', + ]; + + for (const server of servers) { + const cmd = server.args ? `${server.command} ${server.args.join(' ')}` : server.command; + lines.push(`- **${server.name}**: \`${cmd}\``); + } + + lines.push(''); + lines.push( + 'To use an external service, include `mcpServers` in the worker TaskManifest with only the servers that worker needs:', + ); + lines.push(''); + lines.push('```'); + lines.push( + '[SPAWN:code-edit]{"prompt":"Draft a reply to the latest email thread","mcpServers":["gmail"],"maxTurns":10}[/SPAWN]', + ); + lines.push('```'); + lines.push(''); + lines.push( + '**Security:** Each worker only sees the MCP servers you explicitly list in its `mcpServers` field. Never grant a worker more server access than the task requires.', + ); + lines.push(''); + + return lines.join('\n'); +} + +function formatSmartOutputRouterSection( + workspacePath: string, + fastModel: string, + balancedModel: string, +): string { + return [ + '## Smart Output Router', + '', + 'Before generating any output, classify the request to pick the right format and delivery path.', + 'This prevents creating static files when an interactive app is needed, and avoids spawning an', + 'app when a simple inline response would do.', + '', + '### Output Classification', + '', + '| Request type | Output format | Delivery |', + '| --- | --- | --- |', + '| Raw data, records, structured export | `.json` file | SHARE:whatsapp or SHARE:telegram |', + '| Chart, graph, or data visualisation | HTML + Chart.js / D3 | APP:start (interactive) or SHARE:github-pages |', + '| Document, report, or printable output | HTML + print CSS | SHARE:github-pages or SHARE:whatsapp |', + '| Interactive form, wizard, live dashboard | HTML + `openbridge-client.js` | APP:start (relay required) |', + '| Short summary (< 1 KB text) | Inline in response | None — embed directly |', + '', + '### When to Use `openbridge-client.js`', + '', + 'Use the Interaction Relay SDK (`openbridge-client.js`) **only when the app needs to send data back**', + 'to the Master AI or receive live updates from it. Examples:', + '', + '- A form that submits user input for AI processing', + '- A dashboard that fetches a fresh analysis on user request', + '- An interactive wizard that sends step-by-step answers to the Master', + '- Any app displaying AI-generated content that changes over time', + '', + 'Static outputs (charts with fixed data, printable reports, file downloads) do **not** need', + '`openbridge-client.js`. Prefer static HTML unless two-way communication is required.', + '', + '### Example — Data Export (JSON)', + '', + '```', + `[SPAWN:read-only]{"prompt":"Export all orders from the last 30 days to ${workspacePath}/.openbridge/generated/orders.json as a JSON array.","model":"${fastModel}","maxTurns":10}[/SPAWN]`, + '', + `[SHARE:whatsapp]{"path":"${workspacePath}/.openbridge/generated/orders.json"}[/SHARE]`, + '```', + '', + '### Example — Data Visualisation (static chart)', + '', + '```', + `[SPAWN:code-edit]{"prompt":"Create a static chart app in ${workspacePath}/.openbridge/generated/apps/sales-chart/. index.html: render a bar chart of monthly sales using Chart.js from CDN. Pure HTML/JS — no relay needed.","model":"${fastModel}","maxTurns":10}[/SPAWN]`, + '', + `[APP:start]${workspacePath}/.openbridge/generated/apps/sales-chart[/APP]`, + '```', + '', + '### Example — Printable Report (HTML with print CSS)', + '', + '```', + `[SPAWN:code-edit]{"prompt":"Create an HTML report in ${workspacePath}/.openbridge/generated/report.html. Include executive summary, findings table, and recommendations. Add @media print CSS so it prints cleanly. No JavaScript required.","model":"${balancedModel}","maxTurns":10}[/SPAWN]`, + '', + `[SHARE:github-pages]{"path":"${workspacePath}/.openbridge/generated/report.html"}[/SHARE]`, + '```', + '', + '### Example — Interactive Tool (with relay)', + '', + '```', + `[SPAWN:code-edit]{"prompt":"Create a feedback form app in ${workspacePath}/.openbridge/generated/apps/feedback/. index.html: HTML form with name, message, type fields. Include . On submit, call openbridge.submit({name, message, type}) and show \\"Sent to AI!\\". No backend needed — relay handles delivery.","model":"${fastModel}","maxTurns":10}[/SPAWN]`, + '', + `[APP:start]${workspacePath}/.openbridge/generated/apps/feedback[/APP]`, + '```', + '', + '> **Rule:** `openbridge-client.js` + `APP:start` for two-way apps. `SHARE:github-pages` for static reports. `SHARE:whatsapp` / `SHARE:telegram` for file downloads. Inline text for short answers.', + '', + ].join('\n'); +} + +function formatDiscoveredTools(tools: DiscoveredTool[]): string { + if (tools.length === 0) { + return 'No AI tools discovered on this machine.'; + } + + const lines: string[] = []; + for (const tool of tools) { + const role = tool.role ?? 'unknown'; + const version = tool.version ?? 'unknown'; + const caps = tool.capabilities?.length ? ` — ${tool.capabilities.join(', ')}` : ''; + const strengths = TOOL_STRENGTHS[tool.name]; + const bestFor = strengths ? ` | Best for: ${strengths.bestFor}` : ''; + lines.push(`- **${tool.name}** (${role}, v${version})${caps}${bestFor}`); + } + return lines.join('\n'); +} diff --git a/src/master/monorepo-detector.ts b/src/master/monorepo-detector.ts new file mode 100644 index 00000000..84689f28 --- /dev/null +++ b/src/master/monorepo-detector.ts @@ -0,0 +1,181 @@ +/** + * Monorepo Detector + * + * Detects whether a workspace is a monorepo by scanning for multiple project + * manifest files (`package.json`, `.git`, `pom.xml`, `go.mod`, etc.) at + * depth 1–2 from the workspace root. + * + * A workspace is classified as a monorepo when two or more sub-directories + * at depth 1–2 each contain their own project manifest. The function is + * purely filesystem-based — no AI calls, no file-count thresholds. + */ + +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; +import { createLogger } from '../core/logger.js'; + +const logger = createLogger('monorepo-detector'); + +/** Maximum directory depth to scan for project manifests (root = depth 0). */ +const MAX_SCAN_DEPTH = 2; + +/** Directories that are never treated as sub-project roots. */ +const EXCLUDED_DIRS = new Set([ + 'node_modules', + '.git', + 'dist', + 'build', + '.next', + 'coverage', + 'target', + 'vendor', + '__pycache__', + '.venv', + 'venv', + '.openbridge', + '.cache', + 'out', + '.gradle', + '.mvn', + '.tox', + '.pytest_cache', + '.mypy_cache', +]); + +/** + * Manifest files whose presence signals an independent project root, + * mapped to a human-readable project type string. + */ +const MANIFEST_FILES: Record = { + 'package.json': 'node', + 'Cargo.toml': 'rust', + 'go.mod': 'go', + 'pom.xml': 'java', + 'build.gradle': 'java', + 'build.gradle.kts': 'java', + 'pyproject.toml': 'python', + 'setup.py': 'python', + 'setup.cfg': 'python', + Pipfile: 'python', + 'composer.json': 'php', + Gemfile: 'ruby', + 'mix.exs': 'elixir', + 'pubspec.yaml': 'dart', + 'CMakeLists.txt': 'cpp', +}; + +/** A detected sub-project within the monorepo. */ +export interface MonorepoSubProject { + /** Relative path from workspace root (e.g. `packages/ui`). */ + path: string; + /** Project type inferred from the detected manifest (e.g. `node`, `go`). */ + type: string; +} + +/** Result returned by {@link detectMonorepoPattern}. */ +export interface MonorepoDetectionResult { + /** `true` when 2+ sub-directories each contain their own project manifest. */ + isMonorepo: boolean; + /** Detected sub-projects (empty when `isMonorepo` is `false`). */ + subProjects: MonorepoSubProject[]; +} + +/** + * Scan `workspacePath` for monorepo indicators at depth 1–2. + * + * A workspace is considered a monorepo when at least two sub-directories + * (at depth 1 or depth 2 inside a non-excluded parent) each contain a + * recognised project manifest file such as `package.json`, `go.mod`, or + * `pom.xml`. A standalone `.git` directory at depth 1 is also treated as a + * monorepo signal when combined with another manifest-bearing directory. + * + * @param workspacePath Absolute path to the root workspace directory. + * @returns Detection result with `isMonorepo` flag and `subProjects` list. + */ +export async function detectMonorepoPattern( + workspacePath: string, +): Promise { + logger.debug({ workspacePath }, 'Starting monorepo detection'); + + const subProjects: MonorepoSubProject[] = []; + + let topLevelEntries: string[] = []; + try { + const dirents = await fs.readdir(workspacePath, { withFileTypes: true }); + topLevelEntries = dirents + .filter((d) => d.isDirectory() && !EXCLUDED_DIRS.has(d.name)) + .map((d) => d.name); + } catch (err) { + logger.warn({ err, workspacePath }, 'Failed to read workspace root — returning non-monorepo'); + return { isMonorepo: false, subProjects: [] }; + } + + for (const dirName of topLevelEntries) { + const dirAbsPath = path.join(workspacePath, dirName); + const relPath = dirName; + + // Depth 1: check if this directory itself has a manifest + const depth1Type = await detectManifestType(dirAbsPath); + if (depth1Type !== null) { + subProjects.push({ path: relPath, type: depth1Type }); + continue; // treat the whole dir as one sub-project; no need to descend + } + + // Depth 2: scan immediate children of this directory + try { + const childDirents = await fs.readdir(dirAbsPath, { withFileTypes: true }); + for (const child of childDirents) { + if (!child.isDirectory()) continue; + if (EXCLUDED_DIRS.has(child.name)) continue; + + const childAbsPath = path.join(dirAbsPath, child.name); + const childType = await detectManifestType(childAbsPath); + if (childType !== null) { + subProjects.push({ + path: `${relPath}/${child.name}`, + type: childType, + }); + } + } + } catch (err) { + logger.debug({ err, dir: relPath }, 'Failed to read depth-2 directory — skipping'); + } + } + + const isMonorepo = subProjects.length >= 2; + + logger.info( + { isMonorepo, subProjectCount: subProjects.length, paths: subProjects.map((p) => p.path) }, + 'Monorepo detection complete', + ); + + return { isMonorepo, subProjects: isMonorepo ? subProjects : [] }; +} + +/** + * Return the project type string for the first manifest found in `dirPath`, + * or `null` if no recognised manifest exists there. + */ +async function detectManifestType(dirPath: string): Promise { + for (const [manifestFile, projectType] of Object.entries(MANIFEST_FILES)) { + try { + await fs.access(path.join(dirPath, manifestFile)); + return projectType; + } catch { + // file does not exist — try next + } + } + + // Also treat a nested .git directory as a monorepo signal (embedded repo) + try { + await fs.access(path.join(dirPath, '.git')); + return 'git-repo'; + } catch { + // no .git either + } + + return null; +} + +// Export MAX_SCAN_DEPTH for testing +export { MAX_SCAN_DEPTH }; diff --git a/src/master/observation-extractor.ts b/src/master/observation-extractor.ts new file mode 100644 index 00000000..8bfca394 --- /dev/null +++ b/src/master/observation-extractor.ts @@ -0,0 +1,454 @@ +/** + * Observation Extractor + * + * Parses raw worker text output into structured Observation objects. + * Uses regex patterns and heuristics to extract: + * - title: first meaningful summary line from the output + * - narrative: cleaned summary of what happened + * - facts: discrete facts from bullet points / numbered lists + * - concepts: domain keywords and identifiers referenced + * - files_read: file paths detected in read context + * - files_modified: file paths detected in edit/write/create context + * - type: auto-classified from task profile + output content + */ + +import type { Observation } from '../memory/observation-store.js'; +import type { ObservationType } from '../types/agent.js'; + +// --------------------------------------------------------------------------- +// Public Input Type +// --------------------------------------------------------------------------- + +export interface ExtractionInput { + /** Raw text output from the worker agent */ + output: string; + /** Session ID to attach to the produced observation */ + sessionId: string; + /** Worker ID to attach to the produced observation */ + workerId: string; + /** Tool profile the worker ran with (e.g. 'read-only', 'code-edit') */ + profile?: string; + /** Original task prompt sent to the worker (used for type classification) */ + prompt?: string; +} + +// --------------------------------------------------------------------------- +// Title Extraction +// --------------------------------------------------------------------------- + +/** + * Extract a one-line title from worker output. + * Strategy (in order of preference): + * 1. A line that starts with a common summary verb (e.g., "Fixed …", "Updated …") + * 2. A markdown heading line (## or #) + * 3. The first non-empty, non-boilerplate line + * 4. Fallback: truncated first 80 chars of output + */ +function extractTitle(output: string): string { + const lines = output + .split('\n') + .map((l) => l.trim()) + .filter(Boolean); + + // 1. Summary-verb lines + const summaryVerbs = + /^(fixed|updated|added|removed|created|modified|refactored|implemented|resolved|found|wrote|completed|changed|extracted|migrated|improved|optimised|optimized|analyzed|analysed)\b/i; + for (const line of lines) { + if (summaryVerbs.test(line) && line.length <= 120) { + return line.replace(/[.!]+$/, ''); + } + } + + // 2. Markdown headings + for (const line of lines) { + const m = line.match(/^#{1,3}\s+(.+)/); + if (m && m[1] && m[1].length <= 120) { + return m[1].trim(); + } + } + + // 3. First non-empty line that isn't a code fence or separator + for (const line of lines) { + if (line.startsWith('```') || line.startsWith('---') || line.startsWith('===')) continue; + if (line.length >= 10 && line.length <= 120) { + return line.replace(/[.!]+$/, ''); + } + } + + // 4. Fallback + return output.slice(0, 80).replace(/\s+/g, ' ').trim(); +} + +// --------------------------------------------------------------------------- +// Facts Extraction +// --------------------------------------------------------------------------- + +/** + * Extract discrete facts from bullet points, numbered lists, and notable statements. + * Returns up to 20 fact strings. + */ +function extractFacts(output: string): string[] { + const facts = new Set(); + + for (const line of output.split('\n')) { + const trimmed = line.trim(); + + // Bullet points: - item, * item, • item + const bulletMatch = trimmed.match(/^[-*•]\s+(.{10,200})/); + if (bulletMatch && bulletMatch[1]) { + facts.add(bulletMatch[1].trim()); + continue; + } + + // Numbered lists: 1. item, 2. item + const numberedMatch = trimmed.match(/^\d+[.)]\s+(.{10,200})/); + if (numberedMatch && numberedMatch[1]) { + facts.add(numberedMatch[1].trim()); + continue; + } + + // Notable fact keywords on their own line + if (/^(note:|warning:|error:|result:|summary:|conclusion:|finding:)/i.test(trimmed)) { + const factText = trimmed.replace(/^[^:]+:\s*/i, ''); + if (factText.length >= 10) facts.add(factText.trim()); + } + } + + return Array.from(facts).slice(0, 20); +} + +// --------------------------------------------------------------------------- +// Concepts Extraction +// --------------------------------------------------------------------------- + +/** + * Extract domain concepts / keywords from worker output. + * Scans for: + * - CamelCase identifiers (function/class names) + * - Package names (e.g., `better-sqlite3`, `express`) + * - Error names (e.g., `TypeError`, `SqliteError`) + * - Quoted technical terms + */ +function extractConcepts(output: string): string[] { + const concepts = new Set(); + + // CamelCase identifiers (at least 2 words joined, ≥6 chars) + const camelRe = /\b([A-Z][a-z]+(?:[A-Z][a-z]+)+)\b/g; + for (const m of output.matchAll(camelRe)) { + if (m[1] && m[1].length >= 6 && m[1].length <= 50) concepts.add(m[1]); + } + + // Error class names: XxxError, XxxException + const errorRe = /\b([A-Z][A-Za-z]+(?:Error|Exception|Fault|Warning))\b/g; + for (const m of output.matchAll(errorRe)) { + if (m[1]) concepts.add(m[1]); + } + + // npm-style package names in backticks or quotes: `better-sqlite3` + const packageRe = /[`'"]([a-z][\w-]{2,40})[`'"]/g; + for (const m of output.matchAll(packageRe)) { + if (m[1] && /[-]/.test(m[1])) concepts.add(m[1]); + } + + // Function call patterns: functionName() + const funcRe = /\b([a-z][A-Za-z0-9]{3,30})\(\)/g; + for (const m of output.matchAll(funcRe)) { + if (m[1]) concepts.add(`${m[1]}()`); + } + + return Array.from(concepts).slice(0, 30); +} + +// --------------------------------------------------------------------------- +// File Path Extraction — files_read (OB-1626) +// --------------------------------------------------------------------------- + +/** + * FILE PATH REGEX — matches common relative and absolute path formats: + * - src/foo/bar.ts + * - ./relative/path.js + * - ../parent/path.ts + * - /absolute/path/file.ext + * Minimum: 2 segments or leading dot-slash, must end with a recognisable extension. + */ +const FILE_PATH_RE = + /(?:^|[\s(["'`])((\.{1,2}\/[\w./-]+|src\/[\w./-]+|tests?\/[\w./-]+|lib\/[\w./-]+|dist\/[\w./-]+|\/[\w./-]{6,}))(?=[\s)"'`]|$)/gm; + +const READABLE_EXTENSIONS = new Set([ + 'ts', + 'tsx', + 'js', + 'jsx', + 'mjs', + 'cjs', + 'json', + 'yaml', + 'yml', + 'toml', + 'env', + 'md', + 'txt', + 'html', + 'css', + 'scss', + 'sql', + 'sh', + 'bash', + 'py', + 'rb', + 'go', + 'rs', + 'java', +]); + +function hasKnownExtension(p: string): boolean { + const dot = p.lastIndexOf('.'); + if (dot === -1) return false; + return READABLE_EXTENSIONS.has(p.slice(dot + 1).toLowerCase()); +} + +/** Lines that indicate a file was READ (not written) */ +const READ_CONTEXT_RE = + /\b(read|reading|opened?|viewing?|scanned?|checked?|inspected?|loaded?|parsed?|examined?|found in|referenced? in|imported? from|looking at)\b/i; + +/** + * Extract file paths that appear in a read context. + * + * Strategy: + * 1. Lines with explicit read-context keywords (opened, scanned, imported…) → high confidence + * 2. Neutral lines — file path present but no write-context keyword → treated as read + * 3. Tool output patterns ("Reading src/…", "Read file: …") → explicit tool signal + * + * Lines that match WRITE_CONTEXT_RE are skipped here; those paths belong to + * extractFilesModified(). + */ +export function extractFilesRead(output: string): string[] { + const paths = new Set(); + + for (const line of output.split('\n')) { + const allPaths = matchFilePaths(line); + if (allPaths.length === 0) continue; + + // Explicit read context: high confidence + if (READ_CONTEXT_RE.test(line)) { + for (const p of allPaths) paths.add(p); + continue; + } + + // Neutral lines (no write indicator): file is referenced, treat as read + if (!WRITE_CONTEXT_RE.test(line)) { + for (const p of allPaths) paths.add(p); + } + } + + // Explicit tool output patterns: "Reading src/…" / "Read file: src/…" + const readToolRe = /\bread(?:ing)?\s+(?:file\s+)?([./][\w./-]+\.\w+)/gi; + for (const m of output.matchAll(readToolRe)) { + if (m[1] && hasKnownExtension(m[1])) paths.add(m[1]); + } + + return Array.from(paths).slice(0, 50); +} + +// --------------------------------------------------------------------------- +// File Path Extraction — files_modified (OB-1627) +// --------------------------------------------------------------------------- + +/** + * Lines that indicate a file was WRITTEN / MODIFIED / CREATED. + * Covers base, past, progressive, and third-person forms so that + * "edit", "write", "create", "update", "delete", "remove" (and variants) + * are all matched regardless of tense. + */ +const WRITE_CONTEXT_RE = + /\b(edit(?:s?|ed|ing)?|writ(?:e[s]?|ten|ing)|wrote?|creat(?:e[s]?|ed|ing)|modif(?:y|ie[sd]|ying)|updat(?:e[s]?|ed|ing)|delet(?:e[s]?|ed|ing)|remov(?:e[s]?|ed|ing)|overwr(?:ote|itten|iting))\b/i; + +/** + * Claude Code / SDK tool invocation patterns that indicate file modification. + * Matches: Edit(...), Write(...), NotebookEdit(...), MultiEdit(...) + */ +const TOOL_WRITE_RE = + /(?:Edit|Write|NotebookEdit|MultiEdit)\s*\(\s*["'`]?([./]?[\w./-]+\.\w+)["'`]?/g; + +/** + * Explicit executor output lines — highest confidence signal. + * Matches patterns emitted by Claude Code and shell tool wrappers, e.g.: + * "Written to src/foo/bar.ts" + * "Wrote to src/foo/bar.ts" + * "Created file src/foo/bar.ts" + * "Saved to src/foo/bar.ts" + * "Overwrote src/foo/bar.ts" + * "Edit applied to src/foo/bar.ts" + */ +const TOOL_WRITE_OUTPUT_RE = + /(?:written\s+to|wrote\s+to|created\s+file|saved\s+to|overwrote?|edit\s+applied\s+to)\s*:?\s+([./]?[\w./-]+\.\w+)/gi; + +/** + * Extract file paths that appear in a write/edit/create context. + * + * Strategy (in order of confidence): + * 1. Tool invocation patterns: Edit(...), Write(...), NotebookEdit(...), MultiEdit(...) + * 2. Explicit executor output lines: "Written to src/...", "Saved to src/...", etc. + * 3. Lines containing write-context keywords + a recognisable file path + */ +export function extractFilesModified(output: string): string[] { + const paths = new Set(); + + // 1. Tool invocation patterns (most reliable) + for (const m of output.matchAll(TOOL_WRITE_RE)) { + if (m[1] && hasKnownExtension(m[1])) paths.add(m[1]); + } + + // 2. Explicit executor output lines (also very reliable) + for (const m of output.matchAll(TOOL_WRITE_OUTPUT_RE)) { + if (m[1] && hasKnownExtension(m[1])) paths.add(m[1]); + } + + // 3. Lines with write-context keywords + file path + for (const line of output.split('\n')) { + const allPaths = matchFilePaths(line); + if (allPaths.length === 0) continue; + + if (WRITE_CONTEXT_RE.test(line)) { + for (const p of allPaths) paths.add(p); + } + } + + return Array.from(paths).slice(0, 50); +} + +/** Shared helper: extract all file-path tokens from a single line */ +function matchFilePaths(line: string): string[] { + const found: string[] = []; + FILE_PATH_RE.lastIndex = 0; + for (const m of line.matchAll(FILE_PATH_RE)) { + const p = m[1]; + if (p && hasKnownExtension(p)) found.push(p); + } + return found; +} + +// --------------------------------------------------------------------------- +// Type Classification (OB-1628) +// --------------------------------------------------------------------------- + +/** Keyword → observation type mapping (first match wins) */ +const TYPE_KEYWORDS: Array<{ re: RegExp; type: ObservationType }> = [ + { re: /\b(fix(?:ed|ing)?|bug|bugfix|regression|defect|patch|hotfix|repair)\b/i, type: 'bugfix' }, + { + re: /\b(test(?:s|ing|ed)?|spec|assert(?:ion)?|coverage|pass(?:ed)?|fail(?:ed)?|vitest|jest|mocha)\b/i, + type: 'test-result', + }, + { + re: /\b(refactor(?:ing|ed)?|rename(?:d)?|reorgani[sz]e|extract(?:ed)?|clean(?:up|ed)?)\b/i, + type: 'refactor', + }, + { + re: /\b(architect(?:ure|ural)?|design(?:\s+pattern)?|schema|structure|layout|module)\b/i, + type: 'architecture', + }, + { + re: /\b(depend(?:ency|encies|s)|package|npm|yarn|pnpm|install(?:ed|ing)?|upgrade)\b/i, + type: 'dependency', + }, + { + re: /\b(config(?:uration|ured|uring)?|settings?|env(?:ironment)?|\.env|tsconfig|eslint)\b/i, + type: 'config', + }, + { + re: /\b(doc(?:ument(?:ation|ed|ing)?|s)?|readme|comment(?:s|ed)?|jsdoc|changelog)\b/i, + type: 'documentation', + }, + { + re: /\b(perf(?:ormance)?|optimi[sz]e(?:d|ation)?|speed|latency|throughput|memory\s+usage|cache)\b/i, + type: 'performance', + }, + { + re: /\b(security|auth(?:entication|orization)?|vuln(?:erability)?|cve|inject(?:ion)?|xss|csrf|sanitize)\b/i, + type: 'security', + }, +]; + +/** Profile → default observation type when content signals are absent */ +const PROFILE_DEFAULT_TYPE: Record = { + 'read-only': 'investigation', + 'code-audit': 'test-result', + 'code-edit': 'refactor', + master: 'architecture', + 'full-access': 'investigation', +}; + +/** + * Classify observation type from: + * 1. Output content keywords (first matching rule wins) + * 2. Task prompt keywords (same rules) + * 3. Worker tool profile default + * 4. Hard fallback: 'investigation' + */ +export function classifyObservationType( + output: string, + prompt: string | undefined, + profile: string | undefined, +): ObservationType { + const combined = `${prompt ?? ''} ${output}`; + + for (const { re, type } of TYPE_KEYWORDS) { + if (re.test(combined)) return type; + } + + if (profile && profile in PROFILE_DEFAULT_TYPE) { + return PROFILE_DEFAULT_TYPE[profile]!; + } + + return 'investigation'; +} + +// --------------------------------------------------------------------------- +// Narrative Builder +// --------------------------------------------------------------------------- + +/** + * Build a short narrative (≤500 chars) from worker output. + * Prefers the first substantial paragraph; falls back to truncated output. + */ +function buildNarrative(output: string): string { + const paragraphs = output + .split(/\n{2,}/) + .map((p) => p.replace(/\s+/g, ' ').trim()) + .filter((p) => p.length >= 20); + + if (paragraphs.length > 0 && paragraphs[0]) { + return paragraphs[0].slice(0, 500); + } + + return output.replace(/\s+/g, ' ').trim().slice(0, 500); +} + +// --------------------------------------------------------------------------- +// Main Export +// --------------------------------------------------------------------------- + +/** + * Parse a worker's text output into a structured Observation. + * + * All fields are extracted using regex + heuristics — no AI calls. + * The caller is responsible for persisting the result via MemoryManager. + * + * @param input - Worker output + metadata required to build the observation + * @returns A populated Observation ready for insertion into the store + */ +export function extractObservation(input: ExtractionInput): Observation { + const { output, sessionId, workerId, profile, prompt } = input; + + return { + session_id: sessionId, + worker_id: workerId, + type: classifyObservationType(output, prompt, profile), + title: extractTitle(output), + narrative: buildNarrative(output), + facts: extractFacts(output), + concepts: extractConcepts(output), + files_read: extractFilesRead(output), + files_modified: extractFilesModified(output), + }; +} diff --git a/src/master/planning-gate.ts b/src/master/planning-gate.ts new file mode 100644 index 00000000..15360b69 --- /dev/null +++ b/src/master/planning-gate.ts @@ -0,0 +1,734 @@ +/** + * Planning Gate (OB-1775, OB-1776, OB-1777, OB-1778) + * + * Controls the two-phase execution model for Master AI task processing: + * + * 1. Analysis phase — 1–2 read-only workers investigate the codebase, + * gather facts, and surface risks before any code is written. + * Workers are built via buildAnalysisWorkerSpecs() and capped at + * MAX_ANALYSIS_WORKERS (2). + * + * 2. Execution phase — code-edit workers implement the confirmed strategy. + * This phase ONLY starts after ALL analysis workers have returned their + * output AND the Master has explicitly confirmed the approach via + * confirmApproach(). Attempting to complete the analysis phase before all + * workers have finished throws an error (OB-1777). + * + * For simple tasks (single-file edits, FAQ answers), planning is bypassed + * automatically via `shouldBypassPlanning()` (OB-1778). Wiring into + * MasterManager is done by OB-1779; the reasoning checkpoint before + * full-access workers is OB-1780. + */ + +import { randomUUID } from 'node:crypto'; +import { createLogger } from '../core/logger.js'; + +const logger = createLogger('planning-gate'); + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +/** + * Maximum number of read-only workers that may be spawned during the analysis + * phase. Keeping the cap at 2 avoids over-investigation for most tasks while + * still allowing a second worker for risk/dependency assessment. + */ +export const MAX_ANALYSIS_WORKERS = 2; + +/** + * Max agentic turns for each read-only analysis worker. + * Analysis workers explore and report — they don't need many turns. + */ +export const ANALYSIS_WORKER_MAX_TURNS = 10; + +// --------------------------------------------------------------------------- +// Simple-Task Heuristics (OB-1778) +// --------------------------------------------------------------------------- + +/** + * Word-count threshold below which a task is considered too short to need a + * planning phase. Very brief tasks are typically single-step commands. + */ +const BYPASS_WORD_COUNT_THRESHOLD = 15; + +/** + * If a task mentions more than this many distinct file-path-like tokens it is + * considered multi-file and therefore non-trivial (planning is NOT bypassed). + */ +const BYPASS_MAX_FILE_PATHS = 1; + +/** Regex that matches a leading question word at the start of a task. */ +const FAQ_PATTERN = + /^\s*(what|how|why|who|where|when|which|explain|describe|list|show me|tell me|can you tell|is there|are there|does|do you|what is|what are|what does|what's)\b/i; + +/** Regex that matches file-path-like tokens (src/…, ./…, absolute /path/…). */ +const FILE_PATH_PATTERN = /(?:^|[\s"'`(])(?:\.{0,2}\/[\w./-]+|src\/[\w./-]+)/g; + +/** + * Result returned by `shouldBypassPlanning()`. + * + * When `bypass` is `true`, callers should call `gate.bypass(task, reason)` + * immediately and skip the analysis phase entirely. + */ +export interface BypassDecision { + /** Whether to skip the planning/analysis phase. */ + bypass: boolean; + /** Human-readable reason (always set, even when bypass is false). */ + reason: string; +} + +/** + * Analyse a task description and decide whether the analysis phase should be + * bypassed (OB-1778). + * + * A task bypasses planning when **any** of the following are true: + * + * 1. **FAQ / read-only question** — the task starts with a question word + * (what, how, why, explain, …) and contains no write-intent verbs + * (create, add, fix, update, refactor, delete, remove, implement). + * + * 2. **Single-file edit** — the task references exactly one file path token + * (e.g. `src/core/router.ts`) and does not involve many moving parts. + * + * 3. **Very short task** — the task is fewer than `BYPASS_WORD_COUNT_THRESHOLD` + * words, implying a single, atomic action that needs no pre-investigation. + * + * When multiple signals conflict (e.g. a short task with a write verb that + * touches more than one file), the function returns `bypass: false` to keep + * planning enabled. + * + * @param taskDescription Plain-text description of the task to evaluate. + * @returns A `BypassDecision` with a boolean and human-readable reason. + */ +export function shouldBypassPlanning(taskDescription: string): BypassDecision { + const trimmed = taskDescription.trim(); + + if (!trimmed) { + return { bypass: true, reason: 'empty task description — no planning needed' }; + } + + const words = trimmed.split(/\s+/); + const wordCount = words.length; + + // Detect write-intent verbs — these suggest the task will modify files. + const writeIntentPattern = + /\b(create|add|fix|update|refactor|delete|remove|implement|write|rewrite|rename|move|migrate|replace|change|modify|patch|edit|set|enable|disable|configure)\b/i; + const hasWriteIntent = writeIntentPattern.test(trimmed); + + // Count distinct file-path-like tokens in the task. + const filePaths = Array.from(new Set(trimmed.match(FILE_PATH_PATTERN) ?? [])); + const filePathCount = filePaths.length; + + // ── Rule 1: FAQ / read-only question ────────────────────────────────────── + if (FAQ_PATTERN.test(trimmed) && !hasWriteIntent) { + return { + bypass: true, + reason: 'read-only FAQ question — no code changes implied', + }; + } + + // ── Rule 2: Single-file edit ────────────────────────────────────────────── + if (filePathCount === BYPASS_MAX_FILE_PATHS && hasWriteIntent) { + return { + bypass: true, + reason: `single-file edit (${filePaths[0]?.trim()}) — no multi-file analysis required`, + }; + } + + // ── Rule 3: Very short task (no multi-file concern) ─────────────────────── + if (wordCount < BYPASS_WORD_COUNT_THRESHOLD && filePathCount <= BYPASS_MAX_FILE_PATHS) { + return { + bypass: true, + reason: `short task (${wordCount} words) — simple enough to skip planning`, + }; + } + + // ── Default: planning required ──────────────────────────────────────────── + const reasons: string[] = []; + if (filePathCount > BYPASS_MAX_FILE_PATHS) { + reasons.push(`${filePathCount} file paths mentioned`); + } + if (wordCount >= BYPASS_WORD_COUNT_THRESHOLD) { + reasons.push(`${wordCount} words`); + } + if (hasWriteIntent) { + reasons.push('write-intent detected'); + } + + return { + bypass: false, + reason: `complex task — planning required (${reasons.join(', ')})`, + }; +} + +// --------------------------------------------------------------------------- +// Reasoning Checkpoint (OB-1780) +// --------------------------------------------------------------------------- + +/** Risk severity levels for a reasoning checkpoint. */ +export type RiskLevel = 'low' | 'medium' | 'high'; + +/** A single identified risk signal from a task description. */ +export interface RiskSignal { + /** Short label for the risk pattern detected. */ + pattern: string; + /** Human-readable description of what could go wrong. */ + description: string; + /** Severity of this risk. */ + level: RiskLevel; +} + +/** Result of a reasoning checkpoint performed before a full-access worker spawns. */ +export interface ReasoningCheckpoint { + /** The task prompt that was analysed. */ + prompt: string; + /** Zero or more risk signals identified in the prompt. */ + risks: RiskSignal[]; + /** Highest risk level across all signals (or 'low' when none detected). */ + riskLevel: RiskLevel; + /** ISO 8601 timestamp when the checkpoint was performed. */ + performedAt: string; +} + +/** Risk-signal definitions scanned before every full-access worker spawn. */ +const RISK_SIGNAL_DEFS: Array<{ + pattern: RegExp; + label: string; + description: string; + level: RiskLevel; +}> = [ + { + pattern: /\brm\s+-rf?\b|\bdelete\s+all\b|\bremove\s+all\b|\bwipe\b|\btruncate\b/i, + label: 'destructive-delete', + description: 'Task may delete files or data irreversibly', + level: 'high', + }, + { + pattern: /\b(--force|--no-verify|--hard\s+reset|reset\s+--hard)\b/i, + label: 'bypass-safety', + description: 'Task may bypass safety checks or git protections', + level: 'high', + }, + { + pattern: /\b(drop\s+table|drop\s+database|alter\s+table|truncate\s+table)\b/i, + label: 'schema-destructive', + description: 'Task may make destructive database schema changes', + level: 'high', + }, + { + pattern: /\b(all\s+files?|entire\s+(?:codebase|repo|project|directory)|every\s+file)\b/i, + label: 'broad-scope', + description: 'Task scope is unusually broad — may affect many files', + level: 'medium', + }, + { + pattern: /\b(npm\s+install|pip\s+install|cargo\s+add|yarn\s+add|apt[-\s]install)\b/i, + label: 'dependency-install', + description: 'Task may install new dependencies', + level: 'medium', + }, + { + pattern: + /\b(auth(?:entication)?|secret|password|token|credential|api[-_]key|private[-_]key)\b/i, + label: 'security-sensitive', + description: 'Task touches authentication or secret management code', + level: 'medium', + }, + { + pattern: /\b(migration|schema\s+change|alter\s+column|add\s+column)\b/i, + label: 'schema-migration', + description: 'Task may modify the database schema', + level: 'medium', + }, + { + pattern: /\b(config(?:uration)?|env(?:ironment)?\s+(?:variable|var)|\.env\b|dotenv)\b/i, + label: 'config-change', + description: 'Task modifies configuration or environment settings', + level: 'low', + }, +]; + +/** + * Perform a reasoning checkpoint before spawning a full-access worker (OB-1780). + * + * Scans the task prompt for patterns that indicate the worker may perform + * destructive, broad-scope, or security-sensitive operations. Returns a + * structured `ReasoningCheckpoint` with all identified risk signals and an + * overall risk level. + * + * The checkpoint is purely analytical — it does not block execution. Its + * purpose is to surface "what could go wrong" in the log and task record + * so engineers can review the reasoning trail. + * + * @param prompt The task prompt that will be sent to the full-access worker. + * @returns A `ReasoningCheckpoint` with identified risks and an overall risk level. + */ +export function performReasoningCheckpoint(prompt: string): ReasoningCheckpoint { + const risks: RiskSignal[] = []; + for (const def of RISK_SIGNAL_DEFS) { + if (def.pattern.test(prompt)) { + risks.push({ pattern: def.label, description: def.description, level: def.level }); + } + } + + const riskLevel: RiskLevel = risks.some((r) => r.level === 'high') + ? 'high' + : risks.some((r) => r.level === 'medium') + ? 'medium' + : 'low'; + + return { prompt, risks, riskLevel, performedAt: new Date().toISOString() }; +} + +// --------------------------------------------------------------------------- +// Public Types +// --------------------------------------------------------------------------- + +/** The two high-level phases of a planning-gated task. */ +export type PlanningPhase = 'analysis' | 'execution'; + +/** Current status of a PlanningGate session. */ +export type PlanningGateStatus = + | 'idle' // Gate has not started yet + | 'analysis' // Analysis phase is running (read-only workers) + | 'awaiting_confirmation' // Analysis done — waiting for Master to confirm approach + | 'execution' // Execution phase is running (code-edit workers) + | 'complete' // All phases finished + | 'bypassed'; // Gate was bypassed for a simple task + +/** A read-only planning worker spawned during the analysis phase. */ +export interface PlanningWorker { + /** Unique worker identifier. */ + id: string; + /** The prompt sent to this worker. */ + prompt: string; + /** ISO 8601 timestamp when this worker was spawned. */ + spawnedAt: string; + /** Raw output from the worker (set when complete). */ + output?: string; + /** ISO 8601 timestamp when this worker completed. */ + completedAt?: string; +} + +/** + * Specification for a read-only planning worker to be spawned during the + * analysis phase. Callers receive these specs from `buildAnalysisWorkerSpecs()` + * and are responsible for actually spawning the workers via AgentRunner, then + * calling `recordAnalysisWorker()` / `completeAnalysisWorker()` to track them. + */ +export interface AnalysisWorkerSpec { + /** Unique ID — must match the `PlanningWorker.id` passed to `recordAnalysisWorker()`. */ + id: string; + /** Always 'read-only' — analysis workers must not modify files. */ + profile: 'read-only'; + /** Prompt/instructions for the worker. */ + prompt: string; + /** Max agentic turns (bounded for fast analysis). */ + maxTurns: number; +} + +/** Live state for an active PlanningGate session. */ +export interface PlanningGateState { + /** Unique gate session identifier. */ + sessionId: string; + /** One-line summary of the task being gated. */ + taskSummary: string; + /** Current phase/status of the gate. */ + status: PlanningGateStatus; + /** Read-only workers spawned during the analysis phase. */ + analysisWorkers: PlanningWorker[]; + /** Aggregated analysis output from all planning workers. */ + analysisOutput: string; + /** The confirmed execution strategy (set after analysis, before execution). */ + confirmedStrategy: string; + /** When this gate session was created (ISO 8601). */ + createdAt: string; + /** When the analysis phase completed (ISO 8601). */ + analysisCompletedAt?: string; + /** When the execution phase started (ISO 8601). */ + executionStartedAt?: string; + /** Why planning was bypassed (only set when status === 'bypassed'). */ + bypassReason?: string; +} + +// --------------------------------------------------------------------------- +// PlanningGate +// --------------------------------------------------------------------------- + +/** + * PlanningGate — two-phase execution guard for Master AI. + * + * Prevents code-edit workers from being spawned before read-only analysis + * workers have investigated the task and the Master has confirmed a strategy. + * + * Usage: + * + * ```ts + * const gate = new PlanningGate(); + * + * // Start analysis (read-only workers) + * gate.startAnalysis('Refactor authentication module'); + * gate.recordAnalysisWorker({ id: 'w1', prompt: '...', spawnedAt: now }); + * gate.completeAnalysisWorker('w1', workerOutput); + * gate.completeAnalysis(aggregatedOutput); + * + * // Master reviews and confirms approach + * gate.confirmApproach('Extract AuthService, keep JWT logic in place'); + * + * // Execution workers can now run + * if (gate.allowsExecution) { ... } + * gate.completeExecution(); + * ``` + * + * Simple tasks bypass the gate entirely: + * + * ```ts + * gate.bypass('What does foo() do?', 'read-only FAQ — no code changes'); + * ``` + */ +export class PlanningGate { + private _state: PlanningGateState | null = null; + + // ── Phase Control ─────────────────────────────────────────────── + + /** + * Start a new planning gate session for a task. + * Transitions status from unstarted to `analysis`. + * + * @param taskSummary One-line description of the task being gated. + * @returns A snapshot of the new state. + */ + startAnalysis(taskSummary: string): PlanningGateState { + const now = new Date().toISOString(); + this._state = { + sessionId: randomUUID(), + taskSummary, + status: 'analysis', + analysisWorkers: [], + analysisOutput: '', + confirmedStrategy: '', + createdAt: now, + }; + logger.debug( + { sessionId: this._state.sessionId, taskSummary }, + 'Planning gate: analysis phase started', + ); + return { ...this._state, analysisWorkers: [...this._state.analysisWorkers] }; + } + + /** + * Mark the analysis phase as complete with its aggregated output. + * Transitions status from `analysis` to `awaiting_confirmation`. + * + * **Enforcement (OB-1777):** All registered analysis workers must have + * returned (i.e. have a `completedAt` timestamp) before this method + * succeeds. If any worker is still running, an error is thrown to prevent + * the execution phase from starting prematurely. + * + * Use `canCompleteAnalysis` to check readiness before calling this method, + * and `aggregateWorkerOutputs()` to build the combined output automatically. + * + * @param aggregatedOutput Combined output from all analysis workers. + * You can use `aggregateWorkerOutputs()` to generate this automatically. + * @returns A snapshot of the updated state. + * @throws If any registered analysis worker has not yet completed. + */ + completeAnalysis(aggregatedOutput: string): PlanningGateState { + this._requireStatus('analysis', 'completeAnalysis'); + const state = this._state!; + + // OB-1777: Enforce that every registered analysis worker has returned + // before allowing the execution phase to proceed. + const incompleteWorkers = state.analysisWorkers.filter((w) => !w.completedAt); + if (incompleteWorkers.length > 0) { + throw new Error( + `PlanningGate.completeAnalysis(): ${incompleteWorkers.length} analysis worker(s) have not yet returned. ` + + 'Wait for all workers to complete before ending the analysis phase.', + ); + } + + state.analysisOutput = aggregatedOutput; + state.analysisCompletedAt = new Date().toISOString(); + state.status = 'awaiting_confirmation'; + logger.debug( + { sessionId: state.sessionId, workerCount: state.analysisWorkers.length }, + 'Planning gate: analysis phase complete, awaiting confirmation', + ); + return { ...state, analysisWorkers: [...state.analysisWorkers] }; + } + + /** + * Confirm the execution strategy after reviewing analysis output. + * Transitions status from `awaiting_confirmation` to `execution`. + * + * @param strategy Human-readable description of the confirmed approach. + * @returns A snapshot of the updated state. + */ + confirmApproach(strategy: string): PlanningGateState { + this._requireStatus('awaiting_confirmation', 'confirmApproach'); + const state = this._state!; + state.confirmedStrategy = strategy; + state.executionStartedAt = new Date().toISOString(); + state.status = 'execution'; + logger.debug( + { sessionId: state.sessionId }, + 'Planning gate: approach confirmed, execution phase started', + ); + return { ...state, analysisWorkers: [...state.analysisWorkers] }; + } + + /** + * Mark the entire gate as complete after the execution phase finishes. + * Transitions status from `execution` to `complete`. + * + * @returns A snapshot of the final state. + */ + completeExecution(): PlanningGateState { + this._requireStatus('execution', 'completeExecution'); + const state = this._state!; + state.status = 'complete'; + logger.debug({ sessionId: state.sessionId }, 'Planning gate: complete'); + return { ...state, analysisWorkers: [...state.analysisWorkers] }; + } + + /** + * Bypass the planning gate entirely for simple tasks. + * The gate transitions directly to `bypassed`, allowing execution to + * proceed without an analysis phase. + * + * @param taskSummary One-line description of the task. + * @param reason Human-readable explanation of why planning was skipped. + * @returns A snapshot of the bypassed state. + */ + bypass(taskSummary: string, reason: string): PlanningGateState { + const now = new Date().toISOString(); + this._state = { + sessionId: randomUUID(), + taskSummary, + status: 'bypassed', + analysisWorkers: [], + analysisOutput: '', + confirmedStrategy: '', + createdAt: now, + bypassReason: reason, + }; + logger.debug( + { sessionId: this._state.sessionId, reason }, + 'Planning gate: bypassed (simple task)', + ); + return { ...this._state, analysisWorkers: [] }; + } + + // ── Worker Tracking ───────────────────────────────────────────── + + /** + * Register a read-only planning worker spawned during the analysis phase. + * Must be called while status is `analysis`. + * + * At most `MAX_ANALYSIS_WORKERS` (2) workers may be registered per session. + * Attempting to register a third worker throws an error. + * + * @param worker The worker record to register. + */ + recordAnalysisWorker(worker: PlanningWorker): void { + this._requireStatus('analysis', 'recordAnalysisWorker'); + const state = this._state!; + if (state.analysisWorkers.length >= MAX_ANALYSIS_WORKERS) { + throw new Error( + `PlanningGate.recordAnalysisWorker(): cannot register more than ${MAX_ANALYSIS_WORKERS} analysis workers per session`, + ); + } + state.analysisWorkers.push({ ...worker }); + } + + /** + * Update a registered planning worker with its completed output. + * + * @param workerId The ID of the worker to update. + * @param output Raw text output produced by the worker. + * @returns `true` if the worker was found and updated; `false` otherwise. + */ + completeAnalysisWorker(workerId: string, output: string): boolean { + if (!this._state) return false; + const worker = this._state.analysisWorkers.find((w) => w.id === workerId); + if (!worker) return false; + worker.output = output; + worker.completedAt = new Date().toISOString(); + return true; + } + + // ── Output Aggregation ─────────────────────────────────────────── + + /** + * Aggregate the outputs from all completed analysis workers into a single + * string, labelled by worker index (OB-1777). + * + * Returns an empty string if no workers have completed yet. + * Callers may pass the result directly to `completeAnalysis()`. + */ + aggregateWorkerOutputs(): string { + if (!this._state) return ''; + const completed = this._state.analysisWorkers.filter( + (w) => w.completedAt !== undefined && w.output !== undefined, + ); + if (completed.length === 0) return ''; + return completed + .map((w, i) => `## Analysis Worker ${i + 1}\n\n${w.output ?? ''}`) + .join('\n\n---\n\n'); + } + + // ── Analysis Worker Factory ────────────────────────────────────── + + /** + * Build 1–2 read-only worker specifications for the analysis phase. + * + * Always generates two workers: + * + * 1. **Investigation worker** — reads relevant files and summarises what + * already exists in the codebase that relates to the task. + * + * 2. **Risk-assessment worker** — identifies potential breakage points, + * dependencies, and edge cases that the implementation must handle. + * + * Both workers use the `'read-only'` profile (Read, Glob, Grep only) and + * are capped at `ANALYSIS_WORKER_MAX_TURNS` turns so they return quickly. + * + * Callers must spawn the workers themselves (via AgentRunner), then call + * `recordAnalysisWorker()` and `completeAnalysisWorker()` to update state. + * + * @param taskDescription A plain-text description of the task to investigate. + * @returns Exactly two `AnalysisWorkerSpec` objects ready for spawning. + */ + buildAnalysisWorkerSpecs(taskDescription: string): AnalysisWorkerSpec[] { + const investigationId = randomUUID(); + const riskId = randomUUID(); + + const investigationPrompt = [ + 'You are a read-only investigation worker. Your job is to explore the codebase', + 'and gather facts relevant to the following task. Do NOT modify any files.', + '', + `Task: ${taskDescription}`, + '', + 'Investigate:', + '1. Which files and modules are most relevant to this task?', + '2. What does the current implementation look like in those areas?', + '3. Are there existing patterns or conventions to follow?', + '4. What tests currently cover this area?', + '', + 'End your output with a brief summary of your findings (3–8 bullet points).', + ].join('\n'); + + const riskPrompt = [ + 'You are a read-only risk-assessment worker. Your job is to identify what could', + 'go wrong when implementing the following task. Do NOT modify any files.', + '', + `Task: ${taskDescription}`, + '', + 'Identify:', + '1. Which other parts of the codebase depend on the files likely to change?', + '2. What edge cases or inputs might break the implementation?', + '3. Are there any circular dependencies or shared-state hazards?', + '4. What is the minimum safe scope for this change?', + '', + 'End your output with a brief risk summary (3–8 bullet points).', + ].join('\n'); + + return [ + { + id: investigationId, + profile: 'read-only', + prompt: investigationPrompt, + maxTurns: ANALYSIS_WORKER_MAX_TURNS, + }, + { + id: riskId, + profile: 'read-only', + prompt: riskPrompt, + maxTurns: ANALYSIS_WORKER_MAX_TURNS, + }, + ]; + } + + // ── Inspection ────────────────────────────────────────────────── + + /** Current gate state snapshot, or `null` if no session has started. */ + get state(): PlanningGateState | null { + if (!this._state) return null; + return { ...this._state, analysisWorkers: [...this._state.analysisWorkers] }; + } + + /** Current status, or `'idle'` if no session has started. */ + get status(): PlanningGateStatus { + return this._state?.status ?? 'idle'; + } + + /** `true` when the gate is actively running the analysis phase. */ + get isAnalysisPhase(): boolean { + return this._state?.status === 'analysis'; + } + + /** `true` when analysis is done and the Master must confirm before execution. */ + get isAwaitingConfirmation(): boolean { + return this._state?.status === 'awaiting_confirmation'; + } + + /** `true` when the execution phase is active (code-edit workers may run). */ + get isExecutionPhase(): boolean { + return this._state?.status === 'execution'; + } + + /** `true` when code-edit workers are permitted to run. */ + get allowsExecution(): boolean { + return this._state?.status === 'execution' || this._state?.status === 'bypassed'; + } + + /** `true` when the gate has been bypassed for a simple task. */ + get isBypassed(): boolean { + return this._state?.status === 'bypassed'; + } + + /** `true` when the gate session has fully concluded (complete or bypassed). */ + get isComplete(): boolean { + return this._state?.status === 'complete' || this._state?.status === 'bypassed'; + } + + /** + * `true` when the analysis phase is active and all registered workers have + * returned their output (OB-1777). Callers should check this before calling + * `completeAnalysis()` to avoid the incomplete-worker error. + */ + get canCompleteAnalysis(): boolean { + if (this._state?.status !== 'analysis') return false; + if (this._state.analysisWorkers.length === 0) return false; + return this._state.analysisWorkers.every((w) => w.completedAt !== undefined); + } + + /** Number of analysis workers that have finished (have a `completedAt` timestamp). */ + get completedAnalysisWorkerCount(): number { + return this._state?.analysisWorkers.filter((w) => w.completedAt !== undefined).length ?? 0; + } + + /** Total number of analysis workers registered so far. */ + get totalAnalysisWorkerCount(): number { + return this._state?.analysisWorkers.length ?? 0; + } + + // ── Reset ──────────────────────────────────────────────────────── + + /** Reset the gate so it can be reused for a new task. */ + reset(): void { + this._state = null; + } + + // ── Private ────────────────────────────────────────────────────── + + private _requireStatus(expected: PlanningGateStatus, method: string): void { + const actual = this._state?.status ?? 'idle'; + if (actual !== expected) { + throw new Error( + `PlanningGate.${method}(): expected status '${expected}' but current status is '${actual}'`, + ); + } + } +} diff --git a/src/master/prompt-context-builder.ts b/src/master/prompt-context-builder.ts new file mode 100644 index 00000000..1c0a9be0 --- /dev/null +++ b/src/master/prompt-context-builder.ts @@ -0,0 +1,905 @@ +/** PromptContextBuilder — extracted from MasterManager (OB-1282, OB-F158). */ + +import { + PromptAssembler, + PRIORITY_IDENTITY, + PRIORITY_WORKSPACE, + PRIORITY_MEMORY, + PRIORITY_RAG, + PRIORITY_LEARNINGS, + PRIORITY_WORKER_NEXT, + PRIORITY_ANALYSIS, +} from '../core/prompt-assembler.js'; +import { listDocTypes, getDocType } from '../intelligence/doctype-store.js'; +import { getTopSkills } from '../intelligence/skill-creator.js'; +import { buildUserPreferencesSection } from '../intelligence/user-preferences.js'; +import type { CLIAdapter } from '../core/cli-adapter.js'; +import type { SpawnOptions } from '../core/agent-runner.js'; +import { + formatLearnedPatternsSection, + formatWorkerNextStepsSection, + formatPreFetchedKnowledgeSection, + formatTargetedReaderSection, + formatTemplateSelectionSection, +} from './master-system-prompt.js'; +import type { WorkerNextStepsEntry } from './master-system-prompt.js'; +import type { DotFolderManager } from './dotfolder-manager.js'; +import type { MemoryManager } from '../memory/index.js'; +import type { + MasterSession, + ExplorationSummary, + TaskRecord, + LearningEntry, + WorkspaceMap, +} from '../types/master.js'; +import type { BatchManager } from './batch-manager.js'; +import { MESSAGE_MAX_TURNS_QUICK } from './classification-engine.js'; +import { createLogger } from '../core/logger.js'; + +const logger = createLogger('prompt-context-builder'); + +// --------------------------------------------------------------------------- +// Constants (moved from master-manager.ts) +// --------------------------------------------------------------------------- + +/** Maximum number of recent tasks to include in a context summary on restart */ +const RESTART_CONTEXT_TASK_LIMIT = 10; + +// --------------------------------------------------------------------------- +// Section budget constants (OB-1511 — budget-aware prompt assembly) +// --------------------------------------------------------------------------- + +/** Per-section character budgets. Total ~32K to prevent truncation in AgentRunner. */ +// System prompt section budget — generous cap to avoid truncating output routing, SHARE, and APP docs (OB-F216) +export const SECTION_BUDGET_SYSTEM_PROMPT = 120_000; +export const SECTION_BUDGET_MEMORY = 4_000; +export const SECTION_BUDGET_WORKSPACE_MAP = 4_000; +export const SECTION_BUDGET_RAG = 6_000; +export const SECTION_BUDGET_CONVERSATION_HISTORY = 10_000; + +/** + * Format an ISO timestamp as a human-readable "X ago" string. + * Used to show the Master how fresh its workspace knowledge is. + */ +export function formatTimeAgo(isoTimestamp: string): string { + const ms = Date.now() - new Date(isoTimestamp).getTime(); + const seconds = Math.floor(ms / 1000); + const minutes = Math.floor(seconds / 60); + const hours = Math.floor(minutes / 60); + const days = Math.floor(hours / 24); + if (days > 0) return `${days} day${days !== 1 ? 's' : ''} ago`; + if (hours > 0) return `${hours} hour${hours !== 1 ? 's' : ''} ago`; + if (minutes > 0) return `${minutes} minute${minutes !== 1 ? 's' : ''} ago`; + return 'just now'; +} + +// --------------------------------------------------------------------------- +// Helper: trim conversation history keeping most recent messages (OB-1511) +// --------------------------------------------------------------------------- + +/** + * Trim a conversation section to fit within a character budget, + * keeping the header line and as many of the most recent messages as possible. + */ +export function trimKeepingRecentMessages(section: string, budget: number): string { + const lines = section.split('\n'); + // First line is the section header (e.g. "## Recent conversation (this session):") + const header = lines[0] ?? ''; + const messageLines = lines.slice(1); + + // Work backwards from the most recent messages + const kept: string[] = []; + let currentSize = header.length + 1; // +1 for newline after header + for (let i = messageLines.length - 1; i >= 0; i--) { + const line = messageLines[i]!; + const lineSize = line.length + 1; // +1 for newline + if (currentSize + lineSize > budget) break; + kept.unshift(line); + currentSize += lineSize; + } + + if (kept.length < messageLines.length) { + logger.debug( + { + section: 'conversation history', + original: messageLines.length, + kept: kept.length, + budget, + }, + 'Trimmed conversation history to budget, keeping most recent messages', + ); + } + + return header + '\n' + kept.join('\n'); +} + +// --------------------------------------------------------------------------- +// Exported interfaces +// --------------------------------------------------------------------------- + +/** + * Optional pre-formatted context sections to inject into the Master system prompt. + * Passed to buildMasterSpawnOptions() so PromptAssembler can budget-rank them. + */ +export interface MasterContextSections { + conversationContext?: string | null; + learnedPatternsContext?: string | null; + workerNextStepsContext?: string | null; + knowledgeContext?: string | null; + targetedReaderContext?: string | null; + analysisContext?: string | null; + /** Industry template suggestion — only injected when no DocTypes exist (OB-1466). */ + templateSelectionContext?: string | null; + /** Learned skills section — top-10 skills with usage stats (OB-1471). */ + learnedSkillsContext?: string | null; + /** User preferences section — per-sender format/language/hours (OB-1474). */ + userPreferencesContext?: string | null; +} + +// --------------------------------------------------------------------------- +// Dependencies interface — callbacks + shared references from MasterManager +// --------------------------------------------------------------------------- + +export interface PromptContextBuilderDeps { + workspacePath: string; + dotFolder: DotFolderManager; + adapter?: CLIAdapter; + messageTimeout: number; + + // Mutable references via getters + getMemory: () => MemoryManager | null; + getSystemPrompt: () => string | null; + getMasterSession: () => MasterSession | null; + getMapLastVerifiedAt: () => string | null; + getLearningsSummary: () => string | null; + getExplorationSummary: () => ExplorationSummary | null; + getWorkspaceContextSummary: () => string | null; + getBatchManager: () => BatchManager | null; + + // Drain operations — return and clear pending items + drainCancellationNotifications: () => string[]; + drainDeepModeResumeOffers: () => string[]; + + // Store helpers delegated back to MasterManager + readWorkspaceMapFromStore: () => Promise; + readAllTasksFromStore: () => Promise; + + /** + * Optional callback invoked after each `buildMasterSpawnOptions()` call with + * the assembled system prompt length in characters. Use this to wire the + * prompt size signal into `SessionCompactor.notifyPromptSize()` so that + * prompt-size-based early compaction can be triggered (OB-1513). + */ + onPromptSizeReport?: (chars: number) => void; +} + +// --------------------------------------------------------------------------- +// PromptContextBuilder class +// --------------------------------------------------------------------------- + +export class PromptContextBuilder { + private deps: PromptContextBuilderDeps; + + constructor(deps: PromptContextBuilderDeps) { + this.deps = deps; + } + + /** Update mutable dependency references (e.g. after memory init). */ + updateDeps(partial: Partial): void { + this.deps = { ...this.deps, ...partial }; + } + + // ------------------------------------------------------------------------- + // buildMasterSpawnOptions (OB-1246) + // ------------------------------------------------------------------------- + + /** + * Assemble SpawnOptions for a Master AI call — attaches a budget-ranked system prompt. + * Uses PromptAssembler for budget-aware system prompt construction. + * Each context section is prioritized so lower-priority content is truncated + * or dropped when the total budget is exceeded. + */ + buildMasterSpawnOptions( + prompt: string, + timeout?: number, + maxTurns?: number, + contextSections?: MasterContextSections, + skipWorkspaceContext?: boolean, + ): SpawnOptions { + const session = this.deps.getMasterSession(); + if (!session) { + throw new Error('Master session not initialized — call initMasterSession() first'); + } + const opts: SpawnOptions = { + prompt, + workspacePath: this.deps.workspacePath, + allowedTools: [...session.allowedTools], + maxTurns: maxTurns ?? MESSAGE_MAX_TURNS_QUICK, + timeout: timeout ?? this.deps.messageTimeout, + retries: 0, // Master session calls don't auto-retry (caller handles) + }; + + // Retrieve adapter-aware system prompt budget (OB-F147/OB-F148) + const budget = this.deps.adapter?.getPromptBudget?.() ?? { maxSystemPromptChars: 100_000 }; + const assembler = new PromptAssembler(); + + // Base system prompt — identity and rules (highest priority) + // Budget is model-aware: 60% of adapter's system prompt budget, capped at 200K (OB-F216). + // The remaining 40% is reserved for injected sections (conversation context, RAG, learnings). + const systemPromptBudget = Math.min(budget.maxSystemPromptChars * 0.6, 200_000); + const systemPrompt = this.deps.getSystemPrompt(); + if (systemPrompt) { + logger.debug( + { + section: 'System Prompt', + actual: systemPrompt.length, + budget: systemPromptBudget, + maxSystemPromptChars: budget.maxSystemPromptChars, + }, + 'Section size vs budget', + ); + assembler.addSection('System Prompt', systemPrompt, PRIORITY_IDENTITY, systemPromptBudget); + } + + // Drain pending cancellation notifications (OB-884). + const cancellations = this.deps.drainCancellationNotifications(); + if (cancellations.length > 0) { + assembler.addSection( + 'Worker Cancellations', + '## IMPORTANT — Worker Cancellation Events\n\n' + cancellations.join('\n'), + 95, + ); + } + + // Drain pending Deep Mode resume offers (OB-1405). + const resumeOffers = this.deps.drainDeepModeResumeOffers(); + if (resumeOffers.length > 0) { + assembler.addSection( + 'Deep Mode Resume', + '## IMPORTANT — Incomplete Deep Mode Sessions\n\n' + resumeOffers.join('\n\n'), + 93, + ); + } + + // Batch context — important for task continuity (OB-1617) + const batchManager = this.deps.getBatchManager(); + if (batchManager) { + const activeBatchId = batchManager.getCurrentBatchId(); + if (activeBatchId) { + const batchContext = batchManager.buildBatchContextSection(activeBatchId); + if (batchContext) { + assembler.addSection('Batch Context', batchContext, 85); + } + } + } + + // Workspace knowledge — project type, frameworks, structure + const explorationSummary = this.deps.getExplorationSummary(); + if (!skipWorkspaceContext && explorationSummary?.status === 'completed') { + const mapContext = this.deps.getWorkspaceContextSummary(); + if (mapContext) { + let contextText = mapContext; + const mapLastVerifiedAt = this.deps.getMapLastVerifiedAt(); + if (mapLastVerifiedAt) { + contextText += `\n\nMap last verified: ${formatTimeAgo(mapLastVerifiedAt)}`; + } + const wsContent = '## Current Workspace Knowledge\n\n' + contextText; + logger.debug( + { + section: 'Workspace Knowledge', + actual: wsContent.length, + budget: SECTION_BUDGET_WORKSPACE_MAP, + }, + 'Section size vs budget', + ); + assembler.addSection( + 'Workspace Knowledge', + wsContent, + PRIORITY_WORKSPACE, + SECTION_BUDGET_WORKSPACE_MAP, + ); + } + } + + // Available DocTypes — registered business data entities + const docTypesSection = this.buildDocTypesSection(); + if (docTypesSection) { + assembler.addSection('Available DocTypes', docTypesSection, 75); + } + + // Learned skills — top-10 reusable skill patterns (OB-1471) + if (contextSections?.learnedSkillsContext) { + assembler.addSection('Learned Skills', contextSections.learnedSkillsContext, 73); + } + + // User preferences — per-sender format/language/working-hours (OB-1474) + if (contextSections?.userPreferencesContext) { + assembler.addSection('User Preferences', contextSections.userPreferencesContext, 71); + } + + // Industry template suggestion — only when no DocTypes exist (OB-1466) + if (contextSections?.templateSelectionContext) { + assembler.addSection('Template Selection', contextSections.templateSelectionContext, 72); + } + + // Conversation context — memory.md + session history + cross-session FTS5 + if (contextSections?.conversationContext) { + const convBudget = SECTION_BUDGET_MEMORY + SECTION_BUDGET_CONVERSATION_HISTORY; + logger.debug( + { + section: 'Conversation Context', + actual: contextSections.conversationContext.length, + budget: convBudget, + }, + 'Section size vs budget', + ); + assembler.addSection( + 'Conversation Context', + contextSections.conversationContext, + PRIORITY_MEMORY, + convBudget, + ); + } + + // Pre-fetched knowledge (RAG) + if (contextSections?.knowledgeContext) { + const ragContent = formatPreFetchedKnowledgeSection(contextSections.knowledgeContext); + logger.debug( + { section: 'Knowledge Context', actual: ragContent.length, budget: SECTION_BUDGET_RAG }, + 'Section size vs budget', + ); + assembler.addSection('Knowledge Context', ragContent, PRIORITY_RAG, SECTION_BUDGET_RAG); + } + + // Targeted reader results + if (contextSections?.targetedReaderContext) { + assembler.addSection( + 'Targeted Reader', + formatTargetedReaderSection(contextSections.targetedReaderContext), + 55, + ); + } + + // Learned patterns — model success rates, effective prompt templates + if (contextSections?.learnedPatternsContext) { + assembler.addSection( + 'Learned Patterns', + contextSections.learnedPatternsContext, + PRIORITY_LEARNINGS + 5, + ); + } + + // Learnings summary — past task outcomes + const learningsSummary = this.deps.getLearningsSummary(); + if (learningsSummary) { + assembler.addSection( + 'Learnings', + '## Learnings from Past Tasks\n\n' + learningsSummary, + PRIORITY_LEARNINGS, + ); + } + + // Worker next steps — follow-up work from recent workers + if (contextSections?.workerNextStepsContext) { + assembler.addSection( + 'Worker Next Steps', + contextSections.workerNextStepsContext, + PRIORITY_WORKER_NEXT, + ); + } + + // Analysis context — planning gate output + if (contextSections?.analysisContext) { + assembler.addSection('Analysis Context', contextSections.analysisContext, PRIORITY_ANALYSIS); + } + + const assembled = assembler.assemble(budget.maxSystemPromptChars); + if (assembled) { + opts.systemPrompt = assembled; + // Notify compactor of assembled prompt size so it can trigger early + // compaction if the prompt approaches the truncation limit (OB-1513). + if (this.deps.onPromptSizeReport) { + this.deps.onPromptSizeReport(assembled.length); + } + } + + return opts; + } + + // ------------------------------------------------------------------------- + // buildDocTypesSection (OB-1385) + // ------------------------------------------------------------------------- + + /** + * Build the "## Available Business Data (DocTypes)" section for injection into + * the Master system prompt. Lists all registered DocTypes with their fields and + * available state-machine actions. Returns null when no DocTypes are registered + * or the database is unavailable. + */ + private buildDocTypesSection(): string | null { + const memory = this.deps.getMemory(); + if (!memory) return null; + const db = memory.getDb(); + if (!db) return null; + + let doctypes: ReturnType; + try { + doctypes = listDocTypes(db); + } catch { + return null; + } + if (doctypes.length === 0) return null; + + const lines: string[] = ['## Available Business Data (DocTypes)', '']; + + for (const dt of doctypes) { + lines.push(`### ${dt.label_plural} (\`${dt.name}\`)`); + + let full: ReturnType | null = null; + try { + full = getDocType(db, dt.id); + } catch { + // If full detail fails, show minimal info + } + + if (full) { + // Fields + const visibleFields = full.fields.sort((a, b) => a.sort_order - b.sort_order).slice(0, 8); // cap at 8 fields to avoid prompt bloat + if (visibleFields.length > 0) { + const fieldList = visibleFields + .map((f) => { + const req = f.required ? '*' : ''; + return ` - \`${f.name}\` (${f.field_type})${req}`; + }) + .join('\n'); + lines.push('**Fields:**'); + lines.push(fieldList); + } + + // Available actions (transitions) + const uniqueActions = [ + ...new Map(full.transitions.map((t) => [t.action_name, t.action_label])).entries(), + ]; + if (uniqueActions.length > 0) { + const actionList = uniqueActions.map(([, label]) => ` - ${label}`).join('\n'); + lines.push('**Actions:**'); + lines.push(actionList); + } + } + + // Example commands + const singular = dt.label_singular.toLowerCase(); + const plural = dt.label_plural.toLowerCase(); + lines.push('**Example commands:**'); + lines.push(` - "list ${plural}"`); + lines.push(` - "create ${singular} for X"`); + if (full && full.transitions.length > 0) { + const firstAction = full.transitions[0]; + if (firstAction) { + lines.push(` - "${firstAction.action_label} ${singular} #42"`); + } + } + + lines.push(''); + } + + return lines.join('\n'); + } + + // ------------------------------------------------------------------------- + // buildLearnedSkillsContext (OB-1471) + // ------------------------------------------------------------------------- + + /** + * Build the "## Learned Skills" section listing the top 10 skills by effectiveness. + * Returns null when no skills are stored or the database is unavailable. + */ + buildLearnedSkillsContext(): string | null { + const memory = this.deps.getMemory(); + if (!memory) return null; + const db = memory.getDb(); + if (!db) return null; + + let skills: ReturnType; + try { + skills = getTopSkills(db, 10); + } catch { + return null; + } + if (skills.length === 0) return null; + + const lines: string[] = [ + '## Learned Skills', + '', + 'You have built up reusable skill patterns from past tasks. When a user request matches a skill below, **prefer executing that learned skill** over generating a new plan from scratch.', + '', + ]; + + for (const skill of skills) { + const usageLine = + skill.usageCount > 0 + ? ` | used ${skill.usageCount}× | ${Math.round(skill.successRate * 100)}% success` + : ' | not yet executed'; + const durationLine = + skill.avgDurationMs !== null ? ` | avg ${Math.round(skill.avgDurationMs / 1000)}s` : ''; + lines.push(`### ${skill.name}${usageLine}${durationLine}`); + lines.push(skill.description); + if (skill.steps.length > 0) { + lines.push( + `**Steps:** ${skill.steps.slice(0, 5).join(' → ')}${skill.steps.length > 5 ? ` → … (${skill.steps.length} steps total)` : ''}`, + ); + } + if (skill.requiredDocTypes.length > 0) { + lines.push(`**DocTypes:** ${skill.requiredDocTypes.join(', ')}`); + } + lines.push(''); + } + + return lines.join('\n'); + } + + // ------------------------------------------------------------------------- + // buildUserPreferencesContext (OB-1474) + // ------------------------------------------------------------------------- + + /** + * Build the "## User Preferences for {sender}" section for the given sender. + * Returns null when no preference data is stored or the database is unavailable. + */ + buildUserPreferencesContext(sender: string): string | null { + const memory = this.deps.getMemory(); + if (!memory) return null; + const db = memory.getDb(); + if (!db) return null; + + try { + return buildUserPreferencesSection(db, sender); + } catch { + return null; + } + } + + // ------------------------------------------------------------------------- + // buildTemplateSelectionContext (OB-1466) + // ------------------------------------------------------------------------- + + /** + * Build the "## Industry Template Available" section for injection when + * no DocTypes are registered but industry templates exist in the workspace. + * Returns null when DocTypes already exist or no templates are available. + */ + async buildTemplateSelectionContext(): Promise { + const memory = this.deps.getMemory(); + if (!memory) return null; + const db = memory.getDb(); + if (!db) return null; + + // Only suggest templates when the user has no DocTypes yet + let doctypeCount: number; + try { + doctypeCount = listDocTypes(db).length; + } catch { + return null; + } + if (doctypeCount > 0) return null; + + // List templates available in the workspace + const templates = await this.deps.dotFolder.listAvailableTemplates(); + if (templates.length === 0) return null; + + return formatTemplateSelectionSection(templates); + } + + // ------------------------------------------------------------------------- + // buildConversationContext (OB-731, OB-1022, OB-1025) + // ------------------------------------------------------------------------- + + /** + * Retrieve conversation context for the Master's system prompt. + * + * Three layers (in order): + * 1. Recent session messages — last 10 user+master turns from the current session. + * 2. memory.md — Master's curated brain (always small, always relevant). + * 3. Cross-session FTS5 — BM25-ranked hits from past sessions. + */ + async buildConversationContext( + userMessage: string, + sessionId?: string, + sender?: string, + ): Promise { + const sections: string[] = []; + const memory = this.deps.getMemory(); + + // Budget for conversation history layers (session + cross-session) + const historyBudget = SECTION_BUDGET_CONVERSATION_HISTORY; + // Budget for memory.md layer + const memoryBudget = SECTION_BUDGET_MEMORY; + + // Layer 1: Recent conversation messages from the CURRENT session + if (sessionId && memory) { + try { + const sessionMessages = sender + ? await memory.getSessionHistoryForSender(sessionId, sender, 20) + : await memory.getSessionHistory(sessionId, 20); + const relevant = sessionMessages + .filter((e) => e.role === 'user' || e.role === 'master') + .slice(-10); + if (relevant.length > 0) { + const lines = relevant.map((e) => { + const label = e.role === 'user' ? 'User' : 'You'; + const content = e.content.length > 400 ? e.content.slice(0, 400) + '…' : e.content; + return `${label}: ${content}`; + }); + let sessionSection = '## Recent conversation (this session):\n' + lines.join('\n'); + // Trim to history budget, keeping most recent messages + if (sessionSection.length > historyBudget) { + sessionSection = trimKeepingRecentMessages(sessionSection, historyBudget); + } + sections.push(sessionSection); + } + } catch (err) { + logger.warn({ err }, 'Failed to load session history for context injection'); + } + } + + // Layer 2: load memory.md (OB-1022) + try { + const memoryContent = await this.deps.dotFolder.readMemoryFile(); + if (memoryContent && memoryContent.trim().length > 0) { + let memSection = '## Memory:\n' + memoryContent.trim(); + if (memSection.length > memoryBudget) { + logger.debug( + { section: 'memory.md', actual: memSection.length, budget: memoryBudget }, + 'Trimming memory.md to budget', + ); + memSection = memSection.slice(0, memoryBudget); + } + sections.push(memSection); + } + } catch (err) { + logger.warn({ err }, 'Failed to read memory.md for context injection'); + } + + // Layer 3: cross-session FTS5 search via searchConversations() (OB-1025). + // Uses remaining history budget after Layer 1 + const layer1Size = sections.length > 0 ? sections[0]!.length : 0; + const crossSessionBudget = Math.max(0, historyBudget - layer1Size); + + if (memory && crossSessionBudget > 0) { + try { + const crossSession = await memory.searchConversations(userMessage, 5, sender); + const relevant = crossSession.filter((e) => e.role === 'user' || e.role === 'master'); + if (relevant.length > 0) { + const lines = relevant.map((e) => { + const dateStr = e.created_at + ? new Date(e.created_at).toISOString().replace('T', ' ').slice(0, 16) + : ''; + const label = e.role === 'user' ? 'User' : 'Master'; + const snippet = e.content.length > 500 ? e.content.slice(0, 500) + '…' : e.content; + return dateStr ? `[${dateStr}] ${label}: ${snippet}` : `${label}: ${snippet}`; + }); + let crossSection = '## Related past conversations:\n' + lines.join('\n'); + if (crossSection.length > crossSessionBudget) { + crossSection = crossSection.slice(0, crossSessionBudget); + } + sections.push(crossSection); + } + } catch (err) { + logger.warn( + { err }, + 'Failed to retrieve cross-session conversation history for context injection', + ); + } + } + + return sections.length > 0 ? sections.join('\n\n') : null; + } + + // ------------------------------------------------------------------------- + // buildLearnedPatternsContext (OB-735) + // ------------------------------------------------------------------------- + + /** + * Build the "## Learned Patterns" section for injection into the Master system prompt. + * Returns null when there is insufficient data or MemoryManager is unavailable. + */ + async buildLearnedPatternsContext(): Promise { + const memory = this.deps.getMemory(); + if (!memory) return null; + try { + const [allLearnings, effectivePrompts] = await Promise.all([ + memory.getLearnedTaskTypes(), + memory.getHighEffectivenessPrompts(0.7, 5), + ]); + + const modelLearnings = allLearnings + .filter((l) => l.successCount + l.failureCount > 5) + .map((l) => ({ + taskType: l.taskType, + bestModel: l.bestModel, + successRate: l.successRate, + totalTasks: l.successCount + l.failureCount, + })); + + const promptPatterns = effectivePrompts.map((p) => ({ + name: p.name, + effectiveness: p.effectiveness, + usageCount: p.usage_count, + })); + + return formatLearnedPatternsSection({ modelLearnings, effectivePrompts: promptPatterns }); + } catch (err) { + logger.warn({ err }, 'Failed to build learned patterns context'); + return null; + } + } + + // ------------------------------------------------------------------------- + // buildWorkerNextStepsContext (OB-1635) + // ------------------------------------------------------------------------- + + /** + * Read next_steps from the 5 most recent completed worker summaries. + * Returns null when no workers have a summary or none reported follow-up work. + */ + async buildWorkerNextStepsContext(): Promise { + const memory = this.deps.getMemory(); + if (!memory) return null; + try { + const recentWorkers = await memory.getRecentWorkerSpawns(5); + const entries: WorkerNextStepsEntry[] = []; + for (const worker of recentWorkers) { + if (!worker.summary_json) continue; + let parsed: unknown; + try { + parsed = JSON.parse(worker.summary_json); + } catch { + continue; + } + const parsedRecord = + typeof parsed === 'object' && parsed !== null + ? (parsed as Record) + : null; + const nextSteps = + parsedRecord !== null && typeof parsedRecord['next_steps'] === 'string' + ? parsedRecord['next_steps'] + : ''; + entries.push({ + taskSummary: worker.task_summary ?? '', + nextSteps, + }); + } + return formatWorkerNextStepsSection(entries); + } catch (err) { + logger.warn({ err }, 'Failed to build worker next steps context'); + return null; + } + } + + // ------------------------------------------------------------------------- + // buildLearningsSummary + // ------------------------------------------------------------------------- + + /** + * Build a concise summary of past learnings for system prompt injection. + * Groups the most recent entries by task type and computes stats. + * Capped at ~2000 chars to avoid prompt bloat. + */ + buildLearningsSummary(entries: LearningEntry[]): string | null { + const recent = entries.slice(-50); + if (recent.length === 0) return null; + + const byType = new Map(); + for (const entry of recent) { + const group = byType.get(entry.taskType) ?? []; + group.push(entry); + byType.set(entry.taskType, group); + } + + const lines: string[] = [`Based on ${recent.length} recent task executions:`, '']; + + for (const [taskType, group] of byType) { + if (group.length < 3) continue; + + const successes = group.filter((e) => e.success); + const successRate = ((successes.length / group.length) * 100).toFixed(0); + const avgDuration = Math.round( + group.reduce((sum, e) => sum + e.durationMs, 0) / group.length, + ); + + const modelStats = new Map(); + for (const e of group) { + const model = e.modelUsed ?? 'unknown'; + const stats = modelStats.get(model) ?? { total: 0, success: 0 }; + stats.total++; + if (e.success) stats.success++; + modelStats.set(model, stats); + } + const bestModel = [...modelStats.entries()] + .filter(([, s]) => s.total >= 2) + .sort((a, b) => b[1].success / b[1].total - a[1].success / a[1].total)[0]; + + const profileStats = new Map(); + for (const e of group) { + const profile = e.profileUsed ?? 'unknown'; + const stats = profileStats.get(profile) ?? { total: 0, success: 0 }; + stats.total++; + if (e.success) stats.success++; + profileStats.set(profile, stats); + } + const bestProfile = [...profileStats.entries()] + .filter(([, s]) => s.total >= 2) + .sort((a, b) => b[1].success / b[1].total - a[1].success / a[1].total)[0]; + + let line = `- **${taskType}**: ${successRate}% success (${group.length} tasks, avg ${avgDuration}ms)`; + if (bestModel) { + const modelRate = ((bestModel[1].success / bestModel[1].total) * 100).toFixed(0); + line += ` — best model: ${bestModel[0]} (${modelRate}%)`; + } + if (bestProfile) { + line += ` — best profile: ${bestProfile[0]}`; + } + lines.push(line); + } + + if (lines.length <= 2) return null; + + const summary = lines.join('\n'); + if (summary.length > 2000) { + return summary.slice(0, 1997) + '...'; + } + return summary; + } + + // ------------------------------------------------------------------------- + // buildContextSummary (session restart recovery) + // ------------------------------------------------------------------------- + + /** + * Build a context summary for a restarted Master session. + * Loads workspace-map.json and recent task history to seed the new session. + */ + async buildContextSummary(): Promise { + const parts: string[] = []; + + parts.push( + '# Session Context Recovery', + '', + 'Your previous session ended unexpectedly. Here is the accumulated context to resume from:', + '', + ); + + const map = await this.deps.readWorkspaceMapFromStore(); + if (map) { + parts.push('## Workspace Summary'); + parts.push(`- **Project:** ${map.projectName} (${map.projectType})`); + parts.push(`- **Path:** ${map.workspacePath}`); + if (map.frameworks.length > 0) { + parts.push(`- **Frameworks:** ${map.frameworks.join(', ')}`); + } + parts.push(`- **Summary:** ${map.summary}`); + parts.push(''); + } + + const tasks = await this.deps.readAllTasksFromStore(); + if (tasks.length > 0) { + const recentTasks = tasks + .sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime()) + .slice(0, RESTART_CONTEXT_TASK_LIMIT); + + parts.push('## Recent Task History'); + for (const task of recentTasks) { + const status = task.status === 'completed' ? 'completed' : task.status; + parts.push(`- [${status}] "${task.description.slice(0, 100)}" (from ${task.sender})`); + if (task.result) { + parts.push(` Result: ${task.result.slice(0, 200)}`); + } + } + parts.push(''); + } + + parts.push('Continue operating normally. Respond to the next user message as usual.'); + + return parts.join('\n'); + } +} diff --git a/src/master/prompt-evolver.ts b/src/master/prompt-evolver.ts new file mode 100644 index 00000000..19936022 --- /dev/null +++ b/src/master/prompt-evolver.ts @@ -0,0 +1,226 @@ +/** + * Prompt Evolution (OB-734) + * + * Every 50 task completions, query for underperforming prompts (effectiveness < 0.7). + * For each underperforming prompt, spawn a quick Haiku worker to suggest an improved + * version. Save the improved version as a new prompt version with neutral effectiveness + * (0.5). After 20 uses, compare effectiveness: if worse, deactivate and restore the + * previous version. + */ + +import type { AgentRunner } from '../core/agent-runner.js'; +import { TOOLS_READ_ONLY } from '../core/agent-runner.js'; +import type { MemoryManager, PromptRecord } from '../memory/index.js'; +import { createLogger } from '../core/logger.js'; + +const logger = createLogger('prompt-evolver'); + +/** Minimum usage count a prompt must have before being considered for evolution. */ +const MIN_USAGE_BEFORE_EVOLUTION = 10; + +/** Effectiveness threshold below which a prompt is considered underperforming. */ +const UNDERPERFORMING_THRESHOLD = 0.7; + +/** Minimum uses of the new version before we compare it to the previous version. */ +const MIN_USES_FOR_COMPARISON = 20; + +/** If a new version's effectiveness drops below this compared to the old one, revert. */ +const REVERT_EFFECTIVENESS_DELTA = 0.05; + +/** + * Build the prompt text to send to the Haiku worker asking for an improved version. + */ +function buildEvolutionPrompt(record: PromptRecord): string { + const effectivenessPercent = Math.round(record.effectiveness * 100); + return `You are a prompt engineering expert. The following system prompt has a ${effectivenessPercent}% success rate (${record.usage_count} uses, ${record.success_count} successes). Analyze it and suggest a concise, improved version that should increase its success rate. + +Current prompt (name: "${record.name}", version: ${record.version}): +--- +${record.content} +--- + +Respond with ONLY the improved prompt text. Do not add explanations, headers, or markdown code fences. Output the raw improved prompt text directly.`; +} + +/** + * Extract the improved prompt text from the worker's output. + * If the worker wrapped it in a code fence, strip the fence. + */ +function extractImprovedPrompt(output: string): string { + const trimmed = output.trim(); + + // Strip markdown code fences if present + const fenceMatch = trimmed.match(/^```(?:\w+)?\n([\s\S]*?)\n```$/); + if (fenceMatch?.[1]) { + return fenceMatch[1].trim(); + } + + return trimmed; +} + +/** + * Attempt to evolve a single underperforming prompt. + * Returns true if a new version was saved, false otherwise. + */ +async function evolvePrompt( + prompt: PromptRecord, + memory: MemoryManager, + agentRunner: AgentRunner, + workspacePath: string, +): Promise { + logger.info( + { name: prompt.name, version: prompt.version, effectiveness: prompt.effectiveness }, + 'Attempting to evolve underperforming prompt', + ); + + const evolutionPrompt = buildEvolutionPrompt(prompt); + + let result; + try { + result = await agentRunner.spawn({ + prompt: evolutionPrompt, + workspacePath, + model: 'haiku', + allowedTools: [...TOOLS_READ_ONLY], + maxTurns: 3, + retries: 1, + }); + } catch (err) { + logger.warn({ err, name: prompt.name }, 'Prompt evolution worker failed'); + return false; + } + + if (result.exitCode !== 0 || !result.stdout.trim()) { + logger.warn( + { name: prompt.name, exitCode: result.exitCode }, + 'Prompt evolution worker returned empty or failed output', + ); + return false; + } + + const improvedContent = extractImprovedPrompt(result.stdout); + + // Sanity check: improved prompt must be non-trivially different + if (improvedContent === prompt.content || improvedContent.length < 20) { + logger.info( + { name: prompt.name }, + 'Evolution worker returned unchanged or trivial content, skipping', + ); + return false; + } + + try { + await memory.createPromptVersion(prompt.name, improvedContent); + logger.info({ name: prompt.name }, 'New prompt version saved with neutral effectiveness (0.5)'); + return true; + } catch (err) { + logger.warn({ err, name: prompt.name }, 'Failed to save evolved prompt version'); + return false; + } +} + +/** + * Check whether any recently-created prompt versions should be reverted. + * + * A version is reverted when: + * - It has been used at least MIN_USES_FOR_COMPARISON times + * - Its effectiveness is more than REVERT_EFFECTIVENESS_DELTA worse than the + * version that was active before it (version - 1) + * + * On revert: the new version is deactivated by creating a fresh version from + * the previous content (restoring it as the active version). + */ +async function checkAndRevertIfWorse(memory: MemoryManager): Promise { + let allPromptNames: string[]; + try { + // getUnderperformingPrompts returns active prompts below threshold. + // We also need to check recently promoted prompts regardless of effectiveness. + // We use a low threshold (0.0) to get ALL active prompts, then filter by usage. + const allActive = await memory.getUnderperformingPrompts(1.1); // threshold > 1 = all + allPromptNames = [...new Set(allActive.map((p) => p.name))]; + } catch { + return; + } + + for (const name of allPromptNames) { + let versions: PromptRecord[]; + try { + versions = await memory.getPromptStats(name); + } catch { + continue; + } + + // versions are ordered by version DESC; [0] is the latest + const latest = versions[0]; + const previous = versions[1]; + + if (!latest || !previous) continue; + + // Only evaluate if the latest version has enough usage data + if (latest.usage_count < MIN_USES_FOR_COMPARISON) continue; + + // Only revert if the new version is noticeably worse + const delta = previous.effectiveness - latest.effectiveness; + if (delta > REVERT_EFFECTIVENESS_DELTA) { + logger.info( + { + name, + latestVersion: latest.version, + latestEffectiveness: latest.effectiveness, + previousVersion: previous.version, + previousEffectiveness: previous.effectiveness, + delta, + }, + 'New prompt version underperforms predecessor — reverting to previous content', + ); + + try { + // Restore previous content as a new active version + await memory.createPromptVersion(name, previous.content); + logger.info( + { name }, + 'Prompt reverted to previous content (new version created from old content)', + ); + } catch (err) { + logger.warn({ err, name }, 'Failed to revert prompt version'); + } + } + } +} + +/** + * Main entry point for prompt evolution. + * Call this every 50 task completions. + */ +export async function evolvePrompts( + memory: MemoryManager, + agentRunner: AgentRunner, + workspacePath: string, +): Promise { + logger.info('Running prompt evolution cycle'); + + // First, check whether any recently-promoted versions should be reverted + await checkAndRevertIfWorse(memory); + + // Find prompts that have enough usage and are underperforming + let candidates: PromptRecord[]; + try { + const underperforming = await memory.getUnderperformingPrompts(UNDERPERFORMING_THRESHOLD); + candidates = underperforming.filter((p) => p.usage_count >= MIN_USAGE_BEFORE_EVOLUTION); + } catch (err) { + logger.warn({ err }, 'Failed to query underperforming prompts'); + return; + } + + if (candidates.length === 0) { + logger.info('No underperforming prompts eligible for evolution'); + return; + } + + logger.info({ count: candidates.length }, 'Found underperforming prompts eligible for evolution'); + + // Evolve each candidate (sequentially to avoid overwhelming the AI) + for (const candidate of candidates) { + await evolvePrompt(candidate, memory, agentRunner, workspacePath); + } +} diff --git a/src/master/result-parser.ts b/src/master/result-parser.ts new file mode 100644 index 00000000..e958e130 --- /dev/null +++ b/src/master/result-parser.ts @@ -0,0 +1,230 @@ +/** + * Result Parser — Robust JSON extraction from AI output + * + * AI responses aren't always clean JSON. This module implements progressive fallback strategies: + * 1. Direct JSON.parse() on the full stdout + * 2. Extract from markdown code fences (```json ... ```) + * 3. Regex extraction of first {...} block + * 4. Return parse error for retry handling + */ + +import { type ZodSchema } from 'zod/v3'; + +import { createLogger } from '../core/logger.js'; + +const logger = createLogger('result-parser'); + +export interface ParseResult { + success: true; + data: T; + method: 'direct' | 'markdown' | 'regex' | 'markdown-fallback'; +} + +export interface ParseError { + success: false; + error: string; + rawOutput: string; +} + +export type ParsedAIResult = ParseResult | ParseError; + +/** + * Validate parsed data against an optional Zod schema. + * Returns a ParseResult on success, or ParseError on schema validation failure. + */ +function validateParsed( + data: unknown, + schema: ZodSchema | undefined, + stdout: string, + label: string, + method: 'direct' | 'markdown' | 'regex', +): ParsedAIResult { + if (!schema) { + return { success: true, data: data as T, method }; + } + const validation = schema.safeParse(data); + if (!validation.success) { + logger.error( + { label, method, errors: validation.error.errors }, + 'Schema validation failed for parsed JSON', + ); + return { + success: false, + error: `Schema validation failed: ${validation.error.message}`, + rawOutput: stdout, + }; + } + return { success: true, data: validation.data, method }; +} + +/** + * Parse AI output to extract JSON result + * + * @param stdout - Raw output from AI command + * @param label - Human-readable label for logging (e.g., "structure scan") + * @param schema - Optional Zod schema to validate the parsed JSON against + * @returns Parsed result or error + */ +export function parseAIResult( + stdout: string, + label: string, + schema?: ZodSchema, +): ParsedAIResult { + // Strategy 1: Direct JSON.parse() + try { + const data = JSON.parse(stdout) as unknown; + logger.debug({ label, method: 'direct' }, 'AI result parsed successfully (direct)'); + return validateParsed(data, schema, stdout, label, 'direct'); + } catch (directError) { + logger.debug( + { label, error: String(directError) }, + 'Direct JSON parse failed, trying markdown extraction', + ); + } + + // Strategy 2: Extract from markdown code fences + const markdownMatch = stdout.match(/```(?:json)?\s*\n([\s\S]*?)\n```/); + if (markdownMatch?.[1]) { + try { + const data = JSON.parse(markdownMatch[1]) as unknown; + logger.debug({ label, method: 'markdown' }, 'AI result parsed successfully (markdown fence)'); + return validateParsed(data, schema, stdout, label, 'markdown'); + } catch (markdownError) { + logger.debug( + { label, error: String(markdownError) }, + 'Markdown fence extraction failed, trying regex', + ); + } + } + + // Strategy 3: Regex extraction - try all {...} blocks + // Find all potential JSON blocks and try parsing each one + let searchStart = 0; + while (searchStart < stdout.length) { + const braceIndex = stdout.indexOf('{', searchStart); + if (braceIndex === -1) break; + + let braceCount = 0; + let inString = false; + let escapeNext = false; + + for (let i = braceIndex; i < stdout.length; i++) { + const char = stdout[i]; + + if (escapeNext) { + escapeNext = false; + continue; + } + + if (char === '\\') { + escapeNext = true; + continue; + } + + if (char === '"') { + inString = !inString; + continue; + } + + if (!inString) { + if (char === '{') { + braceCount++; + } else if (char === '}') { + braceCount--; + if (braceCount === 0) { + // Found matching closing brace + const jsonCandidate = stdout.slice(braceIndex, i + 1); + try { + const data = JSON.parse(jsonCandidate) as unknown; + logger.debug( + { label, method: 'regex' }, + 'AI result parsed successfully (regex extraction)', + ); + return validateParsed(data, schema, stdout, label, 'regex'); + } catch (regexError) { + logger.debug( + { label, error: String(regexError) }, + 'Regex extraction failed for candidate, trying next', + ); + // Continue searching after this block + searchStart = i + 1; + break; + } + } + } + } + } + + // If we didn't find a closing brace, move past this opening brace + if (braceCount !== 0) { + searchStart = braceIndex + 1; + } + } + + // All strategies failed + const truncatedOutput = stdout.length > 200 ? `${stdout.slice(0, 200)}...` : stdout; + logger.warn( + { label, outputLength: stdout.length, truncatedOutput }, + 'All JSON extraction strategies failed', + ); + + return { + success: false, + error: + 'Could not extract valid JSON from AI output using any strategy (direct, markdown, regex)', + rawOutput: stdout, + }; +} + +/** + * Parse AI result with automatic retry logic + * + * @param executeFn - Function that executes the AI command and returns stdout + * @param label - Human-readable label for logging + * @param maxRetries - Maximum number of retry attempts (default: 3) + * @returns Parsed result or final error after all retries + */ +export async function parseAIResultWithRetry( + executeFn: () => Promise, + label: string, + maxRetries: number = 3, +): Promise> { + let lastError: ParseError | null = null; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + logger.debug({ label, attempt, maxRetries }, 'Executing AI command'); + + try { + const stdout = await executeFn(); + const result = parseAIResult(stdout, label); + + if (result.success) { + return result; + } + + lastError = result; + + if (attempt < maxRetries) { + logger.info({ label, attempt, maxRetries }, 'Parse failed, retrying...'); + // Exponential backoff: 1s, 2s, 4s + await new Promise((resolve) => setTimeout(resolve, 1000 * Math.pow(2, attempt - 1))); + } + } catch (execError) { + logger.error({ label, attempt, error: String(execError) }, 'AI command execution failed'); + + lastError = { + success: false, + error: `AI command execution failed: ${String(execError)}`, + rawOutput: '', + }; + + if (attempt < maxRetries) { + logger.info({ label, attempt, maxRetries }, 'Execution failed, retrying...'); + await new Promise((resolve) => setTimeout(resolve, 1000 * Math.pow(2, attempt - 1))); + } + } + } + + logger.error({ label, maxRetries }, 'All retry attempts exhausted'); + return lastError!; +} diff --git a/src/master/seed-prompts.ts b/src/master/seed-prompts.ts new file mode 100644 index 00000000..c8b28f78 --- /dev/null +++ b/src/master/seed-prompts.ts @@ -0,0 +1,1418 @@ +/** + * Seed Prompts — Initial Prompt Library Templates + * + * This module contains the initial prompt templates that are seeded into + * .openbridge/prompts/ when the Master AI first initializes. + * + * Each prompt template is: + * - Designed for a specific task type (exploration, execution, verification) + * - Optimized for JSON output that matches our Zod schemas + * - Tracked for effectiveness (success rate) in the prompt manifest + * - Editable by the Master AI for self-improvement + */ + +import type { PromptTemplate } from '../types/master.js'; + +/** + * Seed prompt metadata (used for manifest initialization) + */ +interface SeedPrompt { + id: string; + filename: string; + content: string; + description: string; + category: 'exploration' | 'task' | 'verification' | 'other'; + version: string; +} + +/** + * Exploration: Structure Scan + * + * Generates a prompt for scanning workspace structure. + * Expected output: structure-scan.json matching StructureScanSchema + */ +export const EXPLORATION_STRUCTURE_SCAN: SeedPrompt = { + id: 'exploration-structure-scan', + filename: 'exploration-structure-scan.md', + category: 'exploration', + version: '1.0.0', + description: 'Scans workspace structure and returns top-level files/dirs with file counts', + content: `# Task: Workspace Structure Scan + +Scan the workspace at **{{workspacePath}}** and return a JSON object with its structure. + +## Instructions + +1. List all **top-level files** (files directly in the workspace root) +2. List all **top-level directories** (directories directly in the workspace root) +3. For each top-level directory, count how many files it contains (recursively, but skip node_modules/.git/dist/.next/build/coverage/target) +4. Identify **configuration files** (package.json, tsconfig.json, requirements.txt, Cargo.toml, .env.example, etc.) +5. List **skipped directories** (node_modules, .git, dist, etc.) +6. Count **total files** in the workspace (excluding skipped directories) + +## Skip These Directories + +- node_modules +- .git +- dist +- build +- .next +- coverage +- target +- vendor +- __pycache__ +- .venv +- venv + +## Output Format + +Return ONLY valid JSON matching this schema: + +\`\`\`json +{ + "workspacePath": "{{workspacePath}}", + "topLevelFiles": ["README.md", "package.json"], + "topLevelDirs": ["src", "tests", "docs"], + "directoryCounts": { + "src": 42, + "tests": 18, + "docs": 5 + }, + "configFiles": ["package.json", "tsconfig.json"], + "skippedDirs": ["node_modules", ".git", "dist"], + "totalFiles": 65, + "scannedAt": "2026-02-22T...", + "durationMs": 1200 +} +\`\`\` + +**IMPORTANT:** +- Return ONLY the JSON object, no explanations or markdown +- Use ISO 8601 format for scannedAt +- durationMs should reflect actual scan time in milliseconds +- Do NOT read file contents in this phase (just list and count) +`, +}; + +/** + * Exploration: Project Classification + * + * Classifies the project type and detects frameworks/tools. + * Expected output: classification.json matching ClassificationSchema + */ +export const EXPLORATION_CLASSIFICATION: SeedPrompt = { + id: 'exploration-classification', + filename: 'exploration-classification.md', + category: 'exploration', + version: '1.0.0', + description: 'Classifies project type and detects frameworks, commands, dependencies', + content: `# Task: Project Classification + +Classify the project at **{{workspacePath}}** based on the structure scan results. + +## Structure Scan Results + +\`\`\`json +{{structureScan}} +\`\`\` + +## Instructions + +1. **Read configuration files** listed in the structure scan (package.json, requirements.txt, etc.) +2. **Determine project type**: + - Code projects: "node", "python", "rust", "go", "java", "react-app", "api-backend", etc. + - Business workspaces: "cafe-operations", "legal-docs", "accounting-records", "real-estate-listings", etc. + - Mixed: "business-app-with-data", "mixed" +3. **Detect frameworks and tools** (React, Express, Django, TypeScript, Vite, etc.) +4. **Extract commands** from package.json scripts, Makefile, etc. +5. **List dependencies** from package.json, requirements.txt, Cargo.toml, etc. +6. **Identify key insights** (build system, testing framework, deployment targets, etc.) + +## Classification Heuristics + +**Code workspace indicators:** +- Presence of: package.json, requirements.txt, Cargo.toml, go.mod +- Directories: src/, lib/, tests/, components/, api/ +- Extensions: .ts, .js, .py, .rs, .go + +**Business workspace indicators:** +- Extensions: .xlsx, .csv, .pdf, .docx, .txt, .md (without code configs) +- No build configs or dependency files +- Directories: invoices/, reports/, contracts/, inventory/, sales/ + +## Output Format + +Return ONLY valid JSON matching this schema: + +\`\`\`json +{ + "projectType": "node", + "projectName": "openbridge", + "frameworks": ["typescript", "node", "vitest"], + "commands": { + "dev": "npm run dev", + "test": "npm test", + "build": "npm run build" + }, + "dependencies": [ + { "name": "typescript", "version": "^5.7.0", "type": "dev" }, + { "name": "pino", "version": "^9.0.0", "type": "runtime" } + ], + "insights": [ + "TypeScript project with strict mode enabled", + "Uses Vitest for testing", + "ESM-only project (type: module)" + ], + "classifiedAt": "2026-02-22T...", + "durationMs": 1500 +} +\`\`\` + +**IMPORTANT:** +- Return ONLY the JSON object, no explanations +- Read actual config file contents, don't guess +- Be accurate — if you can't determine something, omit it +- Use ISO 8601 format for classifiedAt +`, +}; + +/** + * Task: Execute User Request + * + * General purpose prompt for executing user tasks. + */ +export const TASK_EXECUTE: SeedPrompt = { + id: 'task-execute', + filename: 'task-execute.md', + category: 'task', + version: '1.0.0', + description: 'Executes a user-requested task with workspace context', + content: `# Task: Execute User Request + +Execute the following user request in the context of the workspace. + +## User Request + +{{userMessage}} + +## Workspace Context + +**Project:** {{projectName}} +**Type:** {{projectType}} +**Frameworks:** {{frameworks}} + +**Available Commands:** +{{commands}} + +**Project Structure:** +{{structure}} + +## Instructions + +1. Understand what the user is asking for +2. Use the workspace context to inform your approach +3. If the task requires code changes, make them +4. If the task requires running commands, execute them +5. Provide a clear, concise response about what you did + +## Available Tools + +You have access to the following tools: +{{allowedTools}} + +## Response Format + +Provide your response in natural language. If you completed the task, explain what you did. If you encountered issues, explain what went wrong and what the user should do. + +**Do NOT output JSON unless the user explicitly requested it.** +`, +}; + +/** + * Task: Verify Implementation + * + * Verifies that a completed task meets requirements. + */ +export const TASK_VERIFY: SeedPrompt = { + id: 'task-verify', + filename: 'task-verify.md', + category: 'verification', + version: '1.0.0', + description: 'Verifies that a task implementation meets requirements', + content: `# Task: Verify Implementation + +Verify that the following task was completed successfully. + +## Original Task + +{{taskDescription}} + +## Implementation Notes + +{{implementationNotes}} + +## Verification Steps + +1. Check that all requirements from the original task are met +2. Run tests if applicable (npm test, pytest, cargo test, etc.) +3. Check for errors or warnings +4. Verify that the implementation follows project conventions +5. Confirm that the changes don't break existing functionality + +## Output Format + +Return a JSON object with verification results: + +\`\`\`json +{ + "verified": true, + "requirementsMet": true, + "testsPassed": true, + "errors": [], + "warnings": ["Minor: unused import in file.ts"], + "notes": "Implementation looks good. All tests pass.", + "verifiedAt": "2026-02-22T..." +} +\`\`\` + +**IMPORTANT:** +- Return ONLY the JSON object +- Set verified=true only if ALL checks pass +- List any errors or warnings found +- Use ISO 8601 format for verifiedAt +`, +}; + +/** + * Task: Code Audit + * + * Runs test suites, linters, and type checkers; reports findings with severity, + * file, line, description, and fix suggestion. + */ +export const TASK_CODE_AUDIT: SeedPrompt = { + id: 'task-code-audit', + filename: 'task-code-audit.md', + category: 'task', + version: '1.0.0', + description: + 'Runs tests, linter, and type checker; reports findings with severity, location, and fix suggestions', + content: `# Task: Code Audit + +Perform a code audit on the workspace at **{{workspacePath}}**. + +## Step 1 — Run Verification Commands + +Run each command in order and capture its output: + +1. \`npm test\` — run the full test suite +2. \`npm run lint\` — check for linting errors +3. \`npm run typecheck\` — check for TypeScript type errors + +If a command is not available (missing from package.json scripts), skip it and note it as "not configured". + +## Step 2 — Analyse Failures + +For each failing test: +1. Read the test file to understand what is being tested +2. Read the source file under test to find the root cause +3. Note the file path, line number, error message, and likely fix + +For each lint error: +1. Note the rule name, file path, line number, and description +2. Determine the fix (auto-fixable, minor refactor, or design change) + +For each type error: +1. Note the file path, line number, error message (e.g. TS2345) +2. Determine the fix + +## Step 3 — Report Findings + +Report every finding using the format below. Use **severity levels**: +- \`critical\` — test failure or type error that breaks the build +- \`high\` — lint error that blocks CI or indicates a likely runtime bug +- \`medium\` — lint warning or non-critical type issue +- \`low\` — style issue or informational note + +\`\`\`json +{ + "summary": { + "testsPassed": 42, + "testsFailed": 3, + "testsSkipped": 1, + "lintErrors": 2, + "lintWarnings": 4, + "typeErrors": 1, + "commandsRun": ["npm test", "npm run lint", "npm run typecheck"], + "commandsSkipped": [] + }, + "findings": [ + { + "severity": "critical", + "category": "test", + "file": "src/core/router.ts", + "line": 142, + "description": "Test 'routes /history to handler' fails — handler returns undefined instead of OutboundMessage", + "fixSuggestion": "Return a valid OutboundMessage object from the /history branch (line 142)" + }, + { + "severity": "high", + "category": "lint", + "file": "src/master/master-manager.ts", + "line": 307, + "description": "@typescript-eslint/no-floating-promises — promise not awaited", + "fixSuggestion": "Add await before the call, or explicitly void it with \`void someCall()\`" + }, + { + "severity": "medium", + "category": "typecheck", + "file": "src/memory/chunk-store.ts", + "line": 88, + "description": "TS2345: Argument of type 'string | undefined' is not assignable to parameter of type 'string'", + "fixSuggestion": "Add a null-check guard before passing the value, or widen the parameter type" + } + ], + "auditedAt": "2026-02-22T12:00:00.000Z" +} +\`\`\` + +## Rules + +- Run commands with a reasonable timeout (5 minutes per command). +- Do NOT modify any files — this is a read-and-report audit only. +- If all commands pass with zero errors, report an empty findings array and set a clear summary. +- Report test pass/fail counts even when all tests pass. +- Return ONLY the JSON object — no extra markdown, no preamble. +`, +}; + +/** + * Task: Generate Output + * + * Generates a file output (HTML report, JSON data export, PDF, etc.) and + * places it in .openbridge/generated/ so it can be shared via SHARE markers. + */ +export const TASK_GENERATE_OUTPUT: SeedPrompt = { + id: 'task-generate-output', + filename: 'task-generate-output.md', + category: 'task', + version: '1.0.0', + description: + 'Generates a file output in .openbridge/generated/ and appends a SHARE marker so the Master can deliver it to the user', + content: `# Task: Generate Output File + +Generate a file based on the user request and write it to the output directory. + +## User Request + +{{userMessage}} + +## Workspace Context + +**Project:** {{projectName}} +**Type:** {{projectType}} +**Workspace path:** {{workspacePath}} + +## Output Directory + +Write all generated files to: +\`{{workspacePath}}/.openbridge/generated/\` + +Create the directory if it does not exist. + +## Format Selection + +Choose the output format that best matches the user request: + +| Output type | Format | File extension | +| --- | --- | --- | +| Report, dashboard, or interactive output | HTML | \`.html\` | +| Structured data export | JSON | \`.json\` | +| Tabular data | CSV | \`.csv\` | +| Document / printable report | PDF (or Markdown if PDF tooling unavailable) | \`.pdf\` / \`.md\` | +| Plain text output | Text | \`.txt\` | + +If the user specifies a format, use that format exactly. + +## Instructions + +1. Determine the appropriate output format from the table above (or use the user-specified format). +2. Generate the content based on the user request and workspace context. +3. Write the file to \`{{workspacePath}}/.openbridge/generated/.\`. + - Use a short, descriptive filename (e.g., \`test-report.html\`, \`api-audit.json\`, \`summary.md\`). + - Do NOT overwrite existing files — use a unique name if the file already exists. +4. After writing the file, append a SHARE marker at the very end of your response. + +## SHARE Marker + +After writing the file, end your response with one of these SHARE markers based on format: + +**HTML report → GitHub Pages (public URL):** +\`\`\` +[SHARE:github-pages]{"path":"{{workspacePath}}/.openbridge/generated/.html"}[/SHARE] +\`\`\` + +**PDF, DOC, or binary document → WhatsApp/Telegram attachment:** +\`\`\` +[SHARE:whatsapp]{"path":"{{workspacePath}}/.openbridge/generated/.pdf"}[/SHARE] +\`\`\` + +**JSON, CSV, or data file → WhatsApp/Telegram attachment:** +\`\`\` +[SHARE:whatsapp]{"path":"{{workspacePath}}/.openbridge/generated/.json"}[/SHARE] +\`\`\` + +**Large text or Markdown → WhatsApp/Telegram attachment:** +\`\`\` +[SHARE:whatsapp]{"path":"{{workspacePath}}/.openbridge/generated/.md"}[/SHARE] +\`\`\` + +Replace \`\` with the actual filename you wrote. +Use the SHARE target that matches the active messaging channel (whatsapp, telegram, github-pages, or email). + +## Rules + +- Write only to \`.openbridge/generated/\` — do NOT modify workspace source files. +- Always include a SHARE marker at the end of your response so the Master can deliver the file. +- Keep the generated content accurate and relevant to the user request. +- If you cannot determine the correct format, default to HTML (most universally useful). +- If generating HTML, include basic styling so the report is readable in a browser. +`, +}; + +/** + * Task: Targeted Read + * + * Reads specific files and answers a focused question with bullet-point findings. + * Used by the targeted reader path when RAG confidence is low (< 0.3). + */ +export const TASK_TARGETED_READ: SeedPrompt = { + id: 'task-targeted-read', + filename: 'task-targeted-read.md', + category: 'task', + version: '1.0.0', + description: + 'Reads specified files, extracts relevant information, and summarizes findings in 3-5 bullet points with file paths and line numbers', + content: `# Task: Targeted File Read + +Read the specified files and answer the following question. + +## Files to Read + +{{filePaths}} + +## Question + +{{question}} + +## Instructions + +1. Read each file listed above in full. +2. Identify the sections most relevant to the question. +3. Extract the key information that directly answers the question. +4. Summarize your findings in **3–5 bullet points**. + +## Output Format + +Respond with a short preamble (1 sentence) and then your bullet-point findings. + +Each bullet must include: +- The **file path** and **line number(s)** where the information was found +- A concise description of what was found + +Example: + +Here is what I found in the specified files: + +- \`src/core/router.ts:142\` — The \`/history\` command handler returns the last 20 messages from the conversation store. +- \`src/memory/conversation-store.ts:88\` — \`getRecentMessages()\` accepts a \`limit\` parameter (default 20) and returns messages ordered by timestamp descending. +- \`src/types/message.ts:34\` — \`OutboundMessage\` requires \`to\`, \`content\`, and \`messageId\` fields; \`content\` must be a non-empty string. + +## Rules + +- Read **only** the listed files — do not read additional files unless a listed file imports something critical. +- Do NOT modify any files — this is a read-only task. +- If a file does not exist, note it as missing and continue with the remaining files. +- If none of the files contain relevant information, say so explicitly. +- Keep your response concise — 3–5 bullets is the target. +`, +}; + +/** + * Deep Mode: Investigate Phase + * + * Explores the codebase to identify relevant files, patterns, dependencies, + * and potential issues related to the user's request. + * All examined files are listed. Findings are numbered and categorized by type. + */ +export const DEEP_INVESTIGATE: SeedPrompt = { + id: 'deep-investigate', + filename: 'deep-investigate.md', + category: 'task', + version: '1.0.0', + description: + 'Deep Mode investigation: explore codebase, identify relevant files, patterns, dependencies, and potential issues. List every file examined. Number and categorize all findings.', + content: `# Deep Mode — Investigate Phase + +Thoroughly explore the codebase at **{{workspacePath}}** and identify all issues, patterns, and dependencies relevant to the following request. + +## User Request + +{{userRequest}} + +## Workspace Context + +**Project:** {{projectName}} +**Type:** {{projectType}} +**Frameworks:** {{frameworks}} + +**Known files and structure:** +{{structure}} + +## Instructions + +### Step 1 — Identify Relevant Files + +Determine which files in the workspace are relevant to the request. Consider: + +- Source files that implement the feature, module, or area in question +- Test files covering that area +- Configuration files (tsconfig.json, eslint.config.js, package.json, etc.) +- Documentation files (README, CHANGELOG, architecture docs) +- Type definitions and shared interfaces +- Any file imported by or importing the affected code + +Use glob patterns and grep to locate files. Read each relevant file in full. + +### Step 2 — Examine Each File + +For every file you read: + +1. Note the **file path** and a one-line summary of its purpose. +2. Identify the **relevant sections** (functions, classes, types, configs) related to the request. +3. Note any **issues**: bugs, missing error handling, inconsistencies, deprecated patterns, missing tests. +4. Note any **dependencies**: what this file imports and what imports it. + +### Step 3 — Categorize Findings + +Group your findings into the following categories: + +| Category | Description | +| --- | --- | +| \`bug\` | Logic errors, incorrect behaviour, crashes | +| \`missing-test\` | Untested functionality or missing test cases | +| \`type-error\` | TypeScript type inconsistencies or unsafe casts | +| \`pattern\` | Architectural patterns observed (good or concerning) | +| \`dependency\` | Import graph, circular deps, missing deps | +| \`config\` | Configuration gaps, wrong defaults, missing env vars | +| \`documentation\` | Missing, stale, or incorrect docs/comments | +| \`performance\` | Inefficient algorithms, N+1 queries, unnecessary re-renders | +| \`security\` | Input validation gaps, exposed secrets, unsafe operations | +| \`other\` | Anything that doesn't fit the categories above | + +Assign each finding to exactly one category. + +### Step 4 — Produce Findings List + +Number every finding starting from 1. Each finding must include: + +- **Number:** unique integer for reference in subsequent phases +- **Category:** one of the categories above +- **Severity:** \`critical\` | \`high\` | \`medium\` | \`low\` | \`info\` +- **File:** path relative to workspace root, with line number(s) if applicable +- **Title:** short one-line summary (≤ 80 characters) +- **Description:** detailed explanation — what the issue is, why it matters, evidence from the code +- **Suggestion:** concrete recommended next step (read-only; do NOT implement in this phase) + +## Output Format + +Provide your response in two sections: + +### Files Examined + +List every file you read, one per line: + +\`\`\` +src/core/router.ts — message routing and command handling +src/types/agent.ts — agent and task type definitions +tests/core/router.test.ts — router unit tests +\`\`\` + +### Findings + +List every finding in the following format: + +--- + +**Finding #1** · \`bug\` · severity: \`high\` +**File:** \`src/core/router.ts:142\` +**Title:** /history handler returns undefined instead of OutboundMessage +**Description:** The \`/history\` branch at line 142 falls through without returning a value. This causes the caller to receive \`undefined\` and crash when trying to access \`.content\`. +**Suggestion:** Return a valid \`OutboundMessage\` object from the \`/history\` branch. + +--- + +**Finding #2** · \`missing-test\` · severity: \`medium\` +**File:** \`src/core/router.ts\` +**Title:** No test coverage for /stop-all command +**Description:** The \`/stop-all\` command is implemented at line 310 but has no corresponding test in \`tests/core/router.test.ts\`. +**Suggestion:** Add a test case that calls \`/stop-all\` and asserts workers are terminated. + +--- + +Continue this pattern for all findings, incrementing the number each time. + +## Rules + +- **Read only — do NOT modify any files.** +- List every file you examined, even if it contained no findings. +- If a file path does not exist, note it as missing and continue. +- Do not summarise or skip findings — completeness is more important than brevity. +- Reference specific line numbers wherever possible. +- End your response with the Findings section so the report phase can process it. +`, +}; + +/** + * Deep Mode: Report Phase + * + * Takes the raw findings list from the Investigate phase and organizes + * them into a structured, human-readable report suitable for planning. + * Sections: Executive Summary, Detailed Findings (numbered, with severity), + * Files Affected, Dependencies, Recommendations. + */ +export const DEEP_REPORT: SeedPrompt = { + id: 'deep-report', + filename: 'deep-report.md', + category: 'task', + version: '1.0.0', + description: + 'Deep Mode reporting: organize investigation findings into Executive Summary, Detailed Findings (numbered with severity), Files Affected, Dependencies, and Recommendations.', + content: `# Deep Mode — Report Phase + +Organize the investigation findings below into a structured report that can be used for planning. + +## Original Request + +{{userRequest}} + +## Investigation Findings + +{{investigationFindings}} + +## Instructions + +Read the investigation findings carefully, then produce a structured report in the following format. + +### Section 1 — Executive Summary + +Write 3–5 sentences that answer: + +1. **What area of the codebase is affected?** (which modules, files, features) +2. **What is the overall health?** (stable, concerning, critical) +3. **What are the top 2–3 most important issues?** (brief, high-level) +4. **What is the recommended next step?** (high-level action) + +### Section 2 — Detailed Findings + +For each finding from the investigation, produce a numbered entry. + +Use the original numbering from the investigation where possible. If the investigation contains duplicate or overlapping findings, merge them and note the merge. + +Each entry must include: + +| Field | Description | +| --- | --- | +| **Finding #N** | Unique number (carry over from investigation) | +| **Category** | One of: \`bug\`, \`missing-test\`, \`type-error\`, \`pattern\`, \`dependency\`, \`config\`, \`documentation\`, \`performance\`, \`security\`, \`other\` | +| **Severity** | \`critical\` | \`high\` | \`medium\` | \`low\` | \`info\` | +| **Title** | One-line summary (≤ 80 characters) | +| **Description** | 2–4 sentences: what the issue is, why it matters, evidence | +| **Files** | File path(s) with line numbers if applicable | +| **Recommendation** | Concrete action to resolve the finding | + +Sort findings within each severity level: critical → high → medium → low → info. + +### Section 3 — Files Affected + +List every file mentioned in the investigation findings. + +For each file, note: +- Whether it has findings against it (and how many) +- A one-line description of the file's purpose +- Whether it needs to be modified to resolve findings + +Format: + +\`\`\` +src/core/router.ts — message routing (3 findings — needs modification) +src/types/agent.ts — agent type definitions (1 finding — needs modification) +tests/core/router.test.ts — router unit tests (2 findings — needs modification) +src/core/bridge.ts — main orchestrator (0 findings — referenced only) +\`\`\` + +### Section 4 — Dependencies + +Describe the dependency relationships between findings. + +Identify: +1. **Blocking relationships** — findings that must be resolved before others can be addressed +2. **Parallel groups** — findings that can be resolved independently and simultaneously +3. **Cascade risks** — findings where fixing one may affect others + +Format as a short list: + +- Finding #1 blocks Finding #3 (same function — fixing #1 changes the function signature that #3 depends on) +- Findings #2, #4, #6 are independent and can be worked in parallel +- Fixing Finding #5 may require re-testing Findings #7 and #8 + +If there are no dependency relationships, state: "All findings are independent." + +### Section 5 — Recommendations + +Provide a prioritized list of recommended actions. Order from highest to lowest priority. + +Each recommendation must include: +- **Priority** (1 = highest) +- **Action** — what to do (concise imperative sentence) +- **Rationale** — why this should be done first +- **Findings addressed** — which finding numbers this action resolves +- **Estimated complexity** — \`trivial\` (< 30 min) | \`small\` (< 2 h) | \`medium\` (< 1 day) | \`large\` (> 1 day) + +Example: + +1. **Fix /history handler undefined return** · Findings #1 · complexity: trivial + Return a valid \`OutboundMessage\` from the \`/history\` branch. This is a crash-path fix that unblocks manual testing. + +2. **Add tests for /stop-all command** · Findings #2, #4 · complexity: small + Add 2 test cases to \`tests/core/router.test.ts\`. Unblocks CI for the router module. + +## Output Format + +Write your report in **Markdown** using the section headings above. Do NOT output JSON. + +Your report should be complete and self-contained — a developer reading it should be able to understand all issues without referring back to the raw investigation output. + +## Rules + +- **Read only — do NOT modify any files.** +- Preserve all finding numbers from the investigation — do not renumber unless merging duplicates. +- If the investigation findings are empty or minimal, note that in the Executive Summary and produce a minimal report. +- Severity levels must match those in the investigation — do not upgrade or downgrade without justification. +- Keep the Executive Summary concise — 3–5 sentences, no bullet points. +- The Recommendations section is the most important section for the planning phase — make it actionable. +`, +}; + +/** + * Deep Mode: Plan Phase + * + * Takes the structured report from the Report phase and produces an execution + * plan. For each finding/recommendation: task description, files to modify, + * estimated complexity, dependencies on other tasks, and risk level. + * Tasks are ordered by dependency and priority, and grouped into parallel + * batches so the Execute phase can run independent tasks concurrently. + */ +export const DEEP_PLAN: SeedPrompt = { + id: 'deep-plan', + filename: 'deep-plan.md', + category: 'task', + version: '1.0.0', + description: + 'Deep Mode planning: convert report recommendations into an ordered execution plan with task descriptions, files to modify, complexity, dependencies, risk level, and parallel batches.', + content: `# Deep Mode — Plan Phase + +Convert the report below into a concrete execution plan. Each recommendation becomes one or more tasks. Tasks are ordered by dependency and grouped into parallel batches. + +## Original Request + +{{userRequest}} + +## Report + +{{reportFindings}} + +## Instructions + +### Step 1 — Derive Tasks from Recommendations + +For each recommendation in the report, produce one task. If a recommendation is large, split it into smaller, atomic sub-tasks — each sub-task should be completable in a single worker turn. + +A task is **atomic** if: +- It modifies at most 3 files +- It has a clear, verifiable completion criterion +- It can be completed without waiting for another in-progress task + +### Step 2 — Describe Each Task + +For every task, capture the following fields: + +| Field | Description | +| --- | --- | +| **Task #N** | Unique sequential number, starting from 1 | +| **Title** | One-line summary (≤ 80 characters) | +| **Description** | 2–4 sentences: what to do, how to do it, what the end state looks like | +| **Files to Modify** | List of file paths (relative to workspace root) that this task will change | +| **Complexity** | \`trivial\` (< 30 min) | \`small\` (< 2 h) | \`medium\` (< 1 day) | \`large\` (> 1 day) | +| **Dependencies** | Task numbers that must complete before this task starts (empty = none) | +| **Risk** | \`low\` — safe, reversible change | \`medium\` — modifies shared code | \`high\` — deletes, renames, or restructures | +| **Finding Refs** | Finding numbers from the report that this task addresses | + +### Step 3 — Order by Dependency and Priority + +Sort tasks so that: +1. Tasks with no dependencies come first. +2. Among tasks at the same dependency level, order by severity (critical → high → medium → low → info) then by complexity (trivial → small → medium → large). +3. Tasks that depend on earlier tasks follow after their dependencies. + +### Step 4 — Group into Parallel Batches + +Group tasks into numbered batches. All tasks within a batch have no dependencies on each other and can run simultaneously. + +Format: + +**Batch 1 (parallel):** Tasks #1, #2, #3 +**Batch 2 (parallel):** Tasks #4, #5 +**Batch 3 (sequential):** Task #6 (depends on Batch 2) + +A batch is **sequential** (single task) if the task depends on one or more tasks in the previous batch. + +## Output Format + +Produce your plan in **Markdown** using the structure below. + +### Execution Plan — Summary + +- **Total tasks:** N +- **Parallel batches:** N +- **Estimated total complexity:** (sum across all tasks — e.g., "3 trivial, 2 small, 1 medium") +- **Highest risk task:** Task #N — short title + +--- + +### Tasks + +--- + +**Task #1** · complexity: \`trivial\` · risk: \`low\` +**Title:** Fix /history handler undefined return +**Files to Modify:** \`src/core/router.ts\` +**Dependencies:** none +**Finding Refs:** Finding #1 +**Description:** Add a missing return statement in the \`/history\` branch at line 142 of \`src/core/router.ts\`. The branch currently falls through without returning a value, causing crashes. Return a valid \`OutboundMessage\` object. + +--- + +**Task #2** · complexity: \`small\` · risk: \`low\` +**Title:** Add test coverage for /stop-all command +**Files to Modify:** \`tests/core/router.test.ts\` +**Dependencies:** none +**Finding Refs:** Finding #2, Finding #4 +**Description:** Add 2 test cases to \`tests/core/router.test.ts\` covering the \`/stop-all\` command. One test verifies workers are terminated; one verifies the response message format. No production code changes needed. + +--- + +Continue for all tasks, incrementing the number. + +--- + +### Parallel Batches + +**Batch 1 (parallel):** Tasks #1, #2 +**Batch 2 (parallel):** Tasks #3, #4 +**Batch 3 (sequential):** Task #5 (depends on Tasks #3 and #4) + +## Rules + +- **Read only — do NOT modify any files.** +- Every recommendation from the report must map to at least one task. Do not drop recommendations. +- Keep tasks atomic — if a recommendation requires more than 3 file changes, split it. +- Dependencies must be acyclic — no circular chains. +- Risk must reflect the real danger of the change: prefer \`low\` for new tests, \`medium\` for modifying shared utilities, \`high\` for deletes or large refactors. +- The Parallel Batches section is the most important output — the Execute phase uses it to schedule workers. +`, +}; + +/** + * Deep Mode: Execute Phase + * + * Executes a specific task from the plan. Makes minimum changes to satisfy + * the task description, runs tests after the change, and reports exactly + * what was modified together with pass/fail test results. + */ +export const DEEP_EXECUTE: SeedPrompt = { + id: 'deep-execute', + filename: 'deep-execute.md', + category: 'task', + version: '1.0.0', + description: + 'Deep Mode execution: implement a single plan task with minimum changes, run tests after, and report changes made and test results.', + content: `# Deep Mode — Execute Phase + +Execute the following task from the plan. Make the **minimum changes** required to satisfy the task description. Run tests after your changes and report the outcome. + +## Original Request + +{{userRequest}} + +## Task to Execute + +**Task #{{taskNumber}}** — {{taskTitle}} + +**Files to Modify:** +{{filesToModify}} + +**Description:** +{{taskDescription}} + +**Constraints:** +{{constraints}} + +## Execution Plan (for reference) + +{{planContext}} + +## Instructions + +### Step 1 — Read Before Modifying + +Before making any changes: + +1. Read every file listed in "Files to Modify" in full. +2. Read any files that are directly imported by or import from those files if understanding the interface is required. +3. Identify the exact lines that need to change. +4. Confirm the change is minimal — if the same outcome can be achieved by modifying fewer lines, prefer that. + +### Step 2 — Apply Changes + +Apply the changes described in the task. Rules: + +- **Minimum diff** — change only what the task requires. Do not reformat, refactor, or clean up unrelated code. +- **Do not modify files not listed** in "Files to Modify" unless a type or import in a listed file forces a change elsewhere. If an unlisted file must change, note it in your report. +- **Do not add** docstrings, comments, or type annotations to code you did not change. +- **Preserve existing style** — match surrounding indentation, quote style, and naming conventions. +- If the task description says to add a test, write it in the appropriate test file. +- If the task description says to add a prompt or constant, add it in the appropriate module. + +### Step 3 — Run Tests + +After applying all changes, run the project test suite: + +\`\`\` +npm test +\`\`\` + +If \`npm test\` is not available or fails with a "script not found" error, try: + +\`\`\` +npx vitest run +\`\`\` + +Capture the full output, including pass/fail counts and any error messages. + +If tests fail, determine whether: +- The failure is **caused by your changes** (you must fix it before finishing) +- The failure is **pre-existing** (note it but do not fix unrelated issues) + +Fix only failures introduced by your changes. Do NOT fix pre-existing failures. + +### Step 4 — Report + +Write a concise report covering: + +1. **Changes Made** — list every file changed, with a one-sentence description per change +2. **Lines Changed** — for each file, the approximate line range (e.g. "added lines 142–155") +3. **Test Results** — pass/fail counts and any new failures +4. **Pre-existing Failures** — note any test failures that existed before your changes (do not fix these) +5. **Notes** — any deviations from the plan, unlisted files touched, or important observations + +## Output Format + +Provide your response in the following structure: + +### Changes Made + +| File | Lines Changed | Description | +| --- | --- | --- | +| \`src/core/router.ts\` | 142–155 | Added return statement in /history branch | +| \`tests/core/router.test.ts\` | 310–330 | Added test case for /history undefined return | + +### Test Results + +\`\`\` +✓ 247 tests passed +✗ 0 tests failed +\`\`\` + +(Paste the relevant portion of test output here — pass/fail counts and any failure messages.) + +### Pre-existing Failures + +None. (Or list any test failures that existed before your changes.) + +### Notes + +(Any deviations, unlisted files touched, or observations. If none, write "None.") + +## Rules + +- Make the **minimum change** that satisfies the task — do not improve unrelated code. +- If a change would break other tests, find a different approach or report the conflict. +- Do NOT skip the test step — tests must run after every change. +- Do NOT mark the task as done if tests fail due to your changes. +- Stay within the listed files unless there is no alternative. +- Report honestly — if you could not complete the task, explain why. +`, +}; + +/** + * Deep Mode: Verify Phase + * + * Runs the full project verification suite (npm test, lint, typecheck, build) + * after the Execute phase completes. Reports pass/fail for each command, + * identifies the root cause of any failures, and attributes failures to the + * task that introduced them. + */ +export const DEEP_VERIFY: SeedPrompt = { + id: 'deep-verify', + filename: 'deep-verify.md', + category: 'verification', + version: '1.0.0', + description: + 'Deep Mode verification: run npm test, lint, typecheck, and build. Report pass/fail for each command. Identify the root cause of failures and which executed task introduced them.', + content: `# Deep Mode — Verify Phase + +Run the full verification suite on the workspace to confirm the executed tasks are correct and have not introduced regressions. + +## Original Request + +{{userRequest}} + +## Executed Tasks + +The following tasks were completed in the Execute phase. Use this list to attribute failures to specific tasks. + +{{executedTasks}} + +## Instructions + +### Step 1 — Run Verification Commands + +Run each command in order and capture the **full output** (stdout + stderr): + +1. \`npm test\` — run the full test suite +2. \`npm run lint\` — check for linting errors +3. \`npm run typecheck\` — check for TypeScript type errors +4. \`npm run build\` — compile and build the project + +**Timeout:** Allow up to 5 minutes per command. If a command exceeds the timeout, mark it as \`timeout\` and continue to the next. + +**Unavailable commands:** If a script is not listed in \`package.json\`, mark it as \`not-configured\` and continue. + +### Step 2 — Analyse Failures + +For every command that produced errors, failures, or a non-zero exit code: + +1. Read the error message and stack trace carefully. +2. Identify the **root cause** — the specific file, function, or configuration responsible. +3. Check the list of executed tasks to determine which task most likely **introduced** the failure: + - Cross-reference the failing file path against the "Files to Modify" listed for each task. + - If the failure is in a file changed by Task #N, attribute it to Task #N. + - If no executed task touched the failing file, mark it as \`pre-existing\`. +4. Determine whether the failure is **blocking** (must be fixed before the session is complete) or **informational** (does not prevent the build from working). + +### Step 3 — Report Results + +For each command, report: +- Pass or fail +- Exit code +- Summary counts (tests passed/failed, error count, etc.) +- Any failures with root cause and task attribution + +## Output Format + +Produce your verification report in **Markdown** using the following structure. + +--- + +## Verification Report + +### Command Results + +| Command | Status | Exit Code | Summary | +| --- | --- | --- | --- | +| \`npm test\` | ✅ Pass | 0 | 247 passed, 0 failed | +| \`npm run lint\` | ❌ Fail | 1 | 2 errors, 4 warnings | +| \`npm run typecheck\` | ✅ Pass | 0 | 0 errors | +| \`npm run build\` | ✅ Pass | 0 | Compiled successfully | + +--- + +### Failures + +For each failed command, list every error: + +--- + +**Failure #1** — \`npm run lint\` +**File:** \`src/core/router.ts:142\` +**Error:** \`@typescript-eslint/no-floating-promises — promise not awaited\` +**Root Cause:** The fix applied in Task #3 added a \`processMessage()\` call without awaiting it. +**Introduced By:** Task #3 — Fix /history handler undefined return +**Blocking:** Yes — CI will fail on this error. +**Fix Suggestion:** Add \`await\` before the \`processMessage()\` call at line 142. + +--- + +**Failure #2** — \`npm run lint\` +**File:** \`src/master/master-manager.ts:307\` +**Error:** \`@typescript-eslint/no-explicit-any — avoid using the \`any\` type\` +**Root Cause:** Pre-existing lint warning not related to any executed task. +**Introduced By:** Pre-existing (not caused by executed tasks) +**Blocking:** No — warning only (exit code 1 is from the first error). +**Fix Suggestion:** Type the parameter explicitly instead of using \`any\`. + +--- + +Continue for all failures. + +--- + +### Summary + +- **Total commands run:** N +- **Passed:** N +- **Failed:** N +- **Not configured:** N (list which) +- **Timeout:** N (list which) + +**Failures introduced by executed tasks:** +- Task #3 introduced 1 lint error (Failure #1) + +**Pre-existing failures:** +- 1 lint warning (Failure #2) — existed before this session + +**Overall verdict:** ✅ All executed changes are correct — pre-existing issues only +(or) +**Overall verdict:** ❌ Executed changes introduced N failures — must be fixed + +--- + +### Recommended Fixes + +List only failures introduced by executed tasks. For each: + +1. **Failure #N** — one-sentence fix description + - File: \`path/to/file.ts\` + - Line: N + - Fix: (concrete, actionable instruction) + +If there are no failures introduced by executed tasks, write: "No fixes required — all failures are pre-existing." + +## Rules + +- **Do NOT modify any files** — this is a read-and-report phase. Report failures; do not fix them. +- Run all four commands even if an earlier command fails — capture results for all. +- Pre-existing failures must be clearly labelled — do not attribute them to executed tasks. +- If you cannot determine which task introduced a failure, write "Unknown — further investigation needed". +- The "Overall verdict" line is the most important output — the Deep Mode Manager uses it to decide whether to proceed or roll back. +- If all commands pass, write a brief success summary and mark the verdict as ✅. +`, +}; + +/** + * Task: Build App + * + * Creates a self-contained web app in .openbridge/generated/apps/{name}/. + * Supports both static HTML/CSS/JS apps and simple Node.js server apps. + * Ends with an APP:start marker so the Master can launch the app. + */ +export const TASK_BUILD_APP: SeedPrompt = { + id: 'task-build-app', + filename: 'task-build-app.md', + category: 'task', + version: '1.0.0', + description: + 'Creates a self-contained web app (HTML/CSS/JS or Node.js) in .openbridge/generated/apps/{name}/ and appends an APP:start marker so the Master can launch it', + content: `# Task: Build Web App + +Create a self-contained web app based on the user request and write it to the output directory. + +## User Request + +{{userMessage}} + +## Workspace Context + +**Project:** {{projectName}} +**Type:** {{projectType}} +**Workspace path:** {{workspacePath}} + +## App Name + +{{appName}} + +## Output Directory + +Write all app files to: +\`{{workspacePath}}/.openbridge/generated/apps/{{appName}}/\` + +Create the directory if it does not exist. + +## App Type Selection + +Choose the simplest app type that satisfies the request: + +| Request type | App type | Files needed | +| --- | --- | --- | +| Data visualisation, dashboard, form, or static content | **Static HTML** | \`index.html\`, \`styles.css\`, \`app.js\` (optional) | +| Needs a backend (form handling, dynamic data, API calls) | **Node.js server** | \`server.js\`, \`package.json\`, optional \`public/\` | + +**Default to static HTML** unless the request explicitly requires a backend. + +## Instructions — Static HTML App + +Create the following files: + +### \`index.html\` + +- Self-contained HTML5 document +- Load any libraries (Chart.js, D3, Alpine.js, etc.) from CDN — no npm installs for static apps +- Link to \`styles.css\` and \`app.js\` using relative paths +- Must work when served from a local HTTP server (no file:// protocol assumptions) +- Include a \`\` matching the app name +- Use semantic HTML elements + +### \`styles.css\` + +- Clean, responsive layout using CSS Grid or Flexbox +- Mobile-friendly (viewport meta tag in index.html, responsive breakpoints) +- Use CSS custom properties (variables) for colors and spacing +- No external CSS frameworks required — write styles from scratch + +### \`app.js\` (optional — only if interactivity is needed) + +- Vanilla JavaScript — no build step, no bundler +- Use \`DOMContentLoaded\` to wait for the page to load +- Keep it simple and self-contained + +## Instructions — Node.js Server App + +Create the following files: + +### \`server.js\` + +- Use Express (or built-in \`http\` module for trivial cases) +- Listen on \`process.env.PORT\` with fallback to \`3000\` +- Serve static files from \`public/\` if needed +- Keep routes simple and focused on the request + +### \`package.json\` + +\`\`\`json +{ + "name": "{{appName}}", + "version": "1.0.0", + "scripts": { + "start": "node server.js" + }, + "dependencies": { + "express": "^4.18.0" + } +} +\`\`\` + +After writing the files, run \`npm install\` in the app directory to install dependencies. + +### \`public/\` (optional) + +- Static assets (HTML, CSS, JS) served by the Express server +- Follow the same HTML/CSS/JS guidelines as the static app type above + +## Rules + +- Write only to \`{{workspacePath}}/.openbridge/generated/apps/{{appName}}/\` — do NOT modify workspace source files. +- Keep the app focused — build exactly what the user requested, nothing more. +- All HTML must be valid and render without errors in a modern browser. +- For static apps, do NOT create a \`package.json\` or run \`npm install\`. +- For Node.js apps, always run \`npm install\` before finishing so the app starts immediately. +- Do NOT use TypeScript, React, Vue, or any compile step — keep the app runnable without a build. +- Ensure the app starts on \`PORT\` (Node.js) or serves \`index.html\` at the root (static). + +## After Creating the Files + +Confirm the files were written successfully, then end your response with: + +\`\`\` +APP:start {{workspacePath}}/.openbridge/generated/apps/{{appName}} +\`\`\` + +This marker tells the Master AI to start the app server and return the live URL to the user. +`, +}; + +/** + * Codex-specific worker system prompt prefix. + * + * Prepended to all Codex worker prompts to guide file access behavior. + * Codex workers waste turns on inline bash/Python gymnastics instead of using + * direct file-reading commands — this prefix steers them toward simple, direct + * file operations (OB-F91). + */ +export const CODEX_WORKER_PREFIX = + 'Use file reading commands to read files. Do NOT use complex bash/shell scripts for file operations. Use simple, direct commands.\n\n---\n\n'; + +/** + * Returns a tool-specific system prompt prefix to prepend to a worker prompt. + * Returns an empty string for tools that don't need a prefix (e.g., Claude). + * + * @param toolName - The name of the AI tool being used (e.g., "codex", "claude") + */ +export function applyToolPromptPrefix(prompt: string, toolName: string): string { + if (toolName.toLowerCase().includes('codex')) { + return CODEX_WORKER_PREFIX + prompt; + } + return prompt; +} + +/** + * All seed prompts in order + */ +export const SEED_PROMPTS: SeedPrompt[] = [ + EXPLORATION_STRUCTURE_SCAN, + EXPLORATION_CLASSIFICATION, + TASK_EXECUTE, + TASK_VERIFY, + TASK_CODE_AUDIT, + TASK_GENERATE_OUTPUT, + TASK_TARGETED_READ, + TASK_BUILD_APP, + DEEP_INVESTIGATE, + DEEP_REPORT, + DEEP_PLAN, + DEEP_EXECUTE, + DEEP_VERIFY, +]; + +/** + * Initialize the prompt library by seeding all templates. + * Creates .openbridge/prompts/ directory and writes all seed prompts. + */ +export async function seedPromptLibrary(dotFolderManager: { + writePromptTemplate: ( + filename: string, + content: string, + metadata: Omit<PromptTemplate, 'filePath' | 'createdAt' | 'updatedAt' | 'lastUsedAt'>, + ) => Promise<void>; +}): Promise<void> { + for (const prompt of SEED_PROMPTS) { + await dotFolderManager.writePromptTemplate(prompt.filename, prompt.content, { + id: prompt.id, + description: prompt.description, + category: prompt.category, + version: prompt.version, + usageCount: 0, + successCount: 0, + successRate: 0, + }); + } +} diff --git a/src/master/session-compactor.ts b/src/master/session-compactor.ts new file mode 100644 index 00000000..cd8ccda9 --- /dev/null +++ b/src/master/session-compactor.ts @@ -0,0 +1,829 @@ +/** + * Session Compactor (OB-1666, OB-1667, OB-1668) + * + * Monitors the Master AI session's turn count via `agent_activity` tracking + * and determines when compaction is needed to prevent context window overflow. + * + * Turn count is derived from two sources: + * - `sessions.message_count` — number of messages processed in this session + * - `agent_activity` (type='worker', parent_id=sessionId) — worker spawns + * + * Compaction is triggered when totalTurns >= maxTurns × threshold. + * + * When compaction fires, `compactTurns()` produces a structured + * {@link CompactionSummary} from the conversation history, preserving key + * identifiers (file paths, function names, task IDs, finding IDs) so the next + * session segment can resume without re-reading the full history. + */ + +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; +import type Database from 'better-sqlite3'; +import { createLogger } from '../core/logger.js'; + +const logger = createLogger('session-compactor'); + +// --------------------------------------------------------------------------- +// Public Types +// --------------------------------------------------------------------------- + +/** + * A single conversation turn supplied to `compactTurns()`. + */ +export interface ConversationTurn { + role: 'user' | 'assistant' | 'system'; + content: string; +} + +/** + * Identifiers extracted from a block of text by `extractIdentifiers()`. + * + * Used by `compactTurns()` internally and available as a standalone public + * method so callers can extract identifiers from any text without running a + * full compaction (e.g. for populating `compaction_history.identifiers_preserved`). + */ +export interface ExtractedIdentifiers { + /** Deduplicated file paths found in the text (e.g. `src/master/session-compactor.ts`). */ + filePaths: string[]; + /** Function/method names found in the text, with trailing `()`. */ + functionNames: string[]; + /** OpenBridge task IDs found in the text (e.g. `OB-1669`). */ + taskIds: string[]; + /** OpenBridge finding IDs found in the text (e.g. `OB-F84`). */ + findingIds: string[]; +} + +/** + * Structured compaction summary produced by `compactTurns()`. + * + * Preserves identifiers extracted from the original turns so the next session + * segment can resume work without replaying the full conversation history. + */ +export interface CompactionSummary { + /** Brief text overview of the activity covered in the summarized turns. */ + overview: string; + /** Deduplicated file paths referenced across all turns (e.g. `src/master/session-compactor.ts`). */ + filePaths: string[]; + /** Function/method names mentioned across all turns, with trailing `()`. */ + functionNames: string[]; + /** OpenBridge task IDs found in turns (e.g. `OB-1668`). */ + taskIds: string[]; + /** OpenBridge finding IDs found in turns (e.g. `OB-F84`). */ + findingIds: string[]; + /** Completed work items inferred from explicit completion markers in turn content. */ + completedWork: string[]; + /** Pending or in-progress work items inferred from explicit markers in turn content. */ + pendingWork: string[]; + /** Total number of turns summarized. */ + turnCount: number; + /** ISO timestamp when this summary was produced. */ + compactedAt: string; +} + +export interface CompactorConfig { + /** Max turns for the Master session (mirrors MASTER_MAX_TURNS). */ + maxTurns: number; + /** + * Fraction of maxTurns at which compaction is triggered. + * Default: 0.8 (compaction fires when 80% of maxTurns is consumed). + */ + threshold?: number; + /** + * Maximum number of retries on compaction failure before giving up. + * Default: 2. + */ + maxRetries?: number; + /** + * Maximum prompt character limit used for prompt-size-based compaction. + * Default: `800_000` for Opus 4.6 / Sonnet 4.6 (1M token context window, ~3.4M chars total). + * `32_768` for Haiku 4.5 and all other/unrecognized models (conservative fallback). + * Explicit values override the model-aware default. + */ + promptSizeLimit?: number; + /** + * Fraction of promptSizeLimit at which prompt-size compaction is triggered. + * Default: 0.8 (fires when assembled prompt exceeds 80% of the model-aware limit — + * e.g. 640K chars for Opus 4.6 / Sonnet 4.6, or 26K chars for Haiku 4.5 / others). + */ + promptSizeThreshold?: number; + /** + * Optional model ID used to derive model-aware prompt size defaults. + * When set to `claude-opus-4-6` or `claude-sonnet-4-6` (Opus 4.6 / Sonnet 4.6), + * the default `promptSizeLimit` is raised to `800_000` to match their 1M token + * context windows. All other models keep the conservative `32_768` default. + */ + modelId?: string; +} + +/** + * Async handler invoked when compaction is triggered. + * Receives a point-in-time snapshot describing why compaction fired. + * Implementations should summarize conversation turns and persist the + * summary (e.g. to memory.md) so the next session segment retains context. + */ +export type CompactionHandler = (snapshot: TurnSnapshot) => Promise<void>; + +/** + * Result returned by `triggerIfNeeded()`. + */ +export interface CompactionTriggerResult { + /** Whether compaction was actually triggered (false when below threshold). */ + triggered: boolean; + /** The turn snapshot captured at trigger time. */ + snapshot: TurnSnapshot; + /** + * Reason compaction was skipped (only present when `triggered` is false). + * E.g. "below threshold" or "already compacted this session". + */ + skippedReason?: string; +} + +/** + * A point-in-time snapshot of the session's turn consumption. + * Returned by `snapshotTurns()` — callers can inspect fields or call + * `needsCompaction` directly. + */ +export interface TurnSnapshot { + /** Session ID that was queried. */ + sessionId: string; + /** Messages processed in this session (from `sessions.message_count`). */ + messageCount: number; + /** Worker spawns recorded under this session (from `agent_activity`). */ + workerSpawnCount: number; + /** + * Total observed turn cost: messageCount + workerSpawnCount. + * This is the value compared against the threshold. + */ + totalTurns: number; + /** Whether totalTurns has reached or exceeded the compaction threshold. */ + needsCompaction: boolean; + /** + * Primary reason compaction is needed. + * - `'turn-count'` — turn-count threshold exceeded. + * - `'prompt-size'` — assembled prompt exceeded 80% of the size limit. + * - `undefined` — compaction not needed. + */ + compactionReason?: 'turn-count' | 'prompt-size'; + /** The configured maxTurns value used for this snapshot. */ + maxTurns: number; + /** The fraction threshold in use (e.g. 0.8). */ + threshold: number; + /** The absolute turn count at which compaction fires (Math.floor(maxTurns × threshold)). */ + thresholdTurns: number; + /** Most-recently reported assembled prompt size in chars (0 if not yet reported). */ + lastPromptChars: number; + /** Whether the last reported prompt size exceeded the configured size threshold. */ + promptSizeExceeded: boolean; + /** ISO timestamp when this snapshot was captured. */ + capturedAt: string; +} + +// --------------------------------------------------------------------------- +// SessionCompactor +// --------------------------------------------------------------------------- + +/** + * Monitors a Master session's turn consumption and determines when compaction + * should be triggered to prevent the context window from being exhausted. + * + * Usage: + * ```typescript + * const compactor = new SessionCompactor({ maxTurns: 50 }); + * if (compactor.shouldCompact(db, sessionId)) { + * // ... trigger compaction + * } + * ``` + */ +export class SessionCompactor { + private readonly maxTurns: number; + private readonly threshold: number; + readonly maxRetries: number; + private readonly promptSizeLimit: number; + private readonly promptSizeThreshold: number; + + /** + * Track the totalTurns value at which we last triggered compaction. + * Prevents re-triggering on every subsequent message once the cumulative + * count exceeds the threshold (since message_count never resets). + * Next compaction fires when totalTurns >= lastCompactedAtTurns + thresholdTurns. + */ + private lastCompactedAtTurns = 0; + + /** + * Set to true by `notifyPromptSize()` when the assembled prompt exceeds + * the configured prompt-size threshold. Reset after successful compaction. + */ + private _promptSizeExceeded = false; + + /** Most-recently reported assembled prompt size in chars. */ + private _lastPromptChars = 0; + + constructor(config: CompactorConfig) { + this.maxTurns = config.maxTurns; + this.threshold = config.threshold ?? 0.8; + this.maxRetries = config.maxRetries ?? 2; + // Model-aware default: Opus 4.6 / Sonnet 4.6 get 800_000 (matching their 1M token context + // window); all other models keep the conservative 32_768 fallback. + const modelId = config.modelId; + const isLargeContextModel = + modelId != null && + (/opus.*4[.-]6/i.test(modelId) || + modelId === 'claude-opus-4-6' || + /sonnet.*4[.-]6/i.test(modelId) || + modelId === 'claude-sonnet-4-6'); + const defaultPromptSizeLimit = isLargeContextModel ? 800_000 : 32_768; + this.promptSizeLimit = config.promptSizeLimit ?? defaultPromptSizeLimit; + this.promptSizeThreshold = config.promptSizeThreshold ?? 0.8; + } + + /** + * Notify the compactor of the most-recently assembled prompt size. + * + * Called by `PromptContextBuilder.buildMasterSpawnOptions()` after each + * prompt assembly. When `chars` exceeds `promptSizeLimit × promptSizeThreshold` + * (default: 80% of 32 768), the internal `_promptSizeExceeded` flag is raised + * and the next `triggerIfNeeded()` call will fire early compaction regardless + * of the current turn count. + * + * @param chars - Length of the assembled system prompt in characters. + */ + notifyPromptSize(chars: number): void { + this._lastPromptChars = chars; + const warnAt = Math.floor(this.promptSizeLimit * this.promptSizeThreshold); + const exceeded = chars >= warnAt; + if (exceeded && !this._promptSizeExceeded) { + logger.warn( + { + chars, + warnAt, + promptSizeLimit: this.promptSizeLimit, + promptSizeThreshold: this.promptSizeThreshold, + }, + 'SessionCompactor: prompt size exceeded %d%% threshold (%d/%d chars) — early compaction queued', + Math.round(this.promptSizeThreshold * 100), + chars, + this.promptSizeLimit, + ); + } + this._promptSizeExceeded = exceeded; + } + + /** + * Absolute turn count at which compaction fires. + * Computed as `Math.floor(maxTurns × threshold)`. + */ + get thresholdTurns(): number { + return Math.floor(this.maxTurns * this.threshold); + } + + /** + * Capture a turn-count snapshot for the given session. + * + * Queries: + * - `sessions` table for `message_count` (cumulative interactions) + * - `agent_activity` for worker spawns whose `parent_id` matches `sessionId` + * + * Both counts are combined to estimate total context consumption, since each + * worker spawn contributes additional context into the Master session window. + */ + snapshotTurns(db: Database.Database, sessionId: string): TurnSnapshot { + // --- message_count from sessions table --- + let messageCount = 0; + try { + const row = db + .prepare(`SELECT message_count FROM sessions WHERE id = ? LIMIT 1`) + .get(sessionId) as { message_count: number | null } | undefined; + messageCount = row?.message_count ?? 0; + } catch (err) { + logger.warn({ err, sessionId }, 'SessionCompactor: failed to read sessions.message_count'); + } + + // --- worker spawns from agent_activity --- + let workerSpawnCount = 0; + try { + const row = db + .prepare( + `SELECT COUNT(*) AS count + FROM agent_activity + WHERE parent_id = ? AND type = 'worker'`, + ) + .get(sessionId) as { count: number } | undefined; + workerSpawnCount = row?.count ?? 0; + } catch (err) { + logger.warn( + { err, sessionId }, + 'SessionCompactor: failed to read agent_activity worker count', + ); + } + + const totalTurns = messageCount + workerSpawnCount; + // Compare turns accumulated *since last compaction*, not the absolute total. + // This prevents compaction from re-triggering on every message once the + // cumulative count first exceeds the threshold. + const turnsSinceLastCompaction = totalTurns - this.lastCompactedAtTurns; + const turnCountExceeded = turnsSinceLastCompaction >= this.thresholdTurns; + const needsCompaction = turnCountExceeded || this._promptSizeExceeded; + const compactionReason: TurnSnapshot['compactionReason'] = needsCompaction + ? turnCountExceeded + ? 'turn-count' + : 'prompt-size' + : undefined; + + logger.debug( + { + sessionId, + messageCount, + workerSpawnCount, + totalTurns, + turnsSinceLastCompaction, + lastCompactedAtTurns: this.lastCompactedAtTurns, + thresholdTurns: this.thresholdTurns, + promptSizeExceeded: this._promptSizeExceeded, + lastPromptChars: this._lastPromptChars, + needsCompaction, + compactionReason, + }, + 'SessionCompactor: turn snapshot', + ); + + return { + sessionId, + messageCount, + workerSpawnCount, + totalTurns, + needsCompaction, + compactionReason, + maxTurns: this.maxTurns, + threshold: this.threshold, + thresholdTurns: this.thresholdTurns, + lastPromptChars: this._lastPromptChars, + promptSizeExceeded: this._promptSizeExceeded, + capturedAt: new Date().toISOString(), + }; + } + + /** + * Return `true` when the session's turn count has reached or exceeded the + * compaction threshold. + * + * Equivalent to `snapshotTurns(db, sessionId).needsCompaction` but more + * convenient when only a boolean is required. + */ + shouldCompact(db: Database.Database, sessionId: string): boolean { + return this.snapshotTurns(db, sessionId).needsCompaction; + } + + /** + * Check the session's turn count and trigger compaction if the configurable + * threshold has been reached. + * + * When compaction is needed the provided `handler` is invoked (if any). + * If no handler is supplied the method still returns `triggered: true` so + * the caller can take its own action (e.g. restart the session segment). + * + * Returns a {@link CompactionTriggerResult} describing whether compaction + * fired and why (or why not). + * + * @param db - Open SQLite database handle. + * @param sessionId - The current Master session ID to inspect. + * @param handler - Optional async callback invoked when threshold is exceeded. + */ + async triggerIfNeeded( + db: Database.Database, + sessionId: string, + handler?: CompactionHandler, + ): Promise<CompactionTriggerResult> { + const snapshot = this.snapshotTurns(db, sessionId); + + if (!snapshot.needsCompaction) { + logger.debug( + { + sessionId, + totalTurns: snapshot.totalTurns, + thresholdTurns: snapshot.thresholdTurns, + }, + 'SessionCompactor: below threshold — skipping compaction', + ); + return { triggered: false, snapshot, skippedReason: 'below threshold' }; + } + + logger.info( + { + sessionId, + totalTurns: snapshot.totalTurns, + thresholdTurns: snapshot.thresholdTurns, + maxTurns: snapshot.maxTurns, + threshold: snapshot.threshold, + compactionReason: snapshot.compactionReason, + lastPromptChars: snapshot.lastPromptChars, + promptSizeExceeded: snapshot.promptSizeExceeded, + }, + 'SessionCompactor: threshold exceeded — triggering compaction (reason: %s)', + snapshot.compactionReason ?? 'unknown', + ); + + if (handler) { + let lastError: unknown; + for (let attempt = 0; attempt <= this.maxRetries; attempt++) { + try { + await handler(snapshot); + lastError = undefined; + break; + } catch (err) { + lastError = err; + logger.warn( + { err, sessionId, attempt, maxRetries: this.maxRetries }, + 'SessionCompactor: compaction handler failed — will retry if attempts remain', + ); + } + } + if (lastError !== undefined) { + logger.warn( + { err: lastError, sessionId, maxRetries: this.maxRetries }, + 'SessionCompactor: compaction failed after all retries — continuing session without compaction', + ); + } else { + // Record the turn count at which compaction succeeded so we don't + // re-trigger on every subsequent message. + this.lastCompactedAtTurns = snapshot.totalTurns; + // Reset prompt-size flag — it was the trigger and has now been handled. + this._promptSizeExceeded = false; + logger.info( + { sessionId, lastCompactedAtTurns: this.lastCompactedAtTurns }, + 'SessionCompactor: recorded compaction point — next compaction after %d more turns', + this.thresholdTurns, + ); + } + } else { + // No handler but compaction was triggered — still record the point + this.lastCompactedAtTurns = snapshot.totalTurns; + this._promptSizeExceeded = false; + } + + return { triggered: true, snapshot }; + } + + // --------------------------------------------------------------------------- + // Identifier Extraction (OB-1669) + // --------------------------------------------------------------------------- + + /** + * Extract key identifiers from any block of text using regex patterns. + * + * Scans for: + * - File paths: `src/...`, `tests/...`, `docs/...`, `scripts/...`, `./...`, + * and common absolute paths (`/Users/...`, `/home/...`, etc.) + * - Function/method names: any camelCase/snake_case token followed by `()` + * - OpenBridge task IDs: `OB-NNNN` (four or more digits) + * - OpenBridge finding IDs: `OB-FNNNN` + * + * All results are deduplicated and sorted. This method is the canonical + * extraction implementation — `compactTurns()` delegates to it internally. + * + * @param text - Raw text to scan (may be concatenated conversation content). + */ + extractIdentifiers(text: string): ExtractedIdentifiers { + return { + filePaths: this._extractFilePaths(text), + functionNames: this._extractFunctionNames(text), + taskIds: this._extractTaskIds(text), + findingIds: this._extractFindingIds(text), + }; + } + + // --------------------------------------------------------------------------- + // Compaction Strategy (OB-1668) + // --------------------------------------------------------------------------- + + /** + * Summarize a sequence of conversation turns into a structured + * {@link CompactionSummary} that preserves key identifiers. + * + * The summary captures: + * - File paths (`src/...`, `tests/...`, `docs/...`, `./...`, absolute paths) + * - Function/method names (token followed by `()`) + * - OpenBridge task IDs (`OB-NNNN`) + * - OpenBridge finding IDs (`OB-FNNNN`) + * - Completed work items (from `✅`, `[x]`, `done:`, `completed:`, `fixed:`) + * - Pending work items (from `TODO:`, `NEXT:`, `pending:`, `[ ]`, `needs:`) + * + * @param turns - The conversation turns to compact. + */ + compactTurns(turns: ConversationTurn[]): CompactionSummary { + if (turns.length === 0) { + return { + overview: 'No turns to compact.', + filePaths: [], + functionNames: [], + taskIds: [], + findingIds: [], + completedWork: [], + pendingWork: [], + turnCount: 0, + compactedAt: new Date().toISOString(), + }; + } + + const allContent = turns.map((t) => t.content).join('\n'); + const ids = this.extractIdentifiers(allContent); + + return { + overview: this._buildOverview(turns), + filePaths: ids.filePaths, + functionNames: ids.functionNames, + taskIds: ids.taskIds, + findingIds: ids.findingIds, + completedWork: this._extractCompletedWork(allContent), + pendingWork: this._extractPendingWork(allContent), + turnCount: turns.length, + compactedAt: new Date().toISOString(), + }; + } + + // --------------------------------------------------------------------------- + // Private extraction helpers + // --------------------------------------------------------------------------- + + /** + * Build a one-sentence overview from the first user turn and the last + * assistant turn in the sequence. + */ + private _buildOverview(turns: ConversationTurn[]): string { + const firstUser = turns.find((t) => t.role === 'user')?.content ?? ''; + const lastAssistant = [...turns].reverse().find((t) => t.role === 'assistant')?.content ?? ''; + + const userPreview = firstUser.slice(0, 100).replace(/\n/g, ' ').trim(); + const assistantPreview = lastAssistant.slice(0, 100).replace(/\n/g, ' ').trim(); + + if (userPreview && assistantPreview) { + return `Compacted ${turns.length} turns. First request: "${userPreview}". Last response: "${assistantPreview}".`; + } + if (userPreview) { + return `Compacted ${turns.length} turns. First request: "${userPreview}".`; + } + return `Compacted ${turns.length} turns.`; + } + + /** + * Extract file paths from text. + * Matches `src/`, `tests/`, `docs/` relative paths and absolute paths. + */ + private _extractFilePaths(text: string): string[] { + const found = new Set<string>(); + const patterns: RegExp[] = [ + /\bsrc\/[\w/\-.]+\.\w+\b/g, + /\btests\/[\w/\-.]+\.\w+\b/g, + /\bdocs\/[\w/\-.]+\.\w+\b/g, + /\bscripts\/[\w/\-.]+\.\w+\b/g, + /\.\/[\w/\-.]+\.\w+/g, + /\/(?:Users|home|var|tmp|opt|etc)\/[\w/\-.]+\.\w+/g, + ]; + for (const pattern of patterns) { + for (const match of text.matchAll(pattern)) { + found.add(match[0]); + } + } + return Array.from(found).sort(); + } + + /** + * Extract function/method names (identifier immediately followed by `()`). + * Excludes very short or all-uppercase tokens (likely control-flow keywords). + */ + private _extractFunctionNames(text: string): string[] { + const found = new Set<string>(); + for (const match of text.matchAll(/\b([a-z_][a-zA-Z0-9_]{2,})\(\)/g)) { + found.add(`${match[1]}()`); + } + return Array.from(found).sort(); + } + + /** + * Extract OpenBridge task IDs of the form `OB-NNNN` (digits only, not finding IDs). + */ + private _extractTaskIds(text: string): string[] { + const found = new Set<string>(); + for (const match of text.matchAll(/\bOB-(\d{4,})\b/g)) { + found.add(`OB-${match[1]}`); + } + return Array.from(found).sort(); + } + + /** + * Extract OpenBridge finding IDs of the form `OB-FNNNN`. + */ + private _extractFindingIds(text: string): string[] { + const found = new Set<string>(); + for (const match of text.matchAll(/\bOB-F(\d+)\b/g)) { + found.add(`OB-F${match[1]}`); + } + return Array.from(found).sort(); + } + + /** + * Extract completed work items from explicit completion markers in text. + * Markers: `✅ ...`, `[x] ...`, `done: ...`, `completed: ...`, `fixed: ...` + */ + private _extractCompletedWork(text: string): string[] { + const found: string[] = []; + const markers: RegExp[] = [ + /✅\s+(.+)/g, + /\[x\]\s+(.+)/gi, + /\bdone:\s*(.+)/gi, + /\bcompleted:\s*(.+)/gi, + /\bfixed:\s*(.+)/gi, + ]; + for (const pattern of markers) { + for (const match of text.matchAll(pattern)) { + const item = match[1]?.trim().slice(0, 120) ?? ''; + if (item) found.push(item); + } + } + return [...new Set(found)]; + } + + /** + * Extract pending/in-progress work items from explicit markers in text. + * Markers: `TODO: ...`, `NEXT: ...`, `pending: ...`, `[ ] ...`, `needs: ...` + */ + private _extractPendingWork(text: string): string[] { + const found: string[] = []; + const markers: RegExp[] = [ + /\bTODO:\s*(.+)/gi, + /\bNEXT:\s*(.+)/gi, + /\bpending:\s*(.+)/gi, + /\[ \]\s+(.+)/g, + /\bneeds:\s*(.+)/gi, + ]; + for (const pattern of markers) { + for (const match of text.matchAll(pattern)) { + const item = match[1]?.trim().slice(0, 120) ?? ''; + if (item) found.push(item); + } + } + return [...new Set(found)]; + } + + // --------------------------------------------------------------------------- + // Memory Write (OB-1670) + // --------------------------------------------------------------------------- + + /** + * Format a {@link CompactionSummary} as a Markdown section suitable for + * inclusion in `memory.md`. + * + * The resulting block is a `## Session Compaction` entry that captures: + * - timestamp + turn count + * - brief overview + * - files referenced, task IDs, finding IDs + * - completed and pending work items + * + * @param summary - The compaction summary to format. + */ + formatSummaryAsMarkdown(summary: CompactionSummary): string { + const lines: string[] = [ + `## Session Compaction — ${summary.compactedAt}`, + '', + `**Overview:** ${summary.overview}`, + `**Turns summarized:** ${summary.turnCount}`, + ]; + + if (summary.filePaths.length > 0) { + lines.push('', `**Files referenced:** ${summary.filePaths.join(', ')}`); + } + + if (summary.taskIds.length > 0) { + lines.push(`**Tasks:** ${summary.taskIds.join(', ')}`); + } + + if (summary.findingIds.length > 0) { + lines.push(`**Findings:** ${summary.findingIds.join(', ')}`); + } + + if (summary.completedWork.length > 0) { + lines.push('', '**Completed:**'); + for (const item of summary.completedWork.slice(0, 10)) { + lines.push(`- ${item}`); + } + } + + if (summary.pendingWork.length > 0) { + lines.push('', '**Pending:**'); + for (const item of summary.pendingWork.slice(0, 10)) { + lines.push(`- ${item}`); + } + } + + lines.push(''); + return lines.join('\n'); + } + + /** + * Write a {@link CompactionSummary} to `memory.md` at the given file path, + * ensuring cross-session continuity before starting a new session segment. + * + * Behaviour: + * - Reads existing `memory.md` if present (creates it if missing). + * - Finds or creates a `<!-- compaction-history -->` block at the top of the + * file, immediately after the first `# Memory` heading (or at the very top + * if no heading exists). + * - Prepends the new compaction entry inside that block so the most-recent + * entry appears first. + * - Trims the oldest compaction entries when the total file would exceed 200 + * lines, keeping the rest of the file intact. + * - Writes the result back to disk. + * + * @param summary - The compaction summary to persist. + * @param memoryFilePath - Absolute path to `memory.md`. + */ + async writeCompactionSummaryToMemory( + summary: CompactionSummary, + memoryFilePath: string, + ): Promise<void> { + const MAX_LINES = 200; + const BLOCK_START = '<!-- compaction-history -->'; + const BLOCK_END = '<!-- /compaction-history -->'; + + // Read existing content (tolerate missing file). + let existing = ''; + try { + existing = await fs.readFile(memoryFilePath, 'utf-8'); + } catch { + // File does not exist yet — start with an empty string. + } + + const newEntry = this.formatSummaryAsMarkdown(summary); + + let updated: string; + + if (existing.includes(BLOCK_START) && existing.includes(BLOCK_END)) { + // Inject new entry immediately after the opening marker. + updated = existing.replace(BLOCK_START, `${BLOCK_START}\n${newEntry}`); + } else { + // Insert a new block after the first `# ` heading, or at the top. + const headingMatch = /^# .+$/m.exec(existing); + if (headingMatch) { + const insertPos = headingMatch.index + headingMatch[0].length; + updated = + `${existing.slice(0, insertPos)}\n\n${BLOCK_START}\n${newEntry}${BLOCK_END}\n` + + existing.slice(insertPos); + } else { + updated = `${BLOCK_START}\n${newEntry}${BLOCK_END}\n\n${existing}`; + } + } + + // Enforce 200-line limit by trimming oldest compaction entries. + let lines = updated.split('\n'); + while (lines.length > MAX_LINES) { + // Find the last compaction entry boundary inside the block and remove it. + const blockStartIdx = lines.findIndex((l) => l.trim() === BLOCK_START); + const blockEndIdx = lines.findIndex((l) => l.trim() === BLOCK_END); + + if (blockStartIdx === -1 || blockEndIdx === -1 || blockEndIdx <= blockStartIdx) { + // No managed block found — just hard-truncate. + lines = lines.slice(0, MAX_LINES); + break; + } + + // Find the second `## Session Compaction` header inside the block — + // that marks the start of the oldest entry we can trim. + const blockLines = lines.slice(blockStartIdx + 1, blockEndIdx); + const compactionHeaders = blockLines.reduce<number[]>((acc, l, i) => { + if (/^## Session Compaction/.test(l)) acc.push(i); + return acc; + }, []); + + if (compactionHeaders.length <= 1) { + // Only one entry in the block — hard-truncate instead of removing it. + lines = lines.slice(0, MAX_LINES); + break; + } + + // Remove the last (oldest) compaction entry from inside the block. + // compactionHeaders[] indices are relative to blockLines (blockStartIdx+1). + // The LAST entry in the array is the oldest (at the bottom of the block). + const oldestRelIdx = compactionHeaders[compactionHeaders.length - 1] ?? 0; + const oldestStart = blockStartIdx + 1 + oldestRelIdx; + // The oldest entry extends from oldestStart to blockEndIdx (end of block). + const removeCount = blockEndIdx - oldestStart; + + if (removeCount <= 0) { + // Safety: cannot trim further — hard-truncate to prevent infinite loop. + lines = lines.slice(0, MAX_LINES); + break; + } + + lines.splice(oldestStart, removeCount); + } + + const finalContent = lines.join('\n'); + + await fs.mkdir(path.dirname(memoryFilePath), { recursive: true }); + await fs.writeFile(memoryFilePath, finalContent, 'utf-8'); + + logger.info( + { memoryFilePath, turnCount: summary.turnCount, lines: lines.length }, + 'SessionCompactor: wrote compaction summary to memory.md', + ); + } +} diff --git a/src/master/skill-manager.ts b/src/master/skill-manager.ts new file mode 100644 index 00000000..2bc5a817 --- /dev/null +++ b/src/master/skill-manager.ts @@ -0,0 +1,756 @@ +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; + +import { SkillSchema } from '../types/agent.js'; +import type { Skill } from '../types/agent.js'; +import { createLogger } from '../core/logger.js'; + +const logger = createLogger('skill-manager'); + +/** Directory name inside `.openbridge/` where user-defined skills are stored */ +const SKILLS_DIR = 'skills'; + +/** Supported user-defined skill file extensions */ +const SUPPORTED_EXTENSIONS = ['.json', '.js', '.cjs', '.mjs', '.md']; + +/** Number of successful completions of the same task pattern required before auto-creating a skill */ +export const AUTO_CREATE_THRESHOLD = 3; + +/** Internal metadata file names (dot-prefixed, skipped during skill loading) */ +const STATS_FILE = '.skill-stats.json'; +const PATTERNS_FILE = '.task-patterns.json'; + +// ── Interfaces ──────────────────────────────────────────────────────────── + +/** + * Per-skill usage statistics — tracked across invocations and persisted to + * `.openbridge/skills/.skill-stats.json`. + */ +export interface SkillStats { + name: string; + usageCount: number; + successCount: number; + failureCount: number; + /** Ratio of successful invocations (0–1). */ + successRate: number; + lastUsedAt?: string; +} + +/** Internal: tracks how often a normalised task-prompt pattern has succeeded. */ +interface TaskPattern { + /** Normalised prompt pattern (first 5 significant words, lowercase). */ + pattern: string; + /** Tool profile the task was run with (e.g. 'read-only', 'code-edit'). */ + toolProfile: string; + successCount: number; + failureCount: number; + /** Up to 3 raw prompts that contributed to this pattern (used as examplePrompts). */ + examplePrompts: string[]; + /** Skill name written to disk once the threshold is reached. */ + autoCreatedSkillName?: string; + lastSucceededAt?: string; +} + +// ── Helpers ─────────────────────────────────────────────────────────────── + +/** + * Parse a SKILL.md file into a `Skill` object. + * + * Expected format: + * ```markdown + * # skill-name + * + * One-line description of what this skill does. + * + * ## Tool Profile + * read-only + * + * ## Tools Needed + * - Read + * - Glob + * + * ## Example Prompts + * - "Review my code changes" + * - "Analyze this PR" + * + * ## Constraints + * - Do not modify any files + * + * ## Max Turns + * 10 + * + * ## System Prompt + * You are a specialized code reviewer... + * ``` + * + * Rules: + * - H1 (`#`) → `name` + * - First non-empty paragraph after H1 (before any H2) → `description` + * - `## Tool Profile` section → `toolProfile` (first non-empty line) + * - `## Tools Needed` section → bullet list → `toolsNeeded` + * - `## Example Prompts` section → bullet list → `examplePrompts` (quotes stripped) + * - `## Constraints` section → bullet list → `constraints` + * - `## Max Turns` section → integer → `maxTurns` + * - `## System Prompt` section → multi-line content → `systemPrompt` + */ +export function parseSkillMarkdown(content: string): Partial<Record<string, unknown>> { + const lines = content.split('\n'); + + let name = ''; + let description = ''; + let toolProfile = ''; + const toolsNeeded: string[] = []; + const examplePrompts: string[] = []; + const constraints: string[] = []; + let maxTurns: number | undefined; + let systemPrompt = ''; + + type Section = + | 'preamble' + | 'tool-profile' + | 'tools-needed' + | 'example-prompts' + | 'constraints' + | 'max-turns' + | 'system-prompt'; + + let section: Section = 'preamble'; + const descriptionLines: string[] = []; + const systemPromptLines: string[] = []; + + for (const raw of lines) { + const line = raw.trimEnd(); + + // H1 = skill name + if (/^#\s+/.test(line) && !/^##/.test(line)) { + name = line.replace(/^#\s+/, '').trim(); + section = 'preamble'; + continue; + } + + // H2 = section switch + if (/^##\s+/.test(line)) { + const heading = line + .replace(/^##\s+/, '') + .trim() + .toLowerCase(); + if (heading === 'tool profile') section = 'tool-profile'; + else if (heading === 'tools needed') section = 'tools-needed'; + else if (heading === 'example prompts') section = 'example-prompts'; + else if (heading === 'constraints') section = 'constraints'; + else if (heading === 'max turns') section = 'max-turns'; + else if (heading === 'system prompt') section = 'system-prompt'; + else section = 'preamble'; // unknown section — ignore content + continue; + } + + switch (section) { + case 'preamble': + // Collect description lines (first content block after H1) + if (line.trim() || descriptionLines.length > 0) { + descriptionLines.push(line); + } + break; + + case 'tool-profile': + if (!toolProfile && line.trim()) { + toolProfile = line.trim(); + } + break; + + case 'tools-needed': { + const item = parseBulletItem(line); + if (item) toolsNeeded.push(item); + break; + } + + case 'example-prompts': { + const item = parseBulletItem(line); + if (item) examplePrompts.push(item.replace(/^["']|["']$/g, '')); + break; + } + + case 'constraints': { + const item = parseBulletItem(line); + if (item) constraints.push(item); + break; + } + + case 'max-turns': { + if (maxTurns === undefined && line.trim()) { + const n = parseInt(line.trim(), 10); + if (!isNaN(n) && n > 0) maxTurns = n; + } + break; + } + + case 'system-prompt': + systemPromptLines.push(line); + break; + } + } + + // Trim description to first non-empty paragraph + description = (descriptionLines.join('\n').trim().split(/\n\n+/)[0] ?? '').trim(); + + systemPrompt = systemPromptLines.join('\n').trim(); + + return { + name, + description, + toolProfile, + toolsNeeded, + examplePrompts, + constraints, + ...(maxTurns !== undefined ? { maxTurns } : {}), + ...(systemPrompt ? { systemPrompt } : {}), + }; +} + +/** Extract the text content of a markdown bullet list item (`- foo` or `* foo`). */ +function parseBulletItem(line: string): string | null { + const match = /^[*-]\s+(.+)$/.exec(line.trim()); + return match?.[1]?.trim() ?? null; +} + +/** + * Serialise a `Skill` object to SKILL.md format. + * + * The output is parseable by `parseSkillMarkdown()` — round-trips cleanly. + */ +export function skillToMarkdown(skill: Skill): string { + const lines: string[] = []; + + lines.push(`# ${skill.name}`); + lines.push(''); + lines.push(skill.description); + lines.push(''); + + lines.push('## Tool Profile'); + lines.push(skill.toolProfile); + lines.push(''); + + if (skill.toolsNeeded.length > 0) { + lines.push('## Tools Needed'); + for (const tool of skill.toolsNeeded) { + lines.push(`- ${tool}`); + } + lines.push(''); + } + + if (skill.examplePrompts.length > 0) { + lines.push('## Example Prompts'); + for (const prompt of skill.examplePrompts) { + lines.push(`- "${prompt}"`); + } + lines.push(''); + } + + if (skill.constraints.length > 0) { + lines.push('## Constraints'); + for (const constraint of skill.constraints) { + lines.push(`- ${constraint}`); + } + lines.push(''); + } + + if (skill.maxTurns !== undefined) { + lines.push('## Max Turns'); + lines.push(String(skill.maxTurns)); + lines.push(''); + } + + if (skill.systemPrompt) { + lines.push('## System Prompt'); + lines.push(skill.systemPrompt); + lines.push(''); + } + + return lines.join('\n'); +} + +/** + * Normalise a raw task prompt into a short, reusable pattern key. + * + * Strategy: + * 1. Lowercase everything. + * 2. Strip file paths (`src/...`, `./...`, absolute paths). + * 3. Strip file-extension tokens (e.g. `router.ts`, `config.json`). + * 4. Strip bare numbers. + * 5. Strip punctuation. + * 6. Take the first 5 significant words (length > 2). + * + * Examples: + * "Review src/core/router.ts for security issues" → "review for security issues" + * "Run the tests in tests/core/" → "run tests" + * "Analyse performance of the database queries" → "analyse performance database queries" + */ +export function normalizePattern(prompt: string): string { + return prompt + .toLowerCase() + .replace(/(?:src\/|\.\/|\/[\w/.-]+)\S*/g, '') // strip file paths + .replace(/\b\w+\.\w{1,5}\b/g, '') // strip file.ext tokens + .replace(/\b\d+\b/g, '') // strip bare numbers + .replace(/[^a-z\s]/g, ' ') // strip punctuation + .replace(/\s+/g, ' ') + .trim() + .split(/\s+/) + .filter((w) => w.length > 2) // drop short/stop words + .slice(0, 5) + .join(' '); +} + +// ── SkillManager ────────────────────────────────────────────────────────── + +/** + * Manages skill definitions for OpenBridge. + * + * Skills are reusable task templates that the Master AI can use to spawn + * workers for common development tasks (code review, testing, dependency + * audit, API docs generation, etc.). + * + * Skill definitions are loaded from two sources: + * 1. **Built-in skills** — registered programmatically via `registerBuiltIn()` + * 2. **User-defined skills** — discovered from `<workspacePath>/.openbridge/skills/` + * as `.md`, `.json`, `.js`, `.cjs`, or `.mjs` files + * + * User-defined skills with the same `name` as a built-in skill override the + * built-in, allowing workspace-specific customisation. + * + * ## Success tracking + * + * Call `recordSkillUsage(name, success)` after every skill invocation. + * Stats are persisted to `.openbridge/skills/.skill-stats.json` and can be + * retrieved via `getSkillStats()`. + * + * ## Auto-creation from task patterns + * + * Call `maybeAutoCreateSkill(taskPrompt, toolProfile, success)` after every + * worker completion (even when no explicit skill was selected). When the same + * normalised prompt pattern succeeds `AUTO_CREATE_THRESHOLD` times the manager + * writes a new `<name>.md` file to `.openbridge/skills/` and registers the + * skill in memory. Pattern state is persisted to + * `.openbridge/skills/.task-patterns.json`. + */ +export class SkillManager { + private readonly skillsPath: string; + private readonly statsPath: string; + private readonly patternsPath: string; + + private readonly skills: Map<string, Skill> = new Map(); + private readonly statsMap: Map<string, SkillStats> = new Map(); + private readonly patternMap: Map<string, TaskPattern> = new Map(); + + private loaded = false; + private statsLoaded = false; + private patternsLoaded = false; + + constructor(workspacePath: string) { + this.skillsPath = path.join(workspacePath, '.openbridge', SKILLS_DIR); + this.statsPath = path.join(this.skillsPath, STATS_FILE); + this.patternsPath = path.join(this.skillsPath, PATTERNS_FILE); + } + + /** + * Register a built-in skill definition programmatically. + * Must be called before `load()` to ensure correct override precedence. + */ + registerBuiltIn(skill: Skill): void { + this.skills.set(skill.name, { ...skill, isUserDefined: false }); + } + + /** + * Discover and load user-defined skill definitions from `.openbridge/skills/`. + * + * Supported formats: + * - `.md` — SKILL.md document parsed by `parseSkillMarkdown()` + * - `.json` — plain JSON object validated against `SkillSchema` + * - `.js` / `.cjs` / `.mjs` — ES/CJS module with a default export or any + * named export that validates against `SkillSchema` + * + * Dot-prefixed files (`.skill-stats.json`, `.task-patterns.json`, etc.) are + * skipped automatically. + * + * Safe to call multiple times — subsequent calls are no-ops unless + * `forceReload` is `true`. + * + * @param forceReload - When `true`, re-reads the skills directory even if + * already loaded (useful after the user drops a new skill file). + * @returns Number of user-defined skills successfully loaded. + */ + async load(forceReload = false): Promise<number> { + if (this.loaded && !forceReload) return this.countUserDefined(); + + let entries: string[]; + try { + entries = await fs.readdir(this.skillsPath); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') { + // Directory doesn't exist — normal; only built-ins are used + this.loaded = true; + return 0; + } + logger.warn({ skillsPath: this.skillsPath, err }, 'Error reading skills directory'); + this.loaded = true; + return 0; + } + + // Skip dot-prefixed metadata files (.skill-stats.json, .task-patterns.json, etc.) + const skillFiles = entries.filter( + (e) => !e.startsWith('.') && SUPPORTED_EXTENSIONS.some((ext) => e.endsWith(ext)), + ); + + let loadedCount = 0; + + for (const file of skillFiles) { + const filePath = path.join(this.skillsPath, file); + const skill = await this.loadSkillFile(filePath, file); + if (skill) { + this.skills.set(skill.name, { ...skill, isUserDefined: true }); + loadedCount++; + logger.info({ skill: skill.name, file }, 'Loaded user-defined skill'); + } + } + + this.loaded = true; + return loadedCount; + } + + /** + * Returns all registered skills (built-in + user-defined). + * Callers should `await load()` first to include user-defined skills. + */ + getAll(): Skill[] { + return Array.from(this.skills.values()); + } + + /** + * Find a skill by exact name (case-sensitive). + * Returns `undefined` if no skill with that name is registered. + */ + getByName(name: string): Skill | undefined { + return this.skills.get(name); + } + + /** + * Find all skills whose `toolProfile` matches the given profile name. + * Useful when the Master AI wants to pick a skill for a specific profile. + */ + getByProfile(toolProfile: string): Skill[] { + return this.getAll().filter((s) => s.toolProfile === toolProfile); + } + + /** + * Returns all user-defined skills (loaded from `.openbridge/skills/`). + */ + getUserDefined(): Skill[] { + return this.getAll().filter((s) => s.isUserDefined); + } + + /** + * Returns all built-in skills (registered via `registerBuiltIn()`). + */ + getBuiltIn(): Skill[] { + return this.getAll().filter((s) => !s.isUserDefined); + } + + /** + * Total number of registered skills (built-in + user-defined). + */ + get size(): number { + return this.skills.size; + } + + // ── Success tracking ────────────────────────────────────────────────── + + /** + * Record a skill invocation outcome. + * + * Updates the in-memory stats map and persists to + * `.openbridge/skills/.skill-stats.json`. + * + * @param skillName - The `name` field of the invoked skill. + * @param success - `true` if the worker completed without error. + */ + async recordSkillUsage(skillName: string, success: boolean): Promise<void> { + await this.ensureStatsLoaded(); + + const now = new Date().toISOString(); + const existing = this.statsMap.get(skillName); + + if (existing) { + existing.usageCount++; + if (success) existing.successCount++; + else existing.failureCount++; + existing.successRate = existing.successCount / existing.usageCount; + existing.lastUsedAt = now; + } else { + this.statsMap.set(skillName, { + name: skillName, + usageCount: 1, + successCount: success ? 1 : 0, + failureCount: success ? 0 : 1, + successRate: success ? 1 : 0, + lastUsedAt: now, + }); + } + + await this.saveStats(); + } + + /** + * Returns usage statistics for all skills that have been invoked at least + * once, sorted by `usageCount` descending. + * + * Note: loads persisted stats on first call. + */ + async getSkillStats(): Promise<SkillStats[]> { + await this.ensureStatsLoaded(); + return Array.from(this.statsMap.values()).sort((a, b) => b.usageCount - a.usageCount); + } + + // ── Auto-creation from task patterns ────────────────────────────────── + + /** + * Observe a worker task completion and, if the same normalised prompt + * pattern has succeeded `AUTO_CREATE_THRESHOLD` times, auto-create a new + * skill definition and write it to `.openbridge/skills/<name>.md`. + * + * The newly created skill is also registered in memory so it is immediately + * available via `getAll()` / `getByName()`. + * + * @param taskPrompt - The raw prompt sent to the worker. + * @param toolProfile - The tool profile the worker ran with. + * @param success - Whether the worker completed successfully. + * @returns The newly created `Skill`, or `null` if no skill was created. + */ + async maybeAutoCreateSkill( + taskPrompt: string, + toolProfile: string, + success: boolean, + ): Promise<Skill | null> { + await this.ensurePatternsLoaded(); + + const pattern = normalizePattern(taskPrompt); + if (!pattern) return null; + + const key = `${pattern}::${toolProfile}`; + const now = new Date().toISOString(); + + const existing = this.patternMap.get(key); + + if (existing) { + if (success) { + existing.successCount++; + existing.lastSucceededAt = now; + if (taskPrompt && !existing.examplePrompts.includes(taskPrompt)) { + existing.examplePrompts = [...existing.examplePrompts, taskPrompt].slice(-3); + } + } else { + existing.failureCount++; + } + + // Auto-create once the threshold is hit and a skill hasn't been created yet + if (existing.successCount >= AUTO_CREATE_THRESHOLD && !existing.autoCreatedSkillName) { + const skill = this.buildSkillFromPattern(existing); + if (skill) { + await this.writeSkillFile(skill); + this.skills.set(skill.name, skill); + existing.autoCreatedSkillName = skill.name; + logger.info( + { skill: skill.name, pattern, successCount: existing.successCount }, + 'Auto-created skill from task pattern', + ); + await this.savePatterns(); + return skill; + } + } + } else { + this.patternMap.set(key, { + pattern, + toolProfile, + successCount: success ? 1 : 0, + failureCount: success ? 0 : 1, + examplePrompts: taskPrompt ? [taskPrompt] : [], + lastSucceededAt: success ? now : undefined, + }); + } + + await this.savePatterns(); + return null; + } + + // ── Private helpers ────────────────────────────────────────────── + + private countUserDefined(): number { + return this.getUserDefined().length; + } + + private async loadSkillFile(filePath: string, fileName: string): Promise<Skill | null> { + try { + if (fileName.endsWith('.json')) { + return await this.loadJsonSkill(filePath); + } + if (fileName.endsWith('.md')) { + return await this.loadMarkdownSkill(filePath); + } + return await this.loadModuleSkill(filePath, fileName); + } catch (err) { + logger.warn({ file: fileName, err }, 'Failed to load skill file'); + return null; + } + } + + private async loadMarkdownSkill(filePath: string): Promise<Skill | null> { + const content = await fs.readFile(filePath, 'utf-8'); + const data = parseSkillMarkdown(content); + const parsed = SkillSchema.safeParse(data); + if (!parsed.success) { + logger.warn({ filePath, errors: parsed.error.issues }, 'SKILL.md does not match SkillSchema'); + return null; + } + return parsed.data; + } + + private async loadJsonSkill(filePath: string): Promise<Skill | null> { + const raw = await fs.readFile(filePath, 'utf-8'); + const data = JSON.parse(raw) as unknown; + const parsed = SkillSchema.safeParse(data); + if (!parsed.success) { + logger.warn( + { filePath, errors: parsed.error.issues }, + 'Skill JSON does not match SkillSchema', + ); + return null; + } + return parsed.data; + } + + private async loadModuleSkill(filePath: string, fileName: string): Promise<Skill | null> { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const mod = await import(filePath); + const candidates = Object.values(mod as Record<string, unknown>); + + for (const candidate of candidates) { + const parsed = SkillSchema.safeParse(candidate); + if (parsed.success) { + return parsed.data; + } + } + + logger.warn({ file: fileName }, 'Skill module has no valid Skill export'); + return null; + } + + // ── Stats persistence ───────────────────────────────────────────── + + private async ensureStatsLoaded(): Promise<void> { + if (this.statsLoaded) return; + await this.loadStats(); + } + + private async loadStats(): Promise<void> { + try { + const raw = await fs.readFile(this.statsPath, 'utf-8'); + const data = JSON.parse(raw) as SkillStats[]; + if (Array.isArray(data)) { + for (const s of data) { + if (s && typeof s.name === 'string') { + this.statsMap.set(s.name, s); + } + } + } + } catch { + // File doesn't exist yet — start fresh + } + this.statsLoaded = true; + } + + private async saveStats(): Promise<void> { + await this.ensureSkillsDir(); + const data = Array.from(this.statsMap.values()); + await fs.writeFile(this.statsPath, JSON.stringify(data, null, 2), 'utf-8'); + } + + // ── Pattern persistence ─────────────────────────────────────────── + + private async ensurePatternsLoaded(): Promise<void> { + if (this.patternsLoaded) return; + await this.loadPatterns(); + } + + private async loadPatterns(): Promise<void> { + try { + const raw = await fs.readFile(this.patternsPath, 'utf-8'); + const data = JSON.parse(raw) as TaskPattern[]; + if (Array.isArray(data)) { + for (const p of data) { + if (p && typeof p.pattern === 'string' && typeof p.toolProfile === 'string') { + const key = `${p.pattern}::${p.toolProfile}`; + this.patternMap.set(key, p); + } + } + } + } catch { + // File doesn't exist yet — start fresh + } + this.patternsLoaded = true; + } + + private async savePatterns(): Promise<void> { + await this.ensureSkillsDir(); + const data = Array.from(this.patternMap.values()); + await fs.writeFile(this.patternsPath, JSON.stringify(data, null, 2), 'utf-8'); + } + + // ── Directory helper ────────────────────────────────────────────── + + private async ensureSkillsDir(): Promise<void> { + try { + await fs.mkdir(this.skillsPath, { recursive: true }); + } catch { + // Already exists — ignore + } + } + + // ── Auto-creation helpers ───────────────────────────────────────── + + /** + * Build a minimal `Skill` object from an accumulated task pattern. + * Returns `null` if the pattern is too short or a skill with that name + * already exists. + */ + private buildSkillFromPattern(pattern: TaskPattern): Skill | null { + const words = pattern.pattern.split(/\s+/).filter(Boolean).slice(0, 3); + if (words.length === 0) return null; + + const skillName = `auto-${words.join('-')}`; + + // Don't overwrite an existing skill + if (this.skills.has(skillName)) return null; + + const description = `Auto-created skill for tasks matching "${pattern.pattern}". Generated from ${pattern.successCount} successful completions.`; + + return SkillSchema.parse({ + name: skillName, + description, + toolProfile: pattern.toolProfile, + toolsNeeded: [], + examplePrompts: pattern.examplePrompts.slice(0, 3), + constraints: [], + isUserDefined: true, + }); + } + + /** + * Serialise a `Skill` to SKILL.md format and write it to + * `.openbridge/skills/<name>.md`. + */ + private async writeSkillFile(skill: Skill): Promise<void> { + await this.ensureSkillsDir(); + const content = skillToMarkdown(skill); + const filePath = path.join(this.skillsPath, `${skill.name}.md`); + await fs.writeFile(filePath, content, 'utf-8'); + logger.info({ filePath, skill: skill.name }, 'Wrote auto-created skill file'); + } +} diff --git a/src/master/skill-pack-loader.ts b/src/master/skill-pack-loader.ts new file mode 100644 index 00000000..048743f0 --- /dev/null +++ b/src/master/skill-pack-loader.ts @@ -0,0 +1,868 @@ +import path from 'path'; +import fs from 'fs/promises'; + +import { DocumentSkillSchema, SkillPackSchema } from '../types/agent.js'; +import type { DocumentSkill, SkillPack } from '../types/agent.js'; +import { createLogger } from '../core/logger.js'; +import type { AgentRunner } from '../core/agent-runner.js'; +import { TOOLS_READ_ONLY } from '../core/agent-runner.js'; +import type { MemoryManager } from '../memory/index.js'; +import type { LearningsSummary } from '../memory/task-store.js'; + +import { documentWriterSkill } from './skill-packs/document-writer.js'; +import { presentationMakerSkill } from './skill-packs/presentation-maker.js'; +import { spreadsheetBuilderSkill } from './skill-packs/spreadsheet-builder.js'; +import { reportGeneratorSkill } from './skill-packs/report-generator.js'; +import { BUILT_IN_SKILL_PACKS } from './skill-packs/index.js'; + +const logger = createLogger('skill-pack-loader'); + +/** Built-in skill packs shipped with OpenBridge */ +const BUILT_IN_SKILLS: DocumentSkill[] = [ + documentWriterSkill, + presentationMakerSkill, + spreadsheetBuilderSkill, + reportGeneratorSkill, +]; + +/** Result returned by {@link loadSkillPacks} */ +export interface SkillPackLoaderResult { + /** All available skill packs (built-in + custom), keyed by skill name */ + skills: Map<string, DocumentSkill>; + /** Number of custom skill packs successfully loaded from `.openbridge/skill-packs/` */ + customCount: number; +} + +/** + * Discovers and loads skill packs from two sources: + * + * 1. **Built-in defaults** — the four packs shipped with OpenBridge + * (`document-writer`, `presentation-maker`, `spreadsheet-builder`, `report-generator`). + * 2. **User-defined packs** — `.js` / `.cjs` / `.mjs` files placed in + * `<workspacePath>/.openbridge/skill-packs/`. Each file must export at least + * one value that validates against `DocumentSkillSchema`. + * + * Custom skill packs with the same `name` as a built-in pack **override** the + * built-in (allowing workspace-specific customisation). + * + * @param workspacePath - Absolute path to the target workspace. + * @returns A map of skill packs keyed by their `name` field, plus a count of + * custom packs loaded. + */ +export async function loadSkillPacks(workspacePath: string): Promise<SkillPackLoaderResult> { + const skills = new Map<string, DocumentSkill>(); + + // Seed with built-in skill packs + for (const skill of BUILT_IN_SKILLS) { + skills.set(skill.name, skill); + } + + // Discover custom skill packs from .openbridge/skill-packs/ + const customDir = path.join(workspacePath, '.openbridge', 'skill-packs'); + let customCount = 0; + + let entries: string[]; + try { + entries = await fs.readdir(customDir); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') { + // Directory does not exist — this is normal; only built-ins are used + return { skills, customCount }; + } + logger.warn({ customDir, err }, 'Error reading skill-packs directory'); + return { skills, customCount }; + } + + const jsFiles = entries.filter( + (e) => e.endsWith('.js') || e.endsWith('.cjs') || e.endsWith('.mjs'), + ); + + for (const file of jsFiles) { + const filePath = path.join(customDir, file); + try { + // Dynamic import — custom packs must be pre-compiled to JS + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const mod = await import(filePath); + + // Accept default export or any named export that validates as DocumentSkill + const candidates = Object.values(mod as Record<string, unknown>); + let loaded = false; + + for (const candidate of candidates) { + const parsed = DocumentSkillSchema.safeParse(candidate); + if (parsed.success) { + skills.set(parsed.data.name, parsed.data); + customCount++; + loaded = true; + logger.info({ skill: parsed.data.name, file }, 'Loaded custom skill pack'); + break; // One valid skill per file + } + } + + if (!loaded) { + logger.warn({ file }, 'Custom skill pack file has no valid DocumentSkill export'); + } + } catch (err) { + logger.warn({ file, err }, 'Failed to load custom skill pack'); + } + } + + return { skills, customCount }; +} + +/** + * Returns the built-in skill pack list synchronously (no file I/O). + * Useful for quick lookups when custom skill packs are not needed. + */ +export function getBuiltInSkillPacks(): DocumentSkill[] { + return [...BUILT_IN_SKILLS]; +} + +/** + * Finds the first skill pack that produces the given file format. + * Searches the full skill map returned by {@link loadSkillPacks}. + * + * @param skills - The skill map from `loadSkillPacks()`. + * @param fileFormat - Target format string (e.g., `'docx'`, `'pdf'`, `'pptx'`). + * @returns The matching `DocumentSkill`, or `undefined` if none found. + */ +export function findSkillByFormat( + skills: Map<string, DocumentSkill>, + fileFormat: string, +): DocumentSkill | undefined { + for (const skill of skills.values()) { + if (skill.fileFormat === fileFormat) return skill; + } + return undefined; +} + +// ── SKILLPACK.md format ─────────────────────────────────────────────────────── +// +// SKILLPACK.md files let users define domain-specific instruction bundles for +// workers without writing TypeScript. Each file maps to a `SkillPack` object. +// +// Format: +// +// # <name> +// +// <description — first paragraph> +// +// ## Tool Profile +// <toolProfile> +// +// ## When to Use +// <description override — used only if the first-paragraph description is absent> +// +// ## Required Tools +// - <tool1> +// - <tool2> +// +// ## Tags +// - <tag1> +// - <tag2> +// +// ## Prompt Extension +// <multi-line text injected into the worker system prompt> +// +// ## Example Tasks +// - "<example task description>" +// +// ## Constraints +// - <constraint — appended to Prompt Extension> +// +// Required fields: name (H1), description or ## When to Use, ## Tool Profile. +// All other sections are optional. + +/** + * Extracts bullet-list items from a section body string. + * Handles `- item`, `* item` list markers. + */ +function extractListItems(body: string): string[] { + return body + .split('\n') + .map((line) => + line + .replace(/^[-*]\s+/, '') + .replace(/^["']|["']$/g, '') + .trim(), + ) + .filter(Boolean); +} + +/** + * Parses a `SKILLPACK.md` file content into a partial `SkillPack` object. + * + * The caller should validate the result against `SkillPackSchema` before use. + * + * @param content - Raw markdown string. + * @returns Partial SkillPack fields extracted from the markdown. + */ +export function parseSkillPackMd(content: string): Partial<SkillPack> { + const result: Partial<SkillPack> & { requiredTools: string[]; tags: string[] } = { + requiredTools: [], + tags: [], + }; + + if (!content.trim()) return result; + + const lines = content.split('\n'); + + // Track current H2 section name and its accumulated body lines + let currentSection: string | null = null; + const sectionLines: Record<string, string[]> = {}; + + // Track lines before the first ## heading (after H1) for description + let pastH1 = false; + const preH2Lines: string[] = []; + + for (const line of lines) { + if (!pastH1 && /^#\s+/.test(line)) { + // H1 — extract name + result.name = line.replace(/^#\s+/, '').trim(); + pastH1 = true; + continue; + } + + if (/^##\s+/.test(line)) { + // H2 — start a new section + currentSection = line.replace(/^##\s+/, '').trim(); + if (!sectionLines[currentSection]) { + sectionLines[currentSection] = []; + } + continue; + } + + if (currentSection !== null) { + sectionLines[currentSection]!.push(line); + } else if (pastH1) { + preH2Lines.push(line); + } + } + + // Extract description from first non-empty paragraph before any H2 + const descPara = preH2Lines.join('\n').trim(); + if (descPara && !descPara.startsWith('#')) { + result.description = descPara; + } + + // Process each section + for (const [heading, bodyLines] of Object.entries(sectionLines)) { + const body = bodyLines.join('\n').trim(); + + switch (heading) { + case 'Tool Profile': + result.toolProfile = body.split('\n')[0]!.trim(); + break; + + case 'When to Use': + if (!result.description) { + result.description = body; + } + break; + + case 'Required Tools': + result.requiredTools = extractListItems(body); + break; + + case 'Tags': + result.tags = extractListItems(body); + break; + + case 'Prompt Extension': + result.systemPromptExtension = body; + break; + + case 'Constraints': { + const constraints = extractListItems(body); + if (constraints.length > 0) { + const constraintsSection = `\n\n## Constraints\n${constraints.map((c) => `- ${c}`).join('\n')}`; + result.systemPromptExtension = (result.systemPromptExtension ?? '') + constraintsSection; + } + break; + } + + // 'Example Tasks' — metadata only, not part of SkillPack schema; intentionally ignored + default: + break; + } + } + + return result; +} + +/** + * Keyword sets used to match task prompts to built-in skill packs. + * Ordered by specificity so that more specific matches win over general ones. + */ +const SKILL_PACK_KEYWORDS: Record<string, string[]> = { + 'security-audit': [ + 'security', + 'audit', + 'vulnerability', + 'vulnerabilities', + 'semgrep', + 'codeql', + 'owasp', + 'injection', + 'xss', + 'sql injection', + 'insecure', + 'cve', + 'exploit', + 'penetration', + 'pentest', + 'secure code', + 'security review', + 'threat', + ], + 'code-review': [ + 'review', + 'code review', + 'pull request', + 'pr review', + 'diff', + 'git diff', + 'feedback on', + 'review this', + 'check this code', + 'review the changes', + 'review my', + 'critique', + 'assess the code', + ], + 'test-writer': [ + 'write test', + 'write tests', + 'add test', + 'add tests', + 'create test', + 'create tests', + 'unit test', + 'unit tests', + 'test coverage', + 'tdd', + 'test-driven', + 'vitest', + 'jest', + 'mocha', + 'spec file', + 'test suite', + 'edge case', + 'edge cases', + ], + 'data-analysis': [ + 'analyse data', + 'analyze data', + 'data analysis', + 'csv', + 'json data', + 'ndjson', + 'statistics', + 'statistical', + 'dataset', + 'correlation', + 'distribution', + 'visuali', + 'chart', + 'histogram', + 'aggregate', + 'pivot', + 'parse data', + 'sqlite', + 'database', + '.db', + 'query', + 'supplier', + 'invoice', + 'payment', + 'extract data', + 'best supplier', + 'top supplier', + 'sales data', + 'pos data', + 'export data', + ], + documentation: [ + 'write docs', + 'generate docs', + 'documentation', + 'api docs', + 'api reference', + 'readme', + 'changelog', + 'jsdoc', + 'tsdoc', + 'docstring', + 'document this', + 'document the', + 'write documentation', + 'update docs', + 'update the docs', + 'add docs', + ], + 'web-deploy': [ + 'deploy', + 'deployment', + 'vercel', + 'netlify', + 'cloudflare', + 'wrangler', + 'put this live', + 'go live', + 'publish site', + 'deploy to', + 'host this', + 'hosting', + 'static site', + 'deploy this', + 'ship this', + 'launch site', + 'deploy the app', + 'deploy the site', + ], + 'spreadsheet-handler': [ + 'spreadsheet', + 'excel', + 'xlsx', + 'xls', + 'csv', + 'read spreadsheet', + 'read excel', + 'modify spreadsheet', + 'modify excel', + 'update spreadsheet', + 'filter spreadsheet', + 'sort data', + 'aggregate data', + 'pivot table', + 'google sheets', + 'exceljs', + 'sheetjs', + ], + 'file-converter': [ + 'convert', + 'conversion', + 'convert file', + 'convert to pdf', + 'convert to docx', + 'convert to html', + 'convert to markdown', + 'pandoc', + 'libreoffice', + 'pdf to text', + 'docx to pdf', + 'markdown to pdf', + 'markdown to docx', + 'html to pdf', + 'docx to html', + 'docx to markdown', + 'extract text', + 'ocr', + 'tesseract', + 'mammoth', + 'pdf-parse', + 'format conversion', + 'file format', + 'epub', + 'rtf', + ], +}; + +/** + * Selects the most appropriate skill pack for a worker task based on keyword + * matching against the task prompt. + * + * Scoring: each keyword match increments the pack's score by 1 (phrase matches + * count as 2). The pack with the highest score wins. Returns `undefined` when + * no pack scores above zero. + * + * @param prompt - The worker task prompt text. + * @param packs - Candidate skill packs to match against (typically {@link BUILT_IN_SKILL_PACKS}). + * @returns The best-matching `SkillPack`, or `undefined` if no match. + */ +export function selectSkillPackForTask(prompt: string, packs: SkillPack[]): SkillPack | undefined { + const lower = prompt.toLowerCase(); + let bestPack: SkillPack | undefined; + let bestScore = 0; + + for (const pack of packs) { + const keywords = SKILL_PACK_KEYWORDS[pack.name]; + if (!keywords) continue; + + let score = 0; + for (const kw of keywords) { + if (lower.includes(kw)) { + // Phrase keywords (containing a space) worth 2 points for specificity + score += kw.includes(' ') ? 2 : 1; + } + } + + if (score > bestScore) { + bestScore = score; + bestPack = pack; + } + } + + return bestPack; +} + +/** + * Serialises a `SkillPack` to SKILLPACK.md format. + * Useful for persisting auto-created or edited skill packs back to disk. + * + * @param pack - The SkillPack to serialise. + * @returns A markdown string in SKILLPACK.md format. + */ +export function skillPackToMarkdown(pack: SkillPack): string { + const lines: string[] = []; + + lines.push(`# ${pack.name}`, ''); + lines.push(pack.description, ''); + lines.push('## Tool Profile', pack.toolProfile, ''); + + if (pack.requiredTools.length > 0) { + lines.push('## Required Tools'); + for (const tool of pack.requiredTools) { + lines.push(`- ${tool}`); + } + lines.push(''); + } + + if (pack.tags.length > 0) { + lines.push('## Tags'); + for (const tag of pack.tags) { + lines.push(`- ${tag}`); + } + lines.push(''); + } + + if (pack.systemPromptExtension) { + // Strip any trailing Constraints section before re-serialising (they are + // stored inline in systemPromptExtension and will be re-appended below) + const basePrompt = pack.systemPromptExtension.replace(/\n\n## Constraints[\s\S]*$/, '').trim(); + if (basePrompt) { + lines.push('## Prompt Extension', basePrompt, ''); + } + + // Re-extract constraints if they were appended + const constraintsMatch = pack.systemPromptExtension.match(/\n\n## Constraints\n([\s\S]*)$/); + if (constraintsMatch) { + const items = extractListItems(constraintsMatch[1]!); + if (items.length > 0) { + lines.push('## Constraints'); + for (const item of items) { + lines.push(`- ${item}`); + } + lines.push(''); + } + } + } + + return lines.join('\n'); +} + +/** Result returned by {@link loadSkillPackMarkdown} */ +export interface SkillPackMarkdownLoaderResult { + /** All loaded SkillPack objects keyed by name */ + packs: Map<string, SkillPack>; + /** Number of successfully loaded packs */ + count: number; +} + +/** + * Discovers and loads `SKILLPACK.md` files from + * `<workspacePath>/.openbridge/skill-packs/`. + * + * Files that fail to parse or fail Zod validation are skipped with a warning. + * All loaded packs have `isUserDefined: true`. + * + * @param workspacePath - Absolute path to the target workspace. + * @returns A map of SkillPack objects keyed by their `name` field. + */ +export async function loadSkillPackMarkdown( + workspacePath: string, +): Promise<SkillPackMarkdownLoaderResult> { + const packs = new Map<string, SkillPack>(); + const skillPacksDir = path.join(workspacePath, '.openbridge', 'skill-packs'); + + let entries: string[]; + try { + entries = await fs.readdir(skillPacksDir); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') { + return { packs, count: 0 }; + } + logger.warn( + { skillPacksDir, err }, + 'Error reading skill-packs directory for SKILLPACK.md files', + ); + return { packs, count: 0 }; + } + + const mdFiles = entries.filter((e) => e.endsWith('.md') && !e.startsWith('.')); + + for (const file of mdFiles) { + const filePath = path.join(skillPacksDir, file); + try { + const content = await fs.readFile(filePath, 'utf-8'); + const partial = parseSkillPackMd(content); + + const parsed = SkillPackSchema.safeParse({ ...partial, isUserDefined: true }); + if (parsed.success) { + packs.set(parsed.data.name, parsed.data); + logger.info({ name: parsed.data.name, file }, 'Loaded SKILLPACK.md'); + } else { + logger.warn( + { file, issues: parsed.error.issues }, + 'SKILLPACK.md failed validation — skipping', + ); + } + } catch (err) { + logger.warn({ file, err }, 'Failed to read or parse SKILLPACK.md'); + } + } + + return { packs, count: packs.size }; +} + +/** Result returned by {@link loadAllSkillPacks} */ +export interface AllSkillPacksResult { + /** Merged skill packs (built-in + user-defined) — user-defined overrides built-ins by name */ + packs: SkillPack[]; + /** Number of user-defined packs loaded from .openbridge/skill-packs/ */ + userDefinedCount: number; +} + +/** + * Loads all skill packs — built-in defaults merged with user-defined overrides. + * + * User-defined packs discovered from `<workspacePath>/.openbridge/skill-packs/*.md` + * take precedence over built-ins with the same `name`. + * + * @param workspacePath - Absolute path to the target workspace. + * @returns Merged pack list and count of user-defined packs loaded. + */ +export async function loadAllSkillPacks(workspacePath: string): Promise<AllSkillPacksResult> { + const merged = new Map<string, SkillPack>(); + + for (const pack of BUILT_IN_SKILL_PACKS) { + merged.set(pack.name, pack); + } + + const { packs: userPacks, count: userDefinedCount } = await loadSkillPackMarkdown(workspacePath); + + for (const [name, pack] of userPacks) { + if (merged.has(name)) { + logger.info({ name }, 'User-defined skill pack overrides built-in'); + } + merged.set(name, pack); + } + + return { packs: Array.from(merged.values()), userDefinedCount }; +} + +// ── Skill Pack Synthesis ────────────────────────────────────────────────────── + +/** Minimum number of successful tasks required to trigger skill pack synthesis. */ +const MIN_TASKS_FOR_SYNTHESIS = 5; + +/** Minimum success rate (0–1) for a task type to be eligible for synthesis. */ +const MIN_SUCCESS_RATE_FOR_SYNTHESIS = 0.8; + +/** + * Task types that are too generic to benefit from a dedicated skill pack. + * These are filtered out before synthesis is attempted. + */ +const GENERIC_TASK_TYPES = new Set(['task', 'exploration', 'classification']); + +/** + * Build the AI prompt asking a worker to synthesize a SKILLPACK.md for a task type. + */ +function buildSynthesisPrompt(taskType: string, totalTasks: number, successRate: number): string { + const pct = Math.round(successRate * 100); + return `You are a skill pack author for OpenBridge, an AI bridge system. Based on the task pattern described below, create a SKILLPACK.md that captures domain expertise and best practices for this type of work. + +Task pattern: +- Type: "${taskType}" +- Total successful completions: ${totalTasks} (${pct}% success rate) + +A SKILLPACK.md must follow this exact format: + +# <name> + +<one-paragraph description> + +## Tool Profile +<one of: read-only, code-audit, code-edit, full-access> + +## When to Use +<when to use this skill pack> + +## Required Tools +- <tool1> +- <tool2> + +## Tags +- <tag1> +- <tag2> + +## Prompt Extension +<detailed, actionable instructions for workers performing this type of task — minimum 100 words> + +## Constraints +- <constraint1> +- <constraint2> + +Rules: +- Name must be a lowercase slug (hyphens only, no spaces, no special characters) +- Tool Profile must be exactly one of: read-only, code-audit, code-edit, full-access +- Prompt Extension must be at least 100 words and actionable +- Name must differ from built-ins: security-audit, code-review, test-writer, data-analysis, documentation + +Respond with ONLY the SKILLPACK.md content. No explanations, no code fences.`; +} + +/** + * Synthesizes a single skill pack from a task type pattern using an AI worker. + * Returns the created SkillPack on success, null on failure. + */ +async function synthesizeOneSkillPack( + candidate: LearningsSummary, + workspacePath: string, + agentRunner: AgentRunner, +): Promise<SkillPack | null> { + const { task_type: taskType, total_tasks: totalTasks, success_rate: successRate } = candidate; + + logger.info({ taskType, totalTasks, successRate }, 'Synthesizing skill pack from task pattern'); + + const prompt = buildSynthesisPrompt(taskType, totalTasks, successRate); + + let result; + try { + result = await agentRunner.spawn({ + prompt, + workspacePath, + model: 'haiku', + allowedTools: [...TOOLS_READ_ONLY], + maxTurns: 3, + retries: 1, + }); + } catch (err) { + logger.warn({ err, taskType }, 'Skill pack synthesis worker failed'); + return null; + } + + if (result.exitCode !== 0 || !result.stdout.trim()) { + logger.warn( + { taskType, exitCode: result.exitCode }, + 'Skill pack synthesis worker returned empty or failed output', + ); + return null; + } + + // Strip markdown code fences if the worker wrapped the output + const rawContent = result.stdout.trim(); + const fenceMatch = rawContent.match(/^```(?:\w+)?\n([\s\S]*?)\n```$/); + const content = fenceMatch?.[1] ?? rawContent; + + const partial = parseSkillPackMd(content); + const parsed = SkillPackSchema.safeParse({ ...partial, isUserDefined: true }); + + if (!parsed.success) { + logger.warn( + { taskType, issues: parsed.error.issues }, + 'Synthesized skill pack failed Zod validation — skipping', + ); + return null; + } + + // Persist to .openbridge/skill-packs/<name>.md + const skillPacksDir = path.join(workspacePath, '.openbridge', 'skill-packs'); + try { + await fs.mkdir(skillPacksDir, { recursive: true }); + const filePath = path.join(skillPacksDir, `${parsed.data.name}.md`); + await fs.writeFile(filePath, skillPackToMarkdown(parsed.data), 'utf-8'); + logger.info( + { name: parsed.data.name, taskType, file: filePath }, + 'Synthesized skill pack saved to disk', + ); + } catch (err) { + logger.warn({ err, taskType }, 'Failed to save synthesized skill pack to disk'); + return null; + } + + return parsed.data; +} + +/** + * Synthesizes new skill packs from successful task patterns stored in the learnings table. + * + * Scans learning records for task types that: + * 1. Have at least {@link MIN_TASKS_FOR_SYNTHESIS} total completed tasks + * 2. Have a success rate >= {@link MIN_SUCCESS_RATE_FOR_SYNTHESIS} + * 3. Are not already covered by an existing built-in or user-defined skill pack + * 4. Are not generic catch-all types (e.g., `'task'`, `'exploration'`, `'classification'`) + * + * For each eligible pattern, a Haiku worker is spawned to generate a SKILLPACK.md + * which is validated and saved to `<workspacePath>/.openbridge/skill-packs/`. + * + * This extends the prompt evolution system — call it alongside {@link evolvePrompts} + * every N task completions. + * + * @param memory - MemoryManager for reading learning records. + * @param agentRunner - AgentRunner for spawning the synthesis worker. + * @param workspacePath - Absolute path to the target workspace. + * @returns Number of skill packs successfully synthesized and saved. + */ +export async function synthesizeSkillPacksFromPatterns( + memory: MemoryManager, + agentRunner: AgentRunner, + workspacePath: string, +): Promise<number> { + logger.info('Running skill pack synthesis cycle'); + + // Query task types with high success rates across all models + let candidates: LearningsSummary[]; + try { + candidates = await memory.getHighSuccessLearnings( + MIN_SUCCESS_RATE_FOR_SYNTHESIS, + MIN_TASKS_FOR_SYNTHESIS, + ); + } catch (err) { + logger.warn({ err }, 'Failed to query high-success learnings for skill pack synthesis'); + return 0; + } + + if (candidates.length === 0) { + logger.info('No task patterns eligible for skill pack synthesis'); + return 0; + } + + // Load existing skill pack names (built-in + user-defined) to avoid duplicates + const { packs: existingPacks } = await loadAllSkillPacks(workspacePath); + const existingNames = new Set(existingPacks.map((p) => p.name)); + + // Also map built-in pack names for fast lookup + const builtInNames = new Set(BUILT_IN_SKILL_PACKS.map((p) => p.name)); + + let synthesized = 0; + + for (const candidate of candidates) { + const { task_type: taskType } = candidate; + + // Skip generic types that don't benefit from a skill pack + if (GENERIC_TASK_TYPES.has(taskType)) continue; + + // Skip if a skill pack with the same name already exists + if (builtInNames.has(taskType) || existingNames.has(taskType)) continue; + + const pack = await synthesizeOneSkillPack(candidate, workspacePath, agentRunner); + if (pack) { + existingNames.add(pack.name); + synthesized++; + } + } + + logger.info({ synthesized }, 'Skill pack synthesis cycle complete'); + return synthesized; +} diff --git a/src/master/skill-packs/brand-assets.ts b/src/master/skill-packs/brand-assets.ts new file mode 100644 index 00000000..def49e32 --- /dev/null +++ b/src/master/skill-packs/brand-assets.ts @@ -0,0 +1,459 @@ +import type { SkillPack } from '../../types/agent.js'; + +/** + * Brand Assets skill pack — SVG logo concepts, social media images, favicon generation + * + * Guides a worker agent to produce self-contained brand asset files: SVG logos, + * social media image templates (OG/Twitter/LinkedIn), and favicons. All outputs + * are valid SVG or HTML files — no build tooling required. + */ +export const brandAssetsSkillPack: SkillPack = { + name: 'brand-assets', + description: + 'Creates brand identity assets — SVG logo concepts, social media image templates (Open Graph, Twitter Card, LinkedIn banner), and favicons (SVG + ICO-compatible). Outputs self-contained SVG files or HTML preview pages viewable in any browser. Ideal for quick logo ideation, consistent brand collateral, and launch-ready social imagery.', + toolProfile: 'code-edit', + requiredTools: ['Read', 'Write', 'Bash(cat:*)'], + tags: [ + 'brand', + 'logo', + 'svg', + 'favicon', + 'social-media', + 'og-image', + 'twitter-card', + 'linkedin', + 'identity', + 'marketing', + 'design', + 'icon', + 'typography', + 'visual', + ], + isUserDefined: false, + systemPromptExtension: `## Brand Assets Mode + +You are creating brand identity assets. Your outputs are self-contained SVG files or HTML preview pages — no external build tools, no npm installs. Every file must render correctly in a modern browser. + +### Asset Type Selection Guide + +| Asset Type | Format | Dimensions | Best for | +|-------------------------|-----------------|------------------------|------------------------------------------------| +| Logo (icon only) | SVG | 100×100 viewBox | Favicons, app icons, standalone symbol | +| Logo (wordmark) | SVG | 300×80 viewBox | Full brand lockup, website header | +| Logo (stacked) | SVG | 200×150 viewBox | Print, presentations, centred layouts | +| Favicon | SVG | 32×32 or 16×16 viewBox | Browser tab icon (modern browsers support SVG) | +| Open Graph image | SVG or HTML→SVG | 1200×630 | Facebook, Discord, Slack link previews | +| Twitter/X Card | SVG or HTML→SVG | 1200×675 | Twitter summary_large_image cards | +| LinkedIn Banner | SVG or HTML→SVG | 1584×396 | Company page or personal profile banner | +| Social media post | SVG or HTML | 1080×1080 | Instagram/Facebook square post | + +**Default approach:** SVG for logos and favicons. HTML (with embedded SVG) for social media images — this allows richer typography and layout. + +--- + +### Methodology + +Work through these steps in order: + +1. **Understand the brand** — name, industry, tone (playful/serious/minimal/bold), colour preferences. +2. **Select a logo concept** — abstract mark, lettermark, wordmark, or combination mark. +3. **Choose a palette** — typically 1–3 colours. Use the brand colour if provided, else select from the palette guide below. +4. **Design the mark** — use SVG primitives (path, circle, rect, polygon, text) for crisp, scalable output. +5. **Export variants** — logo (full), logomark (symbol only), favicon (32×32 crop). +6. **Generate social templates** — OG image, Twitter card using the brand colours and logo. +7. **Write all files** to the workspace with clear naming. + +--- + +### Templates & Examples + +#### Minimal SVG Logo (Lettermark) + +\`\`\`svg +<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 200 80" width="200" height="80"> + <defs> + <linearGradient id="grad" x1="0%" y1="0%" x2="100%" y2="100%"> + <stop offset="0%" stop-color="#4f46e5"/> + <stop offset="100%" stop-color="#7c3aed"/> + </linearGradient> + </defs> + + <!-- Icon mark: rounded square with letter --> + <rect x="0" y="10" width="60" height="60" rx="12" fill="url(#grad)"/> + <text x="30" y="50" font-family="system-ui, sans-serif" font-size="32" + font-weight="700" fill="white" text-anchor="middle" dominant-baseline="middle">A</text> + + <!-- Wordmark --> + <text x="72" y="34" font-family="system-ui, sans-serif" font-size="22" + font-weight="700" fill="#1e1b4b">Acme</text> + <text x="72" y="58" font-family="system-ui, sans-serif" font-size="13" + font-weight="400" fill="#6b7280" letter-spacing="2">STUDIO</text> +</svg> +\`\`\` + +--- + +#### SVG Favicon (32×32) + +\`\`\`svg +<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" width="32" height="32"> + <defs> + <linearGradient id="g" x1="0" y1="0" x2="1" y2="1"> + <stop offset="0%" stop-color="#4f46e5"/> + <stop offset="100%" stop-color="#7c3aed"/> + </linearGradient> + </defs> + <rect width="32" height="32" rx="7" fill="url(#g)"/> + <text x="16" y="22" font-family="system-ui, sans-serif" font-size="18" + font-weight="800" fill="white" text-anchor="middle">A</text> +</svg> +\`\`\` + +--- + +#### Open Graph Image (1200×630) + +\`\`\`html +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <title>OG Image Preview + + + +
+
+
+
+
A
+ Acme Studio +
+
Build faster.
Ship smarter.
+
The AI-powered platform for modern teams.
+
acme.studio
+
+ + +\`\`\` + +--- + +#### Twitter/X Card (1200×675) + +\`\`\`html + + + + + Twitter Card Preview + + + +
+
+
✦ Introducing v2.0
+
The next generation
of Acme Studio
+
Redesigned from the ground up for speed, clarity, and collaboration.
+
+
+
A
+
+
+ acme.studio +
+
+ + +\`\`\` + +--- + +#### LinkedIn Banner (1584×396) + +\`\`\`html + + + + + LinkedIn Banner Preview + + + + + + +\`\`\` + +--- + +### Colour Palette Guide + +When no brand colours are specified, choose a palette from the following: + +| Style | Primary | Accent | Background | Use for | +|---------------|---------------|---------------|-------------|----------------------------------| +| Indigo/Violet | \`#4f46e5\` | \`#7c3aed\` | \`#0f0e1a\` | Tech, SaaS, AI products | +| Emerald/Teal | \`#059669\` | \`#0891b2\` | \`#f0fdf4\` | Health, fintech, sustainability | +| Rose/Orange | \`#e11d48\` | \`#ea580c\` | \`#fff1f2\` | Creative, media, entertainment | +| Slate (B&W) | \`#0f172a\` | \`#475569\` | \`#f8fafc\` | Luxury, legal, consulting | +| Amber/Gold | \`#d97706\` | \`#b45309\` | \`#fffbeb\` | Finance, premium products | +| Sky/Blue | \`#0284c7\` | \`#0369a1\` | \`#f0f9ff\` | Corporate, enterprise, logistics | + +**Typography:** Use \`system-ui, -apple-system, sans-serif\` for consistency across OS. For elegant brands use \`Georgia, serif\`. Never embed web fonts — keep outputs self-contained. + +--- + +### SVG Logo Concepts + +#### Geometric Mark (hexagonal) +\`\`\`svg + + + + + + + + + + + + + + +\`\`\` + +#### Abstract Circuit Mark +\`\`\`svg + + + + + + + + + + + + + + + + + + + +\`\`\` + +#### Rounded Triangle (Play/Forward mark) +\`\`\`svg + + + + + + + + + + +\`\`\` + +--- + +### Favicon Guidelines + +SVG favicons are supported by all modern browsers. Provide a \`\` tag in the preview HTML. + +- **32×32 viewBox** for standard favicon detail level +- **16×16 viewBox** for ultra-minimal mark (single letter or dot) +- Keep it readable at tiny sizes: avoid thin strokes < 2px, avoid > 2 colours +- Include a fallback comment noting PNG generation via Inkscape: \`inkscape -w 32 -h 32 favicon.svg -o favicon.png\` + +--- + +### Output Naming Convention + +| Asset type | File name pattern | +|--------------------------|---------------------------------------| +| Logo (combination mark) | \`logo-.svg\` | +| Logo (icon only) | \`logomark-.svg\` | +| Favicon | \`favicon-.svg\` | +| Open Graph image | \`og-image-.html\` | +| Twitter/X Card | \`twitter-card-.html\` | +| LinkedIn Banner | \`linkedin-banner-.html\` | +| Social post (square) | \`social-post-.html\` | +| Brand preview page | \`brand-preview-.html\` | + +--- + +### Brand Preview Page + +When generating multiple assets for the same brand, produce a single \`brand-preview-.html\` that shows all assets side-by-side using \`