diff --git a/.gitignore b/.gitignore index 706332c..cf827bd 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ temp/ build/ **/node_modules/ !tools/loop-audit/dist/ +!tools/loop-cost/dist/ *.egg-info/ # State files that should not be committed in consumer projects (reference repo dogfoods STATE.md) diff --git a/LOOP.md b/LOOP.md index fd795cd..b34af43 100644 --- a/LOOP.md +++ b/LOOP.md @@ -51,6 +51,13 @@ See [docs/multi-loop.md](docs/multi-loop.md). Priority: CI Sweeper → PR Babysi - GitHub MCP read-only for issue/PR discovery - Scope connectors to read + comment until the loop is trusted +## Budget & Observability + +- Token caps: `loop-budget.md` +- Run history: `loop-run-log.md` +- Estimate: `npx @cobusgreyling/loop-cost --pattern daily-triage` +- Kill switch: `loop-pause-all` label or flag in `STATE.md` + ## Safety & Gates (this repo) - No auto-merge on main except trivial dependency patches (allowlist + verifier) diff --git a/README.md b/README.md index fa06c8e..3cd47dc 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,8 @@ A loop is a recursive goal: you define a purpose and the AI iterates (often with | [Patterns](patterns/README.md) | 6 production patterns including the new low-risk Changelog Drafter | | [Starters](starters/) | Clone-and-run kits (Grok, Claude Code, Codex) | | [loop-audit](tools/loop-audit/) | Loop Readiness Score CLI — `npx @cobusgreyling/loop-audit` | -| [loop-init](tools/loop-init/) | Scaffold starters — `npx @cobusgreyling/loop-init` | +| [loop-init](tools/loop-init/) | Scaffold starters + budget/run-log — `npx @cobusgreyling/loop-init` | +| [loop-cost](tools/loop-cost/) | Token spend estimator — `npx @cobusgreyling/loop-cost` | | [Stories](stories/) | Real wins and honest failures | ## Why This Matters @@ -117,13 +118,16 @@ Machine-readable index: [patterns/registry.yaml](patterns/registry.yaml) (now 6 # 1. Scaffold a starter (or copy manually — see starters/) npx @cobusgreyling/loop-init . --pattern daily-triage --tool grok -# 2. Audit readiness +# 2. Estimate token spend for your cadence +npx @cobusgreyling/loop-cost --pattern daily-triage --level L1 + +# 3. Audit readiness (budget + run-log now scored) npx @cobusgreyling/loop-audit . --suggest -# 3. See scores climb: empty → L1 → L2 +# 4. See scores climb: empty → L1 → L2 bash scripts/before-after-demo.sh -# 4. Start report-only (Grok example) +# 5. Start report-only (Grok example) /loop 1d Run loop-triage. Update STATE.md. No auto-fix in week one. ``` @@ -132,6 +136,7 @@ Packages publish from tagged releases — see [docs/RELEASE.md](docs/RELEASE.md) ```bash cd tools/loop-init && npm ci && npm test && node dist/cli.js /path/to/project --pattern daily-triage --tool grok cd tools/loop-audit && npm ci && npm test && node dist/cli.js /path/to/project --suggest +cd tools/loop-cost && npm ci && npm test && node dist/cli.js --pattern ci-sweeper --cadence 15m ``` Phased rollout: **L1 report → L2 assisted fixes → L3 unattended** — see [loop-design-checklist](docs/loop-design-checklist.md). diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 5d1c7db..3641ac0 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -6,6 +6,7 @@ This repo ships two public npm packages from `tools/`: |---------|-----------|-------------| | `@cobusgreyling/loop-audit` | `tools/loop-audit` | `loop-audit-v*` | | `@cobusgreyling/loop-init` | `tools/loop-init` | `loop-init-v*` | +| `@cobusgreyling/loop-cost` | `tools/loop-cost` | `loop-cost-v*` (add workflow when publishing) | ## One-time setup (trusted publishing — recommended) diff --git a/docs/loop-design-checklist.md b/docs/loop-design-checklist.md index 3b85a85..a48e58d 100644 --- a/docs/loop-design-checklist.md +++ b/docs/loop-design-checklist.md @@ -54,7 +54,10 @@ Use this before enabling a loop in production. Score honestly — a loop missing ## 8. Cost & Limits -- [ ] **Token budget** estimated (see [operating-loops.md](./operating-loops.md)) +- [ ] **Token budget** estimated (`npx @cobusgreyling/loop-cost`, [operating-loops.md](./operating-loops.md)) +- [ ] **`loop-budget.md`** with daily caps and kill switch +- [ ] **`loop-run-log.md`** for append-only run history +- [ ] **`loop-budget` skill** checks spend at start/end of each run - [ ] **Max iterations** per item per run - [ ] **Max auto-PRs** per day (cleanup loops) - [ ] **Pause/kill** criteria defined diff --git a/docs/operating-loops.md b/docs/operating-loops.md index 4bd27bd..1e06aff 100644 --- a/docs/operating-loops.md +++ b/docs/operating-loops.md @@ -4,6 +4,15 @@ Running a loop is operations work. This doc covers cost, logging, metrics, and w ## Token & Cost Budgeting +Estimate before scheduling: + +```bash +npx @cobusgreyling/loop-cost --pattern --cadence --level L1 +npx @cobusgreyling/loop-init . --pattern # scaffolds loop-budget.md + loop-run-log.md + loop-budget skill +``` + +`loop-audit` scores cost observability and caps L3 until budget + run log + LOOP.md budget section exist. + Rough planning factors: | Factor | Impact | diff --git a/docs/pattern-picker.md b/docs/pattern-picker.md index e3c3aa8..2d13d1e 100644 --- a/docs/pattern-picker.md +++ b/docs/pattern-picker.md @@ -16,9 +16,29 @@ flowchart TD J -->|yes| K[Post-Merge Cleanup] J -->|no| L{Release notes / changelog stale?} L -->|yes| M[Changelog Drafter] - L -->|no| G + L -->|no| N{Tight token budget?} + N -->|yes| M + N -->|no| G ``` +## Cost-aware picks + +Estimate before you schedule: + +```bash +npx @cobusgreyling/loop-cost --pattern --level L1 +npx @cobusgreyling/loop-init . --pattern daily-triage --tool grok # scaffolds loop-budget.md + loop-run-log.md +``` + +| Situation | Prefer | Avoid (until budget + early-exit) | +|-----------|--------|-----------------------------------| +| Hobby / tight plan | Changelog Drafter, Daily Triage (L1), Post-Merge | CI Sweeper at 5m, PR Babysitter at 5m | +| Active CI fires | CI Sweeper at **15m+** with early-exit | Full triage every 5m when main is green | +| Many open PRs | PR Babysitter at 10–15m, L1 watch first | L2 fix loops on every tick | +| Release week | Changelog Drafter daily | Dependency Sweeper + CI Sweeper unattended | + +`loop-audit` caps **L3** until `loop-budget.md`, `loop-run-log.md`, and a `LOOP.md` budget section exist. + ## Quick reference | Symptom | Pattern | Start with | diff --git a/loop-budget.md b/loop-budget.md new file mode 100644 index 0000000..a2f3086 --- /dev/null +++ b/loop-budget.md @@ -0,0 +1,28 @@ +# Loop Budget — loop-engineering (reference repo) + +> Dogfood file for the patterns that maintain this repository. + +## Daily limits + +| Loop | Max runs/day | Max tokens/day | Max sub-agent spawns/run | +|------|--------------|----------------|--------------------------| +| Daily Triage | 1 | 100k | 0 (L1) | +| Validate/Audit (CI) | 96 | 500k | 0 | +| Changelog Drafter | 1 | 100k | 2 | + +## On budget exceed + +1. Pause schedulers / disable high-cadence workflows +2. Append event to `loop-run-log.md` +3. Open maintainer issue + +## Kill switch + +- Label: `loop-pause-all` +- Resume only after cleared in `STATE.md` + +## Estimate spend + +```bash +npx @cobusgreyling/loop-cost --pattern daily-triage --level L1 +``` \ No newline at end of file diff --git a/loop-run-log.md b/loop-run-log.md new file mode 100644 index 0000000..882edb6 --- /dev/null +++ b/loop-run-log.md @@ -0,0 +1,22 @@ +# Loop Run Log — loop-engineering + +Append one entry per run. Prune entries older than 30 days. + +## Format + +```json +{ + "run_id": "2026-06-09T08:15:00Z", + "pattern": "daily-triage", + "duration_s": 45, + "items_found": 4, + "actions_taken": 1, + "escalations": 0, + "tokens_estimate": 52000, + "outcome": "report-only | fix-proposed | escalated | no-op" +} +``` + +## Recent Runs + + \ No newline at end of file diff --git a/package.json b/package.json index ef1994e..6cc26c7 100644 --- a/package.json +++ b/package.json @@ -7,8 +7,9 @@ "check:loop-init": "node scripts/check-loop-init-sync.mjs", "test:loop-audit": "cd tools/loop-audit && npm test", "test:loop-init": "cd tools/loop-init && npm test", - "test:tools": "npm run test:loop-audit && npm run test:loop-init", - "build:tools": "cd tools/loop-audit && npm run build && cd ../loop-init && npm run build" + "test:loop-cost": "cd tools/loop-cost && npm test", + "test:tools": "npm run test:loop-audit && npm run test:loop-init && npm run test:loop-cost", + "build:tools": "cd tools/loop-audit && npm run build && cd ../loop-init && npm run build && cd ../loop-cost && npm run build" }, "devDependencies": { "ajv": "^8.17.1", diff --git a/patterns/changelog-drafter.md b/patterns/changelog-drafter.md index 502a7d6..95f700a 100644 --- a/patterns/changelog-drafter.md +++ b/patterns/changelog-drafter.md @@ -97,6 +97,22 @@ The loop should prune entries once a release is tagged/published and the draft i | Tone mismatch with project | Provide a short "Release voice" section in AGENTS.md or a project skill that the drafter reads. | | Accidentally publishing | Never grant the loop write access to tags or the live CHANGELOG without an explicit human gate + PR. | +## Cost Profile + +| Scenario | Tokens/run | Notes | +|----------|------------|-------| +| No-op (no new merges) | ~5k | Exit when nothing since last tag | +| Scan + categorize | ~35k | PR/commit scan | +| Draft + verify | ~80k | Categorized release notes draft | + +**Cadence**: 1d · **Tier**: low · **Suggested daily cap**: 100k tokens + +```bash +npx @cobusgreyling/loop-cost --pattern changelog-drafter --level L1 +``` + +One of the cheapest high-value loops. Safe to run alongside others. + ## Success Metrics - Time from "last merge" to "published release notes" (target: < 1 day for patch releases) diff --git a/patterns/ci-sweeper.md b/patterns/ci-sweeper.md index c4eb6e0..f29294f 100644 --- a/patterns/ci-sweeper.md +++ b/patterns/ci-sweeper.md @@ -93,6 +93,22 @@ Use `/goal` for focused "get green" sessions; use scheduled sweeper for ongoing | Token burn on red main | Pause loop after N failures; batch fixes | | Wrong branch targeted | Explicit branch allowlist in skill | +## Cost Profile + +| Scenario | Tokens/run | Notes | +|----------|------------|-------| +| No-op (CI green) | ~5k | **Required** — do not run full sweeper when green | +| Triage / classify | ~50k | Log parse + failure classification | +| Fix attempt (L2) | ~200k | Worktree + implementer + verifier | + +**Cadence**: 5m–15m · **Tier**: very-high · **Suggested daily cap**: 1M tokens · **Early exit required** + +```bash +npx @cobusgreyling/loop-cost --pattern ci-sweeper --cadence 15m --level L2 +``` + +At 15m cadence without early-exit, worst-case spend exceeds 5M tokens/day. Never run full action paths on every tick. + ## Success Metrics - Mean time to first proposed fix after CI goes red diff --git a/patterns/daily-triage.md b/patterns/daily-triage.md index d7636fb..a3cf364 100644 --- a/patterns/daily-triage.md +++ b/patterns/daily-triage.md @@ -91,6 +91,22 @@ Fields the loop must update every run: | Auto-fix on wrong priority | Start report-only; add explicit effort/risk gates | | Missed overnight failures | Add `fireImmediately: true` or run at start of day + mid-day | +## Cost Profile + +| Scenario | Tokens/run | Notes | +|----------|------------|-------| +| No-op | ~5k | Nothing actionable in state | +| Full triage (L1) | ~50k | CI + issues + commits scan | +| Assisted fix (L2) | ~200k | Worktree + implementer + verifier | + +**Cadence**: 1d–2h · **Tier**: low · **Suggested daily cap**: 100k tokens + +```bash +npx @cobusgreyling/loop-cost --pattern daily-triage --cadence 1d --level L1 +``` + +Scaffold `loop-budget.md` and `loop-run-log.md` with `loop-init`. See [operating-loops.md](../docs/operating-loops.md). + ## Success Metrics - Time from "something broke" to "human knows about it" diff --git a/patterns/dependency-sweeper.md b/patterns/dependency-sweeper.md index 3571f51..9248707 100644 --- a/patterns/dependency-sweeper.md +++ b/patterns/dependency-sweeper.md @@ -85,6 +85,22 @@ Prune merged/closed entries on every run. | Update touches dozens of transitive deps at once | Treat as high-risk. Only touch direct deps in the minimal-fix step. | | Notification spam | Only notify human for items in the "needs human" section of state. Everything else is silent or summarized daily. | +## Cost Profile + +| Scenario | Tokens/run | Notes | +|----------|------------|-------| +| No-op (nothing to bump) | ~5k | Exit when scan is clean | +| Triage / scan | ~60k | Audit + Dependabot + lockfile scan | +| Patch + verify (L2) | ~300k | Worktree + full test suite | + +**Cadence**: 6h–1d · **Tier**: medium · **Suggested daily cap**: 500k tokens + +```bash +npx @cobusgreyling/loop-cost --pattern dependency-sweeper --level L2 +``` + +Verifier runs (`npm ci && npm test`) dominate cost — cap attempts per package in `loop-budget.md`. + ## Success Metrics - Median time from "vulnerability published / Dependabot PR opened" → merged (for items the loop touched). diff --git a/patterns/post-merge-cleanup.md b/patterns/post-merge-cleanup.md index add5c8c..aaf35d6 100644 --- a/patterns/post-merge-cleanup.md +++ b/patterns/post-merge-cleanup.md @@ -91,6 +91,22 @@ Last run: 2026-06-09 22:00 UTC | Noise from every TODO | Only act on TODOs with merge context or linked tickets | | Competing with feature work | Run off-peak; cap auto-PRs per day (e.g. 2) | +## Cost Profile + +| Scenario | Tokens/run | Notes | +|----------|------------|-------| +| No-op | ~5k | No recent merges to scan | +| Scan + prioritize | ~40k | Merge list + TODO scan | +| Small fix (L2) | ~150k | Worktree + verifier | + +**Cadence**: 1d–6h · **Tier**: low · **Suggested daily cap**: 200k tokens + +```bash +npx @cobusgreyling/loop-cost --pattern post-merge-cleanup --cadence 1d --level L1 +``` + +Run off-peak. Cap auto-PRs per day in `loop-budget.md`. + ## Success Metrics - Reduction in "we forgot to remove X after merge" incidents diff --git a/patterns/pr-babysitter.md b/patterns/pr-babysitter.md index 33c07bd..2a4f48c 100644 --- a/patterns/pr-babysitter.md +++ b/patterns/pr-babysitter.md @@ -83,6 +83,22 @@ Example state entry: - **Stale state** → The loop should prune closed/merged PRs on every run. - **Notification fatigue** → Use selective notifications (only when human action is truly required). +## Cost Profile + +| Scenario | Tokens/run | Notes | +|----------|------------|-------| +| No-op (empty watchlist) | ~3k | **Target most runs** — exit early | +| Triage pass | ~80k | PR + CI status scan | +| Fix attempt (L2) | ~250k | Worktree + minimal-fix + verifier | + +**Cadence**: 5m–15m · **Tier**: high · **Suggested daily cap**: 2M tokens · **Early exit required** + +```bash +npx @cobusgreyling/loop-cost --pattern pr-babysitter --cadence 10m --level L1 --conservative +``` + +High cadence without early-exit burns tokens fast. Use `loop-budget` skill + `loop-run-log.md`. + ## Success Metrics - Average time from "ready for review" to merge (for PRs the loop touched). diff --git a/patterns/registry.schema.json b/patterns/registry.schema.json index 43e6ea6..0e215dc 100644 --- a/patterns/registry.schema.json +++ b/patterns/registry.schema.json @@ -79,6 +79,24 @@ "token_cost": { "type": "string", "enum": ["low", "medium", "high", "very-high"] + }, + "cost": { + "type": "object", + "required": [ + "tokens_noop", + "tokens_report", + "tokens_action", + "suggested_daily_cap", + "early_exit_required" + ], + "additionalProperties": false, + "properties": { + "tokens_noop": { "type": "integer", "minimum": 1000 }, + "tokens_report": { "type": "integer", "minimum": 1000 }, + "tokens_action": { "type": "integer", "minimum": 1000 }, + "suggested_daily_cap": { "type": "integer", "minimum": 10000 }, + "early_exit_required": { "type": "boolean" } + } } } } diff --git a/patterns/registry.yaml b/patterns/registry.yaml index 0dc7235..bd1c12d 100644 --- a/patterns/registry.yaml +++ b/patterns/registry.yaml @@ -1,5 +1,5 @@ # Machine-readable pattern registry for loop-engineering -# Used by loop-audit, docs site, and future tooling. +# Used by loop-audit, loop-cost, docs site, and tooling. patterns: - id: pr-babysitter @@ -16,6 +16,12 @@ patterns: starter: starters/pr-babysitter week_one_mode: L1 token_cost: high + cost: + tokens_noop: 3000 + tokens_report: 80000 + tokens_action: 250000 + suggested_daily_cap: 2000000 + early_exit_required: true - id: daily-triage name: Daily Triage @@ -31,6 +37,12 @@ patterns: starter: starters/minimal-loop week_one_mode: L1 token_cost: low + cost: + tokens_noop: 5000 + tokens_report: 50000 + tokens_action: 200000 + suggested_daily_cap: 100000 + early_exit_required: false - id: ci-sweeper name: CI Sweeper @@ -46,6 +58,12 @@ patterns: starter: starters/ci-sweeper week_one_mode: L2 token_cost: very-high + cost: + tokens_noop: 5000 + tokens_report: 50000 + tokens_action: 200000 + suggested_daily_cap: 1000000 + early_exit_required: true - id: post-merge-cleanup name: Post-Merge Cleanup @@ -61,6 +79,12 @@ patterns: starter: starters/post-merge-cleanup week_one_mode: L1 token_cost: low + cost: + tokens_noop: 5000 + tokens_report: 40000 + tokens_action: 150000 + suggested_daily_cap: 200000 + early_exit_required: false - id: dependency-sweeper name: Dependency Sweeper @@ -76,6 +100,12 @@ patterns: starter: starters/dependency-sweeper week_one_mode: L2 token_cost: medium + cost: + tokens_noop: 5000 + tokens_report: 60000 + tokens_action: 300000 + suggested_daily_cap: 500000 + early_exit_required: true - id: changelog-drafter name: Changelog Drafter @@ -90,4 +120,10 @@ patterns: human_gates: [breaking-changes, security, major-features, marketing-sensitive] starter: starters/changelog-drafter week_one_mode: L1 - token_cost: low \ No newline at end of file + token_cost: low + cost: + tokens_noop: 5000 + tokens_report: 35000 + tokens_action: 80000 + suggested_daily_cap: 100000 + early_exit_required: false \ No newline at end of file diff --git a/scripts/validate-registry.mjs b/scripts/validate-registry.mjs index acaa016..0bac3a6 100644 --- a/scripts/validate-registry.mjs +++ b/scripts/validate-registry.mjs @@ -44,6 +44,19 @@ function validatePattern(p, index) { if (!Array.isArray(p.human_gates) || p.human_gates.length === 0) fail(`${prefix}.human_gates must be non-empty`); if (p.week_one_mode && !VALID_MODES.has(p.week_one_mode)) fail(`${prefix}.week_one_mode invalid`); if (p.token_cost && !VALID_COST.has(p.token_cost)) fail(`${prefix}.token_cost invalid`); + if (!p.cost) fail(`${prefix} missing required field: cost`); + const costKeys = ['tokens_noop', 'tokens_report', 'tokens_action', 'suggested_daily_cap', 'early_exit_required']; + for (const key of costKeys) { + if (!(key in p.cost)) fail(`${prefix}.cost missing field: ${key}`); + } + for (const key of ['tokens_noop', 'tokens_report', 'tokens_action', 'suggested_daily_cap']) { + if (typeof p.cost[key] !== 'number' || p.cost[key] < 1000) { + fail(`${prefix}.cost.${key} must be a positive integer`); + } + } + if (typeof p.cost.early_exit_required !== 'boolean') { + fail(`${prefix}.cost.early_exit_required must be boolean`); + } } async function main() { diff --git a/skills/loop-budget/SKILL.md b/skills/loop-budget/SKILL.md new file mode 100644 index 0000000..f01c678 --- /dev/null +++ b/skills/loop-budget/SKILL.md @@ -0,0 +1,40 @@ +--- +name: loop-budget +description: Check token budget and run-log spend before and after a loop run. Enforces early exit when over budget or when there is no actionable work. +--- + +# Loop Budget Guard + +Run at the **start** and **end** of every loop iteration. + +## Start of run + +1. Read `loop-budget.md` for daily caps and kill-switch flags. +2. Read recent entries in `loop-run-log.md` (last 24h). +3. Sum `tokens_estimate` for the active pattern today. +4. If spend ≥ 80% of the pattern's daily cap → **report-only mode** (no sub-agents, no auto-fix). +5. If spend ≥ 100% or `loop-pause-all` is set → **exit immediately** with a one-line note in STATE.md. +6. If watchlist/state has no actionable items → **exit in <5k tokens** (do not spawn sub-agents). + +## End of run + +Append one JSON object to `loop-run-log.md`: + +```json +{ + "run_id": "", + "pattern": "", + "duration_s": , + "items_found": , + "actions_taken": , + "escalations": , + "tokens_estimate": , + "outcome": "no-op | report-only | fix-proposed | escalated" +} +``` + +## Rules + +- Never exceed `max sub-agent spawns/run` from `loop-budget.md`. +- High-cadence patterns (CI Sweeper, PR Babysitter) **must** early-exit when nothing is actionable. +- On self-throttle, append a line to `loop-budget.md` under **Alerts This Period**. \ No newline at end of file diff --git a/starters/minimal-loop/LOOP.md b/starters/minimal-loop/LOOP.md index 58e2245..2455074 100644 --- a/starters/minimal-loop/LOOP.md +++ b/starters/minimal-loop/LOOP.md @@ -13,8 +13,11 @@ ## Budget -- Max sub-agent spawns per run: 0 (L1) -- Review STATE.md daily +- Max sub-agent spawns per run: 0 (L1) / 2 (L2) +- Max tokens/day: 100k (see `loop-budget.md`) +- Append each run to `loop-run-log.md`; use `loop-budget` skill at start/end +- Kill switch: `loop-pause-all` — pause schedulers and notify human +- Estimate: `npx @cobusgreyling/loop-cost --pattern daily-triage` ## Links diff --git a/templates/SKILL.md.loop-budget b/templates/SKILL.md.loop-budget new file mode 100644 index 0000000..f01c678 --- /dev/null +++ b/templates/SKILL.md.loop-budget @@ -0,0 +1,40 @@ +--- +name: loop-budget +description: Check token budget and run-log spend before and after a loop run. Enforces early exit when over budget or when there is no actionable work. +--- + +# Loop Budget Guard + +Run at the **start** and **end** of every loop iteration. + +## Start of run + +1. Read `loop-budget.md` for daily caps and kill-switch flags. +2. Read recent entries in `loop-run-log.md` (last 24h). +3. Sum `tokens_estimate` for the active pattern today. +4. If spend ≥ 80% of the pattern's daily cap → **report-only mode** (no sub-agents, no auto-fix). +5. If spend ≥ 100% or `loop-pause-all` is set → **exit immediately** with a one-line note in STATE.md. +6. If watchlist/state has no actionable items → **exit in <5k tokens** (do not spawn sub-agents). + +## End of run + +Append one JSON object to `loop-run-log.md`: + +```json +{ + "run_id": "", + "pattern": "", + "duration_s": , + "items_found": , + "actions_taken": , + "escalations": , + "tokens_estimate": , + "outcome": "no-op | report-only | fix-proposed | escalated" +} +``` + +## Rules + +- Never exceed `max sub-agent spawns/run` from `loop-budget.md`. +- High-cadence patterns (CI Sweeper, PR Babysitter) **must** early-exit when nothing is actionable. +- On self-throttle, append a line to `loop-budget.md` under **Alerts This Period**. \ No newline at end of file diff --git a/tools/loop-audit/README.md b/tools/loop-audit/README.md index c5a7324..d6f8d97 100644 --- a/tools/loop-audit/README.md +++ b/tools/loop-audit/README.md @@ -63,6 +63,12 @@ npm publish --access public | MCP / connectors | Mentions or config files | | Worktree evidence | Isolation patterns in docs | | patterns/registry.yaml | Machine index for tooling | +| loop-budget.md | Token caps and kill switch | +| loop-run-log.md | Append-only run history | +| LOOP.md budget section | Cadence limits documented in config | +| loop-budget skill | Runtime budget guard | + +L3 requires budget doc + run log + LOOP.md budget section (in addition to verifier + state). ## Levels diff --git a/tools/loop-audit/dist/auditor.d.ts b/tools/loop-audit/dist/auditor.d.ts index 65548c9..a0b34e1 100644 --- a/tools/loop-audit/dist/auditor.d.ts +++ b/tools/loop-audit/dist/auditor.d.ts @@ -43,6 +43,12 @@ export interface LoopSignals { registry: { present: boolean; }; + cost: { + budgetDoc: boolean; + runLog: boolean; + loopMdBudget: boolean; + budgetSkill: boolean; + }; } export interface Finding { level: 'ok' | 'warn' | 'fail'; diff --git a/tools/loop-audit/dist/auditor.js b/tools/loop-audit/dist/auditor.js index 24d2e2b..326f202 100644 --- a/tools/loop-audit/dist/auditor.js +++ b/tools/loop-audit/dist/auditor.js @@ -23,6 +23,7 @@ const LOOP_SKILL_NAMES = [ const SAFETY_FILES = ['safety.md', 'docs/safety.md', 'SECURITY.md']; const MCP_FILES = ['.mcp.json', 'mcp.json', '.mcp/config.json']; const WORKTREE_HINTS = ['worktree', 'worktrees', 'git worktree']; +const BUDGET_HINTS = [/budget/i, /max tokens/i, /token cap/i, /kill switch/i, /loop-pause-all/i]; async function fileExists(p) { try { await stat(p); @@ -101,9 +102,20 @@ export function computeScore(signals) { score += 3; if (signals.registry.present) score += 2; + if (signals.cost.budgetDoc) + score += 3; + if (signals.cost.runLog) + score += 3; + if (signals.cost.loopMdBudget) + score += 2; + if (signals.cost.budgetSkill) + score += 2; score = Math.min(100, Math.max(0, score)); + const costReady = signals.cost.budgetDoc && + signals.cost.runLog && + signals.cost.loopMdBudget; let level = 'L0'; - if (score >= 78 && signals.verifier.present && signals.stateFile.present) + if (score >= 78 && signals.verifier.present && signals.stateFile.present && costReady) level = 'L3'; else if (score >= 58 && signals.triage.present) level = 'L2'; @@ -111,10 +123,15 @@ export function computeScore(signals) { level = 'L1'; else level = 'L0'; - const assessment = score >= 82 ? 'Strong loop readiness — good candidate for L3 with explicit gates.' : - score >= 62 ? 'Good foundation — add missing verifier + safety docs for L3.' : - score >= 42 ? 'Early loop setup — focus on L1 state + triage before enabling actions.' : - 'Not loop-ready — start with a starter from this repo (minimal-loop or pr-babysitter).'; + const assessment = score >= 82 && costReady + ? 'Strong loop readiness — good candidate for L3 with explicit gates.' + : score >= 82 && !costReady + ? 'Strong signals but missing cost observability (loop-budget.md, loop-run-log.md, LOOP.md budget) — add before L3.' + : score >= 62 + ? 'Good foundation — add missing verifier + safety docs for L3.' + : score >= 42 + ? 'Early loop setup — focus on L1 state + triage before enabling actions.' + : 'Not loop-ready — start with a starter from this repo (minimal-loop or pr-babysitter).'; return { score, level, assessment }; } export async function auditProject(target) { @@ -182,6 +199,22 @@ export async function auditProject(target) { catch { } } const registryPresent = await fileExists(path.join(root, 'patterns', 'registry.yaml')); + const budgetDoc = await fileExists(path.join(root, 'loop-budget.md')); + const runLog = await fileExists(path.join(root, 'loop-run-log.md')); + const loopMdBudget = BUDGET_HINTS.some((re) => re.test(loopMdContent)); + const budgetSkillDirs = [ + path.join(root, 'skills', 'loop-budget'), + path.join(root, '.grok', 'skills', 'loop-budget'), + path.join(root, '.claude', 'skills', 'loop-budget'), + path.join(root, '.codex', 'skills', 'loop-budget'), + ]; + let budgetSkill = false; + for (const dir of budgetSkillDirs) { + if (await fileExists(path.join(dir, 'SKILL.md'))) { + budgetSkill = true; + break; + } + } const signals = { stateFile: { present: statePaths.length > 0, paths: statePaths }, loopConfig: { present: loopMd, path: loopMd ? 'LOOP.md' : undefined }, @@ -196,6 +229,7 @@ export async function auditProject(target) { mcp: { present: mcpPresent }, worktreeEvidence: { present: worktreeEvidence }, registry: { present: registryPresent }, + cost: { budgetDoc, runLog, loopMdBudget, budgetSkill }, }; if (!signals.stateFile.present) { findings.push({ level: 'fail', message: 'No state file (STATE.md or pattern-specific state).' }); @@ -260,7 +294,41 @@ export async function auditProject(target) { findings.push({ level: 'warn', message: 'No patterns/registry.yaml (machine-readable index for future tools).' }); recommendations.push('Add patterns/registry.yaml following the existing format'); } + if (!signals.cost.budgetDoc) { + findings.push({ level: 'warn', message: 'No loop-budget.md — token caps and kill switch undocumented.' }); + recommendations.push('Scaffold with loop-init or copy templates/loop-budget.md.template'); + } + else { + findings.push({ level: 'ok', message: 'loop-budget.md present.' }); + } + if (!signals.cost.runLog) { + findings.push({ level: 'warn', message: 'No loop-run-log.md — run history not persisted.' }); + recommendations.push('Copy templates/loop-run-log.md.template to loop-run-log.md'); + } + else { + findings.push({ level: 'ok', message: 'loop-run-log.md present.' }); + } + if (!signals.cost.loopMdBudget) { + findings.push({ level: 'warn', message: 'LOOP.md does not mention budget, token caps, or kill switch.' }); + recommendations.push('Add a Budget section to LOOP.md (see starters/*/LOOP.md)'); + } + if (!signals.cost.budgetSkill) { + findings.push({ level: 'warn', message: 'No loop-budget skill — budget checks are not automated at runtime.' }); + recommendations.push('Add loop-budget skill via loop-init or templates/SKILL.md.loop-budget'); + } + else { + findings.push({ level: 'ok', message: 'loop-budget skill present.' }); + } const { score, level, assessment } = computeScore(signals); + const costReady = signals.cost.budgetDoc && + signals.cost.runLog && + signals.cost.loopMdBudget; + if (score >= 78 && signals.verifier.present && signals.stateFile.present && !costReady) { + findings.push({ + level: 'warn', + message: 'Score qualifies for L3 but cost observability is incomplete — capped at L2 until budget + run log + LOOP.md budget exist.', + }); + } return { target: root, score, diff --git a/tools/loop-audit/dist/cli.js b/tools/loop-audit/dist/cli.js index 462d404..4dc76cf 100644 --- a/tools/loop-audit/dist/cli.js +++ b/tools/loop-audit/dist/cli.js @@ -55,6 +55,8 @@ try { console.log(' # All tools:'); console.log(' cp starters/minimal-loop/STATE.md.example STATE.md # or -claude / -codex variant'); console.log(' cp starters/minimal-loop/LOOP.md .'); + console.log(' cp templates/loop-budget.md.template loop-budget.md'); + console.log(' cp templates/loop-run-log.md.template loop-run-log.md'); console.log(''); console.log(' # Maker/checker verifier (Grok / generic skills dir)'); console.log(' mkdir -p .grok/skills/loop-verifier'); @@ -69,6 +71,7 @@ try { console.log(''); console.log(' # Or scaffold automatically:'); console.log(' npx @cobusgreyling/loop-init . --pattern daily-triage --tool grok'); + console.log(' npx @cobusgreyling/loop-cost --pattern daily-triage --level L1'); console.log(''); console.log('See docs/loop-design-checklist.md and patterns/ for full guidance.'); } diff --git a/tools/loop-audit/src/auditor.ts b/tools/loop-audit/src/auditor.ts index de7bad8..015efc3 100644 --- a/tools/loop-audit/src/auditor.ts +++ b/tools/loop-audit/src/auditor.ts @@ -15,6 +15,12 @@ export interface LoopSignals { mcp: { present: boolean }; worktreeEvidence: { present: boolean }; registry: { present: boolean }; + cost: { + budgetDoc: boolean; + runLog: boolean; + loopMdBudget: boolean; + budgetSkill: boolean; + }; } export interface Finding { @@ -57,6 +63,7 @@ const LOOP_SKILL_NAMES = [ const SAFETY_FILES = ['safety.md', 'docs/safety.md', 'SECURITY.md']; const MCP_FILES = ['.mcp.json', 'mcp.json', '.mcp/config.json']; const WORKTREE_HINTS = ['worktree', 'worktrees', 'git worktree']; +const BUDGET_HINTS = [/budget/i, /max tokens/i, /token cap/i, /kill switch/i, /loop-pause-all/i]; async function fileExists(p: string): Promise { try { @@ -121,20 +128,34 @@ export function computeScore(signals: LoopSignals): { score: number; level: 'L0' if (signals.mcp.present) score += 3; if (signals.worktreeEvidence.present) score += 3; if (signals.registry.present) score += 2; + if (signals.cost.budgetDoc) score += 3; + if (signals.cost.runLog) score += 3; + if (signals.cost.loopMdBudget) score += 2; + if (signals.cost.budgetSkill) score += 2; score = Math.min(100, Math.max(0, score)); + const costReady = + signals.cost.budgetDoc && + signals.cost.runLog && + signals.cost.loopMdBudget; + let level: 'L0' | 'L1' | 'L2' | 'L3' = 'L0'; - if (score >= 78 && signals.verifier.present && signals.stateFile.present) level = 'L3'; + if (score >= 78 && signals.verifier.present && signals.stateFile.present && costReady) level = 'L3'; else if (score >= 58 && signals.triage.present) level = 'L2'; else if (score >= 38 && signals.stateFile.present) level = 'L1'; else level = 'L0'; const assessment = - score >= 82 ? 'Strong loop readiness — good candidate for L3 with explicit gates.' : - score >= 62 ? 'Good foundation — add missing verifier + safety docs for L3.' : - score >= 42 ? 'Early loop setup — focus on L1 state + triage before enabling actions.' : - 'Not loop-ready — start with a starter from this repo (minimal-loop or pr-babysitter).'; + score >= 82 && costReady + ? 'Strong loop readiness — good candidate for L3 with explicit gates.' + : score >= 82 && !costReady + ? 'Strong signals but missing cost observability (loop-budget.md, loop-run-log.md, LOOP.md budget) — add before L3.' + : score >= 62 + ? 'Good foundation — add missing verifier + safety docs for L3.' + : score >= 42 + ? 'Early loop setup — focus on L1 state + triage before enabling actions.' + : 'Not loop-ready — start with a starter from this repo (minimal-loop or pr-babysitter).'; return { score, level, assessment }; } @@ -207,6 +228,24 @@ export async function auditProject(target: string): Promise { const registryPresent = await fileExists(path.join(root, 'patterns', 'registry.yaml')); + const budgetDoc = await fileExists(path.join(root, 'loop-budget.md')); + const runLog = await fileExists(path.join(root, 'loop-run-log.md')); + const loopMdBudget = BUDGET_HINTS.some((re) => re.test(loopMdContent)); + + const budgetSkillDirs = [ + path.join(root, 'skills', 'loop-budget'), + path.join(root, '.grok', 'skills', 'loop-budget'), + path.join(root, '.claude', 'skills', 'loop-budget'), + path.join(root, '.codex', 'skills', 'loop-budget'), + ]; + let budgetSkill = false; + for (const dir of budgetSkillDirs) { + if (await fileExists(path.join(dir, 'SKILL.md'))) { + budgetSkill = true; + break; + } + } + const signals: LoopSignals = { stateFile: { present: statePaths.length > 0, paths: statePaths }, loopConfig: { present: loopMd, path: loopMd ? 'LOOP.md' : undefined }, @@ -221,6 +260,7 @@ export async function auditProject(target: string): Promise { mcp: { present: mcpPresent }, worktreeEvidence: { present: worktreeEvidence }, registry: { present: registryPresent }, + cost: { budgetDoc, runLog, loopMdBudget, budgetSkill }, }; if (!signals.stateFile.present) { @@ -291,8 +331,46 @@ export async function auditProject(target: string): Promise { recommendations.push('Add patterns/registry.yaml following the existing format'); } + if (!signals.cost.budgetDoc) { + findings.push({ level: 'warn', message: 'No loop-budget.md — token caps and kill switch undocumented.' }); + recommendations.push('Scaffold with loop-init or copy templates/loop-budget.md.template'); + } else { + findings.push({ level: 'ok', message: 'loop-budget.md present.' }); + } + + if (!signals.cost.runLog) { + findings.push({ level: 'warn', message: 'No loop-run-log.md — run history not persisted.' }); + recommendations.push('Copy templates/loop-run-log.md.template to loop-run-log.md'); + } else { + findings.push({ level: 'ok', message: 'loop-run-log.md present.' }); + } + + if (!signals.cost.loopMdBudget) { + findings.push({ level: 'warn', message: 'LOOP.md does not mention budget, token caps, or kill switch.' }); + recommendations.push('Add a Budget section to LOOP.md (see starters/*/LOOP.md)'); + } + + if (!signals.cost.budgetSkill) { + findings.push({ level: 'warn', message: 'No loop-budget skill — budget checks are not automated at runtime.' }); + recommendations.push('Add loop-budget skill via loop-init or templates/SKILL.md.loop-budget'); + } else { + findings.push({ level: 'ok', message: 'loop-budget skill present.' }); + } + const { score, level, assessment } = computeScore(signals); + const costReady = + signals.cost.budgetDoc && + signals.cost.runLog && + signals.cost.loopMdBudget; + + if (score >= 78 && signals.verifier.present && signals.stateFile.present && !costReady) { + findings.push({ + level: 'warn', + message: 'Score qualifies for L3 but cost observability is incomplete — capped at L2 until budget + run log + LOOP.md budget exist.', + }); + } + return { target: root, score, diff --git a/tools/loop-audit/src/cli.ts b/tools/loop-audit/src/cli.ts index d66181e..4541200 100644 --- a/tools/loop-audit/src/cli.ts +++ b/tools/loop-audit/src/cli.ts @@ -56,6 +56,8 @@ try { console.log(' # All tools:'); console.log(' cp starters/minimal-loop/STATE.md.example STATE.md # or -claude / -codex variant'); console.log(' cp starters/minimal-loop/LOOP.md .'); + console.log(' cp templates/loop-budget.md.template loop-budget.md'); + console.log(' cp templates/loop-run-log.md.template loop-run-log.md'); console.log(''); console.log(' # Maker/checker verifier (Grok / generic skills dir)'); console.log(' mkdir -p .grok/skills/loop-verifier'); @@ -70,6 +72,7 @@ try { console.log(''); console.log(' # Or scaffold automatically:'); console.log(' npx @cobusgreyling/loop-init . --pattern daily-triage --tool grok'); + console.log(' npx @cobusgreyling/loop-cost --pattern daily-triage --level L1'); console.log(''); console.log('See docs/loop-design-checklist.md and patterns/ for full guidance.'); } diff --git a/tools/loop-audit/test/auditor.test.mjs b/tools/loop-audit/test/auditor.test.mjs index 804b030..6a3f3c7 100644 --- a/tools/loop-audit/test/auditor.test.mjs +++ b/tools/loop-audit/test/auditor.test.mjs @@ -20,6 +20,7 @@ function emptySignals() { mcp: { present: false }, worktreeEvidence: { present: false }, registry: { present: false }, + cost: { budgetDoc: false, runLog: false, loopMdBudget: false, budgetSkill: false }, }; } @@ -62,11 +63,29 @@ test('computeScore: L3 requires verifier and high score', () => { s.mcp = { present: true }; s.worktreeEvidence = { present: true }; s.registry = { present: true }; + s.cost = { budgetDoc: true, runLog: true, loopMdBudget: true, budgetSkill: true }; const { level, score } = computeScore(s); assert.equal(level, 'L3'); assert.ok(score >= 78); }); +test('computeScore: L3 blocked without cost observability', () => { + const s = emptySignals(); + s.stateFile = { present: true, paths: ['STATE.md'] }; + s.triage = { present: true }; + s.loopConfig = { present: true, path: 'LOOP.md' }; + s.agentsMd = { present: true }; + s.skills = { count: 3, loopSkills: ['loop-triage', 'minimal-fix', 'loop-verifier'] }; + s.verifier = { present: true }; + s.safety = { loopMdMentionsSafety: true, safetyDocPresent: true }; + s.github = { present: true, workflows: true }; + s.mcp = { present: true }; + s.worktreeEvidence = { present: true }; + s.registry = { present: true }; + const { level } = computeScore(s); + assert.equal(level, 'L2'); +}); + test('auditProject: empty directory scores low', async () => { const dir = await mkdtemp(path.join(tmpdir(), 'loop-audit-empty-')); try { diff --git a/tools/loop-cost/README.md b/tools/loop-cost/README.md new file mode 100644 index 0000000..7395749 --- /dev/null +++ b/tools/loop-cost/README.md @@ -0,0 +1,44 @@ +# loop-cost + +Estimate daily token spend for [loop engineering](https://github.com/cobusgreyling/loop-engineering) patterns by cadence and readiness level (L1–L3). + +Uses cost metadata from `patterns/registry.yaml`. + +## Install & Run + +```bash +npx @cobusgreyling/loop-cost --pattern ci-sweeper --cadence 15m --level L2 +npx @cobusgreyling/loop-cost --pattern daily-triage --level L1 --json +npx @cobusgreyling/loop-cost --list +``` + +**From this repo:** + +```bash +cd tools/loop-cost +npm install +npm test +``` + +## Options + +| Flag | Description | +|------|-------------| +| `--pattern` | Pattern id (see `--list`) | +| `--cadence` | Override cadence (e.g. `15m`, `1d`) | +| `--level` | `L1`, `L2`, or `L3` (default `L1`) | +| `--conservative` | Use slower cadence from ranges | +| `--json` | Machine-readable output | + +## Scenarios + +Each estimate includes: + +- **Early-exit / no-op** — empty watchlist, minimal tokens +- **Full triage** — every run does a full scan +- **Action every run** — implementer + verifier every time (worst case) +- **Realistic blend** — level-based mix (documented in output) + +Pair with `loop-budget.md` (scaffolded by `loop-init`) and `loop-audit` cost observability checks. + +See [docs/operating-loops.md](../../docs/operating-loops.md). \ No newline at end of file diff --git a/tools/loop-cost/dist/cli.d.ts b/tools/loop-cost/dist/cli.d.ts new file mode 100644 index 0000000..b798801 --- /dev/null +++ b/tools/loop-cost/dist/cli.d.ts @@ -0,0 +1,2 @@ +#!/usr/bin/env node +export {}; diff --git a/tools/loop-cost/dist/cli.js b/tools/loop-cost/dist/cli.js new file mode 100644 index 0000000..a999ff9 --- /dev/null +++ b/tools/loop-cost/dist/cli.js @@ -0,0 +1,109 @@ +#!/usr/bin/env node +import { readFile, access } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import yaml from 'yaml'; +import { estimateCost, formatEstimateHuman, } from './estimator.js'; +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const PACKAGE_ROOT = path.resolve(__dirname, '..'); +function parseArgs(argv) { + let pattern = 'daily-triage'; + let cadence; + let level = 'L1'; + let conservative = false; + let json = false; + let list = false; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === '--pattern' || a === '-p') + pattern = argv[++i]; + else if (a === '--cadence' || a === '-c') + cadence = argv[++i]; + else if (a === '--level' || a === '-l') + level = argv[++i]; + else if (a === '--conservative') + conservative = true; + else if (a === '--json') + json = true; + else if (a === '--list') + list = true; + else if (a === '--help' || a === '-h') + return { help: true }; + } + return { help: false, pattern, cadence, level, conservative, json, list }; +} +async function loadRegistry() { + const candidates = [ + path.join(PACKAGE_ROOT, 'registry.json'), + path.resolve(PACKAGE_ROOT, '../../patterns/registry.yaml'), + ]; + for (const p of candidates) { + try { + await access(p); + const raw = await readFile(p, 'utf8'); + if (p.endsWith('.json')) + return JSON.parse(raw); + return yaml.parse(raw); + } + catch { + /* try next */ + } + } + throw new Error('Pattern registry not found. Run from loop-engineering repo or install @cobusgreyling/loop-cost.'); +} +async function main() { + const args = parseArgs(process.argv.slice(2)); + if (args.help) { + console.log(`loop-cost — estimate daily token spend for loop patterns + +Usage: + loop-cost --pattern [options] + +Options: + -p, --pattern Pattern id (default: daily-triage) + -c, --cadence Override cadence (e.g. 15m, 1d, 5m-15m) + -l, --level Readiness level (default: L1) + --conservative Use slower cadence from ranges (e.g. 15m not 5m) + --json Machine-readable output + --list List pattern ids + -h, --help This help + +Examples: + loop-cost --pattern ci-sweeper --cadence 15m --level L2 + loop-cost --pattern daily-triage --level L1 --json + loop-cost --list +`); + process.exit(0); + } + const registry = await loadRegistry(); + if (args.list) { + for (const p of registry.patterns) { + console.log(`${p.id}\t${p.token_cost}\t${p.cadence}`); + } + return; + } + const pattern = registry.patterns.find((p) => p.id === args.pattern); + if (!pattern) { + console.error(`Unknown pattern: ${args.pattern}. Use --list for ids.`); + process.exit(1); + } + if (!pattern.cost) { + console.error(`Pattern ${args.pattern} has no cost block in registry.`); + process.exit(1); + } + const result = estimateCost({ + pattern, + cadence: args.cadence, + level: args.level, + conservative: args.conservative, + }); + if (args.json) + console.log(JSON.stringify(result, null, 2)); + else + console.log(formatEstimateHuman(result)); +} +main().catch((err) => { + const msg = err instanceof Error ? err.message : String(err); + console.error('loop-cost failed:', msg); + process.exit(1); +}); diff --git a/tools/loop-cost/dist/estimator.d.ts b/tools/loop-cost/dist/estimator.d.ts new file mode 100644 index 0000000..5ce001e --- /dev/null +++ b/tools/loop-cost/dist/estimator.d.ts @@ -0,0 +1,64 @@ +export type ReadinessLevel = 'L1' | 'L2' | 'L3'; +export interface PatternCost { + tokens_noop: number; + tokens_report: number; + tokens_action: number; + suggested_daily_cap: number; + early_exit_required: boolean; +} +export interface RegistryPattern { + id: string; + name: string; + cadence: string; + token_cost: string; + cost: PatternCost; +} +export interface RegistryDoc { + patterns: RegistryPattern[]; +} +export interface EstimateInput { + pattern: RegistryPattern; + cadence?: string; + level: ReadinessLevel; + conservative?: boolean; +} +export interface EstimateResult { + patternId: string; + patternName: string; + cadence: string; + level: ReadinessLevel; + runsPerDay: number; + tokenCostTier: string; + suggestedDailyCap: number; + earlyExitRequired: boolean; + scenarios: { + noop: { + tokensPerRun: number; + tokensPerDay: number; + }; + report: { + tokensPerRun: number; + tokensPerDay: number; + }; + action: { + tokensPerRun: number; + tokensPerDay: number; + }; + realistic: { + tokensPerRun: number; + tokensPerDay: number; + assumptions: string; + }; + }; + warnings: string[]; +} +export declare function parseInterval(token: string): number; +/** Runs per day for a single interval like 15m or 1d. */ +export declare function runsPerDayForInterval(interval: string): number; +/** + * Parse pattern cadence (e.g. 5m-15m, 1d-2h) into runs/day. + * conservative=true picks the slower cadence in a range. + */ +export declare function cadenceToRunsPerDay(cadence: string, conservative?: boolean): number; +export declare function estimateCost(input: EstimateInput): EstimateResult; +export declare function formatEstimateHuman(r: EstimateResult): string; diff --git a/tools/loop-cost/dist/estimator.js b/tools/loop-cost/dist/estimator.js new file mode 100644 index 0000000..0437465 --- /dev/null +++ b/tools/loop-cost/dist/estimator.js @@ -0,0 +1,135 @@ +const INTERVAL_MS = { + m: 60_000, + h: 3_600_000, + d: 86_400_000, +}; +export function parseInterval(token) { + const m = token.match(/^(\d+)([mhd])$/); + if (!m) + throw new Error(`Invalid cadence interval: ${token}`); + const unit = m[2]; + return Number(m[1]) * INTERVAL_MS[unit]; +} +/** Runs per day for a single interval like 15m or 1d. */ +export function runsPerDayForInterval(interval) { + const ms = parseInterval(interval); + return Math.floor(86_400_000 / ms); +} +/** + * Parse pattern cadence (e.g. 5m-15m, 1d-2h) into runs/day. + * conservative=true picks the slower cadence in a range. + */ +export function cadenceToRunsPerDay(cadence, conservative = false) { + const parts = cadence.split('-').map((p) => p.trim()); + if (parts.length === 1) + return runsPerDayForInterval(parts[0]); + const runs = parts.map(runsPerDayForInterval); + return conservative ? Math.min(...runs) : Math.max(...runs); +} +function realisticMix(level, earlyExitRequired) { + if (level === 'L1') { + return { + noop: earlyExitRequired ? 0.9 : 0.6, + report: earlyExitRequired ? 0.1 : 0.4, + action: 0, + assumptions: earlyExitRequired + ? 'L1: 90% early-exit, 10% full triage' + : 'L1: 60% no-op, 40% full triage', + }; + } + if (level === 'L2') { + return { + noop: earlyExitRequired ? 0.85 : 0.5, + report: earlyExitRequired ? 0.1 : 0.3, + action: earlyExitRequired ? 0.05 : 0.2, + assumptions: earlyExitRequired + ? 'L2: 85% early-exit, 10% triage, 5% implementer+verifier' + : 'L2: 50% no-op, 30% triage, 20% action', + }; + } + return { + noop: 0.4, + report: 0.35, + action: 0.25, + assumptions: 'L3: 40% no-op, 35% triage, 25% action (unattended — monitor closely)', + }; +} +function formatTokens(n) { + if (n >= 1_000_000) + return `${(n / 1_000_000).toFixed(1)}M`; + if (n >= 1_000) + return `${Math.round(n / 1_000)}k`; + return String(n); +} +export function estimateCost(input) { + const cadence = input.cadence ?? input.pattern.cadence; + const runsPerDay = cadenceToRunsPerDay(cadence, input.conservative); + const { cost, token_cost: tokenCostTier } = input.pattern; + const mix = realisticMix(input.level, cost.early_exit_required); + const noopDay = cost.tokens_noop * runsPerDay; + const reportDay = cost.tokens_report * runsPerDay; + const actionDay = cost.tokens_action * runsPerDay; + const realisticPerRun = cost.tokens_noop * mix.noop + + cost.tokens_report * mix.report + + cost.tokens_action * mix.action; + const realisticDay = Math.round(realisticPerRun * runsPerDay); + const warnings = []; + if (cost.early_exit_required) { + warnings.push('Early-exit triage is required — empty watchlist should exit in <5k tokens.'); + } + if (actionDay > cost.suggested_daily_cap) { + warnings.push(`Worst case (action every run) exceeds suggested cap (${formatTokens(cost.suggested_daily_cap)}/day).`); + } + if (realisticDay > cost.suggested_daily_cap) { + warnings.push(`Realistic estimate exceeds suggested daily cap — slow cadence or tighten scope.`); + } + if (runsPerDay >= 96) { + warnings.push(`High cadence (${runsPerDay} runs/day) — verify early-exit is working.`); + } + return { + patternId: input.pattern.id, + patternName: input.pattern.name, + cadence, + level: input.level, + runsPerDay, + tokenCostTier, + suggestedDailyCap: cost.suggested_daily_cap, + earlyExitRequired: cost.early_exit_required, + scenarios: { + noop: { tokensPerRun: cost.tokens_noop, tokensPerDay: noopDay }, + report: { tokensPerRun: cost.tokens_report, tokensPerDay: reportDay }, + action: { tokensPerRun: cost.tokens_action, tokensPerDay: actionDay }, + realistic: { + tokensPerRun: Math.round(realisticPerRun), + tokensPerDay: realisticDay, + assumptions: mix.assumptions, + }, + }, + warnings, + }; +} +export function formatEstimateHuman(r) { + const lines = []; + lines.push(''); + lines.push(`Loop Cost Estimate — ${r.patternName} (${r.patternId})`); + lines.push('═'.repeat(50)); + lines.push(`Cadence: ${r.cadence} → ${r.runsPerDay} runs/day`); + lines.push(`Level: ${r.level} · Registry tier: ${r.tokenCostTier}`); + lines.push(`Suggested daily cap: ${formatTokens(r.suggestedDailyCap)} tokens`); + lines.push(''); + lines.push('Daily token estimates:'); + lines.push(` Early-exit / no-op: ${formatTokens(r.scenarios.noop.tokensPerDay)} (${formatTokens(r.scenarios.noop.tokensPerRun)}/run)`); + lines.push(` Full triage: ${formatTokens(r.scenarios.report.tokensPerDay)} (${formatTokens(r.scenarios.report.tokensPerRun)}/run)`); + lines.push(` Action every run: ${formatTokens(r.scenarios.action.tokensPerDay)} (${formatTokens(r.scenarios.action.tokensPerRun)}/run)`); + lines.push(` Realistic blend: ${formatTokens(r.scenarios.realistic.tokensPerDay)} (${r.scenarios.realistic.assumptions})`); + if (r.warnings.length) { + lines.push(''); + lines.push('Warnings:'); + for (const w of r.warnings) + lines.push(` ! ${w}`); + } + lines.push(''); + lines.push('Docs: docs/operating-loops.md · Scaffold: npx @cobusgreyling/loop-init'); + lines.push(''); + return lines.join('\n'); +} diff --git a/tools/loop-cost/package-lock.json b/tools/loop-cost/package-lock.json new file mode 100644 index 0000000..7cc7ecc --- /dev/null +++ b/tools/loop-cost/package-lock.json @@ -0,0 +1,72 @@ +{ + "name": "@cobusgreyling/loop-cost", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@cobusgreyling/loop-cost", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "yaml": "^2.8.0" + }, + "bin": { + "loop-cost": "dist/cli.js" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "typescript": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/node": { + "version": "20.19.42", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.42.tgz", + "integrity": "sha512-5L7SUaFC1RyDraj2yRhyBzHTobyXHmohD100CChNtyPyleoq37Mqab5Gn8XEKI04dfN/oqPdpHk38MgcQWHbZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + } + } +} diff --git a/tools/loop-cost/package.json b/tools/loop-cost/package.json new file mode 100644 index 0000000..e9b6c09 --- /dev/null +++ b/tools/loop-cost/package.json @@ -0,0 +1,47 @@ +{ + "name": "@cobusgreyling/loop-cost", + "version": "1.0.0", + "description": "Estimate daily token spend for loop engineering patterns by cadence and readiness level.", + "type": "module", + "bin": { + "loop-cost": "./dist/cli.js" + }, + "files": [ + "dist", + "registry.json", + "README.md" + ], + "scripts": { + "bundle": "node scripts/bundle-registry.mjs", + "build": "npm run bundle && tsc", + "test": "npm run build && node --test test/estimator.test.mjs", + "prepublishOnly": "npm test", + "start": "node dist/cli.js" + }, + "engines": { + "node": ">=18" + }, + "keywords": [ + "loop-engineering", + "ai-agents", + "token-budget", + "cost-estimate" + ], + "author": "Cobus Greyling", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/cobusgreyling/loop-engineering.git", + "directory": "tools/loop-cost" + }, + "publishConfig": { + "access": "public" + }, + "dependencies": { + "yaml": "^2.8.0" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "typescript": "^5.0.0" + } +} \ No newline at end of file diff --git a/tools/loop-cost/registry.json b/tools/loop-cost/registry.json new file mode 100644 index 0000000..53ac6e6 --- /dev/null +++ b/tools/loop-cost/registry.json @@ -0,0 +1,252 @@ +{ + "patterns": [ + { + "id": "pr-babysitter", + "name": "PR Babysitter", + "file": "pr-babysitter.md", + "goal": "Shepherd PRs through review, CI, rebase, and merge", + "cadence": "5m-15m", + "risk": "medium", + "tools": [ + "grok", + "claude-code", + "codex", + "github-actions" + ], + "skills": [ + "pr-review-triage", + "minimal-fix", + "rebase-and-clean" + ], + "state": "pr-babysitter-state.md", + "phases": [ + "discover", + "triage", + "fix", + "verify", + "notify" + ], + "human_gates": [ + "security", + "payments", + "auth", + "max-fix-attempts" + ], + "starter": "starters/pr-babysitter", + "week_one_mode": "L1", + "token_cost": "high", + "cost": { + "tokens_noop": 3000, + "tokens_report": 80000, + "tokens_action": 250000, + "suggested_daily_cap": 2000000, + "early_exit_required": true + } + }, + { + "id": "daily-triage", + "name": "Daily Triage", + "file": "daily-triage.md", + "goal": "Prioritized morning scan of CI, issues, commits, and chat", + "cadence": "1d-2h", + "risk": "low", + "tools": [ + "grok", + "claude-code", + "codex", + "github-actions" + ], + "skills": [ + "loop-triage", + "minimal-fix" + ], + "state": "STATE.md", + "phases": [ + "report", + "act-small-wins", + "escalate" + ], + "human_gates": [ + "design-decisions", + "multi-file-refactors" + ], + "starter": "starters/minimal-loop", + "week_one_mode": "L1", + "token_cost": "low", + "cost": { + "tokens_noop": 5000, + "tokens_report": 50000, + "tokens_action": 200000, + "suggested_daily_cap": 100000, + "early_exit_required": false + } + }, + { + "id": "ci-sweeper", + "name": "CI Sweeper", + "file": "ci-sweeper.md", + "goal": "React to failing CI with minimal fixes and escalation", + "cadence": "5m-15m", + "risk": "medium", + "tools": [ + "grok", + "claude-code", + "codex", + "github-actions" + ], + "skills": [ + "ci-triage", + "minimal-fix" + ], + "state": "ci-sweeper-state.md", + "phases": [ + "detect", + "classify", + "fix", + "verify", + "escalate" + ], + "human_gates": [ + "infra-failures", + "max-attempts", + "security-tests" + ], + "starter": "starters/ci-sweeper", + "week_one_mode": "L2", + "token_cost": "very-high", + "cost": { + "tokens_noop": 5000, + "tokens_report": 50000, + "tokens_action": 200000, + "suggested_daily_cap": 1000000, + "early_exit_required": true + } + }, + { + "id": "post-merge-cleanup", + "name": "Post-Merge Cleanup", + "file": "post-merge-cleanup.md", + "goal": "Follow-up tech debt and cleanup after merges to main", + "cadence": "1d-6h", + "risk": "low", + "tools": [ + "grok", + "claude-code", + "codex", + "github-actions" + ], + "skills": [ + "post-merge-scan", + "minimal-fix" + ], + "state": "post-merge-state.md", + "phases": [ + "scan-merges", + "prioritize", + "fix-small", + "ticket-large" + ], + "human_gates": [ + "architectural-debt", + "feature-flags", + "large-diffs" + ], + "starter": "starters/post-merge-cleanup", + "week_one_mode": "L1", + "token_cost": "low", + "cost": { + "tokens_noop": 5000, + "tokens_report": 40000, + "tokens_action": 150000, + "suggested_daily_cap": 200000, + "early_exit_required": false + } + }, + { + "id": "dependency-sweeper", + "name": "Dependency Sweeper", + "file": "dependency-sweeper.md", + "goal": "Discover, safely apply, and verify dependency + vulnerability updates with human gates on risky changes", + "cadence": "6h-1d", + "risk": "medium", + "tools": [ + "grok", + "claude-code", + "codex", + "github-actions" + ], + "skills": [ + "dependency-triage", + "minimal-fix", + "loop-verifier" + ], + "state": "dependency-sweeper-state.md", + "phases": [ + "scan", + "triage-risk", + "patch-safe", + "verify-worktree", + "escalate-risky" + ], + "human_gates": [ + "major-bumps", + "high-sev-cve", + "denylisted-packages", + "max-attempts" + ], + "starter": "starters/dependency-sweeper", + "week_one_mode": "L2", + "token_cost": "medium", + "cost": { + "tokens_noop": 5000, + "tokens_report": 60000, + "tokens_action": 300000, + "suggested_daily_cap": 500000, + "early_exit_required": true + } + }, + { + "id": "changelog-drafter", + "name": "Changelog Drafter", + "file": "changelog-drafter.md", + "goal": "Scan merged PRs and commits, draft categorized high-quality release notes or CHANGELOG entries for human review", + "cadence": "1d", + "risk": "low", + "tools": [ + "grok", + "claude-code", + "codex", + "github-actions" + ], + "skills": [ + "changelog-scan", + "draft-release-notes", + "loop-verifier" + ], + "state": "changelog-drafter-state.md", + "phases": [ + "scan-merges", + "categorize", + "draft", + "review", + "publish" + ], + "human_gates": [ + "breaking-changes", + "security", + "major-features", + "marketing-sensitive" + ], + "starter": "starters/changelog-drafter", + "week_one_mode": "L1", + "token_cost": "low", + "cost": { + "tokens_noop": 5000, + "tokens_report": 35000, + "tokens_action": 80000, + "suggested_daily_cap": 100000, + "early_exit_required": false + } + } + ] +} \ No newline at end of file diff --git a/tools/loop-cost/scripts/bundle-registry.mjs b/tools/loop-cost/scripts/bundle-registry.mjs new file mode 100644 index 0000000..e81387a --- /dev/null +++ b/tools/loop-cost/scripts/bundle-registry.mjs @@ -0,0 +1,20 @@ +#!/usr/bin/env node +import { readFile, writeFile, access } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import yaml from 'yaml'; + +const PACKAGE_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const REPO_REGISTRY = path.resolve(PACKAGE_ROOT, '../../patterns/registry.yaml'); +const DEST = path.join(PACKAGE_ROOT, 'registry.json'); + +try { + await access(REPO_REGISTRY); +} catch { + console.log('bundle-registry: no monorepo registry — keeping existing registry.json'); + process.exit(0); +} + +const doc = yaml.parse(await readFile(REPO_REGISTRY, 'utf8')); +await writeFile(DEST, JSON.stringify(doc, null, 2)); +console.log('bundled patterns/registry.yaml → tools/loop-cost/registry.json'); \ No newline at end of file diff --git a/tools/loop-cost/src/cli.ts b/tools/loop-cost/src/cli.ts new file mode 100644 index 0000000..9a61c60 --- /dev/null +++ b/tools/loop-cost/src/cli.ts @@ -0,0 +1,118 @@ +#!/usr/bin/env node +import { readFile, access } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import yaml from 'yaml'; +import { + estimateCost, + formatEstimateHuman, + type ReadinessLevel, + type RegistryDoc, +} from './estimator.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const PACKAGE_ROOT = path.resolve(__dirname, '..'); + +function parseArgs(argv: string[]) { + let pattern = 'daily-triage'; + let cadence: string | undefined; + let level: ReadinessLevel = 'L1'; + let conservative = false; + let json = false; + let list = false; + + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === '--pattern' || a === '-p') pattern = argv[++i]; + else if (a === '--cadence' || a === '-c') cadence = argv[++i]; + else if (a === '--level' || a === '-l') level = argv[++i] as ReadinessLevel; + else if (a === '--conservative') conservative = true; + else if (a === '--json') json = true; + else if (a === '--list') list = true; + else if (a === '--help' || a === '-h') return { help: true as const }; + } + + return { help: false as const, pattern, cadence, level, conservative, json, list }; +} + +async function loadRegistry(): Promise { + const candidates = [ + path.join(PACKAGE_ROOT, 'registry.json'), + path.resolve(PACKAGE_ROOT, '../../patterns/registry.yaml'), + ]; + + for (const p of candidates) { + try { + await access(p); + const raw = await readFile(p, 'utf8'); + if (p.endsWith('.json')) return JSON.parse(raw) as RegistryDoc; + return yaml.parse(raw) as RegistryDoc; + } catch { + /* try next */ + } + } + throw new Error('Pattern registry not found. Run from loop-engineering repo or install @cobusgreyling/loop-cost.'); +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + if (args.help) { + console.log(`loop-cost — estimate daily token spend for loop patterns + +Usage: + loop-cost --pattern [options] + +Options: + -p, --pattern Pattern id (default: daily-triage) + -c, --cadence Override cadence (e.g. 15m, 1d, 5m-15m) + -l, --level Readiness level (default: L1) + --conservative Use slower cadence from ranges (e.g. 15m not 5m) + --json Machine-readable output + --list List pattern ids + -h, --help This help + +Examples: + loop-cost --pattern ci-sweeper --cadence 15m --level L2 + loop-cost --pattern daily-triage --level L1 --json + loop-cost --list +`); + process.exit(0); + } + + const registry = await loadRegistry(); + + if (args.list) { + for (const p of registry.patterns) { + console.log(`${p.id}\t${p.token_cost}\t${p.cadence}`); + } + return; + } + + const pattern = registry.patterns.find((p) => p.id === args.pattern); + if (!pattern) { + console.error(`Unknown pattern: ${args.pattern}. Use --list for ids.`); + process.exit(1); + } + + if (!pattern.cost) { + console.error(`Pattern ${args.pattern} has no cost block in registry.`); + process.exit(1); + } + + const result = estimateCost({ + pattern, + cadence: args.cadence, + level: args.level, + conservative: args.conservative, + }); + + if (args.json) console.log(JSON.stringify(result, null, 2)); + else console.log(formatEstimateHuman(result)); +} + +main().catch((err: unknown) => { + const msg = err instanceof Error ? err.message : String(err); + console.error('loop-cost failed:', msg); + process.exit(1); +}); \ No newline at end of file diff --git a/tools/loop-cost/src/estimator.ts b/tools/loop-cost/src/estimator.ts new file mode 100644 index 0000000..d0b30cd --- /dev/null +++ b/tools/loop-cost/src/estimator.ts @@ -0,0 +1,197 @@ +export type ReadinessLevel = 'L1' | 'L2' | 'L3'; + +export interface PatternCost { + tokens_noop: number; + tokens_report: number; + tokens_action: number; + suggested_daily_cap: number; + early_exit_required: boolean; +} + +export interface RegistryPattern { + id: string; + name: string; + cadence: string; + token_cost: string; + cost: PatternCost; +} + +export interface RegistryDoc { + patterns: RegistryPattern[]; +} + +export interface EstimateInput { + pattern: RegistryPattern; + cadence?: string; + level: ReadinessLevel; + conservative?: boolean; +} + +export interface EstimateResult { + patternId: string; + patternName: string; + cadence: string; + level: ReadinessLevel; + runsPerDay: number; + tokenCostTier: string; + suggestedDailyCap: number; + earlyExitRequired: boolean; + scenarios: { + noop: { tokensPerRun: number; tokensPerDay: number }; + report: { tokensPerRun: number; tokensPerDay: number }; + action: { tokensPerRun: number; tokensPerDay: number }; + realistic: { tokensPerRun: number; tokensPerDay: number; assumptions: string }; + }; + warnings: string[]; +} + +const INTERVAL_MS: Record = { + m: 60_000, + h: 3_600_000, + d: 86_400_000, +}; + +export function parseInterval(token: string): number { + const m = token.match(/^(\d+)([mhd])$/); + if (!m) throw new Error(`Invalid cadence interval: ${token}`); + const unit = m[2] as keyof typeof INTERVAL_MS; + return Number(m[1]) * INTERVAL_MS[unit]; +} + +/** Runs per day for a single interval like 15m or 1d. */ +export function runsPerDayForInterval(interval: string): number { + const ms = parseInterval(interval); + return Math.floor(86_400_000 / ms); +} + +/** + * Parse pattern cadence (e.g. 5m-15m, 1d-2h) into runs/day. + * conservative=true picks the slower cadence in a range. + */ +export function cadenceToRunsPerDay(cadence: string, conservative = false): number { + const parts = cadence.split('-').map((p) => p.trim()); + if (parts.length === 1) return runsPerDayForInterval(parts[0]); + + const runs = parts.map(runsPerDayForInterval); + return conservative ? Math.min(...runs) : Math.max(...runs); +} + +function realisticMix(level: ReadinessLevel, earlyExitRequired: boolean): { + noop: number; + report: number; + action: number; + assumptions: string; +} { + if (level === 'L1') { + return { + noop: earlyExitRequired ? 0.9 : 0.6, + report: earlyExitRequired ? 0.1 : 0.4, + action: 0, + assumptions: earlyExitRequired + ? 'L1: 90% early-exit, 10% full triage' + : 'L1: 60% no-op, 40% full triage', + }; + } + if (level === 'L2') { + return { + noop: earlyExitRequired ? 0.85 : 0.5, + report: earlyExitRequired ? 0.1 : 0.3, + action: earlyExitRequired ? 0.05 : 0.2, + assumptions: earlyExitRequired + ? 'L2: 85% early-exit, 10% triage, 5% implementer+verifier' + : 'L2: 50% no-op, 30% triage, 20% action', + }; + } + return { + noop: 0.4, + report: 0.35, + action: 0.25, + assumptions: 'L3: 40% no-op, 35% triage, 25% action (unattended — monitor closely)', + }; +} + +function formatTokens(n: number): string { + if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`; + if (n >= 1_000) return `${Math.round(n / 1_000)}k`; + return String(n); +} + +export function estimateCost(input: EstimateInput): EstimateResult { + const cadence = input.cadence ?? input.pattern.cadence; + const runsPerDay = cadenceToRunsPerDay(cadence, input.conservative); + const { cost, token_cost: tokenCostTier } = input.pattern; + const mix = realisticMix(input.level, cost.early_exit_required); + + const noopDay = cost.tokens_noop * runsPerDay; + const reportDay = cost.tokens_report * runsPerDay; + const actionDay = cost.tokens_action * runsPerDay; + + const realisticPerRun = + cost.tokens_noop * mix.noop + + cost.tokens_report * mix.report + + cost.tokens_action * mix.action; + const realisticDay = Math.round(realisticPerRun * runsPerDay); + + const warnings: string[] = []; + if (cost.early_exit_required) { + warnings.push('Early-exit triage is required — empty watchlist should exit in <5k tokens.'); + } + if (actionDay > cost.suggested_daily_cap) { + warnings.push( + `Worst case (action every run) exceeds suggested cap (${formatTokens(cost.suggested_daily_cap)}/day).`, + ); + } + if (realisticDay > cost.suggested_daily_cap) { + warnings.push(`Realistic estimate exceeds suggested daily cap — slow cadence or tighten scope.`); + } + if (runsPerDay >= 96) { + warnings.push(`High cadence (${runsPerDay} runs/day) — verify early-exit is working.`); + } + + return { + patternId: input.pattern.id, + patternName: input.pattern.name, + cadence, + level: input.level, + runsPerDay, + tokenCostTier, + suggestedDailyCap: cost.suggested_daily_cap, + earlyExitRequired: cost.early_exit_required, + scenarios: { + noop: { tokensPerRun: cost.tokens_noop, tokensPerDay: noopDay }, + report: { tokensPerRun: cost.tokens_report, tokensPerDay: reportDay }, + action: { tokensPerRun: cost.tokens_action, tokensPerDay: actionDay }, + realistic: { + tokensPerRun: Math.round(realisticPerRun), + tokensPerDay: realisticDay, + assumptions: mix.assumptions, + }, + }, + warnings, + }; +} + +export function formatEstimateHuman(r: EstimateResult): string { + const lines: string[] = []; + lines.push(''); + lines.push(`Loop Cost Estimate — ${r.patternName} (${r.patternId})`); + lines.push('═'.repeat(50)); + lines.push(`Cadence: ${r.cadence} → ${r.runsPerDay} runs/day`); + lines.push(`Level: ${r.level} · Registry tier: ${r.tokenCostTier}`); + lines.push(`Suggested daily cap: ${formatTokens(r.suggestedDailyCap)} tokens`); + lines.push(''); + lines.push('Daily token estimates:'); + lines.push(` Early-exit / no-op: ${formatTokens(r.scenarios.noop.tokensPerDay)} (${formatTokens(r.scenarios.noop.tokensPerRun)}/run)`); + lines.push(` Full triage: ${formatTokens(r.scenarios.report.tokensPerDay)} (${formatTokens(r.scenarios.report.tokensPerRun)}/run)`); + lines.push(` Action every run: ${formatTokens(r.scenarios.action.tokensPerDay)} (${formatTokens(r.scenarios.action.tokensPerRun)}/run)`); + lines.push(` Realistic blend: ${formatTokens(r.scenarios.realistic.tokensPerDay)} (${r.scenarios.realistic.assumptions})`); + if (r.warnings.length) { + lines.push(''); + lines.push('Warnings:'); + for (const w of r.warnings) lines.push(` ! ${w}`); + } + lines.push(''); + lines.push('Docs: docs/operating-loops.md · Scaffold: npx @cobusgreyling/loop-init'); + lines.push(''); + return lines.join('\n'); +} \ No newline at end of file diff --git a/tools/loop-cost/test/estimator.test.mjs b/tools/loop-cost/test/estimator.test.mjs new file mode 100644 index 0000000..c481b57 --- /dev/null +++ b/tools/loop-cost/test/estimator.test.mjs @@ -0,0 +1,70 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + cadenceToRunsPerDay, + runsPerDayForInterval, + estimateCost, +} from '../dist/estimator.js'; + +const CI_SWEEPER = { + id: 'ci-sweeper', + name: 'CI Sweeper', + cadence: '5m-15m', + token_cost: 'very-high', + cost: { + tokens_noop: 5000, + tokens_report: 50000, + tokens_action: 200000, + suggested_daily_cap: 1000000, + early_exit_required: true, + }, +}; + +test('runsPerDayForInterval: 15m = 96', () => { + assert.equal(runsPerDayForInterval('15m'), 96); +}); + +test('runsPerDayForInterval: 1d = 1', () => { + assert.equal(runsPerDayForInterval('1d'), 1); +}); + +test('cadenceToRunsPerDay: range uses fastest by default', () => { + assert.equal(cadenceToRunsPerDay('5m-15m'), 288); +}); + +test('cadenceToRunsPerDay: conservative uses slowest', () => { + assert.equal(cadenceToRunsPerDay('5m-15m', true), 96); +}); + +test('estimateCost: ci-sweeper 15m L2 warns on high spend', () => { + const r = estimateCost({ + pattern: CI_SWEEPER, + cadence: '15m', + level: 'L2', + }); + assert.equal(r.runsPerDay, 96); + assert.ok(r.scenarios.action.tokensPerDay > r.suggestedDailyCap); + assert.ok(r.warnings.length > 0); + assert.ok(r.scenarios.realistic.tokensPerDay < r.scenarios.action.tokensPerDay); +}); + +test('estimateCost: daily-triage 1d L1 is cheap', () => { + const r = estimateCost({ + pattern: { + id: 'daily-triage', + name: 'Daily Triage', + cadence: '1d', + token_cost: 'low', + cost: { + tokens_noop: 5000, + tokens_report: 50000, + tokens_action: 200000, + suggested_daily_cap: 100000, + early_exit_required: false, + }, + }, + level: 'L1', + }); + assert.equal(r.runsPerDay, 1); + assert.ok(r.scenarios.realistic.tokensPerDay <= 100000); +}); \ No newline at end of file diff --git a/tools/loop-cost/tsconfig.json b/tools/loop-cost/tsconfig.json new file mode 100644 index 0000000..3aaa0bd --- /dev/null +++ b/tools/loop-cost/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "outDir": "dist", + "rootDir": "src", + "strict": true, + "skipLibCheck": true, + "declaration": true + }, + "include": ["src/**/*"] +} \ No newline at end of file diff --git a/tools/loop-init/README.md b/tools/loop-init/README.md index 8479119..f0fa247 100644 --- a/tools/loop-init/README.md +++ b/tools/loop-init/README.md @@ -25,6 +25,12 @@ See [docs/RELEASE.md](../../docs/RELEASE.md) for npm publish tags. The published L2 patterns (`ci-sweeper`, `dependency-sweeper`) also copy `minimal-fix` and `loop-verifier` templates when missing from the starter. +Every scaffold also creates: + +- `loop-budget.md` — pattern-specific daily caps and kill switch +- `loop-run-log.md` — append-only run history +- `loop-budget` skill — runtime budget guard at start/end of each run + ## Tools - `grok` (default) @@ -40,8 +46,9 @@ cd tools/loop-init && npm ci && npm test node dist/cli.js /path/to/project --pattern daily-triage --tool grok ``` -Pair with `loop-audit` after scaffolding: +Pair with `loop-audit` and `loop-cost` after scaffolding: ```bash +npx @cobusgreyling/loop-cost --pattern daily-triage --level L1 npx @cobusgreyling/loop-audit . --suggest ``` \ No newline at end of file diff --git a/tools/loop-init/dist/cli.js b/tools/loop-init/dist/cli.js index 16d1001..a61b669 100644 --- a/tools/loop-init/dist/cli.js +++ b/tools/loop-init/dist/cli.js @@ -34,6 +34,15 @@ const STATE_FILES = { 'post-merge-cleanup': 'post-merge-state.md', 'changelog-drafter': 'changelog-drafter-state.md', }; +/** Mirrors patterns/registry.yaml cost caps — used when scaffolding observability files. */ +const PATTERN_BUDGET = { + 'daily-triage': { name: 'Daily Triage', maxRunsPerDay: 2, dailyCap: 100_000, maxSpawnsL1: 0, maxSpawnsL2: 2 }, + 'pr-babysitter': { name: 'PR Babysitter', maxRunsPerDay: 288, dailyCap: 2_000_000, maxSpawnsL1: 0, maxSpawnsL2: 3 }, + 'ci-sweeper': { name: 'CI Sweeper', maxRunsPerDay: 96, dailyCap: 1_000_000, maxSpawnsL1: 0, maxSpawnsL2: 3 }, + 'dependency-sweeper': { name: 'Dependency Sweeper', maxRunsPerDay: 4, dailyCap: 500_000, maxSpawnsL1: 0, maxSpawnsL2: 3 }, + 'post-merge-cleanup': { name: 'Post-Merge Cleanup', maxRunsPerDay: 1, dailyCap: 200_000, maxSpawnsL1: 0, maxSpawnsL2: 2 }, + 'changelog-drafter': { name: 'Changelog Drafter', maxRunsPerDay: 1, dailyCap: 100_000, maxSpawnsL1: 0, maxSpawnsL2: 2 }, +}; function parseArgs(argv) { let pattern = 'daily-triage'; let tool = 'grok'; @@ -126,6 +135,62 @@ async function copyL2Templates(pattern, tool, targetDir, templatesRoot, dryRun) await copyTemplateVerifier(templatesRoot, targetDir, tool, dryRun); } } +function formatTokenCap(n) { + if (n >= 1_000_000) + return `${n / 1_000_000}M`; + if (n >= 1_000) + return `${n / 1_000}k`; + return String(n); +} +function buildLoopBudgetMd(pattern) { + const b = PATTERN_BUDGET[pattern]; + return `# Loop Budget — YOUR_PROJECT + +> Primary loop: **${b.name}** (scaffolded by loop-init) + +## Daily limits + +| Loop | Max runs/day | Max tokens/day | Max sub-agent spawns/run | +|------|--------------|----------------|--------------------------| +| ${b.name} | ${b.maxRunsPerDay} | ${formatTokenCap(b.dailyCap)} | ${b.maxSpawnsL1} (L1) / ${b.maxSpawnsL2} (L2) | + +## On budget exceed + +1. Pause schedulers (\`scheduler_delete\` or disable automations) +2. Append event to \`loop-run-log.md\` +3. Notify human (Slack / issue / STATE.md High Priority) + +## Kill switch + +- Command or issue label: \`loop-pause-all\` +- Resume only after human clears the flag in STATE.md + +## Estimate spend + +\`\`\`bash +npx @cobusgreyling/loop-cost --pattern ${pattern} +\`\`\` +`; +} +async function scaffoldObservability(pattern, tool, targetDir, templatesRoot, dryRun) { + const budgetPath = path.join(targetDir, 'loop-budget.md'); + const runLogTemplate = path.join(templatesRoot, 'loop-run-log.md.template'); + const runLogPath = path.join(targetDir, 'loop-run-log.md'); + if (!(await exists(budgetPath))) { + const content = buildLoopBudgetMd(pattern); + if (dryRun) { + console.log(` would write: ${budgetPath}`); + } + else { + await writeFile(budgetPath, content); + console.log(` created: loop-budget.md`); + } + } + if (!(await exists(runLogPath))) { + await copyFile(runLogTemplate, runLogPath, dryRun); + } + await copyTemplateSkill(templatesRoot, 'SKILL.md.loop-budget', targetDir, tool, 'loop-budget', dryRun); +} async function copyFile(src, dest, dryRun) { if (!(await exists(src))) return false; @@ -267,6 +332,7 @@ Examples: await copyFile(loopMd, path.join(targetDir, 'LOOP.md'), dryRun); } await copyL2Templates(pattern, tool, targetDir, templatesRoot, dryRun); + await scaffoldObservability(pattern, tool, targetDir, templatesRoot, dryRun); if (!dryRun && !(await exists(path.join(targetDir, 'AGENTS.md')))) { const agentsTemplate = `# AGENTS.md @@ -283,6 +349,7 @@ npm run lint } console.log('\n=== Next steps ==='); console.log(` npx @cobusgreyling/loop-audit ${target === '.' ? '.' : target} --suggest`); + console.log(` npx @cobusgreyling/loop-cost --pattern ${pattern}`); console.log(` First loop command (${tool}):\n ${firstLoopCommand(pattern, tool)}\n`); } async function readDirNames(dir) { diff --git a/tools/loop-init/registry.yaml b/tools/loop-init/registry.yaml new file mode 100644 index 0000000..bd1c12d --- /dev/null +++ b/tools/loop-init/registry.yaml @@ -0,0 +1,129 @@ +# Machine-readable pattern registry for loop-engineering +# Used by loop-audit, loop-cost, docs site, and tooling. + +patterns: + - id: pr-babysitter + name: PR Babysitter + file: pr-babysitter.md + goal: Shepherd PRs through review, CI, rebase, and merge + cadence: 5m-15m + risk: medium + tools: [grok, claude-code, codex, github-actions] + skills: [pr-review-triage, minimal-fix, rebase-and-clean] + state: pr-babysitter-state.md + phases: [discover, triage, fix, verify, notify] + human_gates: [security, payments, auth, max-fix-attempts] + starter: starters/pr-babysitter + week_one_mode: L1 + token_cost: high + cost: + tokens_noop: 3000 + tokens_report: 80000 + tokens_action: 250000 + suggested_daily_cap: 2000000 + early_exit_required: true + + - id: daily-triage + name: Daily Triage + file: daily-triage.md + goal: Prioritized morning scan of CI, issues, commits, and chat + cadence: 1d-2h + risk: low + tools: [grok, claude-code, codex, github-actions] + skills: [loop-triage, minimal-fix] + state: STATE.md + phases: [report, act-small-wins, escalate] + human_gates: [design-decisions, multi-file-refactors] + starter: starters/minimal-loop + week_one_mode: L1 + token_cost: low + cost: + tokens_noop: 5000 + tokens_report: 50000 + tokens_action: 200000 + suggested_daily_cap: 100000 + early_exit_required: false + + - id: ci-sweeper + name: CI Sweeper + file: ci-sweeper.md + goal: React to failing CI with minimal fixes and escalation + cadence: 5m-15m + risk: medium + tools: [grok, claude-code, codex, github-actions] + skills: [ci-triage, minimal-fix] + state: ci-sweeper-state.md + phases: [detect, classify, fix, verify, escalate] + human_gates: [infra-failures, max-attempts, security-tests] + starter: starters/ci-sweeper + week_one_mode: L2 + token_cost: very-high + cost: + tokens_noop: 5000 + tokens_report: 50000 + tokens_action: 200000 + suggested_daily_cap: 1000000 + early_exit_required: true + + - id: post-merge-cleanup + name: Post-Merge Cleanup + file: post-merge-cleanup.md + goal: Follow-up tech debt and cleanup after merges to main + cadence: 1d-6h + risk: low + tools: [grok, claude-code, codex, github-actions] + skills: [post-merge-scan, minimal-fix] + state: post-merge-state.md + phases: [scan-merges, prioritize, fix-small, ticket-large] + human_gates: [architectural-debt, feature-flags, large-diffs] + starter: starters/post-merge-cleanup + week_one_mode: L1 + token_cost: low + cost: + tokens_noop: 5000 + tokens_report: 40000 + tokens_action: 150000 + suggested_daily_cap: 200000 + early_exit_required: false + + - id: dependency-sweeper + name: Dependency Sweeper + file: dependency-sweeper.md + goal: Discover, safely apply, and verify dependency + vulnerability updates with human gates on risky changes + cadence: 6h-1d + risk: medium + tools: [grok, claude-code, codex, github-actions] + skills: [dependency-triage, minimal-fix, loop-verifier] + state: dependency-sweeper-state.md + phases: [scan, triage-risk, patch-safe, verify-worktree, escalate-risky] + human_gates: [major-bumps, high-sev-cve, denylisted-packages, max-attempts] + starter: starters/dependency-sweeper + week_one_mode: L2 + token_cost: medium + cost: + tokens_noop: 5000 + tokens_report: 60000 + tokens_action: 300000 + suggested_daily_cap: 500000 + early_exit_required: true + + - id: changelog-drafter + name: Changelog Drafter + file: changelog-drafter.md + goal: Scan merged PRs and commits, draft categorized high-quality release notes or CHANGELOG entries for human review + cadence: 1d + risk: low + tools: [grok, claude-code, codex, github-actions] + skills: [changelog-scan, draft-release-notes, loop-verifier] + state: changelog-drafter-state.md + phases: [scan-merges, categorize, draft, review, publish] + human_gates: [breaking-changes, security, major-features, marketing-sensitive] + starter: starters/changelog-drafter + week_one_mode: L1 + token_cost: low + cost: + tokens_noop: 5000 + tokens_report: 35000 + tokens_action: 80000 + suggested_daily_cap: 100000 + early_exit_required: false \ No newline at end of file diff --git a/tools/loop-init/scripts/bundle-assets.mjs b/tools/loop-init/scripts/bundle-assets.mjs index 3ce6744..0158f6e 100644 --- a/tools/loop-init/scripts/bundle-assets.mjs +++ b/tools/loop-init/scripts/bundle-assets.mjs @@ -25,4 +25,11 @@ for (const dir of ['starters', 'templates']) { await rm(dest, { recursive: true, force: true }); await cp(src, dest, { recursive: true }); console.log(`bundled ${dir}/ → tools/loop-init/${dir}/`); +} + +const registrySrc = path.join(REPO_ROOT, 'patterns', 'registry.yaml'); +const registryDest = path.join(PACKAGE_ROOT, 'registry.yaml'); +if (await exists(registrySrc)) { + await cp(registrySrc, registryDest); + console.log('bundled patterns/registry.yaml → tools/loop-init/registry.yaml'); } \ No newline at end of file diff --git a/tools/loop-init/src/cli.ts b/tools/loop-init/src/cli.ts index dbc36e8..4abaefa 100644 --- a/tools/loop-init/src/cli.ts +++ b/tools/loop-init/src/cli.ts @@ -51,6 +51,19 @@ const STATE_FILES: Record = { 'changelog-drafter': 'changelog-drafter-state.md', }; +/** Mirrors patterns/registry.yaml cost caps — used when scaffolding observability files. */ +const PATTERN_BUDGET: Record< + Pattern, + { name: string; maxRunsPerDay: number; dailyCap: number; maxSpawnsL1: number; maxSpawnsL2: number } +> = { + 'daily-triage': { name: 'Daily Triage', maxRunsPerDay: 2, dailyCap: 100_000, maxSpawnsL1: 0, maxSpawnsL2: 2 }, + 'pr-babysitter': { name: 'PR Babysitter', maxRunsPerDay: 288, dailyCap: 2_000_000, maxSpawnsL1: 0, maxSpawnsL2: 3 }, + 'ci-sweeper': { name: 'CI Sweeper', maxRunsPerDay: 96, dailyCap: 1_000_000, maxSpawnsL1: 0, maxSpawnsL2: 3 }, + 'dependency-sweeper': { name: 'Dependency Sweeper', maxRunsPerDay: 4, dailyCap: 500_000, maxSpawnsL1: 0, maxSpawnsL2: 3 }, + 'post-merge-cleanup': { name: 'Post-Merge Cleanup', maxRunsPerDay: 1, dailyCap: 200_000, maxSpawnsL1: 0, maxSpawnsL2: 2 }, + 'changelog-drafter': { name: 'Changelog Drafter', maxRunsPerDay: 1, dailyCap: 100_000, maxSpawnsL1: 0, maxSpawnsL2: 2 }, +}; + function parseArgs(argv: string[]) { let pattern: Pattern = 'daily-triage'; let tool: Tool = 'grok'; @@ -163,6 +176,71 @@ async function copyL2Templates( } } +function formatTokenCap(n: number): string { + if (n >= 1_000_000) return `${n / 1_000_000}M`; + if (n >= 1_000) return `${n / 1_000}k`; + return String(n); +} + +function buildLoopBudgetMd(pattern: Pattern): string { + const b = PATTERN_BUDGET[pattern]; + return `# Loop Budget — YOUR_PROJECT + +> Primary loop: **${b.name}** (scaffolded by loop-init) + +## Daily limits + +| Loop | Max runs/day | Max tokens/day | Max sub-agent spawns/run | +|------|--------------|----------------|--------------------------| +| ${b.name} | ${b.maxRunsPerDay} | ${formatTokenCap(b.dailyCap)} | ${b.maxSpawnsL1} (L1) / ${b.maxSpawnsL2} (L2) | + +## On budget exceed + +1. Pause schedulers (\`scheduler_delete\` or disable automations) +2. Append event to \`loop-run-log.md\` +3. Notify human (Slack / issue / STATE.md High Priority) + +## Kill switch + +- Command or issue label: \`loop-pause-all\` +- Resume only after human clears the flag in STATE.md + +## Estimate spend + +\`\`\`bash +npx @cobusgreyling/loop-cost --pattern ${pattern} +\`\`\` +`; +} + +async function scaffoldObservability( + pattern: Pattern, + tool: Tool, + targetDir: string, + templatesRoot: string, + dryRun: boolean, +) { + const budgetPath = path.join(targetDir, 'loop-budget.md'); + const runLogTemplate = path.join(templatesRoot, 'loop-run-log.md.template'); + const runLogPath = path.join(targetDir, 'loop-run-log.md'); + + if (!(await exists(budgetPath))) { + const content = buildLoopBudgetMd(pattern); + if (dryRun) { + console.log(` would write: ${budgetPath}`); + } else { + await writeFile(budgetPath, content); + console.log(` created: loop-budget.md`); + } + } + + if (!(await exists(runLogPath))) { + await copyFile(runLogTemplate, runLogPath, dryRun); + } + + await copyTemplateSkill(templatesRoot, 'SKILL.md.loop-budget', targetDir, tool, 'loop-budget', dryRun); +} + async function copyFile(src: string, dest: string, dryRun: boolean) { if (!(await exists(src))) return false; if (dryRun) { @@ -318,6 +396,7 @@ Examples: } await copyL2Templates(pattern, tool, targetDir, templatesRoot, dryRun); + await scaffoldObservability(pattern, tool, targetDir, templatesRoot, dryRun); if (!dryRun && !(await exists(path.join(targetDir, 'AGENTS.md')))) { const agentsTemplate = `# AGENTS.md @@ -336,6 +415,7 @@ npm run lint console.log('\n=== Next steps ==='); console.log(` npx @cobusgreyling/loop-audit ${target === '.' ? '.' : target} --suggest`); + console.log(` npx @cobusgreyling/loop-cost --pattern ${pattern}`); console.log(` First loop command (${tool}):\n ${firstLoopCommand(pattern, tool)}\n`); } diff --git a/tools/loop-init/test/cli.test.mjs b/tools/loop-init/test/cli.test.mjs index 5cfb95a..f1ea5d8 100644 --- a/tools/loop-init/test/cli.test.mjs +++ b/tools/loop-init/test/cli.test.mjs @@ -41,6 +41,9 @@ test('loop-init scaffolds ci-sweeper with bundled assets', async () => { await access(path.join(dir, '.grok', 'skills', 'ci-triage', 'SKILL.md')); await access(path.join(dir, '.grok', 'skills', 'minimal-fix', 'SKILL.md')); await access(path.join(dir, '.grok', 'skills', 'loop-verifier', 'SKILL.md')); + await access(path.join(dir, 'loop-budget.md')); + await access(path.join(dir, 'loop-run-log.md')); + await access(path.join(dir, '.grok', 'skills', 'loop-budget', 'SKILL.md')); } finally { await rm(dir, { recursive: true, force: true }); }