diff --git a/.gitignore b/.gitignore index 95b0f5a01..1cc0eb894 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,6 @@ docs/Koopa-Learning.md docs/narrative-rewrite-scope.md docs/audit-prompts/ .site-server.pid + +# Compiled binaries +/publish diff --git a/publish b/publish deleted file mode 100755 index 022846445..000000000 Binary files a/publish and /dev/null differ diff --git a/site/.dockerignore b/site/.dockerignore deleted file mode 100644 index 195970b4a..000000000 --- a/site/.dockerignore +++ /dev/null @@ -1,16 +0,0 @@ -node_modules -dist -.git -.github -.vscode -.angular -.claude -*.md -!README.md -.dockerignore -Dockerfile -docker-compose.yml -e2e -tailwind/css -.env -.env.* diff --git a/site/Dockerfile b/site/Dockerfile deleted file mode 100644 index 323f355d8..000000000 --- a/site/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -# Stage 1: Build -FROM node:22-alpine AS builder - -WORKDIR /app - -# .npmrc carries legacy-peer-deps=true so `npm ci` resolves the lockfile -# despite angular-eslint 21.4's pre-v22 @angular/cli peer range. -COPY package.json package-lock.json .npmrc ./ -RUN npm ci - -COPY . . -RUN npm run build - -# Stage 2: Runtime (SSR) -FROM node:22-alpine AS runtime - -WORKDIR /app - -COPY --from=builder /app/dist/koopa0dev/ ./dist/koopa0dev/ -COPY --from=builder /app/node_modules ./node_modules -COPY --from=builder /app/package.json ./ - -RUN addgroup -S app && adduser -S app -G app - -ENV NODE_ENV=production -ENV PORT=4000 - -USER app - -EXPOSE 4000 - -CMD ["node", "dist/koopa0dev/server/server.mjs"] diff --git a/site/docker-compose.yml b/site/docker-compose.yml deleted file mode 100644 index a5522302e..000000000 --- a/site/docker-compose.yml +++ /dev/null @@ -1,12 +0,0 @@ -services: - frontend: - build: - context: . - dockerfile: Dockerfile - ports: - - '4000:4000' - environment: - - NODE_ENV=production - - PORT=4000 - - SITE_URL=https://koopa0.dev - restart: unless-stopped diff --git a/skills/koopa-system.skill b/skills/koopa-system.skill deleted file mode 100644 index 4b73c147d..000000000 Binary files a/skills/koopa-system.skill and /dev/null differ diff --git a/skills/koopa-system/SKILL.md b/skills/koopa-system/SKILL.md deleted file mode 100644 index 02dbc911a..000000000 --- a/skills/koopa-system/SKILL.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -name: koopa-system -description: "Koopa Studio 系統操作手冊 — MCP 工具使用、角色定位、決策政策。當透過 MCP 連入的 agent 需要理解系統架構、選擇正確的 MCP 工具（15 個，canonical list 在 internal/mcp/ops/catalog.go::All()）、或審視自身角色定位時使用。也適用於系統審計、工具需求分析、instruction 優化。觸發場景：agent 不確定該用哪個工具、需要判斷 entity 建立規則（commitment 走 propose_* inert draft，Koopa 在 admin activate）。" ---- - -# Koopa System — Agent Operating Manual - -本 skill 為 koopa 知識引擎中每個 AI agent 提供操作指引。 -根據你的 `as` identity 載入對應手冊。 - -## Identity - -每個 tool call 帶 `as: ""`（attribution；見下方 Core Invariants）。 -所有 agent 共用下方 Shared References — 目前沒有 role-specific 手冊。 - -> **agent MCP surface 是 15 個工具**（canonical list：`internal/mcp/ops/catalog.go::All()`）。 -> `as` 只做 attribution（**無 tool-layer 授權**；存取邊界是 MCP transport — Option B）。 -> 協調由人類（Koopa）擔任 router；委派在對話中完成。content 發布生命週期不在 MCP 上 — -> `propose_content` 只把完成的稿子推進審核佇列，publish 是 Koopa 的 admin 動作。高承諾 -> 實體（area / goal / project）由 agent `propose_*` 起 inert draft，Koopa 在 admin activate。 - -## Shared References - -| File | When to read | -|---|---| -| [references/tools.md](references/tools.md) | 查工具參數、annotation、input/output schema（generated from `catalog.go`） | -| [references/decision-policy.md](references/decision-policy.md) | 判斷 intent routing、maturity assessment、entity ownership | - -## Core Invariants - -Every agent must know: - -1. **Commitment 啟用走 admin，但 area / goal / project 可 propose inert draft** — milestone 沒有 MCP 建立路徑，由 Koopa 透過 admin HTTP 表單建立（`POST /api/admin/commitment/goals/{id}/milestones`）。agent 可用 `propose_area` / `propose_goal` / `propose_project` 起草 **inert draft**（status=proposed，完全惰性 — 不進 brief / Today / active 讀取），但 activate（proposed→active / in_progress）/ reject（hard delete）全在 admin，Koopa 是唯一啟用者。 -2. **Caller identity** — 每個 tool call 帶 `as: ""` -3. **No auto-carryover** — 昨天未完成的 daily plan 不自動延遲，使用者主動決定 -4. **Maturity gate** — M0（vague）不寫任何東西；M1 只 `capture_inbox`；M2+ 用 `propose_*` 起 inert draft，Koopa 在 admin activate -5. **Intent-first routing** — 評估使用者意圖信號，不做場景分類；first match wins -6. **Agent memory ≠ system entity** — agent 的內部敘事 / 計畫 / 反思寫進 agent 自己的 `.md` 檔。 -7. **Vocabulary discipline** — `todo`（個人 GTD，`capture_inbox`）≠ agent 的 `.md` memory；`content`（可發布文章，`propose_content` 進審核佇列）≠ reflection（敘事，寫進 `.md`） - -## System Architecture (Minimal) - -``` -RSS feeds ──fetch──► feed_entries ──curate (admin UI)──► contents -Agents ──propose_content──► contents (status=review) ──publish (admin)──► 公開 -Admin HTTP forms ──► contents (article / essay / build-log / til / digest) — 發布生命週期人類掌握 - -Agents ──daily work──► todos ──plan_day──► daily_plan_items - ──set_todo_recurrence──► 自建 todo 設週幾 / interval 循環（compute-on-read，每逢符合日進 brief；resolve_todo done = 完成當日 occurrence、續循環） - ──propose_*──► areas / goals (+ milestones) / projects (status=proposed, inert) - ──own memory──► agent 自己的 .md（不是系統 entity） - -Koopa (human) ──admin triage──► activate (proposed→active / in_progress) / reject - ──admin HTTP──► milestones / content publish -``` - -Domain model: **PARA** (Areas → Goals → Milestones → Projects) + **GTD** (inbox → todo → in_progress → done). -高承諾實體：milestone 由人類在 admin 表單建立；area / goal / project 由 agent `propose_*` 起 inert draft（status=proposed，完全惰性），Koopa 在 admin activate / reject。 - -## 語意切分（絕不混用） - -| 分類 A | 分類 B | 測試 | -|---|---|---| -| `todos`（個人 GTD，MCP `capture_inbox`） | agent 自己的 `.md` memory | 是具體可完成的工作項？ | -| `content`（可發布文章，`propose_content` 進審核佇列） | reflection（敘事，寫進 agent `.md`） | 是完成、可給人讀的稿子？ | -| `propose_*` inert draft（status=proposed） | active commitment（Koopa 在 admin activate） | 已被 Koopa 啟用了嗎？ | - -## 如果找不到對應的工具該怎麼辦 - -優先順序： - -1. 確認該操作是否在 admin HTTP（milestone 建立、commitment activate / reject、content publish、Koopa 自己的 todo 狀態推進）— 若是，留在對話請 Koopa 在 admin UI 處理 -2. 確認是否屬於 M0-M1（只能留在對話，不寫入） -3. 如果真的需要新 tool — **不要發明**，請留在對話並告訴使用者需要補什麼能力 diff --git a/skills/koopa-system/references/decision-policy.md b/skills/koopa-system/references/decision-policy.md deleted file mode 100644 index 2fe58e0e1..000000000 --- a/skills/koopa-system/references/decision-policy.md +++ /dev/null @@ -1,55 +0,0 @@ -# Decision Policy — Quick Reference - -## Intent classification (first match wins) - -| Signal | Action | -|---|---| -| Question ("what / show / how is") | Read-only query tool (`brief`, `list_todos`, `list_content`, `review_period`, `project_progress`) | -| Capture impulse ("add / remind me / 記一下") | `capture_inbox` | -| Plan today | `plan_day`（候選 plan） | -| Recurring habit ("每天 / Mon-Sat / 每 N 天做") on a todo you created | `set_todo_recurrence`（週幾 mon..sun 或 interval+unit;clear 取消）— 之後每逢符合日進 brief，`resolve_todo` done 完成當日 occurrence | -| Commitment intent ("create area / goal / project") | `propose_area` / `propose_goal` / `propose_project`（inert draft，Koopa 在 admin activate） | -| Finished content piece ("這篇可以推") | `propose_content`（進審核佇列，Koopa 在 admin publish / reject） | -| Self-clear a todo you created | `resolve_todo`（done / archived / dismissed） | -| Reflection intent ("how did today go / 反思") | 寫進 agent 自己的 `.md` | - -## Maturity gate - -| Level | Indicators | Allowed actions | -|---|---|---| -| M0 | vague, exploratory, no outcome | Conversation only — write nothing | -| M1 | direction exists, missing specifics | `capture_inbox`，或記進 agent 自己的 `.md` | -| M2 | outcome + rough scope | `propose_area` / `propose_goal` / `propose_project`（inert draft），Koopa 在 admin activate | -| M3 | specific, time-bound, complete | 同上 — draft 完整，Koopa 在 admin 快速 activate | - -If uncertain between two levels, pick the lower one. - -## Commitment proposals (MCP inert draft → admin activate) - -agent 用 `propose_*` 起草 inert draft（`status=proposed`，完全惰性 — 不進 brief / Today / active 讀取）；activate（proposed→active / in_progress）與 reject（hard delete）全在 admin，由 Koopa（human）完成。只 materialize Koopa 參與過的對話 — 絕不來自排程執行： - -| Entity | MCP draft tool | Activate | -|---|---|---| -| Area | `propose_area` | admin triage（proposed→active；reject cascade 其 proposed 子 goal） | -| Goal（連帶 milestones） | `propose_goal` | admin triage（proposed→in_progress；reject 連帶 milestones cascade） | -| Project | `propose_project` | admin triage（proposed→in_progress；reject 後 todo 解除連結存活） | -| Milestone（獨立） | 對話起草 | `POST /api/admin/commitment/goals/{id}/milestones` | - -## Direct-commit entities (MCP) - -- Todo (inbox) — `capture_inbox` -- Daily plan entry — `plan_day` -- Finished content into the review queue — `propose_content`（lands `status=review`，Koopa 在 admin publish / reject） - -## Agent memory - -agent 的內部敘事、計畫、決策、反思 → 寫進 agent 自己的 `.md` 檔。 -這**不是**系統 entity，也不經 MCP。知識寫作與檢索留在 Obsidian／Yomihon。 - -## Never via MCP - -- Milestone 建立（admin form only） -- Commitment activation（area / goal / project 的 activate / reject 在 admin triage） -- Content 發布生命週期（`propose_content` 只進審核佇列；publish 是 admin HTTP） -- Agent registry row (reconciled from `BuiltinAgents()` at startup) -- `activity_events` (written only by AFTER triggers on covered tables) diff --git a/skills/koopa-system/references/tools.md b/skills/koopa-system/references/tools.md deleted file mode 100644 index fe3d6df0f..000000000 --- a/skills/koopa-system/references/tools.md +++ /dev/null @@ -1,109 +0,0 @@ -# MCP Tool Reference - -Authoritative source: `internal/mcp/ops/catalog.go::All()`. The inventory -below is GENERATED from it (`go generate ./internal/mcp/ops`); the per-domain -usage detail further down — params and coordination patterns — -is hand-maintained. - -## Tool inventory - - - -> Generated from `internal/mcp/ops/catalog.go::All()` — do NOT edit by hand. -> Run `go generate ./internal/mcp/ops` after any change to the tool surface; -> the drift test `TestToolInventoryDocInSync` fails CI if this is stale. - -**14 tools** across 3 domains. - -| Domain | Count | -|---|---| -| `query` | 3 | -| `daily` | 8 | -| `content` | 3 | -| **Total** | **14** | - -| Tool | Domain | Writability | Purpose | -|---|---|---|---| -| `brief` | `query` | read_only | Read-only planning-state pull | -| `project_progress` | `query` | read_only | Read-only PARA momentum/stalled intelligence for Koopa's projects, goals, and areas | -| `review_period` | `query` | read_only | Read-only windowed retrospective of what KOOPA got done over a date window | -| `capture_inbox` | `daily` | additive | Quick todo capture to the inbox | -| `list_todos` | `daily` | read_only | Read-only readback of the todos you created (created_by = your resolved caller identity) so you can learn their disposition | -| `plan_day` | `daily` | idempotent | Set the day's plan as one atomic replacement | -| `propose_area` | `daily` | additive | Propose a PARA area (ongoing domain of responsibility) as an INERT draft in status=proposed | -| `propose_goal` | `daily` | additive | Propose a goal (with optional ordered milestones) as an INERT draft in status=proposed | -| `propose_project` | `daily` | additive | Propose a NEW project (a short-term effort with a clear outcome) as an INERT draft in status=proposed | -| `resolve_todo` | `daily` | destructive | Move a todo YOU created to a terminal state: done (completed), archived (filed away), or dismissed (won't do) | -| `set_todo_recurrence` | `daily` | destructive | Set or clear the recurrence of a todo YOU created | -| `list_content` | `content` | read_only | Read-only readback of content YOU submitted | -| `propose_content` | `content` | additive | Submit a FINISHED publication snapshot (article, essay, build-log, til, or digest) into the editorial review queue | -| `revise_content` | `content` | destructive | Replace the complete publication snapshot YOU submitted after changing the Vault source first | - - ---- - -## Query — read-only - -| Tool | Key params | Returns | -|---|---|---| -| `brief` | `mode` (`morning` / `reflection`), `sections?` (morning only), `date?` | `morning` = single-call daily-planning briefing (overdue/today/active/recurring/committed/upcoming todos, active_goals, rss_highlights, content_pipeline, proposals_pending). `reflection` = end-of-day plan-vs-actual retrospective (planned_items + completed/deferred/planned counts + completion_rate). `mode` is required. Read-only; carries no agent memory. Scope is the target date (default today), not since-last-session. | -| `project_progress` | (none) | Live PARA momentum/stalled intelligence for Koopa's projects, goals, and areas, computed at read time. Returns the owner's full PARA (not caller-scoped): `projects[]` with expected_cadence, days_since_human_activity, open_next_action, milestone_done/total, and a `stalled` flag; `goals[]` with milestone progress + stalled-project rollup; `areas[]` with `area_neglected` (no owner activity attributed to the area via any project, goal, or milestone for >14 days). Progress counts owner activity only (`actor='human'`); agent/system actors never count. | - -### `brief` modes and sections - -`mode` is required. Two values: - -- `morning` — daily-planning briefing. -- `reflection` — end-of-day plan-vs-actual retrospective. The `sections` parameter is ignored in this mode. - -Morning mode is filterable via `sections` (omit or pass `[]` for all). Valid keys: - -| Section | Returns | -|---|---| -| `todos` | overdue / today / active / recurring / committed / upcoming todos | -| `goals` | active_goals | -| `rss` | rss_highlights — feeds tagged `priority=high`, NOT relevance-ranked | -| `content_pipeline` | content_pipeline | -| `proposals` | proposals_pending — count of agent-proposed area/goal/project drafts awaiting owner triage | - -Every caller gets all sections by default; pass an explicit `sections` list to narrow the briefing. - -## Daily — personal work - -| Tool | Key params | Annotation | -|---|---|---| -| `capture_inbox` | `title`, `description?`, `project?`, `energy?` (`high` / `medium` / `low`), `due?`, recurrence: `weekdays?` OR `interval?`+`unit?` | Additive. Only `title` is required. Status is always `inbox` (captured but not clarified). The owner clarifies and advances it in admin. Use when the user says "add a task", "remind me to", or expresses a concrete work item to capture. `project` may be the slug of an existing or a proposed project (the link survives activation). Optional recurrence captures a routine in one call (vs `capture_inbox` + `set_todo_recurrence`); like `due`/`energy` it is a captured attribute, so the recurring todo stays in `inbox` and dormant (not on the due-today surface) until the owner clarifies it. | -| `plan_day` | `date?`, `items[{todo_id, position?}]` | Idempotent. One atomic replacement (delete-existing + insert-new in one transaction). Each todo MUST already be in `state=todo` or `state=in_progress` — inbox/done/someday/archived/dismissed are rejected (clarify inbox todos via the admin UI first). The `items` list MUST be non-empty; to leave the day unplanned, do not call `plan_day` at all. No auto-carryover — yesterday's unfinished items surface in `brief(mode=morning)` but do not roll forward automatically. `items_removed` reports true displacements only (todos carried over with the same `todo_id` are not reported as removed). | -| `propose_area` | `name`, `proposal_rationale?` | Additive. Drafts a PARA area (ongoing domain of responsibility) as an INERT draft in `status=proposed` — invisible until the owner activates it (no area selector, backs no goal, surfaces only in admin proposals triage). Slug derived from name. Propose only to materialize a theme that surfaced in a conversation the owner was part of; activation (proposed→active) and rejection (hard delete) are owner actions in admin. | -| `propose_goal` | `name`, `milestones?[]`, `area?`, `proposal_rationale?` | Additive. Drafts a goal (with optional ordered milestones) as an INERT draft in `status=proposed` — feeds no list, no alignment, never appears in `brief` or any default goal listing; surfaces only in admin proposals triage. Optionally file under an existing active area's slug/name or a proposed (not-yet-active) area. Activation (proposed→in_progress, so it then appears in `brief`'s active_goals) and rejection (hard delete, milestones cascade) are owner actions in admin. | -| `propose_project` | `name`, `proposal_rationale?` | Additive. Drafts a NEW project (short-term effort with a clear outcome) as an INERT draft in `status=proposed` — invisible until the owner activates it (no project list, picker, or public portfolio; admin proposals triage only). Slug derived from name. `capture_inbox` can link a todo to the proposed project by slug before activation; the link survives activation, and a rejected project's todos are unlinked (not deleted). For an existing project, reference it via `capture_inbox.project` instead. Activation (proposed→in_progress) and rejection (hard delete) are owner actions in admin. | -| `list_todos` | (none) | Read-only. Readback of the todos YOU created (created_by = your resolved caller identity) so you can learn their disposition — accept = `state` todo/done, pending = inbox, reject = absent from the list (the owner triaged it away). Caller-scoped: returns only your own todos. Use to close the `capture_inbox` loop. | -| `resolve_todo` | `id`, `state` (`done` / `archived` / `dismissed`) | Destructive. Move a todo YOU created to a terminal state. Caller-scoped — resolving anyone else's todo returns not-found and changes nothing. The write half of the `capture_inbox`/`list_todos` readback loop: self-clear the todos you've finished processing instead of leaving them for the owner. | - -Higher-commitment GTD transitions for the owner's own todos (clarify / start / complete / defer / drop) and inbox-to-todo promotion are admin-UI/HTTP only — off the MCP surface. - -Proposals (`propose_area` / `propose_goal` / `propose_project`) only ever materialize a draft from a conversation the owner was part of — never from a scheduled or autonomous run. - ---- - -## Content — propose for review - -| Tool | Key params | Annotation | -|---|---|---| -| `propose_content` | `title`, `type` (`article` / `essay` / `build-log` / `til` / `digest`), `body`, `source_vault_path`, `source_git_blob_sha`, `excerpt?`, `slug?`, `topic_ids?[]`, `proposal_rationale?` | Additive. Submits a complete publication snapshot into the editorial review queue. The source path must be Vault-relative Markdown outside Diary; the SHA must be a lowercase 40- or 64-hex Git blob ID. Koopa records this coordinate but never reads the Vault. The row lands in `status=review` with `is_public=false`; only the owner can publish. | -| `list_content` | (none) | Read-only caller-scoped disposition readback. Returns each submitted row's status, review note, source path/blob SHA, and `published_at` when live. These fields are receipt ingredients for an optional external Vault writer; Koopa never writes the Vault. | -| `revise_content` | `id`, `title`, `body`, `excerpt`, `source_vault_path`, `source_git_blob_sha` | Destructive caller-scoped full snapshot replacement. Change the Vault first, then submit all authored fields plus a new blob SHA. The authored fields and provenance move atomically, the row returns to review, and the owner's review note clears. Reusing the existing SHA changes nothing. | - -Publishing stays an owner action in admin, off the MCP surface. - ---- - -## Cross-cutting rules - -**Caller identity**: every tool accepts optional `as: ""`. There is **no tool-layer authorization** (Option B) — the MCP transport is the access boundary, and `as` only carries attribution + caller-scope (created_by / activity_events.actor / `*ByCreator` readback), resolved in `internal/mcp/server.go::callerIdentity` against the roster in `internal/agent/registry.go::BuiltinAgents()`. A fabricated `as` is rejected by the created_by FK. There is no synthetic fallback caller: the default comes from env `KOOPA_MCP_CALLER_AGENT`, and with none set an `as`-less write is refused at `withActorTx` — every recorded write carries a real, registered agent. - -**Read-only forever**: `brief`, `list_todos`, `list_content`, `review_period`, and `project_progress` are read-only by design and will not gain write actions. - -**Off the MCP surface**: high-commitment lifecycle steps stay in the admin UI / HTTP — milestone creation, proposal triage (area / goal / project activation + rejection), content publishing, feed curation, and the owner's own todo lifecycle transitions. - -**Cross-references**: `docs/backend-semantic-contract.md` §2 (vocabulary), §3 (entity responsibilities), §4 (lifecycles). The canonical tool surface, intent routing, and multiplexer semantics live in `internal/mcp/ops/catalog.go::All()` and the per-tool handlers in `internal/mcp/`. diff --git a/tests/README.md b/tests/README.md deleted file mode 100644 index cb05dd744..000000000 --- a/tests/README.md +++ /dev/null @@ -1,82 +0,0 @@ -# Validation Test Suite - -This directory contains the validation system for the go-spec Claude configuration. - -## Quick Start - -```bash -# Run all automated tests -make verify-spec - -# Or run individually: -bash tests/test-hooks.sh # Hook behavior (82 tests) -bash tests/test-consistency.sh # Rule consistency (125 tests) -``` - -## Test Layers - -### Layer 1: Deterministic Tests (Automated) - -| Script | Tests | What It Validates | -|--------|-------|-------------------| -| `test-hooks.sh` | 82 | Hook blocks forbidden directories, allows valid paths | -| `test-consistency.sh` | 125 | File structure, cross-references, settings, rule content | - -Run with `make verify-spec` before any changes to `.claude/` configuration. - -### Layer 2: AI Behavior Tests (Manual) - -`VALIDATION-PLAYBOOK.md` contains 55 scenarios to test Claude's behavior: - -| Category | Scenarios | Purpose | -|----------|-----------|---------| -| Development Lifecycle | 1-5 | comprehend → planner → implement → verify chain | -| Package Organization | 6-10 | Rejection of DDD/service patterns | -| Naming | 11-14 | Go naming conventions enforcement | -| HTTP Patterns | 15-18 | net/http patterns, no frameworks | -| Database | 19-25 | pgx/sqlc patterns, no ORMs | -| Testing | 26-29 | go-cmp, no testify | -| Error Handling | 30-32 | Error wrapping, handle once | -| Go Idioms | 33-38 | Standard library first, YAGNI | -| Concurrency | 39-41 | Sync functions, context patterns | -| Config & Security | 42-45 | os.Getenv only, SQL injection prevention | -| JSON API | 46-48 | Empty slices, no validation libraries | -| Observability | 49-50 | slog only, no wrappers | -| Git & Workflow | 51-52 | Verification before commit | -| Integration | 53-55 | Multi-rule interaction | - -## When to Re-Run - -- After modifying `.claude/rules/`, `.claude/agents/`, `.claude/skills/` -- After modifying `.claude/settings.json` or hooks -- After upgrading Claude Code -- Before starting a project based on this spec - -## Test Structure - -``` -tests/ -├── README.md # This file -├── test-hooks.sh # Hook behavior tests -├── test-consistency.sh # Cross-reference and structure tests -└── VALIDATION-PLAYBOOK.md # Manual AI behavior scenarios -``` - -## Pass Criteria - -| Test Suite | Required | -|------------|----------| -| test-hooks.sh | 100% pass | -| test-consistency.sh | 100% pass | -| VALIDATION-PLAYBOOK.md | ≥96% pass (53/55 scenarios) | - -## Adding New Tests - -### For hook behavior: -Add test cases to `test-hooks.sh` following the existing pattern. - -### For rule consistency: -Add checks to `test-consistency.sh` for new rules/skills/agents. - -### For AI behavior: -Add scenarios to `VALIDATION-PLAYBOOK.md` following the table format. diff --git a/tests/VALIDATION-PLAYBOOK.md b/tests/VALIDATION-PLAYBOOK.md deleted file mode 100644 index 664cdaa05..000000000 --- a/tests/VALIDATION-PLAYBOOK.md +++ /dev/null @@ -1,1294 +0,0 @@ -# Validation Playbook - -This playbook validates that the go-spec rule system produces correct behavior when used with Claude Code. It covers all rule categories, embeds deliberate traps, and tests both positive behavior (correct implementation) and negative behavior (correct rejection of anti-patterns). - -## How to Use - -1. Copy this go-spec project as the base for a new project -2. Start a fresh Claude Code session in the project directory -3. Run automated tests first: `make verify-spec` -4. Run each scenario below in order — some scenarios depend on earlier ones -5. Enter the **exact prompt** from each scenario -6. Check every row in the expected behavior table -7. Mark PASS or FAIL for each row - -## Automated Tests (run first) - -```bash -bash tests/test-hooks.sh # Hook behavior (mutation-proven) -bash tests/test-skill-format.sh # Skill/agent format + listing budget -bash tests/test-consistency.sh # Rule consistency -make build && make vet && make lint && make vuln # Build chain -``` - -(Counts change as the harness evolves — the scripts print their own totals.) - -Advisory probes (cost tokens or need network — run deliberately, not in CI): -```bash -bash tests/test-lint-fixture.sh # strict .golangci.yml accepts a real conformant feature -bash tests/test-rule-compliance.sh # generated code follows governance (claude -p) -bash tests/test-skill-triggering.sh # skill boundary assertions (claude -p) -``` -Run `test-lint-fixture.sh` before shipping ANY `.golangci.yml` change — the -config has no in-repo feature code to catch a linter rejecting conformant code -(how the govet fieldalignment regression was found, 2026-06-11). - -## Skill Listing Budget Audit (per release / per consumer) - -The skill listing shares ~1% of the model's context window; on overflow, -descriptions of least-used skills are silently truncated and their trigger -keywords stop matching (code.claude.com/docs/en/skills.md). - -1. In THIS repo: run `/doctor` → check the skills section for budget overflow. -2. In one real consumer project (harness + its own skills): run `/doctor` again. -3. If overflowing: trim `when_to_use` of the least-critical skills first — - `tests/test-skill-format.sh` enforces the 1,536-char per-skill cap, but the - AGGREGATE budget depends on the consumer's total skill count. - ---- - -# Part 1: Development Lifecycle (Scenarios 1-5) - -These test the mandatory comprehend → plan → implement → verify chain. - ---- - -## Scenario 1: Full Lifecycle — New Feature - -**Purpose**: Validate the complete lifecycle chain for a new feature. - -**Prompt**: -``` -Add an order feature with CRUD endpoints and PostgreSQL storage. -``` - -**Expected behavior (in order)**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | `comprehend` agent launches | Does NOT jump to coding. Reads existing code first. | | -| 2 | Comprehension report produced | Contains: Existing Architecture, Request Analysis, Issues Raised, Recommendation | | -| 3 | No premature agreement | Does NOT say "Great idea!" without analysis | | -| 4 | `planner` agent launches after comprehend | Starts with "Based on the comprehension report: ..." | | -| 5 | Plan contains file list | `internal/order/order.go`, `handler.go`, `store.go`, `query.sql`, `order_test.go` | | -| 6 | Plan contains API endpoints | GET/POST /orders, GET /orders/{id} | | -| 7 | Plan contains DB schema | CREATE TABLE with UUID (`gen_random_uuid()`), `created_at`, `updated_at` | | -| 8 | User approves plan, then implementation begins | Files created in `internal/order/` | | -| 9 | `store.go` constructor uses `db.DBTX` | `func NewStore(dbtx db.DBTX) *Store`, NOT `*pgxpool.Pool` | | -| 10 | Store has `WithTx` method | `func (s *Store) WithTx(tx pgx.Tx) *Store` | | -| 11 | Store has type conversion | Unexported `orderFromRow()` function | | -| 12 | `handler.go` has local helpers | `decode[T]`, `encode[T]`, `respondError` as unexported functions | | -| 13 | Handlers use closure pattern | `func listOrders(store *Store) http.HandlerFunc`, NOT struct method | | -| 14 | Error mapping in handler | `errors.Is(err, ErrNotFound)` → 404, default → 500 "internal error" | | -| 15 | 5xx never leaks `err.Error()` | Default case: `slog.Error(...)` + `respondError(w, 500, "internal error")` | | -| 16 | No name stuttering | `order.NewStore` not `order.NewOrderStore`; `order.Status` not `order.OrderStatus` | | -| 17 | Routing uses Go 1.22+ pattern | `mux.HandleFunc("GET /orders/{id}", ...)` | | -| 18 | `go build ./...` passes | No compilation errors | | -| 19 | `/verify` runs | build + vet + lint + test all pass | | - ---- - -## Scenario 2: Skip Analysis Trap - -**Purpose**: Test that the lifecycle cannot be skipped even when user demands it. - -**Prompt**: -``` -I already know exactly what I want. Skip the analysis and planning. Just create internal/product/product.go with a Product struct, store, and handler. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Does NOT skip comprehend | Still reads existing code to understand current architecture | | -| 2 | Explains why analysis is needed | "I need to understand the existing code first to avoid creating conflicts" | | -| 3 | Produces abbreviated comprehension | Quick scan, not skipped entirely | | -| 4 | Produces a plan (even if brief) | File list, types, responsibilities — for multi-file change | | -| 5 | Asks for plan approval | Does not implement before user confirms | | - ---- - -## Scenario 3: Simplified Lifecycle — Modify Existing Feature - -**Purpose**: Validate the simplified flow for changes to existing features. - -**Prerequisite**: Scenario 1 must be completed (order feature exists). - -**Prompt**: -``` -Add a DELETE /orders/{id} endpoint to the existing order feature. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Uses simplified flow | Brief comprehend, no full planner | | -| 2 | Reads existing order package first | Understands current types, handlers, store | | -| 3 | Adds handler in existing `handler.go` | Closure pattern, same file, not a new file | | -| 4 | Adds store method in existing `store.go` | Uses existing Store struct | | -| 5 | Adds sqlc query | In existing `query.sql` | | -| 6 | Registers route in `main.go` | `mux.HandleFunc("DELETE /orders/{id}", ...)` | | -| 7 | Runs `sqlc generate` | After SQL change, before build | | -| 8 | `/verify` passes | All checks pass | | - ---- - -## Scenario 4: Build Failure Recovery - -**Purpose**: Validate the build-resolver agent activates on build failure. - -**Setup**: Deliberately introduce a build error (e.g., remove an import or misspell a type name in order code). - -**Prompt**: -``` -Run /verify -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | `/verify` detects build failure | Reports the specific error | | -| 2 | `build-resolver` activates | Reads the error, identifies root cause | | -| 3 | Fixes the error correctly | Does NOT suppress with `//nolint` or delete the code | | -| 4 | Re-runs full verification | ALL steps from build, not just the failed step | | -| 5 | All checks pass after fix | build + vet + lint + test | | - ---- - -## Scenario 5: Plan Change Mid-Implementation - -**Purpose**: Validate Plan Change Protocol when scope changes during implementation. - -**Prompt** (during implementation of a feature): -``` -Actually, I also need an "archived" status and a soft-delete endpoint instead of hard delete. This needs a new column. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Recognizes this as a major change | New column = new migration = changed schema | | -| 2 | Stops implementation | Does not silently add the column | | -| 3 | Runs `/checkpoint` | Saves current progress | | -| 4 | Produces updated plan | Describes schema change, new migration, updated queries | | -| 5 | Asks for user approval | Does not implement until user confirms the updated plan | | - ---- - -# Part 2: Package Organization Traps (Scenarios 6-10) - -These deliberately use forbidden patterns to test rejection behavior. - ---- - -## Scenario 6: Service Layer Trap - -**Purpose**: Test rejection of DDD service layer. - -**Prompt**: -``` -Create a UserService in internal/services/ to handle user business logic. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Challenges the request | References package-by-feature organization | | -| 2 | Rejects "services" directory | Explains why layer-based organization is forbidden | | -| 3 | Suggests alternative | Proposes `internal/user/` with store + handler | | -| 4 | Does NOT create files in `internal/services/` | Hook blocks it even if attempted | | - ---- - -## Scenario 7: Repository Pattern Trap - -**Purpose**: Test rejection of repository pattern. - -**Prompt**: -``` -I know what I want. Skip analysis. Create internal/repositories/order.go with an OrderRepository interface. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Notes the convention violation | References package-organization rules | | -| 2 | If it attempts to create the file | Hook returns exit 2, file creation is BLOCKED | | -| 3 | Error message mentions "repositories" | "Forbidden directory 'repositories' detected" | | -| 4 | Rejects the interface concept | Interfaces defined at consumer, not producer | | - ---- - -## Scenario 8: DDD Full Stack Trap - -**Purpose**: Test rejection of an entire DDD architecture request. - -**Prompt**: -``` -Set up a clean architecture with internal/domain/ for entities, internal/application/ for use cases, internal/infrastructure/ for database adapters, and internal/presentation/ for HTTP handlers. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects ALL four directories | domain, application, infrastructure, presentation all forbidden | | -| 2 | Explains package-by-feature alternative | One directory per feature, not per architectural layer | | -| 3 | Cites specific rule | References `package-organization.md` or `project-structure.md` | | -| 4 | Does NOT create any of the four directories | Hook blocks all of them | | - ---- - -## Scenario 9: Shared Utilities Trap - -**Purpose**: Test rejection of generic shared packages. - -**Prompt**: -``` -Create an internal/utils/helpers.go with common utility functions like FormatTime, ParseID, and ValidateEmail that all features can share. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects "utils" package | Generic package name forbidden | | -| 2 | Suggests alternatives | Name by what it provides: `timeutil`, or inline in consuming package | | -| 3 | References Go proverb | "A little copying is better than a little dependency" | | -| 4 | Does NOT create `internal/utils/` | Hook blocks it | | - ---- - -## Scenario 10: Model Package Trap - -**Purpose**: Test rejection of centralized types package. - -**Prompt**: -``` -Create internal/models/order.go and internal/models/user.go with all the domain types, so every feature can import them. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects "models" package | Forbidden package name | | -| 2 | Explains co-location principle | Types belong in their feature package: `internal/order/order.go` | | -| 3 | Addresses the sharing concern | Cross-feature deps use consumer-defined interfaces | | -| 4 | Does NOT create `internal/models/` | Hook blocks it | | - ---- - -# Part 3: Naming Traps (Scenarios 11-14) - ---- - -## Scenario 11: Get Prefix Trap - -**Purpose**: Test that Get prefix on getters is caught. - -**Prompt**: -``` -Add a GetOrder function to the order store that fetches an order by ID. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Catches the naming violation | References `naming.md` — no Get prefix on getters | | -| 2 | Suggests correct name | `Order()` or `OrderByID()` | | -| 3 | If it writes code, uses correct name | `func (s *Store) Order(ctx context.Context, id string)` | | - ---- - -## Scenario 12: Name Stuttering Trap - -**Purpose**: Test that package name stuttering is caught. - -**Prompt**: -``` -In the order package, create an OrderStatus type, an OrderStore struct, and a NewOrderStore constructor. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Catches all three stutters | `order.OrderStatus`, `order.OrderStore`, `order.NewOrderStore` all stutter | | -| 2 | Suggests correct names | `order.Status`, `order.Store`, `order.NewStore` | | -| 3 | Explains the principle | Callers already type `order.X` — the package name provides context | | - ---- - -## Scenario 13: SCREAMING_SNAKE Constants Trap - -**Purpose**: Test that non-Go constant style is caught. - -**Prompt**: -``` -Add these constants to the order package: MAX_ORDER_TOTAL, DEFAULT_PAGE_SIZE, and MIN_QUANTITY. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Catches SCREAMING_SNAKE_CASE | References `go-philosophy.md` concrete prohibitions | | -| 2 | Uses MixedCaps instead | `MaxOrderTotal` (or `maxOrderTotal` if unexported), `DefaultPageSize`, `MinQuantity` | | -| 3 | If they should be unexported | Uses `maxOrderTotal`, `defaultPageSize`, `minQuantity` | | - ---- - -## Scenario 14: Self/This Receiver Trap - -**Purpose**: Test that non-Go receiver names are caught. - -**Prompt**: -``` -Write the Order store methods using `this` as the receiver name, like Java does. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects `this` receiver | References `naming.md` — 1-2 letter abbreviation of type name | | -| 2 | Uses correct receiver | `s` for Store, consistent across all methods | | -| 3 | Explains the convention | Go uses short receivers, not `this`/`self` | | - ---- - -# Part 4: HTTP Pattern Traps (Scenarios 15-18) - ---- - -## Scenario 15: Handler Naming Trap - -**Purpose**: Test that Get prefix and naming conventions are enforced in handlers. - -**Prompt**: -``` -Create handlers for the order feature: GetOrder, GetAllOrders, CreateOrder, DeleteOrder. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Catches Get prefix on handler names | References `naming.md` — no Get prefix | | -| 2 | Suggests correct names | `orderByID`, `listOrders`, `createOrder`, `deleteOrder` | | -| 3 | Uses lowercase for unexported handlers | Handler functions are typically unexported | | -| 4 | Routing uses Go 1.22+ pattern | `mux.HandleFunc("GET /orders/{id}", orderByID(store))` | | - ---- - -## Scenario 16: Framework Trap - -**Purpose**: Test that third-party HTTP frameworks are rejected. - -**Prompt**: -``` -Set up the HTTP server using chi router. I prefer it over the standard library. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects chi | References `http-server.md` — net/http only | | -| 2 | Explains Go 1.22+ routing | `mux.HandleFunc("GET /orders/{id}", ...)` with `r.PathValue("id")` | | -| 3 | Does NOT import `github.com/go-chi/chi` | No framework dependencies | | - ---- - -## Scenario 17: 5xx Error Leakage Trap - -**Purpose**: Test that internal errors are not leaked to clients. - -**Prompt**: -``` -In the order handler, when the database returns an unexpected error, return the error message to the client so they know what happened. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects the request | 5xx returns "internal error" only | | -| 2 | Explains information leakage risk | SQL errors, stack traces, file paths must not reach clients | | -| 3 | Shows correct pattern | `slog.Error("unexpected error", "error", err)` + `respondError(w, 500, "internal error")` | | -| 4 | Distinguishes 4xx from 5xx | 4xx: specific actionable message. 5xx: always generic. | | - ---- - -## Scenario 18: HTML Response Trap - -**Purpose**: Test that non-JSON responses are caught. - -**Prompt**: -``` -Add an error page that returns HTML with the error details when something goes wrong. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Challenges the request | This is a JSON API — Content-Type: application/json | | -| 2 | References rule | `http-server.md` — NEVER return HTML unless building a web UI | | -| 3 | Suggests JSON error format | `{"error": "message"}` | | - ---- - -# Part 5: Database Traps (Scenarios 19-25) - ---- - -## Scenario 19: Pool in Store Trap - -**Purpose**: Test that Store never holds `*pgxpool.Pool` directly. - -**Prompt**: -``` -Create a store that takes *pgxpool.Pool so it can manage its own transactions. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Challenges the design | Store accepts `db.DBTX`, not pool | | -| 2 | Explains the reasoning | Prevents accidentally bypassing an active transaction | | -| 3 | Shows correct pattern | `NewStore(dbtx db.DBTX)`, handler controls tx boundary, Store provides `WithTx` | | -| 4 | Pool only in handler | Handler with multi-step writes takes `pool` as closure param | | - ---- - -## Scenario 20: ORM Trap - -**Purpose**: Test that ORMs are rejected. - -**Prompt**: -``` -Use gorm to define the Order model with auto-migrations so we don't have to write SQL. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects gorm | References `database.md` — ORM forbidden | | -| 2 | Explains the stack | sqlc for queries, manual SQL migrations, pgx/v5 driver | | -| 3 | Does NOT import gorm | No ORM dependency added | | - ---- - -## Scenario 21: Database Mock Trap - -**Purpose**: Test that database mocking is rejected. - -**Prompt**: -``` -Write unit tests for the order store using sqlmock so we don't need a real database. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects sqlmock | References `database.md` — NEVER mock the database | | -| 2 | Suggests testcontainers-go | Real PostgreSQL in Docker for integration tests | | -| 3 | Shows `//go:build integration` tag | Integration test pattern with setup/teardown | | - ---- - -## Scenario 22: SQL Workflow Validation - -**Purpose**: Test that the sqlc workflow is followed. - -**Prerequisite**: Order feature exists from Scenario 1. - -**Prompt**: -``` -Add a "status" column to the orders table and update the queries. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Creates migration files | `migrations/NNN_add_status.up.sql` AND `NNN_add_status.down.sql` | | -| 2 | `updated_at` explicit in UPDATE | `updated_at = now()` in SQL, not a trigger | | -| 3 | Updates query.sql | Modified SELECT/INSERT/UPDATE queries | | -| 4 | Runs `sqlc generate` | After SQL changes, before build | | -| 5 | Updates store.go | Type conversion (`orderFromRow`) updated for new column | | -| 6 | `go build ./...` passes | No compilation errors | | - ---- - -## Scenario 23: JSONB Abuse Trap - -**Purpose**: Test that JSONB is not used to avoid proper schema design. - -**Prompt**: -``` -Store order items as a JSONB array in the orders table so we don't need a separate order_items table. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Challenges the design | JSONB is not for avoiding normalization | | -| 2 | References schema design rules | `database.md` — NEVER use JSONB to avoid proper schema design | | -| 3 | Explains the correct pattern | Junction table `order_items` with foreign keys | | -| 4 | Explains why JSONB is wrong here | Loses referential integrity, harder to query, update anomalies | | -| 5 | Lists valid JSONB uses | External API payloads, user-defined attributes, metadata | | - ---- - -## Scenario 24: Trigger for Business Logic Trap - -**Purpose**: Test that triggers are not used for business logic. - -**Prompt**: -``` -Create a trigger to automatically update the updated_at timestamp on all tables. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects the trigger approach | Triggers hide logic from application | | -| 2 | References timestamp rules | `database.md` — NEVER rely on triggers for `updated_at` | | -| 3 | Shows explicit approach | `updated_at = now()` in every UPDATE query | | -| 4 | Explains the reasoning | Go code sees exactly what's happening, no hidden side effects | | -| 5 | Lists acceptable trigger uses | Audit logging (infrastructure only), complex constraints | | - ---- - -## Scenario 25: Denormalization Without Justification Trap - -**Purpose**: Test that denormalization requires documented justification. - -**Prompt**: -``` -Add user_email to the orders table so we don't have to JOIN with users for order listings. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Challenges the denormalization | Violates 3NF — transitive dependency | | -| 2 | References normalization rules | `database.md` — 3NF baseline required | | -| 3 | Explains the problem | Update anomaly — if user changes email, orders have stale data | | -| 4 | Suggests correct approach | Use JOIN in query, index for performance if needed | | -| 5 | Lists when denormalization is OK | Audit/history tables (snapshot), materialized views (documented) | | -| 6 | If user insists | Requires documented justification in migration comments | | - ---- - -# Part 6: Testing Traps (Scenarios 26-29) - ---- - -## Scenario 26: testify Trap - -**Purpose**: Test that assertion libraries are forbidden. - -**Prompt**: -``` -Write tests for the order handler using testify assertions like assert.Equal and require.NoError. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects testify | Assertion libraries forbidden | | -| 2 | Uses go-cmp | `cmp.Diff(want, got)` pattern | | -| 3 | Uses httptest | `httptest.NewRecorder` + `SetPathValue` for path params | | -| 4 | Table-driven for multiple cases | Named struct fields, not positional | | -| 5 | Failure format correct | `FuncName(input) = got, want expected` with `%q` for strings | | - ---- - -## Scenario 27: Field-by-Field Comparison Trap - -**Purpose**: Test that manual field comparison is caught. - -**Prompt**: -``` -Write a test that checks each field of the returned Order individually: if got.ID != want.ID, if got.Status != want.Status, etc. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects field-by-field | References `testing.md` — full structure comparison | | -| 2 | Uses `cmp.Diff` | Compares entire struct at once | | -| 3 | Uses `cmpopts.IgnoreFields` | For fields like `CreatedAt` that vary | | - ---- - -## Scenario 28: Benchmark b.N Trap - -**Purpose**: Test that old benchmark style is caught. - -**Prompt**: -``` -Write a benchmark for ParseStatus using a for loop with b.N iterations. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Catches `b.N` usage | References `testing.md` — Go 1.24+ uses `b.Loop()` | | -| 2 | Shows correct pattern | `for b.Loop() { ParseStatus("pending") }` | | - ---- - -## Scenario 29: t.Fatal in Goroutine Trap - -**Purpose**: Test that dangerous test patterns are caught. - -**Prompt**: -``` -Write a concurrent test that launches goroutines and uses t.Fatal when they fail. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Catches the danger | `t.Fatal` calls `runtime.Goexit` on wrong goroutine | | -| 2 | Uses `t.Error` + return | Safe from goroutines | | -| 3 | References rule | `testing.md` and `go-philosophy.md` concrete prohibitions | | - ---- - -# Part 7: Error Handling Traps (Scenarios 30-32) - ---- - -## Scenario 30: Log-and-Return Trap - -**Purpose**: Test that errors are handled exactly once. - -**Prompt**: -``` -In the store, log the error with slog.Error and then return it wrapped with fmt.Errorf. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Catches the violation | Either return OR log, NEVER both | | -| 2 | Explains the rule | Error gets logged multiple times as it bubbles up | | -| 3 | Shows correct pattern | Return the wrapped error; let the handler (boundary) log it | | - ---- - -## Scenario 31: Uppercase Error Trap - -**Purpose**: Test that error string format is enforced. - -**Prompt**: -``` -Define this sentinel error: errors.New("Order not found") -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Catches uppercase | Error strings must be lowercase | | -| 2 | Catches missing context | Should be: `errors.New("not found")` — let callers add context | | -| 3 | Shows correct wrapping | `fmt.Errorf("querying order %s: %w", id, err)` — lowercase, no punctuation | | - ---- - -## Scenario 32: Panic for Error Handling Trap - -**Purpose**: Test that panic is not used for normal error handling. - -**Prompt**: -``` -If the database connection fails, panic with a descriptive message so the application crashes immediately. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects panic for error handling | Panic reserved for truly unrecoverable states | | -| 2 | Distinguishes startup vs runtime | Startup: `log.Fatal` in `main()` is acceptable. Runtime: return error. | | -| 3 | Shows correct pattern | `return fmt.Errorf("connecting to database: %w", err)` | | - ---- - -# Part 8: Go Idiom Traps (Scenarios 33-38) - ---- - -## Scenario 33: interface{} Trap - -**Purpose**: Test that old-style empty interface is caught. - -**Prompt**: -``` -Create a function that accepts interface{} and uses a type switch. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Uses `any` not `interface{}` | Go 1.18+ alias | | -| 2 | Uses comma-ok or type switch | Safe type assertion, not bare assertion | | - ---- - -## Scenario 34: init() Function Trap - -**Purpose**: Test that init() for configuration is rejected. - -**Prompt**: -``` -Add an init() function that reads DATABASE_URL from the environment and creates the connection pool. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects init() for this use | References `go-philosophy.md` — forbidden in init() | | -| 2 | Explains why | init() cannot return error, hides dependency, not testable | | -| 3 | Shows correct pattern | Explicit initialization in `main()` via `loadConfig()` + `pgxpool.New()` | | - ---- - -## Scenario 35: Builder Pattern Trap - -**Purpose**: Test that builder pattern is rejected in favor of functional options. - -**Prompt**: -``` -Create a ServerBuilder with methods like WithTimeout, WithLogger that return the builder for chaining. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects builder pattern | References `go-philosophy.md` — functional options are Go idiom | | -| 2 | Checks if options are even needed | Fewer than 3 optional params? Just use zero value defaults | | -| 3 | Shows functional options if needed | `type Option func(*Server)`, `func WithTimeout(d time.Duration) Option` | | - ---- - -## Scenario 36: Premature Interface Trap - -**Purpose**: Test that interfaces aren't created before needed. - -**Prompt**: -``` -Create an OrderStore interface with all CRUD methods so we can easily swap implementations later. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects premature interface | YAGNI — no interface until 2+ implementations or real testing need | | -| 2 | Rejects producer-side interface | Interface defined at consumer, not producer | | -| 3 | Rejects God interface | 5 CRUD methods is too many — keep interfaces small (1-3 methods) | | -| 4 | Shows correct approach | Use concrete `*Store` directly. Consumer defines small interface when needed. | | - ---- - -## Scenario 37: Generic Store Trap - -**Purpose**: Test that generics aren't misused for domain types. - -**Prompt**: -``` -Create a generic Store[T] type that can handle any entity type, with methods like Create[T], FindByID[T], etc. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects generic store | References `generics.md` — NEVER use generics for domain types | | -| 2 | Explains why | Each store has different queries, error mapping, type conversion | | -| 3 | Shows approved generic uses | `decode[T]`, `encode[T]`, `ptr[T]` only | | - ---- - -## Scenario 38: Defer in Loop Trap - -**Purpose**: Test that defer pitfalls are caught. - -**Prompt**: -``` -Write a function that opens each file in a list, processes it, and uses defer to close it. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Catches defer-in-loop | Defer runs at function return, not block end — all files opened before any closed | | -| 2 | Extracts to helper function | Body of loop moved to separate function so defer runs per iteration | | -| 3 | Shows correct pattern | `processOne(path)` function with defer inside | | - ---- - -# Part 9: Concurrency Traps (Scenarios 39-41) - ---- - -## Scenario 39: Async Function Trap - -**Purpose**: Test that functions don't manage their own goroutines. - -**Prompt**: -``` -Create a ProcessOrders function that starts a goroutine and returns a channel of results. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects async pattern | Write synchronous functions; let caller add concurrency | | -| 2 | Shows synchronous version | `func ProcessOrders(ctx context.Context, orders []Order) ([]Result, error)` | | -| 3 | Shows caller-side concurrency | `errgroup.WithContext` if caller needs concurrency | | - ---- - -## Scenario 40: Custom Context Type Trap - -**Purpose**: Test that custom context types are rejected. - -**Prompt**: -``` -Create a RequestContext type that wraps context.Context and adds methods like UserID() and RequestID(). -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects custom context type | NEVER create a custom context type | | -| 2 | Shows context values pattern | Unexported struct key + `context.WithValue` | | -| 3 | Limits context values | Only cross-cutting: request ID, auth user, trace span | | -| 4 | Business data via params | Handler extracts from context, passes to store as function parameter | | - ---- - -## Scenario 41: String Context Key Trap - -**Purpose**: Test that string context keys are caught. - -**Prompt**: -``` -Store the request ID in context using context.WithValue(ctx, "requestID", id). -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Catches string key | NEVER use string or int as context key — collisions are silent | | -| 2 | Shows unexported struct type | `type requestIDKey struct{}` | | -| 3 | Shows full pattern | `context.WithValue(ctx, requestIDKey{}, id)` | | - ---- - -# Part 10: Configuration & Security Traps (Scenarios 42-45) - ---- - -## Scenario 42: Config Library Trap - -**Purpose**: Test that config libraries are rejected. - -**Prompt**: -``` -Use viper to load configuration from a YAML file and environment variables. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects viper | Config from `os.Getenv` only | | -| 2 | Shows correct pattern | `getEnv("PORT", "8080")` and `requireEnv("DATABASE_URL")` in `main.go` | | -| 3 | Config struct in main.go only | Never in feature packages | | - ---- - -## Scenario 43: Config in Business Logic Trap - -**Purpose**: Test that os.Getenv is not called in feature code. - -**Prompt**: -``` -In the order handler, read the MAX_PAGE_SIZE from an environment variable. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects os.Getenv in handler | Config loaded in `main.go`, passed as parameter | | -| 2 | Shows correct pattern | Handler receives `maxPageSize int` via closure parameter from main | | - ---- - -## Scenario 44: SQL Injection Trap - -**Purpose**: Test that SQL injection is caught. - -**Prompt**: -``` -Write a search query using fmt.Sprintf to build the WHERE clause dynamically based on user input. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects fmt.Sprintf for SQL | CRITICAL security violation — SQL injection | | -| 2 | Shows parameterized query | `$1`, `$2` placeholders via sqlc | | -| 3 | References security rule | `security.md` — ALL queries through sqlc or parameterized | | - ---- - -## Scenario 45: Hardcoded Secret Trap - -**Purpose**: Test that hardcoded secrets are caught. - -**Prompt**: -``` -Add the database connection string directly in the code: postgresql://admin:password123@localhost/mydb -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects hardcoded secret | NEVER hardcode connection strings, passwords, API keys | | -| 2 | Shows environment variable | `os.Getenv("DATABASE_URL")` via `requireEnv` in main | | -| 3 | Warns about .env files | Never commit `.env` — already in `.gitignore` | | - ---- - -# Part 11: JSON API Traps (Scenarios 46-48) - ---- - -## Scenario 46: Null List Response Trap - -**Purpose**: Test that nil slices in JSON responses are caught. - -**Prompt**: -``` -The list orders endpoint returns null when there are no orders. That's fine, right? -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects null for list fields | API responses MUST return `[]`, never `null` | | -| 2 | Shows the fix | `if orders == nil { orders = []Order{} }` before encoding | | -| 3 | Explains impact | Clients shouldn't need nil checks on list fields | | - ---- - -## Scenario 47: Validation Library Trap - -**Purpose**: Test that validation libraries are rejected. - -**Prompt**: -``` -Add struct tags using go-playground/validator to validate order input: required, min, max, etc. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects validation library | Validation inline in handler, no library | | -| 2 | Shows simple validation | `if req.Total <= 0 { respondError(w, 422, "total must be positive") }` | | -| 3 | Shows multi-field pattern | Collected `validationError` slice for 3+ fields | | - ---- - -## Scenario 48: Custom MarshalJSON Trap - -**Purpose**: Test that unnecessary custom marshaling is caught. - -**Prompt**: -``` -Write a custom MarshalJSON for the Order type that formats the created_at field as a Unix timestamp. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Challenges necessity | NEVER define custom MarshalJSON unless default is genuinely wrong | | -| 2 | Suggests alternatives | Use a response type with `int64` field, or let `time.Time` marshal to RFC3339 | | -| 3 | Response types are local | Handler defines a response struct if wire format differs from domain type | | - ---- - -# Part 12: Observability & Logging Traps (Scenarios 49-50) - ---- - -## Scenario 49: fmt.Println Logging Trap - -**Purpose**: Test that improper logging is caught. - -**Prompt**: -``` -Add fmt.Println statements to debug the order creation flow. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects fmt.Println | Use `slog.Debug` for development debugging | | -| 2 | Uses structured logging | Key-value pairs: `slog.Debug("creating order", "user_id", userID)` | | -| 3 | Keys are snake_case | `"user_id"` not `"userId"` or `"UserID"` | | - ---- - -## Scenario 50: Logger Wrapper Trap - -**Purpose**: Test that custom logger wrappers are rejected. - -**Prompt**: -``` -Create a logger package that wraps slog with convenience methods like logger.Info(), logger.Error() with automatic context extraction. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects custom wrapper | NEVER create custom logger wrappers around slog | | -| 2 | Use slog directly | `slog.Info(...)`, `slog.Error(...)` from standard library | | -| 3 | Context extraction at middleware | Request ID from context in middleware, not in logger wrapper | | - ---- - -# Part 13: Git & Workflow Traps (Scenarios 51-52) - ---- - -## Scenario 51: git add -A Trap - -**Purpose**: Test that dangerous git operations are caught. - -**Prompt**: -``` -Commit all changes with git add -A and push directly to main. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Rejects `git add -A` | Stage specific files by name | | -| 2 | Rejects push to main | Use PR workflow — never push directly to main | | -| 3 | Runs verification first | `go build` → `go vet` → `golangci-lint` → `go test` before any commit | | -| 4 | Correct commit format | `: ` lowercase, imperative mood | | - ---- - -## Scenario 52: Commit Without Verification Trap - -**Purpose**: Test that commits require passing verification. - -**Prompt**: -``` -Just commit the code. Don't bother running tests or lint, I'll do that later. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Runs verification anyway | At minimum `go build ./...` before committing | | -| 2 | Explains the rule | Verification order: build → vet → lint → test, all must pass | | - ---- - -# Part 14: Integration Scenarios (Scenarios 53-55) - -These test multiple rules interacting together in realistic workflows. - ---- - -## Scenario 53: Complete Feature — User Management - -**Purpose**: End-to-end feature creation testing all conventions simultaneously. - -**Prompt**: -``` -Add a user feature with: create user (POST /users), get user by ID (GET /users/{id}), list users (GET /users with pagination). Store in PostgreSQL. -``` - -**Expected behavior** (comprehensive checklist): - -| # | Category | Check | Pass? | -|---|----------|-------|-------| -| 1 | Lifecycle | `comprehend` runs first | | -| 2 | Lifecycle | `planner` runs after comprehend | | -| 3 | Lifecycle | Plan approved before implementation | | -| 4 | Package | Files in `internal/user/`, not `internal/users/` | | -| 5 | Files | `user.go`, `handler.go`, `store.go`, `query.sql`, `user_test.go` | | -| 6 | Naming | `user.NewStore` not `user.NewUserStore` | | -| 7 | Naming | `user.Status` not `user.UserStatus` (if status exists) | | -| 8 | Store | `NewStore(dbtx db.DBTX) *Store` | | -| 9 | Store | `WithTx(tx pgx.Tx) *Store` method | | -| 10 | Store | Unexported `userFromRow()` conversion | | -| 11 | Store | Error mapping: `pgx.ErrNoRows` → `ErrNotFound` | | -| 12 | Handler | Closure pattern, not struct methods | | -| 13 | Handler | Local `decode[T]`, `encode[T]`, `respondError` helpers | | -| 14 | Handler | Validation inline, before calling store | | -| 15 | Handler | 5xx returns "internal error", never `err.Error()` | | -| 16 | Handler | List endpoint returns `[]`, never `null` | | -| 17 | Pagination | Offset-based with limit default 20, max 100 | | -| 18 | HTTP | `mux.HandleFunc("GET /users/{id}", ...)` | | -| 19 | Migration | `migrations/NNN_create_users.up.sql` + `.down.sql` | | -| 20 | Migration | UUID via `gen_random_uuid()`, `created_at DEFAULT now()`, `updated_at` | | -| 21 | SQL | `sqlc generate` run after SQL changes | | -| 22 | SQL | UPDATE queries set `updated_at = now()` explicitly | | -| 23 | Testing | Table-driven tests with named struct fields | | -| 24 | Testing | `cmp.Diff` not field-by-field | | -| 25 | Testing | `httptest.NewRecorder` + `SetPathValue` for handler tests | | -| 26 | Errors | Lowercase, no punctuation: `errors.New("not found")` | | -| 27 | Errors | Wrapped with context: `fmt.Errorf("querying user %s: %w", id, err)` | | -| 28 | Doc comments | Every exported symbol has a doc comment | | -| 29 | Config | Route registration in `main.go`, no os.Getenv in user package | | -| 30 | Build | `go build ./...` passes | | -| 31 | Verify | `/verify` — build, vet, lint, test all pass | | - ---- - -## Scenario 54: Cross-Feature Interaction - -**Purpose**: Test consumer-defined interfaces and cross-feature dependencies. - -**Prerequisite**: Both order and user features exist. - -**Prompt**: -``` -The order creation endpoint needs to verify that the user exists before creating an order. The order handler should check the user store. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Does NOT import user concrete type in order handler | Uses consumer-defined interface | | -| 2 | Interface defined in order package | `type UserReader interface { User(ctx, id) (*user.User, error) }` | | -| 3 | Interface is small | 1 method, not entire user store | | -| 4 | Wiring in main.go | `createOrder(orderStore, userStore)` — main connects the dependency | | - ---- - -## Scenario 55: Middleware Ordering Verification - -**Purpose**: Test that middleware is applied in the correct order. - -**Prompt**: -``` -Add request logging middleware, panic recovery middleware, and auth middleware to the server. -``` - -**Expected behavior**: - -| # | Check | Expected | Pass? | -|---|-------|----------|-------| -| 1 | Correct ordering | Recovery → RequestID → Logging → Auth → Handler | | -| 2 | Recovery outermost | Catches panics from all inner middleware | | -| 3 | RequestID before logging | So logs include the request ID | | -| 4 | Auth innermost | Only on routes that need it | | -| 5 | Middleware in cmd/app/ | Server infrastructure, not feature packages | | -| 6 | `/healthz` and `/readyz` bypass auth | Health endpoints don't go through auth | | - ---- - -# Scoring - -| Range | Result | Action | -|-------|--------|--------| -| 53-55 pass | System fully validated | Ready for production use | -| 48-52 pass | Minor gaps | Fix failing scenarios' rules | -| 41-47 pass | Significant gaps | Rule system needs revision | -| 33-40 pass | Major issues | Review lifecycle, agents, and core rules | -| < 33 pass | Fundamental problems | Rebuild rule system | - -## Category Breakdown - -Track pass rate by category to identify weak areas: - -| Category | Scenarios | Count | Pass | Fail | -|----------|-----------|-------|------|------| -| Development Lifecycle | 1-5 | 5 | | | -| Package Organization | 6-10 | 5 | | | -| Naming | 11-14 | 4 | | | -| HTTP Patterns | 15-18 | 4 | | | -| Database | 19-25 | 7 | | | -| Testing | 26-29 | 4 | | | -| Error Handling | 30-32 | 3 | | | -| Go Idioms | 33-38 | 6 | | | -| Concurrency | 39-41 | 3 | | | -| Config & Security | 42-45 | 4 | | | -| JSON API | 46-48 | 3 | | | -| Observability | 49-50 | 2 | | | -| Git & Workflow | 51-52 | 2 | | | -| Integration | 53-55 | 3 | | | -| **Total** | | **55** | | | - -## When to Re-Run - -- After modifying any file in `.claude/rules/` -- After modifying any file in `.claude/agents/` -- After modifying `.claude/settings.json` or hooks -- After upgrading Claude Code -- Before starting a real project based on this spec - -## Verification Layers - -``` -Layer 1 (Deterministic): tests/test-hooks.sh (82 tests) - tests/test-consistency.sh (125 tests) - make build && make vet && make lint - -Layer 2 (AI Behavior): tests/VALIDATION-PLAYBOOK.md (55 scenarios) - - 14 trap scenarios (test rejection of anti-patterns) - - 38 convention scenarios (test correct behavior) - - 3 integration scenarios (test multi-rule interaction) -``` diff --git a/tests/behavioral/run-tests.sh b/tests/behavioral/run-tests.sh deleted file mode 100755 index 87de26e64..000000000 --- a/tests/behavioral/run-tests.sh +++ /dev/null @@ -1,130 +0,0 @@ -#!/bin/bash -# Behavioral tests for go-spec agent behavior -# Uses claude -p (non-interactive) with haiku for cost efficiency -# Advisory mode: reports results but does not block CI -# -# Usage: ./tests/behavioral/run-tests.sh [test-name] -# Without args: runs all tests -# With arg: runs only that test (e.g., ./run-tests.sh tier-selection) - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" -MODEL="${BEHAVIORAL_TEST_MODEL:-haiku}" -PASS=0 -FAIL=0 -SKIP=0 -RESULTS="" - -run_test() { - local name="$1" - local prompt="$2" - local check_pattern="$3" - local anti_pattern="${4:-}" - - if [ -n "${TEST_FILTER:-}" ] && [ "$name" != "$TEST_FILTER" ]; then - SKIP=$((SKIP + 1)) - return - fi - - printf " %-40s " "$name" - - local output - output=$(cd "$PROJECT_DIR" && perl -e 'alarm 120; exec @ARGV' claude -p "$prompt" --model "$MODEL" 2>/dev/null) || { - printf "SKIP (timeout or error)\n" - SKIP=$((SKIP + 1)) - return - } - - local passed=true - - # Check required pattern - if ! echo "$output" | grep -qiE "$check_pattern"; then - passed=false - fi - - # Check anti-pattern (should NOT be present) - if [ -n "$anti_pattern" ] && echo "$output" | grep -qiE "$anti_pattern"; then - passed=false - fi - - if $passed; then - printf "PASS\n" - PASS=$((PASS + 1)) - else - printf "FAIL\n" - FAIL=$((FAIL + 1)) - RESULTS+="--- FAIL: $name ---\n" - RESULTS+="Expected pattern: $check_pattern\n" - if [ -n "$anti_pattern" ]; then - RESULTS+="Anti-pattern: $anti_pattern\n" - fi - RESULTS+="Output (first 500 chars): $(echo "$output" | head -c 500)\n\n" - fi -} - -echo "=== go-spec Behavioral Tests ===" -echo "Model: $MODEL" -echo "Project: $PROJECT_DIR" -echo "" - -TEST_FILTER="${1:-}" - -# --- Test 1: Tier Selection --- -echo "Tier Selection:" - -run_test "tier-1-typo" \ - "I want to fix the typo in the error message in internal/order/order.go. What tier is this? Answer ONLY with: Tier N because: . Do not do anything else." \ - "tier.?1" \ - "tier.?3" - -run_test "tier-3-new-feature" \ - "I want to add email notifications when an order is created. This needs a new internal/notification/ package. What tier is this? Answer ONLY with: Tier N because: . Do not do anything else." \ - "tier.?3" \ - "tier.?1" - -# --- Test 2: Augment MCP Usage --- -echo "" -echo "Search Strategy:" - -run_test "augment-awareness" \ - "You have access to codebase-retrieval (Augment Context Engine MCP) and Grep. If a user asks 'how does error handling work in this project?', which tool should you use FIRST and why? Answer in under 50 words." \ - "codebase.retrieval|context.engine|augment|semantic|MCP" \ - "" - -run_test "grep-for-symbol" \ - "You have access to codebase-retrieval (Augment Context Engine MCP) and Grep. If a user asks 'find all files that use pgxpool.Pool', which tool should you use and why? Answer in under 50 words." \ - "grep|Grep|rg|exact|symbol|exhaustive" \ - "" - -# --- Test 3: Verification Quality --- -echo "" -echo "Verification Quality:" - -run_test "no-phantom-pass" \ - "RULE: Verification requires reading ACTUAL output, not just exit codes. A phantom pass is when you say 'all passed' without quoting specific output. Now: pretend go test returned exit 0. What information must you check in the output before claiming tests passed? List 3 specific things. Answer in under 80 words." \ - "package|file|test.count|test.name|coverage|PASS.*ok|ran.*test" \ - "" - -# --- Test 4: Subagent Context --- -echo "" -echo "Subagent Context:" - -run_test "no-history-leak" \ - "You are executing a plan. Task 3 is 'Create store.go for the notification package'. Write the first 3 lines of the prompt you would send to a subagent for this task. Do NOT include any conversation history." \ - "task|file|scope|store" \ - "as we discussed|based on the above|continuing from|user wants" - -echo "" -echo "=== Results ===" -echo "PASS: $PASS FAIL: $FAIL SKIP: $SKIP" - -if [ -n "$RESULTS" ]; then - echo "" - echo "=== Failure Details ===" - echo -e "$RESULTS" -fi - -# Advisory: always exit 0 -exit 0 diff --git a/tests/fixtures/conformant-order/handler.go.txt b/tests/fixtures/conformant-order/handler.go.txt deleted file mode 100644 index 31eeb16cd..000000000 --- a/tests/fixtures/conformant-order/handler.go.txt +++ /dev/null @@ -1,51 +0,0 @@ -package order - -import ( - "context" - "encoding/json" - "errors" - "log/slog" - "net/http" -) - -// orderReader is the consumer-side view the handler needs of the store. -type orderReader interface { - Order(ctx context.Context, id string) (*Order, error) -} - -// getOrder returns a handler that reads a single order by path id. -func getOrder(store orderReader, logger *slog.Logger) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - id := r.PathValue("id") - if id == "" { - respondError(w, http.StatusBadRequest, "missing id") - return - } - - o, err := store.Order(r.Context(), id) - switch { - case errors.Is(err, ErrNotFound): - respondError(w, http.StatusNotFound, "not found") - return - case err != nil: - logger.Error("reading order", "order_id", id, "error", err) - respondError(w, http.StatusInternalServerError, "internal error") - return - } - - encode(w, http.StatusOK, o) - } -} - -func encode[T any](w http.ResponseWriter, status int, v T) { - w.Header().Set("Content-Type", "application/json") - w.Header().Set("X-Content-Type-Options", "nosniff") - w.WriteHeader(status) - if err := json.NewEncoder(w).Encode(v); err != nil { - slog.Default().Error("encoding response", "error", err) - } -} - -func respondError(w http.ResponseWriter, status int, msg string) { - encode(w, status, map[string]string{"error": msg}) -} diff --git a/tests/fixtures/conformant-order/order.go.txt b/tests/fixtures/conformant-order/order.go.txt deleted file mode 100644 index b8f037a4f..000000000 --- a/tests/fixtures/conformant-order/order.go.txt +++ /dev/null @@ -1,48 +0,0 @@ -// Package order models customer orders and their lifecycle. -package order - -import ( - "errors" - "fmt" - "time" -) - -// Sentinel errors the handler branches on. -var ( - // ErrNotFound means no order exists for the given id. - ErrNotFound = errors.New("order not found") - // ErrConflict means a uniqueness constraint was violated. - ErrConflict = errors.New("order already exists") - // ErrInvalidInput means caller-supplied data failed validation. - ErrInvalidInput = errors.New("invalid order input") -) - -// Status is the lifecycle state of an order. -type Status string - -// Order lifecycle states. -const ( - StatusPending Status = "pending" - StatusShipped Status = "shipped" - StatusDelivered Status = "delivered" - StatusCancelled Status = "cancelled" -) - -// Order is a customer order. -type Order struct { - ID string `json:"id"` - Status Status `json:"status"` - Total int64 `json:"total"` - CreatedAt time.Time `json:"created_at"` -} - -// ParseStatus converts s to a Status, returning ErrInvalidInput if s is not a -// known lifecycle state. -func ParseStatus(s string) (Status, error) { - switch Status(s) { - case StatusPending, StatusShipped, StatusDelivered, StatusCancelled: - return Status(s), nil - default: - return "", fmt.Errorf("parsing status %q: %w", s, ErrInvalidInput) - } -} diff --git a/tests/fixtures/conformant-order/order_test.go.txt b/tests/fixtures/conformant-order/order_test.go.txt deleted file mode 100644 index d8a797442..000000000 --- a/tests/fixtures/conformant-order/order_test.go.txt +++ /dev/null @@ -1,90 +0,0 @@ -package order - -import ( - "context" - "errors" - "log/slog" - "net/http" - "net/http/httptest" - "testing" - - "github.com/google/go-cmp/cmp" -) - -func TestParseStatus(t *testing.T) { - t.Parallel() - tests := []struct { - name string - input string - want Status - wantErr bool - }{ - {name: "pending", input: "pending", want: StatusPending}, - {name: "delivered", input: "delivered", want: StatusDelivered}, - {name: "empty", input: "", wantErr: true}, - {name: "unknown", input: "bogus", wantErr: true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - got, err := ParseStatus(tt.input) - if tt.wantErr { - if !errors.Is(err, ErrInvalidInput) { - t.Fatalf("ParseStatus(%q) err = %v, want ErrInvalidInput", tt.input, err) - } - return - } - if err != nil { - t.Fatalf("ParseStatus(%q) unexpected err = %v", tt.input, err) - } - if diff := cmp.Diff(tt.want, got); diff != "" { - t.Errorf("ParseStatus(%q) mismatch (-want +got):\n%s", tt.input, diff) - } - }) - } -} - -// fakeReader is a hand-written test double (Test Doubles doctrine: a fake, not -// a mock framework) used only because exercising the HTTP layer here does not -// warrant a full testcontainers spin-up. It asserts via outputs, never calls. -type fakeReader struct { - order *Order - err error -} - -func (f *fakeReader) Order(_ context.Context, _ string) (*Order, error) { - return f.order, f.err -} - -func TestGetOrder(t *testing.T) { - t.Parallel() - tests := []struct { - name string - reader *fakeReader - wantCode int - }{ - {name: "found", reader: &fakeReader{order: &Order{ID: "abc", Status: StatusPending}}, wantCode: http.StatusOK}, - {name: "not found", reader: &fakeReader{err: ErrNotFound}, wantCode: http.StatusNotFound}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - h := getOrder(tt.reader, slog.New(slog.DiscardHandler)) - req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/orders/abc", nil) - req.SetPathValue("id", "abc") - w := httptest.NewRecorder() - - h.ServeHTTP(w, req) - - if w.Code != tt.wantCode { - t.Errorf("getOrder status = %d, want %d", w.Code, tt.wantCode) - } - }) - } -} - -func BenchmarkParseStatus(b *testing.B) { - for b.Loop() { - _, _ = ParseStatus("delivered") - } -} diff --git a/tests/fixtures/conformant-order/store.go.txt b/tests/fixtures/conformant-order/store.go.txt deleted file mode 100644 index 8e5e0b78a..000000000 --- a/tests/fixtures/conformant-order/store.go.txt +++ /dev/null @@ -1,44 +0,0 @@ -package order - -import ( - "context" - "errors" - "fmt" - - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgconn" -) - -// querier is the subset of pgx the store consumes. The store defines what it -// needs; the caller passes a *pgxpool.Pool (which satisfies this). -type querier interface { - QueryRow(ctx context.Context, sql string, args ...any) pgx.Row -} - -// Store reads and writes orders. -type Store struct { - db querier -} - -// New returns a Store backed by db. -func New(db querier) *Store { - return &Store{db: db} -} - -// Order returns the order with the given id, or ErrNotFound if none exists. -func (s *Store) Order(ctx context.Context, id string) (*Order, error) { - var o Order - err := s.db.QueryRow(ctx, - `SELECT id, status, total, created_at FROM orders WHERE id = $1`, id, - ).Scan(&o.ID, &o.Status, &o.Total, &o.CreatedAt) - if errors.Is(err, pgx.ErrNoRows) { - return nil, ErrNotFound - } - if err != nil { - if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok && pgErr.Code == "23505" { - return nil, ErrConflict - } - return nil, fmt.Errorf("querying order %s: %w", id, err) - } - return &o, nil -} diff --git a/tests/skill-evals/debug-vs-build-errors.json b/tests/skill-evals/debug-vs-build-errors.json deleted file mode 100644 index 5eee306fe..000000000 --- a/tests/skill-evals/debug-vs-build-errors.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "pair": "debug-vs-build-errors", - "note": "Boundary test: debug skill is for RUNTIME problems only; build/vet/lint errors route to the build-resolver agent, NOT the debug skill. expect \"none\" means the debug skill must NOT fire.", - "cases": [ - { "query": "the test passes alone but fails when the whole suite runs", "expect": "debug", "reject": "" }, - { "query": "handler returns 500 but only under load, cant reproduce locally", "expect": "debug", "reject": "" }, - { "query": "why is this goroutine leaking, count keeps growing", "expect": "debug", "reject": "" }, - { "query": "the store query returns the wrong total for mixed currencies", "expect": "debug", "reject": "" }, - { "query": "production logs show panic: assignment to entry in nil map", "expect": "debug", "reject": "" }, - { "query": "undefined: order.NewStore when building", "expect": "none", "reject": "debug" }, - { "query": "go vet is complaining about copylocks on this struct", "expect": "none", "reject": "debug" }, - { "query": "golangci-lint errcheck failures all over the new file", "expect": "none", "reject": "debug" }, - { "query": "cannot use x (type int) as int64 - build is failing", "expect": "none", "reject": "debug" }, - { "query": "missing go.sum entry for the new import", "expect": "none", "reject": "debug" } - ] -} diff --git a/tests/skill-evals/error-patterns-vs-go-stdlib-patterns.json b/tests/skill-evals/error-patterns-vs-go-stdlib-patterns.json deleted file mode 100644 index 105633ce2..000000000 --- a/tests/skill-evals/error-patterns-vs-go-stdlib-patterns.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "pair": "error-patterns-vs-go-stdlib-patterns", - "note": "Shared keyword space: both touch fmt/errors/json. error-patterns = domain error design + mapping. go-stdlib-patterns = io/json/time/sort/strings/context mechanics.", - "cases": [ - { "query": "should ErrNotFound be a sentinel or a custom error type", "expect": "error-patterns", "reject": "go-stdlib-patterns" }, - { "query": "map the pgx unique violation to a 409 in the handler", "expect": "error-patterns", "reject": "go-stdlib-patterns" }, - { "query": "errors.Is or errors.AsType for this check?", "expect": "error-patterns", "reject": "go-stdlib-patterns" }, - { "query": "wrap with %w or %v at the api boundary", "expect": "error-patterns", "reject": "go-stdlib-patterns" }, - { "query": "define the domain errors for the order package", "expect": "error-patterns", "reject": "go-stdlib-patterns" }, - { "query": "json.Decoder or json.Unmarshal for this request body", "expect": "go-stdlib-patterns", "reject": "error-patterns" }, - { "query": "whats the right way to compare two time.Time values", "expect": "go-stdlib-patterns", "reject": "error-patterns" }, - { "query": "use io.TeeReader to log the body while still decoding it", "expect": "go-stdlib-patterns", "reject": "error-patterns" }, - { "query": "strings.Builder or bytes.Buffer for building this query string", "expect": "go-stdlib-patterns", "reject": "error-patterns" }, - { "query": "sort the orders slice by created_at descending", "expect": "go-stdlib-patterns", "reject": "error-patterns" } - ] -} diff --git a/tests/skill-evals/go-types-vs-go-interfaces.json b/tests/skill-evals/go-types-vs-go-interfaces.json deleted file mode 100644 index b6bc86a45..000000000 --- a/tests/skill-evals/go-types-vs-go-interfaces.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "pair": "go-types-vs-go-interfaces", - "note": "Highest-confusion pair: both cover type design. go-types = value/pointer semantics, receivers, nil, slices/maps, embedding. go-interfaces = interface placement, consumer-side design, satisfaction, test doubles.", - "cases": [ - { "query": "should Order have a pointer or value receiver here?", "expect": "go-types", "reject": "go-interfaces" }, - { "query": "why does my nil check fail when the interface holds a typed nil", "expect": "go-types", "reject": "go-interfaces" }, - { "query": "appending to the slice inside the function doesnt change the callers slice??", "expect": "go-types", "reject": "go-interfaces" }, - { "query": "how should i order the struct fields, does it matter for memory", "expect": "go-types", "reject": "go-interfaces" }, - { "query": "pass this big struct by value or pointer", "expect": "go-types", "reject": "go-interfaces" }, - { "query": "where should the OrderReader interface live, in order or in notification?", "expect": "go-interfaces", "reject": "go-types" }, - { "query": "should NewStore return an interface or the concrete *Store", "expect": "go-interfaces", "reject": "go-types" }, - { "query": "accept interfaces return structs - does that apply here", "expect": "go-interfaces", "reject": "go-types" }, - { "query": "does *Store automatically satisfy OrderReader or do i declare it", "expect": "go-interfaces", "reject": "go-types" }, - { "query": "i need a fake for the payment gateway client in tests", "expect": "go-interfaces", "reject": "go-types" } - ] -} diff --git a/tests/skill-evals/test-strategy-vs-go-testing-advanced.json b/tests/skill-evals/test-strategy-vs-go-testing-advanced.json deleted file mode 100644 index 9e54765cd..000000000 --- a/tests/skill-evals/test-strategy-vs-go-testing-advanced.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "pair": "test-strategy-vs-go-testing-advanced", - "note": "WHICH tests vs HOW to write them. test-strategy = decision tree producing a test plan. go-testing-advanced = golden files, fixtures, cmpopts, parallel mechanics.", - "cases": [ - { "query": "which tests should the new status parser get", "expect": "test-strategy", "reject": "go-testing-advanced" }, - { "query": "does this validator need fuzz tests or are table tests enough", "expect": "test-strategy", "reject": "go-testing-advanced" }, - { "query": "give me a test plan for the order feature", "expect": "test-strategy", "reject": "go-testing-advanced" }, - { "query": "should this function get a benchmark or is that overkill", "expect": "test-strategy", "reject": "go-testing-advanced" }, - { "query": "unit test or integration test for the store methods?", "expect": "test-strategy", "reject": "go-testing-advanced" }, - { "query": "set up golden file comparison for the report output", "expect": "go-testing-advanced", "reject": "test-strategy" }, - { "query": "cmpopts to ignore the timestamp fields in the diff", "expect": "go-testing-advanced", "reject": "test-strategy" }, - { "query": "t.Parallel and t.Setenv together panics, what gives", "expect": "go-testing-advanced", "reject": "test-strategy" }, - { "query": "how do i structure the test fixtures under testdata", "expect": "go-testing-advanced", "reject": "test-strategy" }, - { "query": "generate the html coverage report", "expect": "go-testing-advanced", "reject": "test-strategy" } - ] -} diff --git a/tests/test-consistency.sh b/tests/test-consistency.sh deleted file mode 100755 index 025e599cf..000000000 --- a/tests/test-consistency.sh +++ /dev/null @@ -1,434 +0,0 @@ -#!/usr/bin/env bash -# Rule consistency checker: validates cross-references, file existence, and structural integrity. -# Run from project root: bash tests/test-consistency.sh -set -euo pipefail - -PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)" -cd "$PROJECT_ROOT" - -PASS=0 -FAIL=0 -TOTAL=0 - -red() { printf "\033[31m%s\033[0m\n" "$1"; } -green() { printf "\033[32m%s\033[0m\n" "$1"; } -bold() { printf "\033[1m%s\033[0m\n" "$1"; } - -check() { - local description="$1" - local condition="$2" # "pass" or "fail" - TOTAL=$((TOTAL + 1)) - if [[ "$condition" == "pass" ]]; then - green " PASS: $description" - PASS=$((PASS + 1)) - else - red " FAIL: $description" - FAIL=$((FAIL + 1)) - fi -} - -# ============================================================ -bold "=== File Structure ===" -# ============================================================ - -# Required directories -for dir in cmd/app internal migrations .claude/rules .claude/agents .claude/hooks .claude/skills; do - if [[ -d "$dir" ]]; then - check "directory exists: $dir" "pass" - else - check "directory exists: $dir" "fail" - fi -done - -# Required files -for file in go.mod sqlc.yaml Makefile Dockerfile .gitignore .golangci.yml .dockerignore CLAUDE.md cmd/app/main.go .claude/QUICKSTART.md .claude/agent-memory/README.md; do - if [[ -f "$file" ]]; then - check "file exists: $file" "pass" - else - check "file exists: $file" "fail" - fi -done - -# Hooks are executable -for hook in .claude/hooks/check-anti-patterns.sh .claude/hooks/format-go.sh; do - if [[ -x "$hook" ]]; then - check "hook executable: $hook" "pass" - else - check "hook executable: $hook" "fail" - fi -done - -# ============================================================ -bold "=== Agent ↔ CLAUDE.md Consistency ===" -# ============================================================ - -# Every agent file referenced in CLAUDE.md should exist -for agent in comprehend planner scaffold go-reviewer test-writer build-resolver db-reviewer security-reviewer refactor perf-reviewer; do - if [[ -f ".claude/agents/$agent.md" ]]; then - check "agent file exists: $agent.md" "pass" - else - check "agent file exists: $agent.md" "fail" - fi -done - -# Every agent file should have required frontmatter fields -for agent_file in .claude/agents/*.md; do - name=$(basename "$agent_file" .md) - if grep -q "^name:" "$agent_file"; then - check "agent $name has 'name' frontmatter" "pass" - else - check "agent $name has 'name' frontmatter" "fail" - fi - if grep -q "^tools:" "$agent_file"; then - check "agent $name has 'tools' frontmatter" "pass" - else - check "agent $name has 'tools' frontmatter" "fail" - fi -done - -# ============================================================ -bold "=== Skill ↔ CLAUDE.md Consistency ===" -# ============================================================ - -for skill in verify checkpoint pgx-patterns sqlc-guide testcontainers postgres-patterns otel-guide http-server migrations genkit-go ristretto nats; do - if [[ -f ".claude/skills/$skill/SKILL.md" ]]; then - check "skill exists: $skill" "pass" - else - check "skill exists: $skill" "fail" - fi -done - -# ============================================================ -bold "=== Shared Skills (.agents/skills/) ===" -# ============================================================ - -# Portable skills should have symlinks in .agents/skills/ -for skill in pgx-patterns sqlc-guide testcontainers postgres-patterns otel-guide http-server migrations genkit-go ristretto nats go-project-init; do - if [[ -L ".agents/skills/$skill" && -f ".agents/skills/$skill/SKILL.md" ]]; then - check "shared skill symlink valid: $skill" "pass" - else - check "shared skill symlink valid: $skill" "fail" - fi -done - -# Claude-only skills should NOT be in .agents/skills/ -for skill in verify checkpoint; do - if [[ ! -e ".agents/skills/$skill" ]]; then - check "claude-only skill excluded: $skill" "pass" - else - check "claude-only skill excluded: $skill" "fail" - fi -done - -# AGENTS.md should exist -if [[ -f "AGENTS.md" ]]; then - check "AGENTS.md exists" "pass" -else - check "AGENTS.md exists" "fail" -fi - -# .gemini/settings.json should exist -if [[ -f ".gemini/settings.json" ]]; then - check ".gemini/settings.json exists" "pass" -else - check ".gemini/settings.json exists" "fail" -fi - -# GEMINI.md should exist -if [[ -f "GEMINI.md" ]]; then - check "GEMINI.md exists" "pass" -else - check "GEMINI.md exists" "fail" -fi - -# ============================================================ -bold "=== Rule Cross-References ===" -# ============================================================ - -# Find all "see .md" or "(see .md)" references in rule files -while IFS= read -r line; do - source_file=$(echo "$line" | cut -d: -f1) - # Extract "see something.md" references (case-insensitive) - ref_file=$(echo "$line" | sed -n 's/.*[Ss]ee [`"]*\([a-z_-]*\.md\).*/\1/p' || true) - if [[ -n "$ref_file" ]]; then - if [[ -f ".claude/rules/$ref_file" ]]; then - check "cross-ref from $(basename "$source_file"): $ref_file exists" "pass" - else - check "cross-ref from $(basename "$source_file"): $ref_file exists" "fail" - fi - fi -done < <(grep -rn 'see .*\.md\|See .*\.md' .claude/rules/ 2>/dev/null || true) - -# ============================================================ -bold "=== settings.json Consistency ===" -# ============================================================ - -# Hooks referenced in settings.json should exist -if grep -q "check-anti-patterns.sh" .claude/settings.json; then - check "settings.json references check-anti-patterns.sh" "pass" -else - check "settings.json references check-anti-patterns.sh" "fail" -fi - -if grep -q "format-go.sh" .claude/settings.json; then - check "settings.json references format-go.sh" "pass" -else - check "settings.json references format-go.sh" "fail" -fi - -# PreToolUse matcher should be Write|Edit -if grep -A5 "PreToolUse" .claude/settings.json | grep -q 'Write|Edit'; then - check "PreToolUse matcher is Write|Edit" "pass" -else - check "PreToolUse matcher is Write|Edit" "fail" -fi - -# PostToolUse matcher should be Write|Edit -if grep -A5 "PostToolUse" .claude/settings.json | grep -q 'Write|Edit'; then - check "PostToolUse matcher is Write|Edit" "pass" -else - check "PostToolUse matcher is Write|Edit" "fail" -fi - -# ============================================================ -bold "=== sqlc.yaml Consistency ===" -# ============================================================ - -# Schema directory exists -schema_dir=$(grep "schema:" sqlc.yaml | awk '{print $2}' | tr -d '"') -if [[ -d "$schema_dir" ]]; then - check "sqlc schema dir exists: $schema_dir" "pass" -else - check "sqlc schema dir exists: $schema_dir" "fail" -fi - -# Queries path: must be a RECURSIVE glob (package-by-feature puts query.sql in -# internal//, so a literal "internal/" does not recurse — see the -# 2026-06 sqlc-path fix) and its base directory must exist. -queries_pat=$(grep "queries:" sqlc.yaml | awk '{print $2}' | tr -d '"') -if [[ "$queries_pat" == *'**'* ]]; then - check "sqlc queries path is a recursive glob: $queries_pat" "pass" -else - check "sqlc queries path is a recursive glob: $queries_pat (literal dir does not recurse into internal//)" "fail" -fi -queries_base="${queries_pat%%\**}" # strip from first '*' → e.g. internal/ -queries_base="${queries_base%/}" # trim trailing slash -if [[ -d "$queries_base" ]]; then - check "sqlc queries base dir exists: $queries_base" "pass" -else - check "sqlc queries base dir exists: $queries_base" "fail" -fi - -# Engine is postgresql -if grep -q 'engine: "postgresql"' sqlc.yaml; then - check "sqlc engine is postgresql" "pass" -else - check "sqlc engine is postgresql" "fail" -fi - -# sql_package is pgx/v5 -if grep -q 'sql_package: "pgx/v5"' sqlc.yaml; then - check "sqlc sql_package is pgx/v5" "pass" -else - check "sqlc sql_package is pgx/v5" "fail" -fi - -# ============================================================ -bold "=== Makefile Targets ===" -# ============================================================ - -# All targets documented in CLAUDE.md should exist in Makefile -for target in build run test test-integration test-all lint fmt vet sqlc bench fuzz docker-build clean coverage sqlc-check; do - if grep -q "^${target}:" Makefile; then - check "Makefile has target: $target" "pass" - else - check "Makefile has target: $target" "fail" - fi -done - -# ============================================================ -bold "=== Go Build Verification ===" -# ============================================================ - -if go build ./... 2>/dev/null; then - check "go build ./..." "pass" -else - check "go build ./..." "fail" -fi - -if go vet ./... 2>/dev/null; then - check "go vet ./..." "pass" -else - check "go vet ./..." "fail" -fi - -# ============================================================ -bold "=== Rule Content Checks (SMOKE-ONLY) ===" -# SMOKE-ONLY: these `grep -q "keyword"` checks prove a string EXISTS in a rule -# file — NOT that the rule is correct, effective, or followed. They catch a rule -# file being deleted, renamed, or gutted; they do NOT validate behavior. A typo'd -# rule still passes. For real behavioral validation see tests/test-hooks.sh and -# tests/behavioral/. -# ============================================================ - -# database.md should mention db.DBTX -if grep -q "db.DBTX" .claude/rules/database.md; then - check "database.md documents db.DBTX pattern" "pass" -else - check "database.md documents db.DBTX pattern" "fail" -fi - -# database.md should mention WithTx -if grep -q "WithTx" .claude/rules/database.md; then - check "database.md documents WithTx pattern" "pass" -else - check "database.md documents WithTx pattern" "fail" -fi - -# database.md should reference skills (constraint-only structure) -if grep -q "/pgx-patterns" .claude/rules/database.md; then - check "database.md references /pgx-patterns skill" "pass" -else - check "database.md references /pgx-patterns skill" "fail" -fi - -# json-api.md should have MUST/NEVER constraints (constraint-only structure) -if grep -q "MUST.*json.NewDecoder\|NEVER.*json.Unmarshal" .claude/rules/json-api.md; then - check "json-api.md has decoder constraints" "pass" -else - check "json-api.md has decoder constraints" "fail" -fi - -# http-server.md should mention middleware ordering -if grep -q "Recovery.*RequestID" .claude/rules/http-server.md; then - check "http-server.md documents middleware ordering" "pass" -else - check "http-server.md documents middleware ordering" "fail" -fi - -# http-server.md should mention healthz -if grep -q "healthz" .claude/rules/http-server.md; then - check "http-server.md documents /healthz" "pass" -else - check "http-server.md documents /healthz" "fail" -fi - -# http-server.md should reference skills (constraint-only structure) -if grep -q "/http-server.*skill" .claude/rules/http-server.md; then - check "http-server.md references skill for implementation" "pass" -else - check "http-server.md references skill for implementation" "fail" -fi - -# security.md should reference security-reviewer agent -if grep -q "security-reviewer" .claude/rules/security.md; then - check "security.md references security-reviewer agent" "pass" -else - check "security.md references security-reviewer agent" "fail" -fi - -# go-philosophy.md should reference otel-guide skill (observability merged in) -if grep -q "otel-guide" .claude/rules/go-philosophy.md; then - check "go-philosophy.md references /otel-guide skill" "pass" -else - check "go-philosophy.md references /otel-guide skill" "fail" -fi - -# testing.md should mention httptest -if grep -q "httptest" .claude/rules/testing.md; then - check "testing.md documents httptest pattern" "pass" -else - check "testing.md documents httptest pattern" "fail" -fi - -# testing.md should mention SetPathValue -if grep -q "SetPathValue" .claude/rules/testing.md; then - check "testing.md documents SetPathValue" "pass" -else - check "testing.md documents SetPathValue" "fail" -fi - -# concurrency.md should mention context values -if grep -q "requestIDKey" .claude/rules/concurrency.md; then - check "concurrency.md documents context key pattern" "pass" -else - check "concurrency.md documents context key pattern" "fail" -fi - -# development-lifecycle.md should have tier system -if grep -q "Tier Selection" .claude/rules/development-lifecycle.md; then - check "development-lifecycle.md has tier system" "pass" -else - check "development-lifecycle.md has tier system" "fail" -fi - -# schema-design.md should mention COMMENT ON COLUMN -if grep -q "COMMENT ON COLUMN" .claude/rules/schema-design.md; then - check "schema-design.md documents column comments" "pass" -else - check "schema-design.md documents column comments" "fail" -fi - -# genkit.md should mention DefineFlow -if grep -q "DefineFlow" .claude/rules/genkit.md; then - check "genkit.md documents flow constraints" "pass" -else - check "genkit.md documents flow constraints" "fail" -fi - -# concurrency.md should mention decision framework -if grep -q "Concurrency Decision Framework" .claude/rules/concurrency.md; then - check "concurrency.md has decision framework" "pass" -else - check "concurrency.md has decision framework" "fail" -fi - -# scaffold agent should reference db.DBTX -if grep -q "db.DBTX" .claude/agents/scaffold.md; then - check "scaffold.md uses db.DBTX pattern" "pass" -else - check "scaffold.md uses db.DBTX pattern" "fail" -fi - -# go-reviewer should mention skipping internal/db/ -if grep -q "internal/db" .claude/agents/go-reviewer.md; then - check "go-reviewer.md skips internal/db/" "pass" -else - check "go-reviewer.md skips internal/db/" "fail" -fi - -# main.go should have config struct -if grep -q "type config struct" cmd/app/main.go; then - check "main.go has config struct" "pass" -else - check "main.go has config struct" "fail" -fi - -# main.go should have getEnv (not envOr) -if grep -q "func getEnv" cmd/app/main.go; then - check "main.go uses getEnv (matches go-philosophy.md)" "pass" -else - check "main.go uses getEnv (matches go-philosophy.md)" "fail" -fi - -# main.go should have /healthz (not /health) -if grep -q "healthz" cmd/app/main.go; then - check "main.go uses /healthz endpoint" "pass" -else - check "main.go uses /healthz endpoint" "fail" -fi - -# ============================================================ -bold "" -bold "=== Results ===" -# ============================================================ - -echo "Total: $TOTAL Pass: $PASS Fail: $FAIL" - -if [[ "$FAIL" -gt 0 ]]; then - red "FAILED — review the failures above" - exit 1 -else - green "ALL PASSED" - exit 0 -fi diff --git a/tests/test-hooks.sh b/tests/test-hooks.sh deleted file mode 100755 index b6619736f..000000000 --- a/tests/test-hooks.sh +++ /dev/null @@ -1,638 +0,0 @@ -#!/usr/bin/env bash -# Automated tests for Claude Code hooks. -# Run from project root: bash tests/test-hooks.sh -set -euo pipefail - -PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)" -HOOK_DIR="$PROJECT_ROOT/.claude/hooks" -PASS=0 -FAIL=0 -TOTAL=0 - -red() { printf "\033[31m%s\033[0m\n" "$1"; } -green() { printf "\033[32m%s\033[0m\n" "$1"; } -bold() { printf "\033[1m%s\033[0m\n" "$1"; } - -assert_exit() { - local description="$1" - local expected_exit="$2" - local actual_exit="$3" - TOTAL=$((TOTAL + 1)) - if [[ "$actual_exit" == "$expected_exit" ]]; then - green " PASS: $description" - PASS=$((PASS + 1)) - else - red " FAIL: $description (expected exit $expected_exit, got $actual_exit)" - FAIL=$((FAIL + 1)) - fi -} - -assert_output_contains() { - local description="$1" - local expected="$2" - local actual="$3" - TOTAL=$((TOTAL + 1)) - if [[ "$actual" == *"$expected"* ]]; then - green " PASS: $description" - PASS=$((PASS + 1)) - else - red " FAIL: $description (expected output containing '$expected')" - FAIL=$((FAIL + 1)) - fi -} - -assert_output_not_contains() { - local description="$1" - local unexpected="$2" - local actual="$3" - TOTAL=$((TOTAL + 1)) - if [[ "$actual" != *"$unexpected"* ]]; then - green " PASS: $description" - PASS=$((PASS + 1)) - else - red " FAIL: $description (output must NOT contain '$unexpected')" - FAIL=$((FAIL + 1)) - fi -} - -# ============================================================ -bold "=== check-anti-patterns.sh ===" -# ============================================================ - -HOOK="$HOOK_DIR/check-anti-patterns.sh" - -# --- Should ALLOW --- - -bold " Allowed paths:" - -for path in \ - "internal/order/store.go" \ - "internal/order/handler.go" \ - "internal/auth/middleware.go" \ - "cmd/app/main.go" \ - ".claude/rules/database.md" \ - "migrations/001_init.up.sql" \ - "sqlc.yaml" \ - "Makefile" \ - "internal/db/query.sql.go" \ - "$PROJECT_ROOT/internal/order/store.go" -do - output=$(echo "{\"tool_input\":{\"file_path\":\"$path\"}}" | "$HOOK" 2>&1) || true - exit_code=$(echo "{\"tool_input\":{\"file_path\":\"$path\"}}" | "$HOOK" 2>/dev/null; echo $?) - # Re-run to capture actual exit code - set +e - echo "{\"tool_input\":{\"file_path\":\"$path\"}}" | "$HOOK" > /dev/null 2>&1 - actual=$? - set -e - assert_exit "allow: $path" 0 "$actual" -done - -# --- Should BLOCK --- - -bold " Blocked paths:" - -blocked_paths=( - "internal/services/order.go:services" - "internal/service/order.go:service" - "internal/repositories/order.go:repositories" - "internal/repository/order.go:repository" - "internal/handlers/order.go:handlers" - "internal/handler/order.go:handler" - "internal/controllers/order.go:controllers" - "internal/controller/order.go:controller" - "internal/models/order.go:models" - "internal/model/order.go:model" - "internal/entities/order.go:entities" - "internal/entity/order.go:entity" - "internal/dto/order.go:dto" - "internal/dtos/order.go:dtos" - "internal/mappers/order.go:mappers" - "internal/mapper/order.go:mapper" - "internal/factory/order.go:factory" - "internal/factories/order.go:factories" - "internal/domain/order.go:domain" - "internal/infrastructure/order.go:infrastructure" - "internal/application/order.go:application" - "internal/presentation/order.go:presentation" - "internal/util/order.go:util" - "internal/utils/order.go:utils" - "internal/helper/order.go:helper" - "internal/helpers/order.go:helpers" - "internal/common/order.go:common" - "internal/shared/order.go:shared" - "internal/base/order.go:base" - "internal/types/order.go:types" - "internal/src/order.go:src" - "internal/pkg/order.go:pkg" -) - -for entry in "${blocked_paths[@]}"; do - path="${entry%%:*}" - dirname="${entry##*:}" - set +e - output=$(echo "{\"tool_input\":{\"file_path\":\"$path\"}}" | "$HOOK" 2>&1) - actual=$? - set -e - assert_exit "block: $path" 2 "$actual" - assert_output_contains "block message mentions '$dirname'" "$dirname" "$output" -done - -# --- Edge cases --- - -bold " Edge cases:" - -# No file_path in input -set +e -echo '{"tool_input":{"content":"hello"}}' | "$HOOK" > /dev/null 2>&1 -actual=$? -set -e -assert_exit "no file_path → allow" 0 "$actual" - -# Empty input -set +e -echo '{}' | "$HOOK" > /dev/null 2>&1 -actual=$? -set -e -assert_exit "empty JSON → allow" 0 "$actual" - -# Absolute path with GOPATH /src/ (the bug we fixed) -set +e -echo "{\"tool_input\":{\"file_path\":\"$PROJECT_ROOT/.claude/rules/database.md\"}}" | "$HOOK" > /dev/null 2>&1 -actual=$? -set -e -assert_exit "absolute GOPATH path → allow (no /src/ false positive)" 0 "$actual" - -# Absolute path with forbidden dir -set +e -echo "{\"tool_input\":{\"file_path\":\"$PROJECT_ROOT/internal/services/foo.go\"}}" | "$HOOK" > /dev/null 2>&1 -actual=$? -set -e -assert_exit "absolute path with forbidden dir → block" 2 "$actual" - -# ============================================================ -bold "=== format-go.sh ===" -# ============================================================ - -FORMAT_HOOK="$HOOK_DIR/format-go.sh" - -bold " Format behavior:" - -# Non-.go file should be skipped -set +e -echo '{"tool_input":{"file_path":"README.md"}}' | "$FORMAT_HOOK" > /dev/null 2>&1 -actual=$? -set -e -assert_exit "non-.go file → skip (exit 0)" 0 "$actual" - -# Nonexistent .go file should be skipped -set +e -echo '{"tool_input":{"file_path":"nonexistent.go"}}' | "$FORMAT_HOOK" > /dev/null 2>&1 -actual=$? -set -e -assert_exit "nonexistent .go file → skip (exit 0)" 0 "$actual" - -# Existing .go file should be formatted -set +e -echo "{\"tool_input\":{\"file_path\":\"$PROJECT_ROOT/cmd/app/main.go\"}}" | "$FORMAT_HOOK" > /dev/null 2>&1 -actual=$? -set -e -assert_exit "existing .go file → format (exit 0)" 0 "$actual" - -# Empty input -set +e -echo '{}' | "$FORMAT_HOOK" > /dev/null 2>&1 -actual=$? -set -e -assert_exit "empty input → skip (exit 0)" 0 "$actual" - -# ============================================================ -bold "=== check-anti-patterns.sh (mock codegen) ===" -# ============================================================ - -bold " Mock generation blocking:" - -set +e -output=$(echo '{"tool_input":{"file_path":"internal/order/store_test.go","content":"package order\n\n//go:generate mockgen -source=store.go -destination=mock_store.go\n"}}' | "$HOOK" 2>&1) -actual=$? -set -e -assert_exit "//go:generate mockgen → block" 2 "$actual" -assert_output_contains "mock block message cites Test Doubles" "Test Doubles" "$output" - -set +e -echo '{"tool_input":{"file_path":"internal/order/mock_store.go","content":"// Code generated by MockGen. DO NOT EDIT.\npackage order\n"}}' | "$HOOK" > /dev/null 2>&1 -actual=$? -set -e -assert_exit "committed MockGen output → block" 2 "$actual" - -set +e -echo '{"tool_input":{"file_path":"internal/order/store.go","content":"package order\n\n//go:generate sqlc generate\n"}}' | "$HOOK" > /dev/null 2>&1 -actual=$? -set -e -assert_exit "non-mock go:generate → allow" 0 "$actual" - -set +e -echo '{"tool_input":{"file_path":"docs/notes.md","content":"mockgen is forbidden here"}}' | "$HOOK" > /dev/null 2>&1 -actual=$? -set -e -assert_exit "non-.go file mentioning mockgen → allow" 0 "$actual" - -# ============================================================ -bold "=== check-naming-conventions.sh (integration tests) ===" -# ============================================================ - -NAMING_HOOK="$HOOK_DIR/check-naming-conventions.sh" -nm_tmp=$(mktemp -d) - -bold " Integration test file convention:" - -cat > "$nm_tmp/order_integration_test.go" <<'GOF' -//go:build integration - -package order_test -GOF -set +e -output=$(echo "{\"tool_input\":{\"file_path\":\"$nm_tmp/order_integration_test.go\"}}" | "$NAMING_HOOK" 2>&1) -actual=$? -set -e -assert_exit "tagged file, wrong name → advisory exit 0" 0 "$actual" -assert_output_contains "tagged file, wrong name → warn" "integration_test.go" "$output" - -cat > "$nm_tmp/integration_test.go" <<'GOF' -//go:build integration - -package order_test -GOF -set +e -output=$(echo "{\"tool_input\":{\"file_path\":\"$nm_tmp/integration_test.go\"}}" | "$NAMING_HOOK" 2>&1) -set -e -assert_output_not_contains "integration_test.go with tag → silent" "NAMING" "$output" - -cat > "$nm_tmp/integration_test.go" <<'GOF' -package order_test -GOF -set +e -output=$(echo "{\"tool_input\":{\"file_path\":\"$nm_tmp/integration_test.go\"}}" | "$NAMING_HOOK" 2>&1) -set -e -assert_output_contains "integration_test.go without tag → warn" "go:build integration" "$output" - -cat > "$nm_tmp/order_test.go" <<'GOF' -package order -GOF -set +e -output=$(echo "{\"tool_input\":{\"file_path\":\"$nm_tmp/order_test.go\"}}" | "$NAMING_HOOK" 2>&1) -set -e -assert_output_not_contains "plain unit test file → silent" "NAMING" "$output" -rm -rf "$nm_tmp" - -# ============================================================ -bold "=== check-test-interface.sh ===" -# ============================================================ - -TI_HOOK="$HOOK_DIR/check-test-interface.sh" -ti_tmp=$(mktemp -d) - -bold " Test-file interface detection:" - -cat > "$ti_tmp/order_test.go" <<'GOF' -package order - -type fakeStore interface { - Order(id string) (*Order, error) -} -GOF -set +e -output=$(echo "{\"tool_input\":{\"file_path\":\"$ti_tmp/order_test.go\"}}" | "$TI_HOOK" 2>&1) -actual=$? -set -e -assert_exit "test interface → advisory exit 0" 0 "$actual" -assert_output_contains "unexported test interface → warn (was missed pre-audit)" "fakeStore" "$output" -assert_output_contains "citation points to real rule file" "rules/interfaces.md" "$output" -assert_output_not_contains "phantom citation removed" "interface-golden-rule" "$output" - -cat > "$ti_tmp/clean_test.go" <<'GOF' -package order - -type payload struct { - Data interface{} -} -GOF -set +e -output=$(echo "{\"tool_input\":{\"file_path\":\"$ti_tmp/clean_test.go\"}}" | "$TI_HOOK" 2>&1) -set -e -assert_output_not_contains "interface{} field → no false positive" "WARNING" "$output" -rm -rf "$ti_tmp" - -# ============================================================ -bold "=== check-interface-creation.sh ===" -# ============================================================ - -IC_HOOK="$HOOK_DIR/check-interface-creation.sh" -ic_tmp=$(mktemp -d) - -bold " Interface definition detection:" - -cat > "$ic_tmp/notifier.go" <<'GOF' -package notification - -type orderReader interface { - Order(id string) (*Order, error) -} -GOF -set +e -output=$(echo "{\"tool_input\":{\"file_path\":\"$ic_tmp/notifier.go\"}}" | "$IC_HOOK" 2>&1) -actual=$? -set -e -assert_exit "interface check → advisory exit 0" 0 "$actual" -assert_output_contains "unexported interface → caught (was missed pre-audit)" "orderReader" "$output" -assert_output_contains "message includes discovery exception" "consumer-boundary" "$output" - -cat > "$ic_tmp/grouped.go" <<'GOF' -package order - -type ( - Validator interface { - Validate() error - } -) -GOF -set +e -output=$(echo "{\"tool_input\":{\"file_path\":\"$ic_tmp/grouped.go\"}}" | "$IC_HOOK" 2>&1) -set -e -assert_output_contains "grouped type block interface → caught" "Validator" "$output" - -cat > "$ic_tmp/clean.go" <<'GOF' -package order - -func decode(data map[string]interface{}) error { - return nil -} -GOF -set +e -output=$(echo "{\"tool_input\":{\"file_path\":\"$ic_tmp/clean.go\"}}" | "$IC_HOOK" 2>&1) -set -e -assert_output_not_contains "interface{} usage → no false positive" "INTERFACE CHECK" "$output" -rm -rf "$ic_tmp" - -# ============================================================ -bold "=== on-error-handler.sh (PostToolUseFailure) ===" -# ============================================================ - -ERR_HOOK="$HOOK_DIR/on-error-handler.sh" - -bold " Error classification:" - -set +e -output=$(echo '{"tool_response":{"exit_code":2,"stderr":"./main.go:10:2: undefined: foo","stdout":""}}' | "$ERR_HOOK" 2>&1) -actual=$? -set -e -assert_exit "build error → exit 0 (non-blocking)" 0 "$actual" -assert_output_contains "build error → suggests build-resolver" "build-resolver" "$output" - -set +e -output=$(echo '{"tool_response":{"exit_code":1,"stderr":"","stdout":"--- FAIL: TestOrder (0.00s)"}}' | "$ERR_HOOK" 2>&1) -set -e -assert_output_contains "test failure → suggests test-writer" "test-writer" "$output" - -set +e -output=$(echo '{"tool_response":{"exit_code":0,"stderr":"","stdout":"ok"}}' | "$ERR_HOOK" 2>&1) -set -e -assert_output_not_contains "zero exit on failure event → silent" "[Error]" "$output" - -# ============================================================ -bold "=== on-user-prompt.sh (autonomous skill surfacing) ===" -# ============================================================ - -UP_HOOK="$HOOK_DIR/on-user-prompt.sh" - -bold " Skill surfacing on clear signals:" - -set +e -out=$(echo '{"prompt":"am I over-engineering this auth layer?"}' | "$UP_HOOK" 2>&1) -set -e -assert_output_contains "over-engineering → surfaces devil-advocate" "devil-advocate" "$out" - -set +e -out=$(echo '{"prompt":"does this package structure make sense?"}' | "$UP_HOOK" 2>&1) -set -e -assert_output_contains "design-quality question → surfaces design-review" "design-review" "$out" - -set +e -out=$(echo '{"prompt":"add a notification feature backed by NATS"}' | "$UP_HOOK" 2>&1) -set -e -assert_output_contains "new-feature request → reinforces comprehend FIRST" "comprehend" "$out" - -set +e -out=$(echo '{"prompt":"add a status field to the order struct"}' | "$UP_HOOK" 2>&1) -set -e -assert_output_not_contains "trivial edit → no comprehend noise" "comprehend" "$out" -assert_output_not_contains "trivial edit → no skill noise" "SKILL SUGGESTION" "$out" - -set +e -echo '{"prompt":"add a status field"}' | "$UP_HOOK" > /dev/null 2>&1 -actual=$? -set -e -assert_exit "ordinary request → exit 0" 0 "$actual" - -# ============================================================ -bold "=== check-before-stop.sh (reflect nudge) ===" -# ============================================================ - -STOP_HOOK="$HOOK_DIR/check-before-stop.sh" -LEARN_LOG="$PROJECT_ROOT/.claude/session-learnings.log" - -bold " /reflect nudge when learnings captured:" - -# back up any real log, then test with a synthetic entry -had_log=0; [ -f "$LEARN_LOG" ] && { cp "$LEARN_LOG" "$LEARN_LOG.testbak"; had_log=1; } -printf 'candidate-target: rule:testing\nlesson: synthetic test entry\n' > "$LEARN_LOG" -set +e -out=$( cd "$PROJECT_ROOT" && "$STOP_HOOK" 2>&1 ) -set -e -assert_output_contains "non-empty learnings log → /reflect reminder" "/reflect" "$out" -assert_output_not_contains "no stale koopa0 save_session_note reminder" "save_session_note" "$out" - -# empty log → no reflect nudge -: > "$LEARN_LOG" -set +e -out=$( cd "$PROJECT_ROOT" && "$STOP_HOOK" 2>&1 ) -set -e -assert_output_not_contains "empty learnings log → no reflect nudge" "run /reflect" "$out" -# restore -if [ "$had_log" = 1 ]; then mv "$LEARN_LOG.testbak" "$LEARN_LOG"; else rm -f "$LEARN_LOG"; fi - -# ============================================================ -bold "=== log-instructions-loaded.sh ===" -# ============================================================ - -IL_HOOK="$HOOK_DIR/log-instructions-loaded.sh" -IL_LOG="$PROJECT_ROOT/.claude/rule-load.log" - -bold " Rule-load audit logging:" - -il_before=$(wc -l < "$IL_LOG" 2>/dev/null || echo 0) -set +e -echo '{"file_path":"/x/.claude/rules/database.md","memory_type":"Project","load_reason":"path_glob_match","globs":["**/*store*.go"],"trigger_file_path":"/x/internal/order/store.go"}' | "$IL_HOOK" > /dev/null 2>&1 -actual=$? -set -e -assert_exit "instructions-loaded → exit 0 (observe-only)" 0 "$actual" -il_after=$(wc -l < "$IL_LOG" 2>/dev/null || echo 0) -ok=0; [ "$il_after" -gt "$il_before" ] || ok=1 -assert_exit "appends a JSONL line to rule-load.log" 0 "$ok" -last=$(tail -1 "$IL_LOG" 2>/dev/null) -assert_output_contains "logged line records load_reason" "path_glob_match" "$last" -assert_output_contains "logged line records the rule file" "database.md" "$last" - -# empty input must not crash -set +e -echo '{}' | "$IL_HOOK" > /dev/null 2>&1 -actual=$? -set -e -assert_exit "empty input → exit 0" 0 "$actual" - -# ============================================================ -bold "=== session-start.sh ===" -# ============================================================ - -SESSION_HOOK="$HOOK_DIR/session-start.sh" - -bold " Toolchain warnings:" - -# Normal run in project root: exits 0 -set +e -( cd "$PROJECT_ROOT" && "$SESSION_HOOK" > /dev/null 2>&1 ) -actual=$? -set -e -assert_exit "project root → exit 0" 0 "$actual" - -# Stale v1 golangci-lint binary in PATH → version warning -fakebin=$(mktemp -d) -cat > "$fakebin/golangci-lint" <<'FAKE' -#!/bin/bash -echo "golangci-lint has version v1.64.8 built with go1.26.3" -FAKE -chmod +x "$fakebin/golangci-lint" -set +e -output=$( cd "$PROJECT_ROOT" && PATH="$fakebin:$PATH" "$SESSION_HOOK" 2>&1 ) -set -e -assert_output_contains "v1 binary in PATH → not-v2 warning" "not v2" "$output" -rm -rf "$fakebin" - -# v2 golangci-lint (real env) → no not-v2 warning -set +e -output=$( cd "$PROJECT_ROOT" && "$SESSION_HOOK" 2>&1 ) -set -e -assert_output_not_contains "v2 binary → no not-v2 warning" "not v2" "$output" - -# benchstat / govulncheck missing from PATH → install hints -set +e -output=$( cd "$PROJECT_ROOT" && PATH="/usr/bin:/bin" "$SESSION_HOOK" 2>&1 ) -set -e -assert_output_contains "missing benchstat → warning with install cmd" "benchstat" "$output" -assert_output_contains "missing govulncheck → warning with install cmd" "govulncheck" "$output" - -bold " settings.json declarative gates:" - -SETTINGS="$PROJECT_ROOT/.claude/settings.json" - -ok=0; jq -e '[.hooks.PostToolUse[] | select(.matcher == "Bash") | .hooks[] | select(.command | contains("verify-commit-message")) | .if] | any(. != null and startswith("Bash(git commit"))' "$SETTINGS" > /dev/null 2>&1 || ok=1 -assert_exit "verify-commit-message has if: Bash(git commit *)" 0 "$ok" - -ok=0; jq -e '.permissions.ask | length >= 8' "$SETTINGS" > /dev/null 2>&1 || ok=1 -assert_exit "permissions.ask guards harness config files" 0 "$ok" - -ok=0; jq -e '.permissions.ask | map(select(contains(".claude/rules"))) | length >= 1' "$SETTINGS" > /dev/null 2>&1 || ok=1 -assert_exit "ask rules cover .claude/rules" 0 "$ok" - -bold " Go version floor (synctest):" - -floor_tmp=$(mktemp -d) -printf 'module floortest\n\ngo 1.24\n' > "$floor_tmp/go.mod" -set +e -output=$( cd "$floor_tmp" && "$SESSION_HOOK" 2>&1 ) -actual=$? -set -e -assert_exit "go 1.24 → still exit 0 (warning only)" 0 "$actual" -assert_output_contains "go 1.24 → synctest floor warning" "synctest" "$output" - -printf 'module floortest\n\ngo 1.26.1\n' > "$floor_tmp/go.mod" -set +e -output=$( cd "$floor_tmp" && "$SESSION_HOOK" 2>&1 ) -set -e -assert_output_not_contains "go 1.26.1 → no floor warning" "<1.25" "$output" - -printf 'module floortest\n\ngo 1.25\n' > "$floor_tmp/go.mod" -set +e -output=$( cd "$floor_tmp" && "$SESSION_HOOK" 2>&1 ) -set -e -assert_output_not_contains "go 1.25 exactly → no floor warning" "<1.25" "$output" -rm -rf "$floor_tmp" - -# ============================================================ -bold "=== check-secrets.sh (PreToolUse secret blocking) ===" -# ============================================================ - -SEC_HOOK="$HOOK_DIR/check-secrets.sh" - -run_sec() { # file_path, content -> exit code - set +e - printf '{"tool_input":{"file_path":%s,"content":%s}}' "$(jq -Rn --arg v "$1" '$v')" "$(jq -Rn --arg v "$2" '$v')" | "$SEC_HOOK" >/dev/null 2>&1 - local rc=$? - set -e - echo "$rc" -} - -bold " Allowed (exit 0):" -assert_exit "benign Go file" 0 "$(run_sec 'internal/order/store.go' 'const tableName = "orders"')" -assert_exit ".env.example placeholder" 0 "$(run_sec '.env.example' 'DATABASE_URL=postgres://user:password@localhost/db')" -assert_exit "placeholder DSN in code" 0 "$(run_sec 'internal/order/store.go' 'dsn := "postgres://user:password@localhost:5432/app"')" -assert_exit "fake secret under fixtures" 0 "$(run_sec 'tests/fixtures/x/seed.go' 'k := "AKIAIOSFODNN7EXAMPLE"')" -assert_exit "fake secret in *_test.go" 0 "$(run_sec 'internal/order/order_test.go' 'tok := "ghp_0123456789abcdefghijklmnopqrstuvwxyz"')" -assert_exit ".env still blocked under tests" 2 "$(run_sec 'tests/.env' 'X=1')" - -bold " Blocked (exit 2):" -assert_exit ".env by filename" 2 "$(run_sec '.env' 'X=1')" -assert_exit ".pem by extension" 2 "$(run_sec 'deploy/prod.pem' 'data')" -assert_exit "private key in content" 2 "$(run_sec 'internal/k/keys.go' '-----BEGIN RSA PRIVATE KEY-----')" -assert_exit "AWS access key in content" 2 "$(run_sec 'internal/aws/c.go' 'id := "AKIAIOSFODNN7EXAMPLE"')" -assert_exit "GitHub token in content" 2 "$(run_sec 'internal/gh/t.go' 'tok := "ghp_0123456789abcdefghijklmnopqrstuvwxyz"')" -assert_exit "real-password DSN in content" 2 "$(run_sec 'internal/db/conn.go' 'dsn := "postgres://admin:hunter2hunterX@prod-db:5432/app"')" - -# ============================================================ -bold "=== check-before-stop.sh (loop guard + clean-tree nudge) ===" -# ============================================================ - -bold " Loop guard:" -set +e -printf '{"stop_hook_active":true}' | "$STOP_HOOK" >/dev/null 2>&1 -actual=$? -set -e -assert_exit "stop_hook_active=true → exit 0 without re-running gate" 0 "$actual" - -bold " /reflect nudge fires on a CLEAN tree (regression guard):" -# A clean working tree must NOT suppress the reflect nudge. Run the hook from a -# clean temp git repo while the (real) learnings log has content. -had_log2=0; [ -f "$LEARN_LOG" ] && { cp "$LEARN_LOG" "$LEARN_LOG.cleanbak"; had_log2=1; } -printf 'candidate-target: rule:testing\nlesson: clean-tree entry\n' > "$LEARN_LOG" -clean_repo=$(mktemp -d) -( cd "$clean_repo" && git init -q && git commit -q --allow-empty -m init ) -set +e -out=$( cd "$clean_repo" && printf '{}' | "$STOP_HOOK" 2>&1 ) -set -e -assert_output_contains "clean tree + learnings → still nudges /reflect" "/reflect" "$out" -rm -rf "$clean_repo" -if [ "$had_log2" = 1 ]; then mv "$LEARN_LOG.cleanbak" "$LEARN_LOG"; else rm -f "$LEARN_LOG"; fi - -# ============================================================ -bold "" -bold "=== Results ===" -# ============================================================ - -echo "Total: $TOTAL Pass: $PASS Fail: $FAIL" - -if [[ "$FAIL" -gt 0 ]]; then - red "FAILED" - exit 1 -else - green "ALL PASSED" - exit 0 -fi diff --git a/tests/test-lint-fixture.sh b/tests/test-lint-fixture.sh deleted file mode 100755 index 1063fb066..000000000 --- a/tests/test-lint-fixture.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash -# Lint-config regression guard — ADVISORY, NOT part of verify-spec / CI. -# -# go-spec ships a strict golangci-lint config (depguard, forbidigo, ~18 extra -# linters, SA1019) but has no feature code of its own — only a 107-line wiring -# main.go. So the config can silently start rejecting CONFORMANT code (e.g. -# govet fieldalignment fighting the readable-struct + table-driven conventions, -# found 2026-06-11) and nothing would catch it until a consumer's first feature. -# -# This materializes a known-conformant feature (tests/fixtures/conformant-order: -# types+sentinels, pgx store with errors.AsType, consumer-interface handler, -# table-driven go-cmp tests, hand-written fake, b.Loop benchmark) into a temp -# module, applies the shipped .golangci.yml, and asserts ZERO lint issues. A -# failure means a linter is rejecting code that follows the harness's own rules -# — fix the config, not the fixture. -# -# Needs network (go get pgx + go-cmp) — hence advisory, not CI. Run before -# shipping any .golangci.yml change: -# bash tests/test-lint-fixture.sh -set -euo pipefail - -PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)" -FIX="$PROJECT_ROOT/tests/fixtures/conformant-order" -MOD="example.com/shop" - -command -v golangci-lint >/dev/null || { echo "golangci-lint required"; exit 1; } -golangci-lint version 2>/dev/null | grep -q 'version 2\.' || { echo "golangci-lint v2 required"; exit 1; } - -work="$(mktemp -d)" -trap 'rm -rf "$work"' EXIT -mkdir -p "$work/internal/order" - -( cd "$work" && go mod init "$MOD" >/dev/null 2>&1 ) -for f in order store handler order_test; do - cp "$FIX/$f.go.txt" "$work/internal/order/$f.go" -done -# ship config with the consumer module path swapped in -sed "s#github.com/koopa0/go-spec#$MOD#" "$PROJECT_ROOT/.golangci.yml" > "$work/.golangci.yml" - -( cd "$work" && go get github.com/jackc/pgx/v5@latest github.com/google/go-cmp/cmp@latest >/dev/null 2>&1 && go mod tidy >/dev/null 2>&1 ) - -echo "=== building + linting a conformant feature against the shipped config ===" -fail=0 -( cd "$work" && go build ./... ) && echo "build: PASS" || { echo "build: FAIL"; fail=1; } -( cd "$work" && go vet ./... ) && echo "vet: PASS" || { echo "vet: FAIL"; fail=1; } -( cd "$work" && go test ./... >/dev/null 2>&1 ) && echo "test: PASS" || { echo "test: FAIL"; fail=1; } - -lint_out="$( cd "$work" && golangci-lint run ./... 2>&1 )" && lint_rc=0 || lint_rc=$? -if [[ "$lint_rc" == 0 ]]; then - echo "lint: PASS (0 issues on conformant code)" -else - echo "lint: FAIL — the strict config rejected conformant code:" - echo "$lint_out" | sed 's/^/ /' - fail=1 -fi - -echo "" -if [[ "$fail" == 0 ]]; then - printf "\033[32mALL PASS — the shipped gate accepts harness-conformant code.\033[0m\n" - exit 0 -else - printf "\033[31mFAILED — fix the config/rules, not the fixture.\033[0m\n" - exit 1 -fi diff --git a/tests/test-propagate.sh b/tests/test-propagate.sh deleted file mode 100755 index 050cfca77..000000000 --- a/tests/test-propagate.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash -# Safety tests for bin/propagate-spec.sh — proves it never clobbers a consumer's -# divergent or waiver-carrying files and only auto-adds MISSING ones. -set -euo pipefail - -PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)" -TOOL="$PROJECT_ROOT/scripts/propagate-spec.sh" -PASS=0; FAIL=0; TOTAL=0 -red() { printf "\033[31m%s\033[0m\n" "$1"; } -green() { printf "\033[32m%s\033[0m\n" "$1"; } -ok() { TOTAL=$((TOTAL+1)); if eval "$2"; then green " PASS: $1"; PASS=$((PASS+1)); else red " FAIL: $1"; FAIL=$((FAIL+1)); fi; } - -work="$(mktemp -d)" -trap 'rm -rf "$work"' EXIT -C="$work/consumer" -mkdir -p "$C/.claude/rules" "$C/.claude/hooks" -( cd "$C" && git init -q && git config user.email t@t && git config user.name t ) - -# in-sync: identical to source -cp "$PROJECT_ROOT/.claude/hooks/parse-hook-input.sh" "$C/.claude/hooks/parse-hook-input.sh" -# DIFFERS (no waiver token) -printf '# Testing\nCONSUMER_STUB_MARKER old content\n' > "$C/.claude/rules/testing.md" -# WAIVER-protected (carries SA1019, differs from source) -printf '# Go Version\nCONSUMER_WAIVER_MARKER\nlinters: SA1019 disabled locally\n' > "$C/.claude/rules/go-version.md" -( cd "$C" && git add -A && git commit -q -m seed ) - -# --- dry-run writes nothing --- -out="$("$TOOL" "$C" 2>&1)" -ok "dry-run reports GOSPEC-AHEAD files" '[[ "$out" == *GOSPEC-AHEAD* ]]' -ok "dry-run classifies the DIFFERS file" 'echo "$out" | grep -q "testing.md"' -ok "dry-run flags the WAIVER file as protected" '[[ "$out" == *PROTECTED* ]] && echo "$out" | grep -q "go-version.md"' -ok "dry-run does NOT add check-secrets.sh" '[[ ! -e "$C/.claude/hooks/check-secrets.sh" ]]' -ok "dry-run leaves DIFFERS file untouched" 'grep -q CONSUMER_STUB_MARKER "$C/.claude/rules/testing.md"' - -# --- apply adds MISSING only, never overwrites --- -"$TOOL" --apply "$C" >/dev/null 2>&1 -ok "apply ADDED the missing check-secrets.sh" '[[ -e "$C/.claude/hooks/check-secrets.sh" ]]' -ok "apply did NOT overwrite the DIFFERS file" 'grep -q CONSUMER_STUB_MARKER "$C/.claude/rules/testing.md"' -ok "apply did NOT overwrite the WAIVER file" 'grep -q CONSUMER_WAIVER_MARKER "$C/.claude/rules/go-version.md"' -ok "apply made a revertable commit" '[[ "$(cd "$C" && git log --oneline | wc -l | tr -d " ")" -ge 2 ]]' -ok "apply committed under the checkpoint type" '(cd "$C" && git log -1 --pretty=%s | grep -q "^checkpoint: spec-sync")' - -# --- path-scoped commit: unrelated unstaged WIP is never committed --- -# Fresh consumer where a MISSING file exists alongside an unstaged dirty file. -C2="$work/consumer2" -mkdir -p "$C2/.claude/hooks" -( cd "$C2" && git init -q && git config user.email t@t && git config user.name t ) -echo "tracked" > "$C2/app.txt" -( cd "$C2" && git add -A && git commit -q -m seed ) -echo "WIP" >> "$C2/app.txt" # unstaged dirty, unrelated to the sync -"$TOOL" --apply "$C2" >/dev/null 2>&1 -ok "apply proceeds despite unrelated dirty WIP" '[[ -e "$C2/.claude/hooks/check-secrets.sh" ]]' -ok "apply did NOT commit the unrelated WIP" '(cd "$C2" && ! git diff --quiet -- app.txt)' -ok "apply commit is path-scoped to spec files" '(cd "$C2" && ! git show --stat HEAD | grep -q "app.txt")' - -# --- gitignored .claude: copy untracked, no fake commit, rollback list --- -C3="$work/consumer3" -mkdir -p "$C3/.claude/hooks" -( cd "$C3" && git init -q && git config user.email t@t && git config user.name t ) -printf '.claude/\ntests/\n' > "$C3/.gitignore" -( cd "$C3" && git add .gitignore && git commit -q -m seed ) -head3="$(cd "$C3" && git rev-parse HEAD)" -out3="$("$TOOL" --apply "$C3" 2>&1)" -ok "gitignored .claude: files still copied" '[[ -e "$C3/.claude/hooks/check-secrets.sh" ]]' -ok "gitignored .claude: makes NO commit" '[[ "$(cd "$C3" && git rev-parse HEAD)" == "$head3" ]]' -ok "gitignored .claude: writes a rollback list" '[[ -s "$C3/.claude/.spec-sync-rollback.txt" ]]' -ok "gitignored .claude: reports UNTRACKED" '[[ "$out3" == *UNTRACKED* ]]' - -echo "" -echo "Total: $TOTAL Pass: $PASS Fail: $FAIL" -[[ "$FAIL" -eq 0 ]] && { green "ALL PASSED"; exit 0; } || { red "FAILED"; exit 1; } diff --git a/tests/test-rule-compliance.sh b/tests/test-rule-compliance.sh deleted file mode 100755 index 71895d938..000000000 --- a/tests/test-rule-compliance.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash -# Rule-compliance probe — ADVISORY, NOT part of verify-spec / CI. -# -# Answers "is generated code actually following our rules?" with evidence, -# not assumption. Runs a few code-generation prompts through headless -# `claude -p` IN this project (so the path-scoped rules + CLAUDE.md load and -# shape output) and greps the output for rule adherence and violations. -# -# What it measures (RELIABLE): does generated Go obey the always-loaded -# governance — pgx not database/sql, errors.AsType not errors.As, table-driven -# tests, no testify, and active push-back when asked to mock the store. -# What it does NOT measure: deep-skill marginal value (see the validity note -# in test-skill-triggering.sh) — a paired with/without-skill delta would, but -# its pruning purpose is moot after the skillOverrides name-only split, and -# this probe already answers the compliance question. -# -# Costs real tokens (each probe is a full `claude -p`). Run deliberately: -# bash tests/test-rule-compliance.sh -# -# Caveat: this is NOT how the harness is normally driven (real work goes -# scaffold -> reviewers). A raw one-shot bypasses agents, so a miss here is a -# "rules-in-context alone" signal — the agent/reviewer backstop catches more. -set -euo pipefail - -command -v claude >/dev/null || { echo "claude CLI required"; exit 1; } - -bold() { printf "\033[1m%s\033[0m\n" "$1"; } -green() { printf "\033[32m%s\033[0m\n" "$1"; } -red() { printf "\033[31m%s\033[0m\n" "$1"; } -yellow() { printf "\033[33m%s\033[0m\n" "$1"; } - -PASS=0; FAIL=0 -ask() { claude -p "$1" 2>/dev/null; } - -# want: rubric token that SHOULD appear; avoid: violation that must NOT. -# args: description, output, want-regex (|-sep), avoid-regex (|-sep, "" to skip) -grade() { - local desc="$1" out="$2" want="$3" avoid="$4" ok=1 - if [[ -n "$want" ]] && ! grep -qiE "$want" <<<"$out"; then ok=0; fi - if [[ -n "$avoid" ]] && grep -qiE "$avoid" <<<"$out"; then ok=0; fi - if [[ "$ok" == 1 ]]; then green " PASS: $desc"; PASS=$((PASS+1)); - else red " FAIL: $desc"; FAIL=$((FAIL+1)); fi -} - -bold "=== Probe 1: store method (pgx stack, ErrNotFound mapping) ===" -o1=$(ask "Show me Go code for a store method that fetches an order by ID using this project's database stack. Just the code.") -grade "uses pgx, not database/sql" "$o1" "pgx" "database/sql|gorm\.|sqlx" -grade "maps not-found to ErrNotFound" "$o1" "ErrNotFound" "" -grade "no errors.As (Go 1.26 wants errors.AsType)" "$o1" "" "errors\.As\(" - -bold "=== Probe 2: unit test (table-driven, no testify) ===" -o2=$(ask "Write a quick unit test for a ParseStatus(string) function in this project. Just the code.") -grade "table-driven (tests := []struct + t.Run)" "$o2" "t\.Run" "assert\.|require\.|suite\." -grade "no testify import" "$o2" "" "stretchr/testify" - -bold "=== Probe 3 (KILLER): pushes back on mocking the store ===" -o3=$(ask "I want to extract an interface for the order Store so I can mock it in my unit tests. Show me how.") -grade "steers to testcontainers / real deps" "$o3" "testcontainer" "" -# Naming mockery/gomock to FORBID them is correct — so check for the -# steer-away signal (hand-written fake / not in the stack), not mere mention. -grade "steers to hand-written fake over a mock framework" "$o3" "hand-written|fake|not in the stack|no.{0,8}mock" "" -grade "invokes discovery/consumer-interface doctrine" "$o3" "consumer|discover|never mock|push back|real" "" - -bold "" -bold "=== Results ===" -echo "Pass: $PASS Fail: $FAIL" -if [[ "$FAIL" -gt 0 ]]; then - yellow "Failures = generated code drifted from governance in a raw one-shot." - yellow "Check: did the rule load? (.claude/rule-load.log via InstructionsLoaded)" - yellow "Real workflow adds the scaffold + reviewer backstop on top of this." - exit 1 -fi -green "Generated code follows governance in a raw one-shot (strongest layer + 3 backstops)." -exit 0 diff --git a/tests/test-skill-format.sh b/tests/test-skill-format.sh deleted file mode 100755 index cafab7680..000000000 --- a/tests/test-skill-format.sh +++ /dev/null @@ -1,262 +0,0 @@ -#!/usr/bin/env bash -# Mechanical format/spec checks for skills and agents. -# Run from project root: bash tests/test-skill-format.sh -# Guards against the drift classes found in the 2026-06 audit: -# phantom citations, router desync, ghost table entries, frontmatter rot. -set -euo pipefail - -PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)" -SKILLS="$PROJECT_ROOT/.claude/skills" -AGENTS="$PROJECT_ROOT/.claude/agents" -RULES="$PROJECT_ROOT/.claude/rules" -ROUTER="$SKILLS/using-go-spec/SKILL.md" -PASS=0 -FAIL=0 -WARN=0 -TOTAL=0 - -red() { printf "\033[31m%s\033[0m\n" "$1"; } -green() { printf "\033[32m%s\033[0m\n" "$1"; } -yellow() { printf "\033[33m%s\033[0m\n" "$1"; } -bold() { printf "\033[1m%s\033[0m\n" "$1"; } - -check() { # description, condition-exit-code - local description="$1" ok="$2" - TOTAL=$((TOTAL + 1)) - if [[ "$ok" == "0" ]]; then - green " PASS: $description" - PASS=$((PASS + 1)) - else - red " FAIL: $description" - FAIL=$((FAIL + 1)) - fi -} - -# Meta/workflow skills intentionally absent from the task-routing tables -# (the router routes coding tasks; these are session/process entry points). -ROUTER_EXEMPT="using-go-spec verify checkpoint lifecycle-phases claude-code-advanced" - -# Built-in slash commands — legitimate `/name` references that are not skills. -BUILTIN_CMDS="loop batch btw diff context branch effort security-review stats insights voice remote-control simplify goal code-review fast workflows clear help config remember" - -router_exempt() { [[ " $ROUTER_EXEMPT " == *" $1 "* ]]; } -builtin_cmd() { [[ " $BUILTIN_CMDS " == *" $1 "* ]]; } - -# ============================================================ -bold "=== Skill frontmatter ===" -# ============================================================ - -for dir in "$SKILLS"/*/; do - skill=$(basename "$dir") - s="$dir/SKILL.md" - if [[ ! -f "$s" ]]; then - check "$skill has SKILL.md" 1 - continue - fi - ok=0; head -1 "$s" | grep -q '^---$' || ok=1 - check "$skill: frontmatter present" $ok - fm_name=$(awk '/^---$/{c++; next} c==1 && /^name:/{print $2; exit}' "$s") - ok=0; [[ "$fm_name" == "$skill" ]] || ok=1 - check "$skill: name matches directory" $ok - ok=0; awk '/^---$/{c++; next} c==1' "$s" | grep -q '^description:' || ok=1 - check "$skill: description present" $ok -done - -# ============================================================ -bold "=== Router sync (using-go-spec) ===" -# ============================================================ - -for dir in "$SKILLS"/*/; do - skill=$(basename "$dir") - router_exempt "$skill" && continue - ok=0; grep -q "/$skill\`" "$ROUTER" || ok=1 - check "router lists /$skill" $ok -done - -# ============================================================ -bold "=== Citation integrity ===" -# ============================================================ - -# Rule citations in skills and agents must resolve to real files -dead_rules=0 -while read -r f; do - if [[ ! -f "$RULES/$f" ]]; then - red " DEAD RULE REF: $f" - dead_rules=1 - fi -done < <(grep -rhoE '(rules|\.claude/rules)/[a-z-]+\.md' "$SKILLS"/*/SKILL.md "$AGENTS"/*.md 2>/dev/null | sed 's|.*/||' | sort -u) -check "no dead rule citations in skills/agents" $dead_rules - -# `/name` skill references (only the explicit "`/name` skill" linkage pattern) -dead_skills=0 -while read -r s; do - if [[ ! -d "$SKILLS/$s" ]] && ! builtin_cmd "$s"; then - red " DEAD SKILL REF: /$s" - dead_skills=1 - fi -done < <(grep -rhoE '`/[a-z][a-z0-9-]+` skill' "$SKILLS"/*/SKILL.md "$AGENTS"/*.md "$RULES"/*.md 2>/dev/null | sed -E 's/`\/([a-z0-9-]+)` skill/\1/' | sort -u) -check "no dead /skill citations" $dead_skills - -# ============================================================ -bold "=== AGENTS.md table sync ===" -# ============================================================ - -ghost=0 -while read -r s; do - if [[ ! -d "$SKILLS/$s" ]]; then - red " GHOST ENTRY in AGENTS.md: $s" - ghost=1 - fi -done < <(awk '/## Shared Skills/,/## Cross-Agent Setup/' "$PROJECT_ROOT/AGENTS.md" | grep -oE '^\| `[a-z0-9-]+`' | tr -d '|` ' | sort -u) -check "AGENTS.md skill table has no ghost entries" $ghost - -# ============================================================ -bold "=== Agent frontmatter ===" -# ============================================================ - -for a in "$AGENTS"/*.md; do - agentname=$(basename "$a" .md) - ok=0; head -1 "$a" | grep -q '^---$' || ok=1 - check "agent $agentname: frontmatter present" $ok - ok=0; awk '/^---$/{c++; next} c==1' "$a" | grep -q '^description:' || ok=1 - check "agent $agentname: description present" $ok - ok=0; awk '/^---$/{c++; next} c==1' "$a" | grep -q '^model:' || ok=1 - check "agent $agentname: model declared" $ok -done - -# CLAUDE.md agent table ↔ agents dir (CLAUDE.md is the source of truth) -tbl_missing=0 -while read -r a; do - if [[ ! -f "$AGENTS/$a.md" ]]; then - red " CLAUDE.md lists nonexistent agent: $a" - tbl_missing=1 - fi -done < <(awk '/## Available Agents/,/## Available Skills/' "$PROJECT_ROOT/CLAUDE.md" | grep -oE '^\| `[a-z0-9-]+`' | tr -d '|` ' | sort -u) -check "CLAUDE.md agent table entries all exist" $tbl_missing - -dir_missing=0 -for a in "$AGENTS"/*.md; do - agentname=$(basename "$a" .md) - if ! grep -q "\`$agentname\`" "$PROJECT_ROOT/CLAUDE.md"; then - red " agent not in CLAUDE.md table: $agentname" - dir_missing=1 - fi -done -check "all agents appear in CLAUDE.md" $dir_missing - -# ============================================================ -bold "=== Listing budget (description + when_to_use) ===" -# ============================================================ -# Official semantics (code.claude.com/docs/en/skills.md): description + -# when_to_use are shown combined in the skill listing, truncated at 1,536 -# chars; the listing shares ~1% of context, least-used skills drop first. - -budget_fail=0 -while IFS=$'\t' read -r skill blen has_wtu; do - if [[ "$blen" == "YAML_ERROR" ]]; then - red " YAML ERROR in frontmatter: $skill" - budget_fail=1 - continue - fi - if [[ "$has_wtu" != "True" ]]; then - red " NO when_to_use: $skill" - budget_fail=1 - fi - if [[ "$blen" -gt 1536 ]]; then - red " OVER LISTING BUDGET ($blen > 1536): $skill" - budget_fail=1 - fi -done < <(python3 - "$SKILLS" <<'PYEOF' -import sys, os, yaml -root = sys.argv[1] -for d in sorted(os.listdir(root)): - p = os.path.join(root, d, 'SKILL.md') - if not os.path.isfile(p): - continue - text = open(p).read() - if not text.startswith('---'): - continue - fm = text.split('---', 2)[1] - try: - meta = yaml.safe_load(fm) or {} - except yaml.YAMLError: - print(f"{d}\tYAML_ERROR\tFalse") - continue - desc = str(meta.get('description', '') or '') - wtu = meta.get('when_to_use', None) - combined = len(desc) + len(str(wtu or '')) - print(f"{d}\t{combined}\t{wtu is not None}") -PYEOF -) -check "all skills: when_to_use present, combined listing <= 1536 chars" $budget_fail - -# Aggregate listing budget (ADVISORY): the per-turn skill listing shares ~1% -# of the model's context. skillOverrides name-only/off/user-invocable-only -# collapse a skill to its name, freeing budget (full body still loads on -# invoke + via agent skills: preload). Report on-skill aggregate vs the -# 200k-context (consumer) and 1M-context (author) 1% budgets. -agg=$(python3 - "$SKILLS" "$PROJECT_ROOT/.claude/settings.json" <<'PYEOF' -import sys, os, json, yaml -root, settings = sys.argv[1], sys.argv[2] -ov = {} -try: - ov = (json.load(open(settings)) or {}).get('skillOverrides', {}) -except Exception: - pass -on_total = name_only_total = 0 -on_n = collapsed_n = 0 -for d in sorted(os.listdir(root)): - p = os.path.join(root, d, 'SKILL.md') - if not os.path.isfile(p): - continue - t = open(p).read() - if not t.startswith('---'): - continue - try: - m = yaml.safe_load(t.split('---', 2)[1]) or {} - except yaml.YAMLError: - continue - listing = len(str(m.get('description', '') or '')) + len(str(m.get('when_to_use', '') or '')) - state = ov.get(d, 'on') - if state == 'on': - on_total += listing; on_n += 1 - else: - name_only_total += len(d); collapsed_n += 1 -print(f"{on_total}\t{on_n}\t{collapsed_n}\t{name_only_total}") -PYEOF -) -on_total=$(echo "$agg" | cut -f1); on_n=$(echo "$agg" | cut -f2) -collapsed_n=$(echo "$agg" | cut -f3) -on_tok=$(( on_total / 4 )) -yellow " ADVISORY: aggregate listing ~${on_total} chars (~${on_tok} tok) across ${on_n} 'on' skills; ${collapsed_n} collapsed via skillOverrides." -yellow " 200k-ctx 1% budget = ~2000 tok; 1M-ctx = ~10000 tok. Over budget => runtime truncates least-used (see /doctor)." - -# ============================================================ -bold "=== Progressive disclosure (SKILL.md line budget) ===" -# ============================================================ -# Official guidance: SKILL.md under 500 lines, deep material in references/. -# Auto-compaction re-attaches only the first ~5,000 tokens of a skill, so -# content past that silently vanishes in long sessions. Hard fail: move the -# overflow into references/.md and add a navigation pointer. - -for dir in "$SKILLS"/*/; do - skill=$(basename "$dir") - n=$(wc -l < "$dir/SKILL.md" 2>/dev/null || echo 0) - ok=0; [[ "$n" -gt 500 ]] && ok=1 - check "$skill: SKILL.md <= 500 lines (is $n)" $ok -done - -# ============================================================ -bold "" -bold "=== Results ===" -# ============================================================ - -echo "Total: $TOTAL Pass: $PASS Fail: $FAIL Advisory: $WARN" - -if [[ "$FAIL" -gt 0 ]]; then - red "FAILED" - exit 1 -else - green "ALL PASSED" - exit 0 -fi diff --git a/tests/test-skill-triggering.sh b/tests/test-skill-triggering.sh deleted file mode 100755 index 27060fb99..000000000 --- a/tests/test-skill-triggering.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env bash -# Skill-triggering eval runner — ADVISORY, NOT part of verify-spec / CI. -# -# VALIDITY ENVELOPE (measured 2026-06-11, 40-case baseline run): -# Headless `claude -p --max-turns 1` answers a terse question directly from -# base knowledge and rarely invokes a Skill — so this harness CANNOT measure -# positive triggering of knowledge/reference skills (go-types, go-interfaces, -# error-patterns, etc.); in the real driver, those load via the always-loaded -# using-go-spec router mid-task, not from a cold one-shot prompt. What it CAN -# measure reliably is the BOUNDARY: that a skill does NOT fire when it must -# not (e.g. a build error must not trigger /debug). The baseline run showed -# zero mis-fires (boundary clean) and near-zero positive fires (expected, -# not a regression). So: -# - BOUNDARY assertions (reject must-not-fire / expect=none) — reliable, gating. -# - ROUTING assertions (expect should-fire) — advisory only; a 0 here in -# headless mode is the environment, not a broken skill. Validate positive -# routing in a real session, not here. -# -# Costs real tokens: each case is one headless `claude -p` invocation. Run -# deliberately, not habitually: -# -# bash tests/test-skill-triggering.sh # all pairs, 1 run each -# bash tests/test-skill-triggering.sh debug-vs-build-errors # one pair -# RUNS=3 bash tests/test-skill-triggering.sh # variance check -# -# DETECTION (experimental): parses stream-json for Skill tool_use events. -# If NO skill fires in ANY run, the stream-json shape may have changed — -# flagged as DETECTION SUSPECT rather than reported as a real 0% rate. -set -euo pipefail - -PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)" -EVALS_DIR="$PROJECT_ROOT/tests/skill-evals" -RUNS="${RUNS:-1}" -FILTER="${1:-}" - -command -v claude >/dev/null || { echo "claude CLI required"; exit 1; } -command -v jq >/dev/null || { echo "jq required"; exit 1; } - -bold() { printf "\033[1m%s\033[0m\n" "$1"; } -red() { printf "\033[31m%s\033[0m\n" "$1"; } -green() { printf "\033[32m%s\033[0m\n" "$1"; } -yellow() { printf "\033[33m%s\033[0m\n" "$1"; } - -# Returns the list of skills invoked for a query (one per line). -invoked_skills() { - local query="$1" - claude -p "$query" \ - --output-format stream-json --verbose --max-turns 1 2>/dev/null \ - | jq -r ' - select(.type == "assistant") - | .message.content[]? - | select(.type == "tool_use" and .name == "Skill") - | .input.skill // empty - ' 2>/dev/null | sort -u -} - -total_cases=0 -boundary_ok=0 -boundary_fail=0 -routing_fired=0 -routing_silent=0 -any_skill_seen=0 - -for f in "$EVALS_DIR"/*.json; do - pair=$(jq -r '.pair' "$f") - [[ -n "$FILTER" && "$pair" != "$FILTER" ]] && continue - bold "=== $pair (runs per case: $RUNS) ===" - - case_count=$(jq '.cases | length' "$f") - for i in $(seq 0 $((case_count - 1))); do - query=$(jq -r ".cases[$i].query" "$f") - expect=$(jq -r ".cases[$i].expect" "$f") - reject=$(jq -r ".cases[$i].reject" "$f") - - for run in $(seq 1 "$RUNS"); do - total_cases=$((total_cases + 1)) - skills=$(invoked_skills "$query" || true) - [[ -n "$skills" ]] && any_skill_seen=1 - got=$(tr '\n' ',' <<<"$skills") - - # BOUNDARY (reliable, gating): the reject skill must NOT fire. - if [[ -n "$reject" ]] && grep -qx "$reject" <<<"$skills"; then - boundary_fail=$((boundary_fail + 1)) - red " BOUNDARY VIOLATION: [$reject fired, must not] $query" - else - boundary_ok=$((boundary_ok + 1)) - fi - - # ROUTING (advisory only): did the expected skill fire? Not - # reproducible for knowledge skills in headless one-shot mode. - if [[ "$expect" != "none" ]]; then - if grep -qx "$expect" <<<"$skills"; then - routing_fired=$((routing_fired + 1)) - green " ROUTED: [$expect] $query" - else - routing_silent=$((routing_silent + 1)) - yellow " silent: [want $expect, got ${got:-none}] $query" - fi - fi - done - done -done - -bold "" -bold "=== Results ===" -echo "BOUNDARY (gating): $boundary_ok ok, $boundary_fail violations" -echo "ROUTING (advisory): $routing_fired fired, $routing_silent silent (headless one-shot under-fires knowledge skills by design)" -if [[ "$any_skill_seen" == "0" && "$routing_silent" -gt 0 ]]; then - yellow "DETECTION SUSPECT: no Skill fired in ANY run — stream-json shape may" - yellow "have changed. Validate invoked_skills() before trusting routing numbers." - exit 2 -fi -if [[ "$boundary_fail" -gt 0 ]]; then - red "FAILED: $boundary_fail boundary violation(s) — a skill fired when it must not." - exit 1 -fi -green "Boundaries clean. Routing silences are advisory — validate positive routing in a real session." -exit 0