fix(story-automator): compute story test counts from JUnit truth

skills/bmad-story-automator/data/orchestration-policy.json

@@ -14,6 +14,10 @@
       "arrays": "replace"
     }
   },
+  "test": {
+    "command": "",
+    "junitPath": ""
+  },
   "workflow": {
     "sequence": ["create", "dev", "auto", "review", "retro"],
     "repeat": {

skills/bmad-story-automator/data/prompts/dev.md

@@ -2,3 +2,7 @@ Execute the BMAD dev-story workflow for story {{story_id}}.
 
 {{skill_line}}{{workflow_line}}{{instructions_line}}{{checklist_line}}Story file: `{{implementation_artifacts}}/{{story_prefix}}-*.md`
 Implement all tasks marked [ ]. Run tests. Update checkboxes.
+Emit a JUnit XML report from the test run if the runner supports it, and always
+print the full test summary (total / failures / errors / skipped). Do NOT
+transcribe test counts into the story prose — the orchestrator records them in a
+machine-owned section from the JUnit report.

skills/bmad-story-automator/src/story_automator/cli.py

@@ -12,6 +12,7 @@
     cmd_list_sessions,
     cmd_reconcile_story,
     cmd_stop_hook,
+    cmd_test_counts,
 )
 from .commands.orchestrator import cmd_orchestrator_helper
 from .commands.state import cmd_build_state_doc, cmd_sprint_compare, cmd_state_metrics, cmd_validate_state
@@ -42,6 +43,7 @@ def main(argv: list[str] | None = None) -> int:
         "build-state-doc": cmd_build_state_doc,
         "commit-story": cmd_commit_story,
         "reconcile-story": cmd_reconcile_story,
+        "test-counts": cmd_test_counts,
         "parse-epic": _cmd_parse_epic,
         "parse-story": _cmd_parse_story,
         "parse-story-range": _cmd_parse_story_range,
@@ -79,6 +81,7 @@ def _usage(stream: object) -> None:
         "build-state-doc",
         "commit-story",
         "reconcile-story",
+        "test-counts",
         "parse-epic",
         "parse-story",
         "parse-story-range",

skills/bmad-story-automator/src/story_automator/commands/basic.py

@@ -8,10 +8,13 @@
 from pathlib import Path
 
 from ..core.artifact_paths import implementation_artifacts_dir, implementation_artifacts_relpath
+from ..core.junit import parse_junit
 from ..core.runtime_layout import active_marker_path, runtime_provider
+from ..core.runtime_policy import PolicyError, load_policy_unresolved, test_config
 from ..core.stop_hooks import HookConfigError, ensure_stop_hook
 from ..core.story_keys import normalize_story_key
 from ..core.utils import (
+    ensure_dir,
     get_project_slug,
     run_cmd,
     write_json,
@@ -247,10 +250,11 @@ def _git_changed_files(repo: str, extra_excludes: tuple[str, ...] = ()) -> list[
     return sorted(files)
 
 
-def _file_list_bounds(lines: list[str]) -> tuple[int, int] | None:
+def _section_bounds(lines: list[str], heading: str) -> tuple[int, int] | None:
+    target = heading.strip().lower()
     start = None
     for idx, line in enumerate(lines):
-        if line.strip().lower() == "### file list":
+        if line.strip().lower() == target:
             start = idx
             break
     if start is None:
@@ -263,6 +267,10 @@ def _file_list_bounds(lines: list[str]) -> tuple[int, int] | None:
     return start, end
 
 
+def _file_list_bounds(lines: list[str]) -> tuple[int, int] | None:
+    return _section_bounds(lines, "### File List")
+
+
 # Known dev-record status annotations to strip when parsing a hand-written File
 # List — a closed set, so we never mangle a real filename that ends in ")".
 _FILE_LIST_ANNOTATIONS = ("(new)", "(modified)", "(deleted)", "(added)", "(renamed)", "(updated)")
@@ -305,6 +313,23 @@ def _reconcile_section(text: str, git_files: list[str]) -> tuple[str, list[str]]
     return "\n".join(new_lines) + suffix, current
 
 
+# Map a story id to its artifact file (resolved key first, prefix glob fallback).
+# Returns (story_file, None) on success or (None, error_payload) so callers can
+# emit the error verbatim and bail.
+def _resolve_story_file(repo: str, story: str) -> tuple[Path | None, dict | None]:
+    norm = normalize_story_key(repo, story)
+    if norm is None:
+        return None, {"ok": False, "error": "story_key_invalid", "input": story}
+    artifacts = implementation_artifacts_dir(repo)
+    exact = artifacts / f"{norm.key}.md"
+    if norm.key and exact.is_file():  # disambiguate via the resolved key before falling back to prefix glob
+        return exact, None
+    matches = sorted(artifacts.glob(f"{norm.prefix}-*.md"))
+    if not matches:
+        return None, {"ok": False, "error": "story_file_not_found", "prefix": norm.prefix}
+    return matches[0], None
+
+
 def cmd_reconcile_story(args: list[str]) -> int:
     repo = ""
     story = ""
@@ -322,20 +347,10 @@ def cmd_reconcile_story(args: list[str]) -> int:
     if not Path(repo).is_dir():
         write_json({"ok": False, "error": "repo_not_found"})
         return 1
-    norm = normalize_story_key(repo, story)
-    if norm is None:
-        write_json({"ok": False, "error": "story_key_invalid", "input": story})
+    story_file, err = _resolve_story_file(repo, story)
+    if story_file is None:
+        write_json(err)
         return 1
-    artifacts = implementation_artifacts_dir(repo)
-    exact = artifacts / f"{norm.key}.md"
-    if norm.key and exact.is_file():  # disambiguate via the resolved key before falling back to prefix glob
-        story_file = exact
-    else:
-        matches = sorted(artifacts.glob(f"{norm.prefix}-*.md"))
-        if not matches:
-            write_json({"ok": False, "error": "story_file_not_found", "prefix": norm.prefix})
-            return 1
-        story_file = matches[0]
     # Exclude the resolved artifacts dir (may be _bmad-output/ OR docs/bmad/...) so the
     # story file and its siblings never pollute the reconciled File List.
     git_files = _git_changed_files(repo, (implementation_artifacts_relpath(repo) + "/",))
@@ -365,6 +380,140 @@ def cmd_reconcile_story(args: list[str]) -> int:
     return 0
 
 
+TEST_COUNTS_HEADING = "### Test Counts"
+
+
+def _render_test_counts(counts: dict) -> list[str]:
+    body = [
+        f"- Tests: {counts['tests']}",
+        f"- Failures: {counts['failures']}",
+        f"- Errors: {counts['errors']}",
+        f"- Skipped: {counts['skipped']}",
+    ]
+    if counts.get("assertions") is not None:  # PHPUnit-only; omit the line entirely otherwise
+        body.append(f"- Assertions: {counts['assertions']}")
+    return body
+
+
+# Rewrite `heading`'s body deterministically (leaving other sections untouched),
+# appending the section at EOF when absent. Mirrors the File List reconcile so a
+# re-run with identical counts is a byte-for-byte no-op.
+def _replace_or_append_section(text: str, heading: str, body: list[str]) -> str:
+    lines = text.splitlines()
+    suffix = "\n" if text.endswith("\n") else ""
+    bounds = _section_bounds(lines, heading)
+    if bounds is None:
+        tail = ["", heading, "", *body]
+        new_lines = [*lines, *tail] if lines else [heading, "", *body]
+        return "\n".join(new_lines) + suffix
+    start, end = bounds
+    rest = lines[end:]
+    block = ["", *body]
+    if rest:  # blank separator only when another section follows
+        block.append("")
+    new_lines = [*lines[: start + 1], *block, *rest]
+    return "\n".join(new_lines) + suffix
+
+
+def cmd_test_counts(args: list[str]) -> int:
+    if args and args[0] in {"--help", "-h"}:
+        print("Usage: test-counts --repo PATH --story KEY [--since EPOCH] [--write]")
+        return 0
+    repo = ""
+    story = ""
+    since: float | None = None
+    do_write = False
+    idx = 0
+    while idx < len(args):
+        arg = args[idx]
+        if arg == "--repo" and idx + 1 < len(args):
+            repo = args[idx + 1]
+            idx += 2
+        elif arg == "--story" and idx + 1 < len(args):
+            story = args[idx + 1]
+            idx += 2
+        elif arg == "--since" and idx + 1 < len(args):
+            # --since is always machine-supplied (date +%s); a non-numeric value
+            # is a contract break. Fail loud rather than silently dropping the
+            # staleness gate, which would let a stale artifact pass as fresh.
+            try:
+                since = float(args[idx + 1])
+            except ValueError:
+                write_json({"ok": False, "error": "since_invalid", "value": args[idx + 1]})
+                return 1
+            idx += 2
+        elif arg == "--write":
+            do_write = True
+            idx += 1
+        else:
+            idx += 1
+    if not repo or not story:
+        write_json({"ok": False, "error": "missing_args"})
+        return 1
+    if not Path(repo).is_dir():
+        write_json({"ok": False, "error": "repo_not_found"})
+        return 1
+    story_file, err = _resolve_story_file(repo, story)
+    if story_file is None:
+        write_json(err)
+        return 1
+    try:
+        cfg = test_config(load_policy_unresolved(repo))
+    except PolicyError:
+        write_json({"ok": False, "error": "policy_invalid"})
+        return 1
+    junit_rel = cfg["junitPath"]
+    command = cfg["command"]
+    if not junit_rel:  # Tier 3: nothing configured — File List reconcile still ran independently
+        write_json({"ok": True, "skipped": True, "reason": "test_not_configured", "test_counts": None, "wrote": False})
+        return 0
+    junit_path = Path(repo) / junit_rel.replace("{story}", story_file.stem)
+    fresh = junit_path.is_file() and (since is None or junit_path.stat().st_mtime >= since)
+    rerun_exit: int | None = None
+    if fresh:  # Tier 1: trust the artifact emitted by this dev run
+        source = "capture"
+    elif command:  # Tier 2: deterministic floor — re-run and parse what it emits
+        # Shell-quote substitutions: the placeholders must be left UNquoted in the
+        # command template (paths with spaces/metacharacters would break bash -c).
+        resolved = command.replace("{junit}", shlex.quote(str(junit_path))).replace("{story}", shlex.quote(story_file.stem))
+        ensure_dir(junit_path.parent)
+        rerun_exit = run_cmd("bash", "-c", resolved, cwd=repo).exit_code  # non-zero is expected when tests fail
+        if not junit_path.is_file():
+            write_json(
+                {"ok": True, "skipped": True, "reason": "test_artifact_not_emitted", "command_exit": rerun_exit, "test_counts": None, "wrote": False}
+            )
+            return 0
+        source = "rerun"
+    else:  # Tier 3: stale/missing artifact and no runner to fall back on
+        reason = "test_artifact_stale" if junit_path.is_file() else "test_artifact_missing"
+        write_json({"ok": True, "skipped": True, "reason": reason, "test_counts": None, "wrote": False})
+        return 0
+    try:
+        counts = parse_junit(junit_path)
+    except ValueError:
+        write_json({"ok": False, "error": "junit_parse_failed", "junit_path": str(junit_path)})
+        return 1
+    text = story_file.read_text(encoding="utf-8")
+    new_text = _replace_or_append_section(text, TEST_COUNTS_HEADING, _render_test_counts(counts))
+    wrote = False
+    if do_write and new_text != text:
+        story_file.write_text(new_text, encoding="utf-8")
+        wrote = True
+    payload = {
+        "ok": True,
+        "skipped": False,
+        "story_file": str(story_file),
+        "source": source,
+        "junit_path": str(junit_path),
+        "test_counts": counts,
+        "wrote": wrote,
+    }
+    if rerun_exit is not None:
+        payload["command_exit"] = rerun_exit
+    write_json(payload)
+    return 0
+
+
 def cmd_list_sessions(args: list[str]) -> int:
     if args and args[0] in {"--help", "-h"}:
         print("Usage: list-sessions --slug SLUG")

skills/bmad-story-automator/src/story_automator/core/junit.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Any
+
+# Universal across phpunit/pytest/jest/gotestsum/etc. `assertions` is PHPUnit-only,
+# so it is reported separately and stays nullable when no suite carries it.
+_COUNT_ATTRS = ("tests", "failures", "errors", "skipped")
+
+
+# Aggregate {tests, failures, errors, skipped, assertions|None} from a JUnit
+# report. Reads only standard testsuite/testsuites attrs (the project decides HOW
+# its runner emits them); raises ValueError on unreadable/non-JUnit XML so the
+# caller can degrade cleanly.
+def parse_junit(path: str | Path) -> dict[str, Any]:
+    try:
+        root = ET.parse(str(path)).getroot()
+    except (ET.ParseError, OSError) as exc:
+        raise ValueError(f"junit parse failed: {path}") from exc
+    if root.tag == "testsuite":
+        suites = [root]
+    elif root.tag == "testsuites":
+        # Sum DIRECT children only: phpunit nests <testsuite> groups whose parent
+        # already carries subtree totals, so a recursive sum would double-count.
+        # A bare aggregate-only <testsuites> (no children) is read directly.
+        suites = root.findall("testsuite")
+        if not suites and root.get("tests") is not None:
+            suites = [root]
+    else:
+        raise ValueError(f"not a junit report (root <{root.tag}>): {path}")
+    counts: dict[str, Any] = {key: 0 for key in _COUNT_ATTRS}
+    assertions = 0
+    has_assertions = False
+    for suite in suites:
+        for key in _COUNT_ATTRS:
+            counts[key] += _int_attr(suite.get(key))
+        raw = suite.get("assertions")
+        if raw is not None:
+            has_assertions = True
+            assertions += _int_attr(raw)
+    counts["assertions"] = assertions if has_assertions else None
+    return counts
+
+
+def _int_attr(value: str | None) -> int:
+    if value is None:
+        return 0
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return 0

skills/bmad-story-automator/src/story_automator/core/runtime_policy.py

@@ -9,7 +9,7 @@
 from .runtime_layout import active_marker_path, bundled_story_skill_root, resolve_portable_path, resolve_skill_dir
 from .utils import ensure_dir, get_project_root, iso_now, md5_hex8, read_text, write_atomic
 
-VALID_TOP_LEVEL_KEYS = {"version", "snapshot", "runtime", "workflow", "steps"}
+VALID_TOP_LEVEL_KEYS = {"version", "snapshot", "runtime", "workflow", "steps", "test"}
 VALID_STEP_NAMES = {"create", "dev", "auto", "review", "retro"}
 VALID_VERIFIERS = {"create_story_artifact", "session_exit", "review_completion", "epic_complete"}
 VALID_ASSET_NAMES = {"skill", "workflow", "instructions", "checklist", "template"}
@@ -48,6 +48,22 @@ def load_effective_policy(project_root: str | None = None, *, resolve_assets: bo
     return policy
 
 
+# Effective policy (bundled + override, deep-merged and validated) WITHOUT
+# resolving step assets/success contracts: reading a project-wide block like
+# `test` must not require sibling skills to be installed, which full resolution
+# would otherwise demand.
+def load_policy_unresolved(project_root: str | None = None) -> dict[str, Any]:
+    root = Path(project_root or get_project_root()).resolve()
+    bundle_root = bundled_skill_root(root)
+    bundled = _read_json(bundle_root / "data" / "orchestration-policy.json")
+    override_path = root / "_bmad" / "bmm" / "story-automator.policy.json"
+    override = _read_json(override_path) if override_path.is_file() else {}
+    policy = _deep_merge(bundled, override)
+    _apply_legacy_env(policy)
+    _validate_policy_shape(policy)
+    return policy
+
+
 def load_runtime_policy(
     project_root: str | None = None,
     state_file: str | Path | None = None,
@@ -208,6 +224,14 @@ def parser_runtime_config(policy: dict[str, Any]) -> dict[str, object]:
     return {"provider": provider, "model": model, "timeoutSeconds": timeout}
 
 
+def test_config(policy: dict[str, Any]) -> dict[str, str]:
+    test = _expect_optional_dict(policy, "test")
+    return {
+        "command": str(test.get("command") or "").strip(),
+        "junitPath": str(test.get("junitPath") or "").strip(),
+    }
+
+
 def bundled_skill_root(project_root: str | Path | None = None) -> Path:
     root = Path(project_root or get_project_root()).resolve()
     try:
@@ -290,6 +314,10 @@ def _validate_policy_shape(policy: dict[str, Any]) -> None:
     runtime = _expect_optional_dict(policy, "runtime")
     _expect_optional_nested_dict(runtime, "merge", "runtime")
     parser_runtime_config(policy)
+    test = _expect_optional_dict(policy, "test")
+    for key in ("command", "junitPath"):
+        if key in test and not isinstance(test.get(key), str):
+            raise PolicyError(f"test.{key} must be a string")
     workflow = _expect_optional_dict(policy, "workflow")
     repeat = _expect_optional_nested_dict(workflow, "repeat", "workflow")
     review = _expect_optional_nested_dict(repeat, "review", "workflow.repeat")

skills/bmad-story-automator/steps-c/step-03-execute.md

@@ -167,6 +167,9 @@ if should_apply_primary_model "$current_agent"; then
 else
   built_cmd=$("$scripts" tmux-wrapper build-cmd dev {story_id} --agent "$current_agent" --state-file "$state_file")
 fi
+# Mark dev start so test-counts can tell a JUnit artifact from THIS run (Tier-1
+# capture) apart from a stale one left by an earlier story.
+dev_started=$(date -u +%s)
 session=$("$scripts" tmux-wrapper spawn dev {epic} {story_id} \
   --agent "$current_agent" \
   --command "$built_cmd")