diff --git a/devtools/test-skill/README.md b/devtools/test-skill/README.md new file mode 100644 index 000000000..3db1f4117 --- /dev/null +++ b/devtools/test-skill/README.md @@ -0,0 +1,158 @@ +# forge test-skill — Local Skill Testing + +Test Forge skills locally without Jira, GitHub, or the hosted beta. +Uses Forge's own deepagents + FilesystemBackend — the same code path +as hosted Forge planning agents. + +## Quick Start + +```bash +# Via forge CLI +forge test-skill run \ + --skill generate-prd \ + --skill-dir skills/myproject/generate-prd \ + --project myproject \ + --input test-case.yaml \ + --output output/ + +# Or directly +python3 devtools/test-skill/run.py \ + --skill generate-prd \ + --skill-dir skills/myproject/generate-prd \ + --project myproject \ + --input test-case.yaml \ + --output output/ +``` + +## What It Reproduces + +Uses Forge's own deepagents library — the same agent, backend, and +middleware as hosted Forge. Skills are discovered via SkillsMiddleware, +not manually injected. + +| Component | How | +|-----------|-----| +| Agent | `create_deep_agent()` — same as `ForgeAgent._create_agent_async()` | +| Backend | `FilesystemBackend(virtual_mode=True)` — file tools match production | +| Skills | SkillsMiddleware auto-discovers from `/opt/forge/skills/{project}/` | +| System prompt | `forge.prompts.load_prompt("system")` — same templates as production | +| User message | `load_prompt("{skill-name}")` — same per-skill templates | +| Model | Configurable in `config.yaml` or `--model` flag | +| References | Injected via `--references` (same format as `forge.references` property) | + +**Not simulated:** shell/command execution (`LocalShellBackend`), MCP tools, +Jira/GitHub integrations, conversation summarization thresholds. + +## Input Format + +```yaml +jira_key: PROJ-1234 +title: "Feature Title" +prompt: | + # PROJ-1234: Feature Title + + ## Description + The full Jira feature description goes here. + Copy it from Jira — no live access needed at runtime. +``` + +If a `gold-prd.md` file exists alongside `input.yaml`, it's automatically +appended to the prompt as an approved PRD (useful for generate-spec). + +## CLI Reference + +### forge test-skill run + +| Flag | Description | +|------|-------------| +| `--skill NAME` | Skill name, e.g., `generate-prd` (required) | +| `--skill-dir PATH` | Path to skill directory containing SKILL.md (required) | +| `--input FILE` | Single input.yaml test case | +| `--dataset DIR` | Directory of test cases (runs all) | +| `--output DIR` | Output directory (required) | +| `--project NAME` | Project name for skill path (overrides config.yaml) | +| `--model MODEL` | Override model (default: `claude-opus-4-6`) | +| `--references FILE` | JSON file with reference docs (same format as `forge.references`) | +| `--repos DIR [DIR...]` | Local repo directories to copy into workspace | +| `--mlflow URI` | MLflow tracking URI for auto-tracing | +| `--mlflow-experiment NAME` | MLflow experiment name (default: `forge-skill-eval`) | + +### forge test-skill eval + +| Flag | Description | +|------|-------------| +| `--criteria FILE` | Path to criteria YAML (required) | +| `--generated FILE` | Path to generated artifact | +| `--gold FILE` | Path to gold standard artifact | +| `--dataset DIR` | Dataset directory (batch mode) | +| `--results-dir DIR` | Runner output directory (batch mode) | +| `--output DIR` | Output directory for reports (required) | +| `--mlflow URI` | MLflow tracking URI | + +## Evaluator + +Judges generated artifacts against gold standards using an LLM judge +(Sonnet by default). Criteria are defined per skill in YAML: + +```yaml +# evaluators/criteria/generate-prd.yaml +skill: generate-prd +judge_model: claude-sonnet-4-6 +gold_standard_file: gold-prd.md + +criteria: + - id: scope-accuracy + name: "Scope Accuracy" + weight: critical + prompt: | + Compare In Scope and Out of Scope items against the gold standard... +``` + +Reports: terminal (colored), JSON (`results.json`), HTML (`report.html`). + +## Adding a New Skill + +1. Verify the prompt template exists at `src/forge/prompts/v1/{skill-name}.md` +2. Create `input.yaml` with pre-fetched Jira content +3. Point `--skill-dir` to the skill directory (must contain `SKILL.md`) +4. Run it — output files and `trace.json` go to `--output` +5. Optionally create `evaluators/criteria/{skill-name}.yaml` for automated grading + +## References + +To inject reference documentation (matching `forge.references` project property): + +```bash +# Export from Forge config +forge get-config MYPROJECT --property forge.references > refs.json + +# Use in test runner +forge test-skill run \ + --skill generate-prd \ + --skill-dir skills/myproject/generate-prd \ + --project myproject \ + --references refs.json \ + --input test-case.yaml \ + --output output/ +``` + +## Configuration + +`devtools/test-skill/config.yaml`: + +```yaml +model: claude-opus-4-6 +max_tokens: 16384 +project: default # override with --project +``` + +## Requirements + +Requires `deepagents`, `langchain-anthropic`, and `langgraph` (all in +Forge's `pyproject.toml`). Uses Vertex AI when `ANTHROPIC_VERTEX_PROJECT_ID` +is set, otherwise direct Anthropic API. + +## Related + +- PR: https://github.com/forge-sdlc/forge/pull/297 +- Issue: https://github.com/forge-sdlc/forge/issues/296 diff --git a/devtools/test-skill/config.yaml b/devtools/test-skill/config.yaml new file mode 100644 index 000000000..01cd548ba --- /dev/null +++ b/devtools/test-skill/config.yaml @@ -0,0 +1,11 @@ +# Model used for skill execution (not for judging — judge model is in criteria YAML) +model: claude-opus-4-6 +max_tokens: 16384 + +# Project name — determines skill path: /opt/forge/skills/{project}/{skill-name}/ +# Override per project or use --project CLI flag (e.g., myproject) +project: default + +# Paths inside the temp workspace (match Forge's container layout) +skill_base_path: /opt/forge/skills +workspace_path: /home/user diff --git a/devtools/test-skill/evaluate.py b/devtools/test-skill/evaluate.py new file mode 100644 index 000000000..93b3c9941 --- /dev/null +++ b/devtools/test-skill/evaluate.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +""" +Forge skill output evaluator — judges generated artifacts against gold standards. + +Usage: + # Evaluate a single generated artifact against its gold standard + python3 devtools/test-skill/evaluate.py \ + --criteria devtools/test-skill/evaluators/criteria/generate-prd.yaml \ + --generated output/enhancements/OSAC-1234/prd.md \ + --gold gold-prd.md \ + --output output/eval/ + + # Evaluate after a runner batch (all cases in a dataset) + python3 devtools/test-skill/evaluate.py \ + --criteria devtools/test-skill/evaluators/criteria/generate-prd.yaml \ + --dataset eval/dataset/cases/ \ + --results-dir output/ \ + --output output/eval/ +""" + +import argparse +import sys +from pathlib import Path + +SCRIPT_DIR = Path(__file__).parent +sys.path.insert(0, str(SCRIPT_DIR)) + +from evaluators.judge import evaluate, load_criteria +from evaluators.reports import print_terminal, save_json, save_html + +try: + import mlflow + import mlflow.anthropic + + HAS_MLFLOW = True +except ImportError: + HAS_MLFLOW = False + + +def find_generated_file(output_dir: Path, criteria_config: dict) -> Path | None: + for pattern in criteria_config.get("generated_file_patterns", []): + matches = list(output_dir.glob(pattern)) + if matches: + return matches[0] + for f in output_dir.rglob("prd.md"): + return f + return None + + +_mlflow_enabled = False + + +def run_single( + criteria_path: Path, + generated_path: Path, + gold_path: Path, + output_dir: Path, +): + case_name = generated_path.parent.name or generated_path.stem + + print(f"Evaluating: {generated_path.name}") + print(f" Generated: {generated_path}") + print(f" Gold: {gold_path}") + + if HAS_MLFLOW and _mlflow_enabled: + with mlflow.start_run(run_name=f"eval — {case_name}"): + report = evaluate(criteria_path, generated_path, gold_path) + + print_terminal(report) + save_json(report, output_dir) + save_html(report, output_dir) + + mlflow.set_tag("case", case_name) + mlflow.set_tag("type", "evaluation") + mlflow.set_tag("skill", report.skill) + mlflow.set_tag("grade", report.grade) + + mlflow.log_metric("total_score", report.total_score) + mlflow.log_metric("max_score", report.max_score) + mlflow.log_metric("score_pct", round(report.total_score / report.max_score * 100, 1)) + mlflow.log_metric("criteria_passed", report.total_passed) + mlflow.log_metric("criteria_total", report.total_criteria) + mlflow.log_metric("critical_failures", len(report.critical_failures)) + + for r in report.results: + mlflow.log_metric(f"c_{r.id}", r.score) + + try: + results_json = output_dir / "results.json" + if results_json.exists(): + mlflow.log_artifact(str(results_json), "eval") + except Exception: + pass + + print(f" MLflow: logged eval for {case_name}") + else: + report = evaluate(criteria_path, generated_path, gold_path) + print_terminal(report) + save_json(report, output_dir) + save_html(report, output_dir) + + return report + + +def run_batch( + criteria_path: Path, + dataset_dir: Path, + results_dir: Path, + output_dir: Path, +): + config = load_criteria(criteria_path) + reports = [] + + for case_dir in sorted(dataset_dir.iterdir()): + if not case_dir.is_dir(): + continue + + gold_file = case_dir / config.get("gold_standard_file", "gold-prd.md") + if not gold_file.exists(): + print(f"Skipping {case_dir.name}: no gold standard") + continue + + case_results = results_dir / case_dir.name + if not case_results.exists(): + print(f"Skipping {case_dir.name}: no run results at {case_results}") + continue + + generated = find_generated_file(case_results, config) + if not generated: + print(f"Skipping {case_dir.name}: no generated artifact found") + continue + + case_output = output_dir / case_dir.name + report = run_single(criteria_path, generated, gold_file, case_output) + reports.append((case_dir.name, report)) + + if reports: + print(f"\n{'='*55}") + print(f"Batch Summary: {len(reports)} cases evaluated") + print(f"{'='*55}") + for name, r in reports: + status = "PASS" if r.overall_pass else "FAIL" + print(f" {name:<30} {r.grade} {status} {r.total_passed}/{r.total_criteria} score {r.total_score}/{r.max_score}") + + +def main(): + global _mlflow_enabled + + parser = argparse.ArgumentParser(description="Forge skill output evaluator") + parser.add_argument("--criteria", required=True, help="Path to criteria YAML file") + parser.add_argument("--generated", help="Path to generated artifact") + parser.add_argument("--gold", help="Path to gold standard artifact") + parser.add_argument("--dataset", help="Path to dataset directory (batch mode)") + parser.add_argument("--results-dir", help="Path to runner output directory (batch mode)") + parser.add_argument("--output", required=True, help="Output directory for reports") + parser.add_argument( + "--mlflow", + metavar="URI", + help="MLflow tracking URI (e.g., http://host:5000). Logs eval scores as MLflow runs.", + ) + parser.add_argument( + "--mlflow-experiment", + default="forge-skill-eval", + help="MLflow experiment name (default: forge-skill-eval)", + ) + args = parser.parse_args() + + if args.mlflow and HAS_MLFLOW: + import logging + logging.getLogger("mlflow.tracing.export").setLevel(logging.ERROR) + mlflow.set_tracking_uri(args.mlflow) + mlflow.set_experiment(args.mlflow_experiment) + mlflow.anthropic.autolog() + _mlflow_enabled = True + print(f"MLflow: tracking to {args.mlflow}, experiment '{args.mlflow_experiment}'") + + criteria_path = Path(args.criteria) + output_dir = Path(args.output) + + if args.generated and args.gold: + run_single(criteria_path, Path(args.generated), Path(args.gold), output_dir) + elif args.dataset and args.results_dir: + run_batch(criteria_path, Path(args.dataset), Path(args.results_dir), output_dir) + else: + print("Error: provide either --generated + --gold, or --dataset + --results-dir") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/devtools/test-skill/evaluators/__init__.py b/devtools/test-skill/evaluators/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/devtools/test-skill/evaluators/criteria/generate-prd.yaml b/devtools/test-skill/evaluators/criteria/generate-prd.yaml new file mode 100644 index 000000000..c0ce03986 --- /dev/null +++ b/devtools/test-skill/evaluators/criteria/generate-prd.yaml @@ -0,0 +1,90 @@ +skill: generate-prd +description: "PRD generation quality criteria based on 7 gaps identified in the Forge experiment" + +gold_standard_file: gold-prd.md +generated_file_patterns: + - "enhancements/*/prd.md" + - ".artifacts/prd/*/03-prd.md" + +judge_model: claude-sonnet-4-6 + +criteria: + - id: persona-coverage + name: "Persona Coverage" + weight: critical + prompt: | + Check if all 4 OSAC personas are addressed (Cloud Provider Admin, + Cloud Infrastructure Admin, Tenant Admin, Tenant User). Each must + have user stories or an explicit "Not affected" note. + Pay special attention to Cloud Infrastructure Admin — if the feature + automates a process they currently perform, they MUST have stories. + + - id: persona-alignment + name: "Persona-Story Alignment" + weight: critical + prompt: | + For each user story, verify the capability matches the persona's role: + - Infrastructure ops (sanitization, hardware lifecycle) → Cloud Infrastructure Admin + - Tenant management (quotas, catalogs, cross-tenant visibility) → Cloud Provider Admin + - Self-service provisioning → Tenant User + - Org config, IDP, org users → Tenant Admin + - Internal system stories ("As the CaaS system...") should not exist + A sanitization story under Cloud Provider Admin is a misattribution. + + - id: scope-accuracy + name: "In Scope / Out of Scope Accuracy" + weight: important + prompt: | + Compare In Scope and Out of Scope items against the gold standard. + Flag items in the wrong section (e.g., billing In Scope when gold + has it Out of Scope). Flag missing items that appear in the gold. + Flag items the generated added that the gold doesn't have — are they + accurate or scope creep? + + - id: design-leakage + name: "Design Leakage" + weight: critical + prompt: | + Check for internal implementation details that don't belong in a PRD: + controller names, reconciler logic, finalizer behavior, playbook + parameters, CRD field names, internal conditions, agent/InfraEnv + terminology, AAP job parameters. Platform vocabulary (ClusterOrder, + BareMetalInstance, Hosted Control Planes) is acceptable. + + - id: problem-statement + name: "Problem Statement Quality" + weight: important + prompt: | + The Problem Statement should describe user pain and cost of inaction + only. It should NOT describe the solution, what the feature introduces, + or how it works. Check for solution language — sentences starting with + "This feature introduces...", "The X eliminates...", etc. + + - id: status-visibility + name: "Async Status Visibility" + weight: important + prompt: | + If the feature involves asynchronous resource creation, check that + status/progress visibility is addressed in In Scope or User Stories. + The user should be able to see the current state and failure reasons. + + - id: template-compliance + name: "Template Compliance" + weight: important + prompt: | + The OSAC PRD template has exactly 6 sections: Problem Statement, + In Scope, Out of Scope, User Stories, Assumptions, Dependencies. + Check for extra sections (Risks, Acceptance Criteria, Open Questions, + Terminology, Milestone) or missing required sections. + + - id: completeness + name: "Content Completeness" + weight: important + prompt: | + Compare the generated PRD against the gold standard section by section. + What key requirements are present in the gold but missing in the generated? + What did the generated add that the gold doesn't have? + Focus on substantive content gaps, not wording differences. + +pass_threshold: 6 +fail_on_critical: true diff --git a/devtools/test-skill/evaluators/judge.py b/devtools/test-skill/evaluators/judge.py new file mode 100644 index 000000000..bacf8a6c4 --- /dev/null +++ b/devtools/test-skill/evaluators/judge.py @@ -0,0 +1,199 @@ +"""LLM judge for evaluating Forge skill outputs against gold standards.""" + +import json +import os +import re +from dataclasses import dataclass, field +from pathlib import Path + +import anthropic +import yaml + + +def _create_client(): + vertex_project = os.environ.get("ANTHROPIC_VERTEX_PROJECT_ID") + vertex_region = os.environ.get("ANTHROPIC_VERTEX_REGION", "us-east5") + if vertex_project: + return anthropic.AnthropicVertex(project_id=vertex_project, region=vertex_region) + return anthropic.Anthropic() + + +@dataclass +class CriterionResult: + id: str + name: str + weight: str + passed: bool + score: int + reasoning: str + quotes: list[str] + + +@dataclass +class EvalReport: + skill: str + generated_path: str + gold_path: str + results: list[CriterionResult] + pass_threshold: int = 6 + fail_on_critical: bool = True + + @property + def total_passed(self) -> int: + return sum(1 for r in self.results if r.passed) + + @property + def total_criteria(self) -> int: + return len(self.results) + + @property + def total_score(self) -> int: + return sum(r.score for r in self.results) + + @property + def max_score(self) -> int: + return len(self.results) * 2 + + @property + def critical_failures(self) -> list[CriterionResult]: + return [r for r in self.results if r.weight == "critical" and not r.passed] + + @property + def overall_pass(self) -> bool: + if self.fail_on_critical and self.critical_failures: + return False + return self.total_passed >= self.pass_threshold + + @property + def grade(self) -> str: + pct = self.total_score / self.max_score if self.max_score else 0 + if pct >= 0.9 and not self.critical_failures: + return "A" + if pct >= 0.75 and not self.critical_failures: + return "B" + if pct >= 0.6: + return "C" + return "D" + + +def load_criteria(criteria_path: Path) -> dict: + with open(criteria_path) as f: + return yaml.safe_load(f) + + +def _extract_json(text: str) -> dict | None: + """Try to extract a JSON object from text that may contain preamble.""" + if text.startswith("```"): + parts = text.split("\n", 1) + if len(parts) == 2: + text = parts[1].rsplit("```", 1)[0] + + try: + return json.loads(text.strip()) + except json.JSONDecodeError: + pass + + match = re.search(r'\{[^{}]*"pass"\s*:', text) + if match: + start = match.start() + depth = 0 + for i in range(start, len(text)): + if text[i] == '{': + depth += 1 + elif text[i] == '}': + depth -= 1 + if depth == 0: + try: + return json.loads(text[start:i + 1]) + except json.JSONDecodeError: + break + + return None + + +def judge_criterion( + client, + model: str, + criterion: dict, + generated: str, + gold: str, +) -> CriterionResult: + system = ( + "You are a document quality judge. Score the generated document against " + "the gold standard for the specific criterion described.\n\n" + "CRITICAL: Return ONLY a JSON object. No thinking, no analysis, no preamble.\n" + "Do NOT explain your reasoning before the JSON. Start your response with {.\n\n" + "JSON schema:\n" + '{"pass": true/false, "score": 0-2, "reasoning": "one sentence", "quotes": ["relevant quote"]}\n\n' + "Scoring: 0 = fails the criterion, 1 = partially meets, 2 = fully meets.\n" + "Keep reasoning to one sentence. Keep quotes short (under 100 chars each, max 3)." + ) + + user = ( + f"## Criterion: {criterion['name']}\n" + f"{criterion['prompt']}\n\n" + f"## Generated Document\n```\n{generated}\n```\n\n" + f"## Gold Standard Document\n```\n{gold}\n```" + ) + + response = client.messages.create( + model=model, + max_tokens=2048, + system=system, + messages=[{"role": "user", "content": user}], + ) + + text = response.content[0].text.strip() + data = _extract_json(text) + + if data is None: + return CriterionResult( + id=criterion["id"], + name=criterion["name"], + weight=criterion.get("weight", "important"), + passed=False, + score=0, + reasoning=f"Judge returned unparseable response: {text[:200]}", + quotes=[], + ) + + return CriterionResult( + id=criterion["id"], + name=criterion["name"], + weight=criterion.get("weight", "important"), + passed=data.get("pass", False), + score=data.get("score", 0), + reasoning=data.get("reasoning", ""), + quotes=data.get("quotes", []), + ) + + +def evaluate( + criteria_path: Path, + generated_path: Path, + gold_path: Path, +) -> EvalReport: + config = load_criteria(criteria_path) + model = config.get("judge_model", "claude-sonnet-4-6") + + generated = generated_path.read_text() + gold = gold_path.read_text() + + client = _create_client() + results = [] + + for criterion in config.get("criteria", []): + print(f" Judging: {criterion['name']}...", end="", flush=True) + result = judge_criterion(client, model, criterion, generated, gold) + status = "PASS" if result.passed else "FAIL" + print(f" {status} {result.score}/2") + results.append(result) + + return EvalReport( + skill=config.get("skill", "unknown"), + generated_path=str(generated_path), + gold_path=str(gold_path), + results=results, + pass_threshold=config.get("pass_threshold", 6), + fail_on_critical=config.get("fail_on_critical", True), + ) diff --git a/devtools/test-skill/evaluators/reports.py b/devtools/test-skill/evaluators/reports.py new file mode 100644 index 000000000..d17219508 --- /dev/null +++ b/devtools/test-skill/evaluators/reports.py @@ -0,0 +1,123 @@ +"""Report generators for evaluation results.""" + +import html as html_mod +import json +from pathlib import Path + +from .judge import EvalReport + + +def print_terminal(report: EvalReport): + GREEN = "\033[92m" + RED = "\033[91m" + YELLOW = "\033[93m" + RESET = "\033[0m" + BOLD = "\033[1m" + + print(f"\n{BOLD}PRD Evaluation: {report.skill}{RESET}") + print("=" * 55) + + for r in report.results: + color = GREEN if r.passed else RED + status = "PASS" if r.passed else "FAIL" + weight_marker = " *" if r.weight == "critical" else "" + reasoning_short = r.reasoning[:60] if r.reasoning else "" + print(f" {r.name:<28} {color}{status}{RESET} {r.score}/2 {reasoning_short}{weight_marker}") + + print("=" * 55) + overall = f"{GREEN}PASS{RESET}" if report.overall_pass else f"{RED}FAIL{RESET}" + grade_colors = {"A": GREEN, "B": GREEN, "C": YELLOW, "D": RED} + gc = grade_colors.get(report.grade, RESET) + print( + f" Total: {report.total_passed}/{report.total_criteria} passed | " + f"Score: {report.total_score}/{report.max_score} | " + f"Grade: {gc}{report.grade}{RESET} | {overall}" + ) + + if report.critical_failures: + print(f"\n {RED}Critical failures:{RESET}") + for r in report.critical_failures: + print(f" - {r.name}: {r.reasoning[:80]}") + + print() + + +def save_json(report: EvalReport, output_dir: Path): + output_dir.mkdir(parents=True, exist_ok=True) + data = { + "skill": report.skill, + "generated_path": report.generated_path, + "gold_path": report.gold_path, + "overall_pass": report.overall_pass, + "grade": report.grade, + "total_passed": report.total_passed, + "total_criteria": report.total_criteria, + "total_score": report.total_score, + "max_score": report.max_score, + "results": [ + { + "id": r.id, + "name": r.name, + "weight": r.weight, + "passed": r.passed, + "score": r.score, + "reasoning": r.reasoning, + "quotes": r.quotes, + } + for r in report.results + ], + } + path = output_dir / "results.json" + with open(path, "w") as f: + json.dump(data, f, indent=2) + print(f"JSON report: {path}") + + +def save_html(report: EvalReport, output_dir: Path): + output_dir.mkdir(parents=True, exist_ok=True) + + rows = [] + for r in report.results: + color = "#4eca8b" if r.passed else "#e85c5c" + status = "PASS" if r.passed else "FAIL" + weight = f' *' if r.weight == "critical" else "" + quotes_html = "" + if r.quotes: + quotes_html = "
".join(f'{html_mod.escape(q[:100])}' for q in r.quotes[:3]) + rows.append( + f'{html_mod.escape(r.name)}{weight}' + f'{status}' + f'{r.score}/2' + f'{html_mod.escape(r.reasoning)}' + f'{quotes_html}' + ) + + overall_color = "#4eca8b" if report.overall_pass else "#e85c5c" + overall_text = "PASS" if report.overall_pass else "FAIL" + + html = f""" + +Eval: {html_mod.escape(report.skill)} + +

Evaluation: {html_mod.escape(report.skill)}

+

Generated: {html_mod.escape(report.generated_path)}
Gold: {html_mod.escape(report.gold_path)}

+
{overall_text} — Grade: {report.grade} | {report.total_passed}/{report.total_criteria} passed, score {report.total_score}/{report.max_score}
+ + +{''.join(rows)} +
CriterionResultScoreReasoningEvidence
+""" + + path = output_dir / "report.html" + with open(path, "w") as f: + f.write(html) + print(f"HTML report: {path}") diff --git a/devtools/test-skill/run.py b/devtools/test-skill/run.py new file mode 100644 index 000000000..5ba14864e --- /dev/null +++ b/devtools/test-skill/run.py @@ -0,0 +1,549 @@ +#!/usr/bin/env python3 +""" +Forge skill test runner — simulates Forge's agent context locally. + +Uses Forge's own prompt templates (src/forge/prompts/) to reproduce the +exact system prompt and user message format, without needing Jira, GitHub, +or the hosted beta. + +Usage: + python3 devtools/test-skill/run.py \ + --skill generate-prd \ + --skill-dir skills/osac/generate-prd \ + --input test-case.yaml \ + --output output/ + + python3 devtools/test-skill/run.py \ + --skill generate-prd \ + --skill-dir skills/osac/generate-prd \ + --dataset eval/dataset/cases/ \ + --output output/ +""" + +import argparse +import asyncio +import contextlib +import json +import os +import shutil +import sys +import tempfile +import time +import uuid +from datetime import date +from pathlib import Path + +# Add Forge source to path so we can import forge.prompts +FORGE_ROOT = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(FORGE_ROOT / "src")) + +import yaml + +from forge.prompts import load_prompt + +try: + import mlflow + import mlflow.anthropic + + HAS_MLFLOW = True +except ImportError: + HAS_MLFLOW = False + +from deepagents import create_deep_agent +from deepagents.backends.filesystem import FilesystemBackend +from langchain_anthropic import ChatAnthropic as LCChatAnthropic +from langgraph.checkpoint.memory import MemorySaver + +try: + from langchain_google_vertexai.model_garden import ( + ChatAnthropicVertex as LCChatAnthropicVertex, + ) +except ImportError: + LCChatAnthropicVertex = None + + +SCRIPT_DIR = Path(__file__).parent + + +def load_config(): + with open(SCRIPT_DIR / "config.yaml") as f: + return yaml.safe_load(f) + + +def _format_references(references: list[dict]) -> str: + if not references: + return "" + lines = ["\n\n## Reference Documentation\n"] + for ref in references: + title = ref.get("title", "Untitled") + url = ref.get("url", "") + tags = ref.get("tags", []) + tag_str = f" [{', '.join(tags)}]" if tags else "" + lines.append(f"- [{title}]({url}){tag_str}\n") + return "".join(lines) + + + +def build_user_message( + skill_name: str, + requirements: str, + project_key: str, + summary: str, +) -> str: + prompt_name = skill_name # e.g., "generate-prd" + context_str = str({"project_key": project_key, "summary": summary}) + try: + return load_prompt( + prompt_name, + raw_requirements=requirements, + context=context_str, + ) + except FileNotFoundError: + return f"Please complete the following task:\n\n{requirements}" + + +def setup_workspace( + skill_dir: Path, + skill_name: str, + project: str, + repo_dirs: list[Path] | None = None, +) -> Path: + workspace = Path(tempfile.mkdtemp(prefix="forge-test-")) + skill_target = workspace / "opt" / "forge" / "skills" / project / skill_name + skill_target.mkdir(parents=True) + shutil.copytree(skill_dir, skill_target, dirs_exist_ok=True) + user_dir = workspace / "home" / "user" + user_dir.mkdir(parents=True) + if repo_dirs: + for repo_path in repo_dirs: + repo_path = Path(repo_path).resolve() + if not repo_path.is_dir(): + print(f" Warning: repo dir not found, skipping: {repo_path}") + continue + target = user_dir / repo_path.name + shutil.copytree( + repo_path, target, dirs_exist_ok=True, + ignore=shutil.ignore_patterns( + ".git", "__pycache__", "node_modules", ".venv", "vendor", + ), + ) + print(f" Repo: {repo_path.name} -> {target}") + return workspace + + +def build_system_prompt_text( + ticket_key: str, + project_key: str, + references: list[dict] | None = None, +) -> str: + system_text = load_prompt("system", current_date=str(date.today())) + system_text += f"\n\nContext:\n- ticket_key: {ticket_key}\n- project_key: {project_key}\n" + system_text += _format_references(references or []) + return system_text + + +async def run_agent_deepagents( + system_prompt: str, + user_message: str, + workspace: Path, + config: dict, + _skill_name: str, + project: str, +) -> dict: + vertex_project = os.environ.get("ANTHROPIC_VERTEX_PROJECT_ID") + vertex_region = os.environ.get("ANTHROPIC_VERTEX_REGION", "us-east5") + model_name = config.get("model", "claude-opus-4-6") + max_tokens = config.get("max_tokens", 16384) + + if vertex_project: + if LCChatAnthropicVertex is None: + raise RuntimeError( + "ANTHROPIC_VERTEX_PROJECT_ID is set but langchain-google-vertexai is not installed. " + "Install with: pip install langchain-google-vertexai" + ) + model = LCChatAnthropicVertex( + model_name=model_name, + project=vertex_project, + location=vertex_region, + max_tokens=max_tokens, + ) + else: + model = LCChatAnthropic( + model=model_name, + max_tokens=max_tokens, + ) + + backend = FilesystemBackend(root_dir=str(workspace), virtual_mode=True) + + skill_paths = [f"/opt/forge/skills/{project}/"] + + checkpointer = MemorySaver() + agent = create_deep_agent( + model=model, + backend=backend, + skills=skill_paths, + system_prompt=system_prompt, + checkpointer=checkpointer, + ) + + thread_id = str(uuid.uuid4()) + result = await agent.ainvoke( + {"messages": [{"role": "user", "content": user_message}]}, + config={"configurable": {"thread_id": thread_id}}, + ) + + messages = result.get("messages", []) if isinstance(result, dict) else [] + trace = [] + total_input = 0 + total_output = 0 + ai_iteration = 0 + + for msg in messages: + msg_type = type(msg).__name__ + if msg_type not in ("AIMessage", "AIMessageChunk"): + continue + + ai_iteration += 1 + content = msg.content + text_blocks = [] + + if isinstance(content, str): + if content.strip(): + text_blocks.append(content) + elif isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + text_blocks.append(block.get("text", "")) + + tool_calls = [ + {"name": tc.get("name", ""), "input": tc.get("args", {})} + for tc in getattr(msg, "tool_calls", []) + ] + + usage = getattr(msg, "usage_metadata", None) or {} + input_tokens = usage.get("input_tokens", 0) if isinstance(usage, dict) else 0 + output_tokens = usage.get("output_tokens", 0) if isinstance(usage, dict) else 0 + total_input += input_tokens + total_output += output_tokens + + resp_meta = getattr(msg, "response_metadata", {}) or {} + stop_reason = resp_meta.get("stop_reason", "") + + trace.append({ + "iteration": ai_iteration, + "stop_reason": stop_reason, + "text": text_blocks, + "tool_calls": tool_calls, + "input_tokens": input_tokens, + "output_tokens": output_tokens, + }) + + if tool_calls: + tc_names = [tc["name"] for tc in tool_calls] + print(f" deepagents: {', '.join(tc_names)}") + + final_text = "" + for entry in trace: + for t in entry.get("text", []): + if t.strip(): + final_text = t + + return { + "trace": trace, + "final_text": final_text, + "total_input_tokens": total_input, + "total_output_tokens": total_output, + "iterations": ai_iteration, + } + + +def collect_output_files( + workspace: Path, + repo_dirs: list[Path] | None = None, + written_paths: set[str] | None = None, +) -> dict[str, str]: + """Collect files the agent wrote during execution. + + When *written_paths* is provided (a set of virtual paths the agent + passed to write_file), only those files are collected — this avoids + capturing pre-existing repo and skill files. Falls back to the + heuristic exclude-list when the set is not available. + """ + repo_names = {Path(r).resolve().name for r in (repo_dirs or [])} + files = {} + for search_root in [workspace / "home" / "user", workspace / "opt" / "forge"]: + if not search_root.exists(): + continue + for fpath in search_root.rglob("*"): + if fpath.is_file() and fpath.suffix != ".pyc": + rel = str(fpath.relative_to(search_root)) + if written_paths is not None: + virt = "/" + str(fpath.relative_to(workspace)) + if virt not in written_paths: + continue + else: + top_dir = rel.split("/")[0] if "/" in rel else "" + if top_dir in repo_names or top_dir == "skills": + continue + if rel not in files: + with contextlib.suppress(UnicodeDecodeError, PermissionError): + files[rel] = fpath.read_text() + return files + + +def _setup_mlflow(tracking_uri: str, experiment_name: str): + """Configure MLflow tracking and Anthropic auto-instrumentation.""" + if not HAS_MLFLOW: + print("Warning: mlflow not installed, skipping MLflow integration") + return False + mlflow.set_tracking_uri(tracking_uri) + mlflow.set_experiment(experiment_name) + mlflow.anthropic.autolog() + print(f"MLflow: tracking to {tracking_uri}, experiment '{experiment_name}'") + return True + + +def _save_outputs(result, workspace, output_dir, config, repo_dirs=None): + """Save output files and trace JSON.""" + output_dir.mkdir(parents=True, exist_ok=True) + + output_files = collect_output_files(workspace, repo_dirs=repo_dirs) + for rel_path, content in output_files.items(): + out_path = output_dir / rel_path + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(content) + print(f"Output: {out_path}") + + final_text = result.get("final_text", "") + skill_name = result.get("_skill_name", "") + filename = "design.md" if "spec" in skill_name else "prd.md" + artifact_path = output_dir / filename + if not artifact_path.exists() and final_text.strip(): + artifact_path.write_text(final_text) + print(f"Output (inline): {artifact_path}") + + trace_path = output_dir / "trace.json" + with open(trace_path, "w") as f: + json.dump( + { + "ticket_key": result["_ticket_key"], + "skill": result["_skill_name"], + "model": config.get("model"), + "elapsed_seconds": result["_elapsed"], + "iterations": result["iterations"], + "total_input_tokens": result["total_input_tokens"], + "total_output_tokens": result["total_output_tokens"], + "trace": result["trace"], + }, + f, + indent=2, + ) + print(f"Trace: {trace_path}") + + +def _log_mlflow_metrics(result, config, output_dir): + """Log metrics and artifacts to the current active MLflow run.""" + mlflow.set_tag("model", config.get("model", "claude-opus-4-6")) + + mlflow.log_metric("elapsed_seconds", result["_elapsed"]) + mlflow.log_metric("iterations", result["iterations"]) + mlflow.log_metric("input_tokens", result["total_input_tokens"]) + mlflow.log_metric("output_tokens", result["total_output_tokens"]) + total = result["total_input_tokens"] + result["total_output_tokens"] + mlflow.log_metric("total_tokens", total) + cost = (result["total_input_tokens"] * 15 + result["total_output_tokens"] * 75) / 1e6 + mlflow.log_metric("cost_usd", round(cost, 2)) + + try: + trace_path = output_dir / "trace.json" + if trace_path.exists(): + mlflow.log_artifact(str(trace_path), "trace") + for f in output_dir.rglob("prd.md"): + if "skills" not in str(f): + mlflow.log_artifact(str(f), "generated") + break + except Exception: + pass + + +def _run_agent( + system_text: str, + user_message: str, + workspace: Path, + config: dict, + skill_name: str, + project: str, +) -> dict: + return asyncio.run( + run_agent_deepagents(system_text, user_message, workspace, config, skill_name, project) + ) + + +def run_single_case( + skill_name: str, + skill_dir: Path, + input_path: Path, + output_dir: Path, + config: dict, + repo_dirs: list[Path] | None = None, +): + with open(input_path) as f: + input_data = yaml.safe_load(f) + + ticket_key = input_data.get("jira_key", input_data.get("ticket_key", "TEST-0000")) + project_key = config.get("project", "default").upper() + project = config.get("project", "default") + summary = input_data.get("title", input_data.get("summary", "")) + requirements = input_data.get("prompt", input_data.get("requirements", "")) + + if not requirements: + print(f"Error: No requirements found in {input_path}") + return + + prd_file = input_path.parent / "gold-prd.md" + if prd_file.exists(): + prd_content = prd_file.read_text() + requirements += f"\n\n## Approved PRD\n\n{prd_content}" + print(f" PRD: loaded {prd_file.name} ({len(prd_content)} chars)") + + print(f"\n{'='*60}") + print(f"Running: {ticket_key} — {summary}") + print(f"Skill: {skill_name} from {skill_dir}") + print(f"{'='*60}") + + workspace = setup_workspace(skill_dir, skill_name, project, repo_dirs=repo_dirs) + print(f"Workspace: {workspace}") + + references = config.get("references", []) + system_text = build_system_prompt_text(ticket_key, project_key, references) + user_message = build_user_message(skill_name, requirements, project_key, summary) + + def _execute(): + return _run_agent( + system_text, user_message, + workspace, config, skill_name, project, + ) + + if HAS_MLFLOW and config.get("mlflow_enabled"): + with mlflow.start_run(run_name=f"{ticket_key} — {summary}"): + mlflow.set_tag("case", ticket_key) + mlflow.set_tag("feature", summary) + mlflow.set_tag("skill", skill_name) + + start = time.time() + result = _execute() + elapsed = round(time.time() - start, 1) + + result["_ticket_key"] = ticket_key + result["_skill_name"] = skill_name + result["_elapsed"] = elapsed + + _save_outputs(result, workspace, output_dir, config, repo_dirs=repo_dirs) + _log_mlflow_metrics(result, config, output_dir) + + print(f"\nDone in {elapsed}s — {result['iterations']} iterations") + print(f"Tokens: {result['total_input_tokens']} input, " + f"{result['total_output_tokens']} output") + print(f"MLflow: logged run for {ticket_key}") + else: + start = time.time() + result = _execute() + elapsed = round(time.time() - start, 1) + + result["_ticket_key"] = ticket_key + result["_skill_name"] = skill_name + result["_elapsed"] = elapsed + + _save_outputs(result, workspace, output_dir, config) + + print(f"\nDone in {elapsed}s — {result['iterations']} iterations") + print(f"Tokens: {result['total_input_tokens']} input, " + f"{result['total_output_tokens']} output") + + shutil.rmtree(workspace) + + +def main(): + parser = argparse.ArgumentParser(description="Forge skill test runner") + parser.add_argument("--skill", required=True, help="Skill name (e.g., generate-prd)") + parser.add_argument("--skill-dir", required=True, help="Path to skill directory") + parser.add_argument("--input", help="Path to a single input.yaml test case") + parser.add_argument("--dataset", help="Path to dataset directory (runs all cases)") + parser.add_argument("--output", required=True, help="Output directory") + parser.add_argument("--model", help="Override model from config") + parser.add_argument( + "--mlflow", + metavar="URI", + help="MLflow tracking URI (e.g., http://host:5000). Enables auto-tracing of all API calls.", + ) + parser.add_argument( + "--mlflow-experiment", + default="forge-skill-eval", + help="MLflow experiment name (default: forge-skill-eval)", + ) + parser.add_argument( + "--repos", + nargs="+", + metavar="DIR", + help="Local repo directories to copy into the workspace (e.g., /path/to/myproject /path/to/enhancement-proposals). " + "Gives the agent codebase access via read/grep tools.", + ) + parser.add_argument( + "--project", + help="Project name for skill path (e.g., osac). Overrides config.yaml project setting.", + ) + parser.add_argument( + "--references", + metavar="FILE", + help="JSON file with reference documentation (same format as forge.references project property).", + ) + args = parser.parse_args() + + config = load_config() + if args.model: + config["model"] = args.model + if args.project: + config["project"] = args.project + if args.references: + refs_path = Path(args.references) + if not refs_path.exists(): + print(f"Error: references file not found: {refs_path}") + sys.exit(1) + with open(refs_path) as f: + config["references"] = json.load(f) + + if args.mlflow: + import logging + logging.getLogger("mlflow.tracing.export").setLevel(logging.ERROR) + config["mlflow_enabled"] = _setup_mlflow(args.mlflow, args.mlflow_experiment) + else: + config["mlflow_enabled"] = False + + skill_dir = Path(args.skill_dir).resolve() + if not (skill_dir / "SKILL.md").exists(): + print(f"Error: No SKILL.md found in {skill_dir}") + sys.exit(1) + + output_dir = Path(args.output).resolve() + + repo_dirs = [Path(r) for r in args.repos] if args.repos else None + + if args.input: + run_single_case(args.skill, skill_dir, Path(args.input), output_dir, config, repo_dirs=repo_dirs) + elif args.dataset: + dataset_dir = Path(args.dataset) + for case_dir in sorted(dataset_dir.iterdir()): + if not case_dir.is_dir(): + continue + input_yaml = case_dir / "input.yaml" + if not input_yaml.exists(): + continue + case_output = output_dir / case_dir.name + run_single_case(args.skill, skill_dir, input_yaml, case_output, config, repo_dirs=repo_dirs) + else: + print("Error: Provide either --input or --dataset") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/docs/dev/skill-evaluation.md b/docs/dev/skill-evaluation.md new file mode 100644 index 000000000..a374b6c48 --- /dev/null +++ b/docs/dev/skill-evaluation.md @@ -0,0 +1,140 @@ +# Skill Evaluation + +Evaluate Forge skill outputs against gold standards using an LLM judge. + +## How It Works + +The evaluator sends each criterion to an LLM judge (Sonnet by default) with +the generated artifact and a gold standard. The judge returns a structured +score (0-2) with reasoning and evidence quotes. + +## Criteria Files + +Criteria are defined per skill in YAML at +`devtools/test-skill/evaluators/criteria/{skill-name}.yaml`: + +```yaml +skill: generate-prd +description: "PRD quality criteria" +judge_model: claude-sonnet-4-6 +gold_standard_file: gold-prd.md + +generated_file_patterns: + - "enhancements/*/prd.md" + +criteria: + - id: scope-accuracy + name: "Scope Accuracy" + weight: critical + prompt: | + Compare In Scope and Out of Scope items against the gold standard. + Flag items in the wrong section or missing items. + + - id: template-compliance + name: "Template Compliance" + weight: important + prompt: | + Check that the document follows the required template sections. + +pass_threshold: 6 +fail_on_critical: true +``` + +### Fields + +| Field | Description | +|-------|-------------| +| `skill` | Skill name for labeling | +| `judge_model` | LLM model for judging (default: `claude-sonnet-4-6`) | +| `gold_standard_file` | Expected gold file name in each dataset case directory | +| `generated_file_patterns` | Glob patterns to find the generated artifact in output | +| `criteria` | List of evaluation criteria (see below) | +| `pass_threshold` | Minimum criteria that must pass for overall PASS | +| `fail_on_critical` | If `true`, any critical failure causes overall FAIL | + +### Criterion Fields + +| Field | Description | +|-------|-------------| +| `id` | Short identifier (used in JSON reports and MLflow metrics) | +| `name` | Human-readable name | +| `weight` | `critical` or `important` | +| `prompt` | Instructions for the judge — what to check and how | + +### Weights + +| Weight | Effect | +|--------|--------| +| `critical` | When `fail_on_critical: true`, any critical failure causes overall FAIL | +| `important` | Counts toward `pass_threshold` but doesn't auto-fail | + +### Scoring + +| Score | Meaning | +|-------|---------| +| 0 | Fails the criterion | +| 1 | Partially meets | +| 2 | Fully meets | + +### Grades + +| Grade | Condition | +|-------|-----------| +| A | Score >= 90% and no critical failures | +| B | Score >= 75% and no critical failures | +| C | Score >= 60% | +| D | Below 60% | + +## Running Evaluations + +### Single artifact + +```bash +forge test-skill eval \ + --criteria devtools/test-skill/evaluators/criteria/generate-prd.yaml \ + --generated output/prd.md \ + --gold gold-prd.md \ + --output output/eval/ +``` + +### Batch (full dataset) + +```bash +forge test-skill eval \ + --criteria devtools/test-skill/evaluators/criteria/generate-prd.yaml \ + --dataset eval/dataset/cases/ \ + --results-dir output/ \ + --output output/eval/ +``` + +Dataset structure: + +``` +eval/dataset/cases/ +├── PROJ-1234/ +│ ├── input.yaml +│ └── gold-prd.md +├── PROJ-5678/ +│ ├── input.yaml +│ └── gold-prd.md +``` + +## Reports + +Three output formats per evaluation: + +| Format | File | Use | +|--------|------|-----| +| Terminal | stdout | Quick pass/fail check | +| JSON | `results.json` | CI integration, programmatic analysis | +| HTML | `report.html` | Visual review with evidence quotes | + +## Writing Good Criteria + +- One specific thing per criterion — don't combine multiple checks +- Include concrete examples of what to look for in the prompt +- Use `critical` weight sparingly — only for requirements that indicate + a fundamentally wrong output +- The judge sees both generated and gold documents — phrase prompts as + comparisons ("Compare X against the gold standard") +- Keep prompts focused: vague criteria produce inconsistent scores diff --git a/docs/dev/test-skill.md b/docs/dev/test-skill.md new file mode 100644 index 000000000..bb4147f22 --- /dev/null +++ b/docs/dev/test-skill.md @@ -0,0 +1,152 @@ +# Testing Skills Locally + +Test Forge skills locally using `forge test-skill` — the same deepagents +code path as hosted Forge, without needing Jira, GitHub, or the hosted beta. + +## Why + +Iterating via the hosted beta costs $3-17 per run and requires real Jira +tickets. `forge test-skill` runs the exact same agent locally against +pre-fetched input, so you can iterate in minutes. + +## Prerequisites + +- Forge installed from source (`pip install -e .` or `uv sync`) +- `deepagents`, `langchain-anthropic`, `langgraph` (included in Forge's dependencies) +- `ANTHROPIC_API_KEY` set, or `ANTHROPIC_VERTEX_PROJECT_ID` for Vertex AI + +## Quick Start + +### Run a skill + +```bash +forge test-skill run \ + --skill generate-prd \ + --skill-dir skills/myproject/generate-prd \ + --project myproject \ + --input test-cases/PROJ-1234/input.yaml \ + --output output/PROJ-1234/ +``` + +### Evaluate the output + +```bash +forge test-skill eval \ + --criteria devtools/test-skill/evaluators/criteria/generate-prd.yaml \ + --generated output/PROJ-1234/enhancements/PROJ-1234/prd.md \ + --gold test-cases/PROJ-1234/gold-prd.md \ + --output output/PROJ-1234/eval/ +``` + +## How It Works + +The runner: + +1. Loads the system prompt from `forge.prompts` (same templates as production) +2. Loads the user message from `src/forge/prompts/v1/{skill-name}.md` +3. Creates a temp workspace mimicking the container layout + (`/opt/forge/skills/{project}/`, `/home/user/`) +4. Creates a deepagents agent with `FilesystemBackend` and `SkillsMiddleware` +5. Invokes the agent and collects output files + trace + +This is the same `create_deep_agent()` + `FilesystemBackend` code path +used by `ForgeAgent` in production — not a simulation. + +## Preparing Test Input + +Create an `input.yaml` with pre-fetched Jira content: + +```yaml +jira_key: PROJ-1234 +title: "Feature Title" +prompt: | + # PROJ-1234: Feature Title + + ## Description + The full Jira feature description goes here. + Copy it from Jira — no live access needed at runtime. + + ## User Stories + ... +``` + +If a `gold-prd.md` file exists alongside `input.yaml`, it's automatically +appended to the prompt as an approved PRD (useful for `generate-spec` testing). + +## Adding Reference Documentation + +If your skill behavior depends on reference URLs configured in +`forge.references`, export and pass them: + +```bash +# Export from Forge project config +forge get-config MYPROJECT --property forge.references > refs.json + +# Pass to test runner +forge test-skill run \ + --skill generate-prd \ + --skill-dir skills/myproject/generate-prd \ + --project myproject \ + --references refs.json \ + --input test-case.yaml \ + --output output/ +``` + +## Adding Repository Context + +To give the agent access to codebase files (for skills that read code): + +```bash +forge test-skill run \ + --skill generate-spec \ + --skill-dir skills/myproject/generate-spec \ + --project myproject \ + --repos /path/to/myrepo /path/to/docs-repo \ + --input test-case.yaml \ + --output output/ +``` + +Repos are copied into the workspace at `/home/user/{repo-name}/`, +excluding `.git`, `__pycache__`, `node_modules`, `.venv`, and `vendor`. + +## MLflow Integration + +Track runs and evaluations in MLflow: + +```bash +forge test-skill run \ + --skill generate-prd \ + --skill-dir skills/myproject/generate-prd \ + --project myproject \ + --input test-case.yaml \ + --output output/ \ + --mlflow http://mlflow-host:5000 + +forge test-skill eval \ + --criteria devtools/test-skill/evaluators/criteria/generate-prd.yaml \ + --dataset eval/dataset/cases/ \ + --results-dir output/ \ + --output output/eval/ \ + --mlflow http://mlflow-host:5000 +``` + +Logged metrics: elapsed time, token counts, cost estimate, iteration count, +and per-criterion eval scores. + +## Configuration + +`devtools/test-skill/config.yaml`: + +```yaml +model: claude-opus-4-6 # override with --model +max_tokens: 16384 +project: default # override with --project +``` + +## What's Not Simulated + +- Shell/command execution (`LocalShellBackend`) — the test runner uses + `FilesystemBackend`, which provides file read/write/grep but no shell +- MCP tools — not loaded in the test runner +- Jira/GitHub integrations — the runner is offline by design +- Conversation summarization thresholds — may differ from production diff --git a/docs/reference/config.md b/docs/reference/config.md index f13fd5f29..a5c1702a0 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -322,3 +322,40 @@ These variables are used by `docker-compose.yml`, `devtools/docker-compose.dev.y ### MCP Servers MCP server configuration lives in `mcp-servers.json`, not `.env`. See the [MCP servers section](https://github.com/forge-sdlc/forge/blob/main/mcp-servers.json) of the repository. + +## test-skill Commands + +Local skill testing and evaluation. See [Testing Skills Locally](../dev/test-skill.md) for the full guide. + +### forge test-skill run + +Run a skill against test cases using deepagents. + +| Flag | Required | Description | +|------|----------|-------------| +| `--skill NAME` | Yes | Skill name (e.g., `generate-prd`) | +| `--skill-dir PATH` | Yes | Path to skill directory containing `SKILL.md` | +| `--output DIR` | Yes | Output directory for results and trace | +| `--input FILE` | One of input/dataset | Single `input.yaml` test case | +| `--dataset DIR` | One of input/dataset | Directory of test cases (runs all) | +| `--project NAME` | No | Project name for skill path (overrides `config.yaml`) | +| `--model MODEL` | No | Override model (default: from `config.yaml`) | +| `--references FILE` | No | JSON file with reference docs (same format as `forge.references`) | +| `--repos DIR [DIR...]` | No | Local repo directories to copy into workspace | +| `--mlflow URI` | No | MLflow tracking URI for auto-tracing | +| `--mlflow-experiment NAME` | No | MLflow experiment name (default: `forge-skill-eval`) | + +### forge test-skill eval + +Evaluate skill outputs against gold standards. See [Skill Evaluation](../dev/skill-evaluation.md) for criteria format. + +| Flag | Required | Description | +|------|----------|-------------| +| `--criteria FILE` | Yes | Path to criteria YAML file | +| `--output DIR` | Yes | Output directory for reports | +| `--generated FILE` | Single mode | Path to generated artifact | +| `--gold FILE` | Single mode | Path to gold standard artifact | +| `--dataset DIR` | Batch mode | Dataset directory | +| `--results-dir DIR` | Batch mode | Runner output directory | +| `--mlflow URI` | No | MLflow tracking URI | +| `--mlflow-experiment NAME` | No | MLflow experiment name | diff --git a/docs/skills/authoring.md b/docs/skills/authoring.md index 6dcb86e7a..0a0fac322 100644 --- a/docs/skills/authoring.md +++ b/docs/skills/authoring.md @@ -118,6 +118,55 @@ The reviewer issues APPROVED or REJECTED verdicts. On rejection, the skill re-ru See the [Auto-Review Guide](../guide/auto-review.md) for configuration options and writing effective review instructions. +## Testing Your Skill + +Use `forge test-skill` to iterate on skills locally without the hosted beta. + +### 1. Prepare a test case + +Save pre-fetched Jira content as `input.yaml`: + +```yaml +jira_key: MYTEAM-100 +title: "Feature Title" +prompt: | + # MYTEAM-100: Feature Title + ## Description + ... +``` + +### 2. Run the skill + +```bash +forge test-skill run \ + --skill generate-prd \ + --skill-dir skills/myteam/generate-prd \ + --project myteam \ + --input test-cases/MYTEAM-100/input.yaml \ + --output output/ +``` + +### 3. Check the output + +Output files land in the `--output` directory. A `trace.json` captures +every agent iteration, tool call, and token usage. + +### 4. Add automated grading (optional) + +Create a criteria YAML and evaluate against a gold standard: + +```bash +forge test-skill eval \ + --criteria devtools/test-skill/evaluators/criteria/generate-prd.yaml \ + --generated output/enhancements/MYTEAM-100/prd.md \ + --gold test-cases/MYTEAM-100/gold.md \ + --output output/eval/ +``` + +See [Testing Skills Locally](../dev/test-skill.md) for the full guide and +[Skill Evaluation](../dev/skill-evaluation.md) for criteria file format +and grading details. + ## Using your skills with Forge See [Customize Forge for your project](../dev/contributing.md#customize-forge-for-your-project) for how to point a Jira project at your skills repo using `forge project-setup` and the skill installer. diff --git a/src/forge/cli.py b/src/forge/cli.py index 0e58d1fa0..ab81caa31 100644 --- a/src/forge/cli.py +++ b/src/forge/cli.py @@ -1196,6 +1196,123 @@ async def cmd_health(_args: argparse.Namespace) -> int: return 0 +def cmd_test_skill_run(args: argparse.Namespace) -> int: + """Run a skill against test cases using deepagents.""" + import importlib.util + from pathlib import Path + + project_root = Path(__file__).resolve().parent.parent.parent + run_module_path = project_root / "devtools" / "test-skill" / "run.py" + if not run_module_path.exists(): + print(f"Error: test-skill runner not found at {run_module_path}", file=sys.stderr) + return 1 + + spec = importlib.util.spec_from_file_location("test_skill_run", run_module_path) + if spec is None or spec.loader is None: + print(f"Error: cannot load {run_module_path}", file=sys.stderr) + return 1 + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + + config = mod.load_config() + if args.model: + config["model"] = args.model + if getattr(args, "project", None): + config["project"] = args.project + if getattr(args, "references", None): + import json as _json + + refs_path = Path(args.references) + if not refs_path.exists(): + print(f"Error: references file not found: {refs_path}", file=sys.stderr) + return 1 + with open(refs_path) as f: + config["references"] = _json.load(f) + + if args.mlflow: + import logging + + logging.getLogger("mlflow.tracing.export").setLevel(logging.ERROR) + config["mlflow_enabled"] = mod._setup_mlflow(args.mlflow, args.mlflow_experiment) + else: + config["mlflow_enabled"] = False + + skill_dir = Path(args.skill_dir).resolve() + if not (skill_dir / "SKILL.md").exists(): + print(f"Error: No SKILL.md found in {skill_dir}", file=sys.stderr) + return 1 + + output_dir = Path(args.output).resolve() + + repo_dirs = [Path(r) for r in args.repos] if getattr(args, "repos", None) else None + + if args.input: + mod.run_single_case( + args.skill, skill_dir, Path(args.input), output_dir, config, repo_dirs=repo_dirs + ) + elif args.dataset: + dataset_dir = Path(args.dataset) + for case_dir in sorted(dataset_dir.iterdir()): + if not case_dir.is_dir(): + continue + input_yaml = case_dir / "input.yaml" + if not input_yaml.exists(): + continue + case_output = output_dir / case_dir.name + mod.run_single_case( + args.skill, skill_dir, input_yaml, case_output, config, repo_dirs=repo_dirs + ) + else: + print("Error: Provide either --input or --dataset", file=sys.stderr) + return 1 + + return 0 + + +def cmd_test_skill_eval(args: argparse.Namespace) -> int: + """Evaluate skill outputs against gold standards.""" + import importlib.util + from pathlib import Path + + project_root = Path(__file__).resolve().parent.parent.parent + eval_module_path = project_root / "devtools" / "test-skill" / "evaluate.py" + if not eval_module_path.exists(): + print(f"Error: test-skill evaluator not found at {eval_module_path}", file=sys.stderr) + return 1 + + spec = importlib.util.spec_from_file_location("test_skill_evaluate", eval_module_path) + if spec is None or spec.loader is None: + print(f"Error: cannot load {eval_module_path}", file=sys.stderr) + return 1 + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + + if args.mlflow and mod.HAS_MLFLOW: + import logging + + logging.getLogger("mlflow.tracing.export").setLevel(logging.ERROR) + mod.mlflow.set_tracking_uri(args.mlflow) + mod.mlflow.set_experiment(args.mlflow_experiment) + mod.mlflow.anthropic.autolog() + mod._mlflow_enabled = True + + criteria_path = Path(args.criteria) + output_dir = Path(args.output) + + if args.generated and args.gold: + mod.run_single(criteria_path, Path(args.generated), Path(args.gold), output_dir) + elif args.dataset and args.results_dir: + mod.run_batch(criteria_path, Path(args.dataset), Path(args.results_dir), output_dir) + else: + print( + "Error: provide either --generated + --gold, or --dataset + --results-dir", + file=sys.stderr, + ) + return 1 + + return 0 + + async def cmd_smoke_test(_args: argparse.Namespace) -> int: """Run an end-to-end smoke test to verify Forge runtime connectivity and execution.""" from forge.config import get_settings @@ -1340,6 +1457,78 @@ def main(argv: list[str] | None = None) -> int: help="Print the installed Forge package version", ) + # test-skill subparser group + test_skill_parser = subparsers.add_parser( + "test-skill", + help="Test and evaluate Forge skills locally", + ) + test_skill_subparsers = test_skill_parser.add_subparsers( + dest="test_skill_command", + help="Test-skill commands", + ) + + # test-skill run + ts_run_parser = test_skill_subparsers.add_parser( + "run", + help="Run a skill against test cases", + ) + ts_run_parser.add_argument("--skill", required=True, help="Skill name (e.g., generate-prd)") + ts_run_parser.add_argument("--skill-dir", required=True, help="Path to skill directory") + ts_run_parser.add_argument("--input", help="Path to a single input.yaml test case") + ts_run_parser.add_argument("--dataset", help="Path to dataset directory (runs all cases)") + ts_run_parser.add_argument("--output", required=True, help="Output directory") + ts_run_parser.add_argument("--model", help="Override model from config") + ts_run_parser.add_argument( + "--mlflow", + metavar="URI", + help="MLflow tracking URI (e.g., http://host:5000)", + ) + ts_run_parser.add_argument( + "--mlflow-experiment", + default="forge-skill-eval", + help="MLflow experiment name (default: forge-skill-eval)", + ) + ts_run_parser.add_argument( + "--project", + help="Project name for skill path (e.g., osac). Overrides config.yaml.", + ) + ts_run_parser.add_argument( + "--references", + metavar="FILE", + help="JSON file with reference documentation (same format as forge.references).", + ) + ts_run_parser.add_argument( + "--repos", + nargs="+", + metavar="DIR", + help="Local repo directories to copy into the workspace, giving the agent " + "codebase access via read/grep tools.", + ) + + # test-skill eval + ts_eval_parser = test_skill_subparsers.add_parser( + "eval", + help="Evaluate skill outputs against gold standards", + ) + ts_eval_parser.add_argument("--criteria", required=True, help="Path to criteria YAML") + ts_eval_parser.add_argument("--generated", help="Path to generated artifact") + ts_eval_parser.add_argument("--gold", help="Path to gold standard artifact") + ts_eval_parser.add_argument("--dataset", help="Path to dataset directory (batch mode)") + ts_eval_parser.add_argument( + "--results-dir", help="Path to runner output directory (batch mode)" + ) + ts_eval_parser.add_argument("--output", required=True, help="Output directory for reports") + ts_eval_parser.add_argument( + "--mlflow", + metavar="URI", + help="MLflow tracking URI (e.g., http://host:5000)", + ) + ts_eval_parser.add_argument( + "--mlflow-experiment", + default="forge-skill-eval", + help="MLflow experiment name (default: forge-skill-eval)", + ) + # skills subparser group skills_parser = subparsers.add_parser( "skills", @@ -1589,6 +1778,22 @@ def main(argv: list[str] | None = None) -> int: parser.print_help() return 0 + # Handle test-skill subcommands (sync handlers — no asyncio.run wrapper) + if args.command == "test-skill": + test_skill_handlers = { + "run": cmd_test_skill_run, + "eval": cmd_test_skill_eval, + } + ts_cmd = getattr(args, "test_skill_command", None) + if ts_cmd is None: + test_skill_parser.print_help() + return 0 + ts_handler = test_skill_handlers.get(ts_cmd) + if ts_handler: + return ts_handler(args) + test_skill_parser.print_help() + return 0 + # Handle skills subcommands if args.command == "skills": skills_handlers = {