diff --git a/CHANGELOG.md b/CHANGELOG.md index c9230aa..3c44d25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,38 @@ Until `1.0.0` the public API may change between minor versions. ## [Unreleased] +### Added — OWASP breadth in the detection corpus (52 → 60 cases) + +- **XXE (CWE-611)** in Java and PHP — `LANG-53`, `LANG-54` (eval#11). +- **Path traversal (CWE-22)** in Go and Java — `LANG-56`, `LANG-57` (eval#12). +- **SSRF (CWE-918)** in Go — `LANG-59` (eval#29), giving the class a second + language alongside Python. +- Three SAFE decoys probing the precision distinctions these rules must make: + default-safe PHP XML parsing (entities are off by default on PHP 8+), a constant + filesystem path, and a **constant host with a user-supplied query string** + (`LANG-55`, `LANG-58`, `LANG-60`). +- Two **pinned** real-repo cases — OWASP WebGoat (Java) and OWASP RailsGoat + (Ruby), the first JVM/Ruby targets here; every prior case is Python or + JavaScript (eval#13). + +The `LANG-60` decoy earned its keep immediately: it caught a false positive in +signetry-core's new Go SSRF rule, fixed in Signetry/core#97 before this landed. + +### Fixed — pinned real-repo cases were not actually pinned + +- `scan_real_repo` cloned with `--depth 1` and then ran `git checkout ` with + `check=False`. On a shallow clone the object is absent, so the checkout failed + (`fatal: unable to read tree`), the failure was swallowed, and the scan silently + ran against the **default-branch tip** — a case documented as "pinned for + reproducibility" was not pinned. Now fetches the specific object first, and if + pinning genuinely cannot be honoured it says so in the result note rather than + reporting an unpinned scan as pinned. + +### Changed + +- Pin `signetry-core` at `v0.7.0`; the corpus additions above depend on its new + Go SSRF / Go+Java path-traversal / PHP XXE rules. + ### Changed — Signetry rename (breaking) - Distribution `signetry-eval` and import package `signetry_eval`. The console diff --git a/pyproject.toml b/pyproject.toml index d1ae142..f4f76b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ classifiers = [ "Topic :: Software Development :: Quality Assurance", ] dependencies = [ - "signetry-core @ git+https://github.com/Signetry/core@v0.6.0", + "signetry-core @ git+https://github.com/Signetry/core@v0.7.0", ] [project.urls] diff --git a/signetry_eval/detection/corpus/multilang.py b/signetry_eval/detection/corpus/multilang.py index b84c1fe..67918eb 100644 --- a/signetry_eval/detection/corpus/multilang.py +++ b/signetry_eval/detection/corpus/multilang.py @@ -251,6 +251,182 @@ def backup(params) ps.executeQuery(); } } +'''}, + expected=[], # SAFE + ), + + # --- OWASP breadth: XXE, path traversal, SSRF (Signetry/eval#11, #12, #29) --- + Case( + id="LANG-53-java-xxe", + family=Family.MULTILANG, + language="java", + title="Java: XML parsed with a default DocumentBuilderFactory (XXE)", + provenance="OWASP A05:2021 Security Misconfiguration; CWE-611. Pattern per the " + "OWASP XXE Prevention cheat sheet: the JAXP default factory resolves " + "external entities unless DOCTYPE processing is explicitly disabled.", + files={"XmlLoader.java": '''\ +import javax.xml.parsers.DocumentBuilderFactory; +import org.xml.sax.InputSource; +import java.io.StringReader; + +public class XmlLoader { + public void load(String xml) throws Exception { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.newDocumentBuilder().parse(new InputSource(new StringReader(xml))); + } +} +'''}, + expected=[ExpectedFinding("CWE-611", "xxe", "XmlLoader.java")], + ), + Case( + id="LANG-54-php-xxe", + family=Family.MULTILANG, + language="php", + title="PHP: loadXML with LIBXML_NOENT|LIBXML_DTDLOAD (XXE)", + provenance="OWASP A05:2021; CWE-611. Since PHP 8 / libxml 2.9 external entities " + "are off by default, so the vulnerable pattern is code that explicitly " + "re-enables them via LIBXML_NOENT / LIBXML_DTDLOAD.", + files={"import.php": '''\ +loadXML($xml, LIBXML_NOENT | LIBXML_DTDLOAD); +echo $doc->saveXML(); +'''}, + expected=[ExpectedFinding("CWE-611", "xxe", "import.php")], + ), + Case( + id="LANG-55-SAFE-php-xml-default", + family=Family.MULTILANG, + language="php", + title="SAFE: PHP loadXML without entity flags (default-safe on modern PHP)", + provenance="Crafted SAFE decoy: parsing untrusted XML is not itself XXE on PHP 8+ " + "(entities off by default) — probes whether the rule keys on the flag " + "rather than on 'parses XML'.", + files={"safe_import.php": '''\ +loadXML($xml); +echo $doc->saveXML(); +'''}, + expected=[], # SAFE + ), + Case( + id="LANG-56-go-path-traversal", + family=Family.MULTILANG, + language="go", + title="Go: file path built from a query parameter (traversal)", + provenance="OWASP A01:2021 Broken Access Control; CWE-22. Unconfined path join " + "from a request parameter, per the OWASP Path Traversal description.", + files={"files.go": '''\ +package main + +import ( + "net/http" + "os" +) + +func download(w http.ResponseWriter, r *http.Request) { + name := r.URL.Query().Get("file") + data, err := os.ReadFile("/var/data/" + name) + if err != nil { + http.Error(w, "not found", 404) + return + } + w.Write(data) +} +'''}, + expected=[ExpectedFinding("CWE-22", "path_traversal", "files.go")], + ), + Case( + id="LANG-57-java-path-traversal", + family=Family.MULTILANG, + language="java", + title="Java: FileInputStream opened on a request parameter (traversal)", + provenance="OWASP A01:2021; CWE-22. Servlet parameter concatenated into a " + "filesystem path with no canonicalisation or base-dir confinement.", + files={"Download.java": '''\ +import javax.servlet.http.HttpServletRequest; + +public class Download { + public void send(HttpServletRequest req) throws Exception { + String name = req.getParameter("file"); + java.io.FileInputStream in = new java.io.FileInputStream("/var/data/" + name); + in.close(); + } +} +'''}, + expected=[ExpectedFinding("CWE-22", "path_traversal", "Download.java")], + ), + Case( + id="LANG-58-SAFE-go-constant-path", + family=Family.MULTILANG, + language="go", + title="SAFE: Go reads a compiled-in constant path", + provenance="Crafted SAFE decoy: a constant filesystem path is not traversal " + "(false-positive probe for the traversal sink).", + files={"config.go": '''\ +package main + +import "os" + +func loadConfig() ([]byte, error) { + return os.ReadFile("/etc/app/config.yaml") +} +'''}, + expected=[], # SAFE + ), + Case( + id="LANG-59-go-ssrf", + family=Family.MULTILANG, + language="go", + title="Go: outbound HTTP request to a user-controlled URL (SSRF)", + provenance="OWASP A10:2021 Server-Side Request Forgery; CWE-918. A fetch-by-URL " + "handler forwards a request parameter straight to http.Get, reaching " + "internal services and cloud metadata endpoints.", + files={"proxy.go": '''\ +package main + +import ( + "io" + "net/http" +) + +func fetch(w http.ResponseWriter, r *http.Request) { + target := r.URL.Query().Get("url") + resp, err := http.Get(target) + if err != nil { + http.Error(w, "fetch failed", 502) + return + } + defer resp.Body.Close() + io.Copy(w, resp.Body) +} +'''}, + expected=[ExpectedFinding("CWE-918", "ssrf", "proxy.go")], + ), + Case( + id="LANG-60-SAFE-go-constant-url", + family=Family.MULTILANG, + language="go", + title="SAFE: Go fetches a constant URL with a user-supplied query string", + provenance="Crafted SAFE decoy: the destination host is compiled in and only the " + "query string is user-controlled — not SSRF. Probes the same " + "constant-URL-with-tainted-parameters distinction that the Python SSRF " + "rule is measured on.", + files={"client.go": '''\ +package main + +import ( + "net/http" + "net/url" +) + +func search(r *http.Request) (*http.Response, error) { + q := r.URL.Query().Get("q") + return http.Get("https://api.example.com/search?q=" + url.QueryEscape(q)) +} '''}, expected=[], # SAFE ), diff --git a/signetry_eval/detection/real_repo_benchmark.py b/signetry_eval/detection/real_repo_benchmark.py index c389b37..33d5194 100644 --- a/signetry_eval/detection/real_repo_benchmark.py +++ b/signetry_eval/detection/real_repo_benchmark.py @@ -40,6 +40,31 @@ def _git_available() -> bool: return False +def _checkout_pinned(root: Path, commit: str) -> str: + """Check out a pinned commit in a (possibly shallow) clone. + + ``resolve_scan_target`` clones with ``--depth 1``, so the pinned object is not + present and a plain ``git checkout `` fails. Previously that failure was + swallowed (``check=False``) and the scan silently ran against the default-branch + tip — so a "pinned for reproducibility" case was not actually pinned. + + Fetch the specific object first (GitHub permits fetching an arbitrary SHA), and + if pinning genuinely cannot be honoured, say so in the note rather than + reporting an unpinned scan as pinned. + """ + fetched = subprocess.run( + ["git", "fetch", "--quiet", "--depth", "1", "origin", commit], + cwd=root, capture_output=True, check=False, + ) + ref = "FETCH_HEAD" if fetched.returncode == 0 else commit + checked = subprocess.run( + ["git", "checkout", "-q", ref], cwd=root, capture_output=True, check=False, + ) + if checked.returncode != 0: + return f"NOT pinned (commit {commit[:12]} unreachable; scanned default branch)" + return f"pinned @ {commit[:12]}" + + def scan_real_repo(case: RealRepoCase, *, use_semgrep: bool = False, depth: int = 1) -> RealRepoResult: """Clone + scan one real repo. Returns a result; never raises (records notes).""" from signetry_core import scan_repository @@ -47,22 +72,25 @@ def scan_real_repo(case: RealRepoCase, *, use_semgrep: bool = False, depth: int if not _git_available(): return RealRepoResult(case.id, case.url, ran=False, note="git unavailable") + pin_note = "" try: with resolve_scan_target(case.url, depth=depth) as root: root = Path(root) if case.commit: - subprocess.run(["git", "checkout", "-q", case.commit], cwd=root, - capture_output=True, check=False) + pin_note = _checkout_pinned(root, case.commit) report = scan_repository(root, use_semgrep=use_semgrep) except RuntimeError as exc: return RealRepoResult(case.id, case.url, ran=False, note=f"clone/scan failed: {exc}") by_cat: dict[str, int] = {} for f in report.findings: by_cat[f.category] = by_cat.get(f.category, 0) + 1 + note = f"layers: {', '.join(report.layers)}" + if pin_note: + note = f"{pin_note} · {note}" return RealRepoResult( case.id, case.url, ran=True, files_scanned=report.files_scanned, total_findings=len(report.findings), by_category=by_cat, - note=f"layers: {', '.join(report.layers)}", + note=note, ) diff --git a/signetry_eval/detection/real_repos.py b/signetry_eval/detection/real_repos.py index 900d3cc..b65c4c0 100644 --- a/signetry_eval/detection/real_repos.py +++ b/signetry_eval/detection/real_repos.py @@ -77,4 +77,29 @@ class RealRepoCase: "Large; scanned shallowly. Expected: assorted JS/TS sinks.", expect_at_least=[], ), + # The cases above pin no commit, so they drift with their default branch. The two + # below are pinned, and are the first JVM/Ruby targets here — every case above is + # Python or JavaScript, which under-exercises the multi-language tier. + RealRepoCase( + id="REAL-webgoat", + url="https://github.com/WebGoat/WebGoat.git", + commit="7517acca95d9851da706452454c223dd13545ef4", + languages=["java"], + provenance="OWASP WebGoat — the reference deliberately-insecure Java teaching " + "app, maintained by OWASP. Pinned for reproducibility. Exercises the " + "Java tier (JDBC concatenation, XXE, native deserialization, path " + "traversal) that no other real-repo case here reaches.", + expect_at_least=[], # reported, not asserted (see the module docstring) + ), + RealRepoCase( + id="REAL-railsgoat", + url="https://github.com/OWASP/railsgoat.git", + commit="0222f7da3406ba3ab637bc6d24ae9366b5f0a680", + languages=["ruby"], + provenance="OWASP RailsGoat — deliberately vulnerable Rails app covering the " + "OWASP Top 10. Pinned for reproducibility. First Ruby target here, so " + "it exercises the Ruby rules (interpolated SQL/command, YAML/Marshal " + "load) against real application code rather than snippets.", + expect_at_least=[], + ), ] diff --git a/tests/test_corpus_benchmark.py b/tests/test_corpus_benchmark.py index e68aded..6ed1ddd 100644 --- a/tests/test_corpus_benchmark.py +++ b/tests/test_corpus_benchmark.py @@ -193,3 +193,58 @@ def test_corpus_markdown_renders(): assert "public test cases" in md assert "Recall by language" in md assert "signetry-core" in md + + +# --- OWASP breadth additions (eval#11 XXE, #12 path traversal, #29 Go SSRF) --- + + +def test_corpus_includes_xxe_traversal_ssrf_cases(): + ids = {c.id for c in ALL_CASES} + assert {"LANG-53-java-xxe", "LANG-54-php-xxe", "LANG-56-go-path-traversal", + "LANG-57-java-path-traversal", "LANG-59-go-ssrf"} <= ids + + +def test_corpus_covers_xxe_traversal_ssrf_in_multiple_languages(): + """Each of the three classes must be represented in at least two languages, so + the corpus measures breadth rather than one language's rule.""" + by_class: dict[str, set[str]] = {} + for c in ALL_CASES: + for e in c.expected: + by_class.setdefault(e.category, set()).add(c.language) + assert {"java", "php"} <= by_class.get("xxe", set()) + assert {"go", "java"} <= by_class.get("path_traversal", set()) + assert {"go", "python"} <= by_class.get("ssrf", set()) + + +@requires_engine +def test_signetry_detects_new_owasp_breadth_cases(): + score = run_corpus_benchmark("signetry-core", signetry_corpus_adapter()) + for cid in ("LANG-53-java-xxe", "LANG-54-php-xxe", "LANG-56-go-path-traversal", + "LANG-57-java-path-traversal", "LANG-59-go-ssrf"): + c = next(x for x in score.cases if x.case_id == cid) + assert c.detected == c.expected >= 1, f"{cid} not detected" + + +@requires_engine +def test_signetry_no_fp_on_new_safe_decoys(): + """The safe decoys probe the precision distinctions these rules must make: + default-safe PHP XML parsing, a constant filesystem path, and a constant host + with a user-supplied query string.""" + score = run_corpus_benchmark("signetry-core", signetry_corpus_adapter()) + for cid in ("LANG-55-SAFE-php-xml-default", "LANG-58-SAFE-go-constant-path", + "LANG-60-SAFE-go-constant-url"): + c = next(x for x in score.cases if x.case_id == cid) + assert c.false_positives == 0, f"false positive on {cid}" + + +def test_pinned_real_repo_cases_exist_and_are_pinned(): + from signetry_eval.detection.real_repos import REAL_REPO_CASES + + pinned = {c.id: c for c in REAL_REPO_CASES if c.commit} + assert {"REAL-webgoat", "REAL-railsgoat"} <= set(pinned) + for c in pinned.values(): + assert len(c.commit) == 40, f"{c.id}: pin must be a full 40-char SHA" + assert c.provenance and len(c.provenance) > 20 + # These are the first JVM/Ruby real-repo targets. + langs = {lang for c in REAL_REPO_CASES for lang in c.languages} + assert {"java", "ruby"} <= langs diff --git a/uv.lock b/uv.lock index 5a88297..58c3c44 100644 --- a/uv.lock +++ b/uv.lock @@ -308,8 +308,8 @@ wheels = [ [[package]] name = "signetry-core" -version = "0.6.0" -source = { git = "https://github.com/Signetry/core?rev=v0.6.0#b1f437ec33ca3e57586abc56accda7de529ec519" } +version = "0.7.0" +source = { git = "https://github.com/Signetry/core?rev=v0.7.0#0d39eb34f32152f5a3015ce9282245e38c25d9fc" } dependencies = [ { name = "cryptography" }, { name = "pyyaml" }, @@ -317,7 +317,7 @@ dependencies = [ [[package]] name = "signetry-eval" -version = "0.2.2" +version = "0.2.3" source = { editable = "." } dependencies = [ { name = "signetry-core" }, @@ -333,6 +333,6 @@ dev = [ requires-dist = [ { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.6.0" }, - { name = "signetry-core", git = "https://github.com/Signetry/core?rev=v0.6.0" }, + { name = "signetry-core", git = "https://github.com/Signetry/core?rev=v0.7.0" }, ] provides-extras = ["dev"]