From f9d3348791e47be36c32182af89d246d83e03e6c Mon Sep 17 00:00:00 2001 From: Binay <150876063+bkd-dotcom@users.noreply.github.com> Date: Wed, 12 Aug 2026 14:33:48 -0400 Subject: [PATCH] brand: sweep remaining umbra brand refs -> Signetry (cli strings, test names) --- signetry_eval/cli.py | 18 +++++++++--------- tests/test_corpus_benchmark.py | 24 ++++++++++++------------ tests/test_detection_benchmark.py | 18 +++++++++--------- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/signetry_eval/cli.py b/signetry_eval/cli.py index 7201de1..c6da7f5 100644 --- a/signetry_eval/cli.py +++ b/signetry_eval/cli.py @@ -66,9 +66,9 @@ def cmd_benchmark(args: argparse.Namespace) -> int: else: print(render_text(scores)) - umbra = next((s for s in scores if s.name == "signetry-core"), None) - if umbra is not None and umbra.recall < args.min_recall: - print(f"FAIL: umbra recall {umbra.recall:.0%} < required {args.min_recall:.0%}", + signetry = next((s for s in scores if s.name == "signetry-core"), None) + if signetry is not None and signetry.recall < args.min_recall: + print(f"FAIL: Signetry recall {signetry.recall:.0%} < required {args.min_recall:.0%}", file=sys.stderr) return 1 return 0 @@ -98,14 +98,14 @@ def cmd_corpus(args: argparse.Namespace) -> int: else: print(render_corpus_text(scores)) - umbra = next((s for s in scores if s.name == "signetry-core"), None) - if umbra is not None: - if umbra.recall < args.min_recall: - print(f"FAIL: umbra recall {umbra.recall:.0%} < required {args.min_recall:.0%}", + signetry = next((s for s in scores if s.name == "signetry-core"), None) + if signetry is not None: + if signetry.recall < args.min_recall: + print(f"FAIL: Signetry recall {signetry.recall:.0%} < required {args.min_recall:.0%}", file=sys.stderr) return 1 - if umbra.false_positive_total > args.max_fp: - print(f"FAIL: umbra false positives {umbra.false_positive_total} > allowed {args.max_fp}", + if signetry.false_positive_total > args.max_fp: + print(f"FAIL: Signetry false positives {signetry.false_positive_total} > allowed {args.max_fp}", file=sys.stderr) return 1 return 0 diff --git a/tests/test_corpus_benchmark.py b/tests/test_corpus_benchmark.py index acfbfd2..e68aded 100644 --- a/tests/test_corpus_benchmark.py +++ b/tests/test_corpus_benchmark.py @@ -64,7 +64,7 @@ def test_families_all_present(): @requires_engine -def test_umbra_full_recall_zero_fp_on_corpus(): +def test_signetry_full_recall_zero_fp_on_corpus(): """With cross-file taint + multi-language rules, the deterministic engine now reaches full recall on the corpus at ZERO false positives.""" score = run_corpus_benchmark("signetry-core", signetry_corpus_adapter()) @@ -79,14 +79,14 @@ def test_umbra_full_recall_zero_fp_on_corpus(): @requires_engine -def test_umbra_detects_cross_file_taint(): +def test_signetry_detects_cross_file_taint(): score = run_corpus_benchmark("signetry-core", signetry_corpus_adapter()) xfile = next(c for c in score.cases if c.case_id == "HARD-21-crossfile-taint-python") assert xfile.detected == xfile.expected == 1 @requires_engine -def test_umbra_covers_all_seven_languages(): +def test_signetry_covers_all_seven_languages(): score = run_corpus_benchmark("signetry-core", signetry_corpus_adapter()) by_lang = score.by_language() for lang in ("go", "java", "ruby", "php", "csharp"): @@ -95,7 +95,7 @@ def test_umbra_covers_all_seven_languages(): @requires_engine -def test_umbra_detects_multivariable_lang_taint(): +def test_signetry_detects_multivariable_lang_taint(): """The multi-variable (source->local->sink) cases across Go/Java/PHP/C# must be detected — the gap the pure per-line regex tier had.""" score = run_corpus_benchmark("signetry-core", signetry_corpus_adapter()) @@ -106,7 +106,7 @@ def test_umbra_detects_multivariable_lang_taint(): @requires_engine -def test_umbra_no_fp_on_parameterized_safe_decoys(): +def test_signetry_no_fp_on_parameterized_safe_decoys(): """Parameterised / prepared-statement SAFE decoys across languages must not trip (the false-positive axis LLMs trade recall to control).""" score = run_corpus_benchmark("signetry-core", signetry_corpus_adapter()) @@ -117,7 +117,7 @@ def test_umbra_no_fp_on_parameterized_safe_decoys(): @requires_engine -def test_umbra_detects_crossfile_taint_in_non_python_langs(): +def test_signetry_detects_crossfile_taint_in_non_python_langs(): score = run_corpus_benchmark("signetry-core", signetry_corpus_adapter()) for cid in ("XLANG-46-go-crossfile-sqli", "XLANG-47-java-crossfile-sqli", "XLANG-48-php-crossfile-sqli", "XLANG-50-ruby-crossfile-sqli", @@ -127,14 +127,14 @@ def test_umbra_detects_crossfile_taint_in_non_python_langs(): @requires_engine -def test_umbra_detects_taint_through_helper(): +def test_signetry_detects_taint_through_helper(): score = run_corpus_benchmark("signetry-core", signetry_corpus_adapter()) craft = next(c for c in score.cases if c.case_id == "CRAFT-15-taint-through-helper-python") assert craft.detected == craft.expected == 1 @requires_engine -def test_umbra_no_fp_on_safe_decoys(): +def test_signetry_no_fp_on_safe_decoys(): score = run_corpus_benchmark("signetry-core", signetry_corpus_adapter()) for c in score.cases: if c.is_safe: @@ -164,14 +164,14 @@ def test_head_to_head_includes_committed_claude_capture(): @requires_engine -def test_umbra_beats_or_matches_claude_on_corpus(): +def test_signetry_beats_or_matches_claude_on_corpus(): """The headline claim: on the harder corpus, the deterministic engine's recall is at least as high as the captured Opus 4.8 baseline, at zero false positives.""" scores = run_corpus_head_to_head() - umbra = next(s for s in scores if s.name == "signetry-core") + signetry = next(s for s in scores if s.name == "signetry-core") claude = next(s for s in scores if s.name == "claude-code-security-review") - assert umbra.recall >= claude.recall - assert umbra.false_positive_total == 0 + assert signetry.recall >= claude.recall + assert signetry.false_positive_total == 0 def test_committed_capture_file_exists(): diff --git a/tests/test_detection_benchmark.py b/tests/test_detection_benchmark.py index b061a95..e9692eb 100644 --- a/tests/test_detection_benchmark.py +++ b/tests/test_detection_benchmark.py @@ -27,12 +27,12 @@ @requires_engine -def test_umbra_achieves_full_recall_zero_fp(): +def test_signetry_achieves_full_recall_zero_fp(): scores = run_detection_benchmark() - umbra = next(s for s in scores if s.name == "signetry-core") - assert umbra.ran is True - assert umbra.recall == 1.0, f"expected full recall, got {umbra.recall} (missed {umbra.missed})" - assert umbra.false_positives == 0 + signetry = next(s for s in scores if s.name == "signetry-core") + assert signetry.ran is True + assert signetry.recall == 1.0, f"expected full recall, got {signetry.recall} (missed {signetry.missed})" + assert signetry.false_positives == 0 @requires_engine @@ -73,7 +73,7 @@ def test_markdown_renders_table(): scores = run_detection_benchmark() md = render_markdown(scores) assert "Recall" in md and "signetry-core" in md - assert "100%" in md # umbra full recall + assert "100%" in md # signetry full recall def test_ground_truth_excludes_open_redirect_from_in_scope(): @@ -86,6 +86,6 @@ def test_ground_truth_excludes_open_redirect_from_in_scope(): def test_semgrep_layer_optional_does_not_break(tmp_path): # use_semgrep=True must not error even if semgrep is absent. scores = run_detection_benchmark(use_semgrep=True) - umbra = next(s for s in scores if s.name == "signetry-core") - assert umbra.ran is True - assert umbra.recall == 1.0 + signetry = next(s for s in scores if s.name == "signetry-core") + assert signetry.ran is True + assert signetry.recall == 1.0