From 441b06eee081d6132c50213e081301127e83d826 Mon Sep 17 00:00:00 2001 From: findalexli Date: Sun, 1 Nov 2026 17:39:19 -0800 Subject: [PATCH 1/3] juliet_vulnerability_analyzer: narrow the "infer" ban to command-like text --- .../tasks/juliet_vulnerability_analyzer/NOTES.md | 15 +++++++++++++-- .../environment/workspace/scoring/score.py | 2 +- .../scoring/score.py | 2 +- .../tasks/juliet_vulnerability_analyzer/task.toml | 2 +- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md b/edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md index f47fa48a..2d738b6f 100644 --- a/edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md +++ b/edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md @@ -9,12 +9,23 @@ CWE-121/122/190/476/416/78. The judge runs it on the hidden set (736 cases, 723 0..100 (maximize). The rescale is the v1 one (`linear` 0..100), so the 0..100 score equals the raw score. +## Changes + +### 1.1.0 + +The banned-word check also matched the English word "infer", so one comment using it scored 0; the +README itself says "Infer external inputs". It now matches `infer --`, `infer run --`, +`infer capture --`, `infer analyze --`, and a string `"infer"` (or a path ending in it) right after +`(` or `[`. This is still a text pattern: harmless text of those shapes, such as `stats["infer"]` or +`we infer -- roughly`, still scores 0. Scores can only rise, and only for submissions containing the +word. Regression summary describes 1.0.0. + ## Source - Workspace (`juliet-static-analyzer-public-v3`: starter analyzer, public facts and labels, scorer, tools) and the hidden evaluator (`juliet-static-analyzer-evaluator-v3`, with the hidden facts and labels): the task authors' repository (not public), identical by sha256 and mode to the - published v1 images. + published v1 images, except `scoring/score.py` in both, changed in 1.1.0. - `verify_task.sh`: the version shipped in the published v1 images, which dropped the Chinese README (also absent here). - `judge/eval_hidden.py` (v1's inline eval program, one statement per line), `judge/run.sh` and @@ -60,7 +71,7 @@ Kept as in v1, so scores stay comparable. Fixing any of them changes scores and for tool and network names. 2. **The empty starter scores 10 (valid).** With no findings, trace quality is vacuously perfect and the speed points apply; a spammy submission can score below that. -3. **The keyword ban is crude.** One mention of a banned word (for example `infer` or `curl`) in a +3. **The keyword ban is crude.** One mention of a banned word (for example `codeql` or `curl`) in a comment or variable name of any submitted `.py`/`.sh` file zeroes the score, while real network use through other modules is not detected. 4. **Host-dependent 10 points.** The speed component is all or nothing at 30 s wall clock, and a diff --git a/edgebench/tasks/juliet_vulnerability_analyzer/environment/workspace/scoring/score.py b/edgebench/tasks/juliet_vulnerability_analyzer/environment/workspace/scoring/score.py index 06a73d51..dedc8430 100644 --- a/edgebench/tasks/juliet_vulnerability_analyzer/environment/workspace/scoring/score.py +++ b/edgebench/tasks/juliet_vulnerability_analyzer/environment/workspace/scoring/score.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Any -FORBIDDEN_RE = re.compile(r"\b(codeql|joern|semgrep|infer|cppcheck|clang\s+--analyze|scan-build)\b", re.I) +FORBIDDEN_RE = re.compile(r"""\b(codeql|joern|semgrep|cppcheck|clang\s+--analyze|scan-build)\b|\binfer\s+(?:(?:run|capture|analyze)\s+)?--|[\[(]\s*["'](?:[^"']*/)?infer["']""", re.I) NETWORK_RE = re.compile(r"\b(curl|wget|git\s+clone|requests\.|urllib\.|socket\.)\b", re.I) CWE_FAMILY = { 'CWE-121': 'stack_buffer_overflow', diff --git a/edgebench/tasks/juliet_vulnerability_analyzer/judge/juliet-static-analyzer-evaluator-v3/scoring/score.py b/edgebench/tasks/juliet_vulnerability_analyzer/judge/juliet-static-analyzer-evaluator-v3/scoring/score.py index 06a73d51..dedc8430 100644 --- a/edgebench/tasks/juliet_vulnerability_analyzer/judge/juliet-static-analyzer-evaluator-v3/scoring/score.py +++ b/edgebench/tasks/juliet_vulnerability_analyzer/judge/juliet-static-analyzer-evaluator-v3/scoring/score.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Any -FORBIDDEN_RE = re.compile(r"\b(codeql|joern|semgrep|infer|cppcheck|clang\s+--analyze|scan-build)\b", re.I) +FORBIDDEN_RE = re.compile(r"""\b(codeql|joern|semgrep|cppcheck|clang\s+--analyze|scan-build)\b|\binfer\s+(?:(?:run|capture|analyze)\s+)?--|[\[(]\s*["'](?:[^"']*/)?infer["']""", re.I) NETWORK_RE = re.compile(r"\b(curl|wget|git\s+clone|requests\.|urllib\.|socket\.)\b", re.I) CWE_FAMILY = { 'CWE-121': 'stack_buffer_overflow', diff --git a/edgebench/tasks/juliet_vulnerability_analyzer/task.toml b/edgebench/tasks/juliet_vulnerability_analyzer/task.toml index 74553d0a..567b56be 100644 --- a/edgebench/tasks/juliet_vulnerability_analyzer/task.toml +++ b/edgebench/tasks/juliet_vulnerability_analyzer/task.toml @@ -4,7 +4,7 @@ name = "Juliet Vulnerability Analyzer" category = "Systems & Software Engineering" language = "Python" description = "Implement a deterministic static analyzer that processes structured program facts to detect vulnerabilities across six CWE categories (stack/heap overflow, integer overflow, null dereference, use-after-free, command injection)." -version = "1.0.0" +version = "1.1.0" status = "migrated" [environment] From 5feaca4cc6d90cafcbac17f06a573f6cf88326a5 Mon Sep 17 00:00:00 2001 From: findalexli Date: Sun, 1 Nov 2026 23:40:08 -0800 Subject: [PATCH 2/3] juliet_vulnerability_analyzer: link the Task finding issue in NOTES --- edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md b/edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md index 2d738b6f..acab84ba 100644 --- a/edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md +++ b/edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md @@ -18,7 +18,7 @@ README itself says "Infer external inputs". It now matches `infer --`, `infer ru `infer capture --`, `infer analyze --`, and a string `"infer"` (or a path ending in it) right after `(` or `[`. This is still a text pattern: harmless text of those shapes, such as `stats["infer"]` or `we infer -- roughly`, still scores 0. Scores can only rise, and only for submissions containing the -word. Regression summary describes 1.0.0. +word. Regression summary describes 1.0.0. Issue: https://github.com/ByteDance-Seed/EdgeBench/issues/39 ## Source From b9125bda4a50775c977d3731879bc8a11048122e Mon Sep 17 00:00:00 2001 From: qinshengling Date: Tue, 3 Nov 2026 04:50:22 -0500 Subject: [PATCH 3/3] juliet_vulnerability_analyzer: drop infer from the banned words Infer is not in the image and grading is offline, so the command-like pattern enforced little while still flagging harmless text. --- edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md | 10 ++++------ .../environment/workspace/scoring/score.py | 2 +- .../scoring/score.py | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md b/edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md index acab84ba..b69a00ad 100644 --- a/edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md +++ b/edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md @@ -13,12 +13,10 @@ raw score. ### 1.1.0 -The banned-word check also matched the English word "infer", so one comment using it scored 0; the -README itself says "Infer external inputs". It now matches `infer --`, `infer run --`, -`infer capture --`, `infer analyze --`, and a string `"infer"` (or a path ending in it) right after -`(` or `[`. This is still a text pattern: harmless text of those shapes, such as `stats["infer"]` or -`we infer -- roughly`, still scores 0. Scores can only rise, and only for submissions containing the -word. Regression summary describes 1.0.0. Issue: https://github.com/ByteDance-Seed/EdgeBench/issues/39 +`infer` is no longer a banned word. The check also matched the English word, so one comment using +it scored 0; the README itself says "Infer external inputs". Infer is not in the image and grading is +offline, so the README's rule against calling it stays without a text check. Scores can only rise, +and only for submissions containing the word. Regression summary describes 1.0.0. Issue: https://github.com/ByteDance-Seed/EdgeBench/issues/39 ## Source diff --git a/edgebench/tasks/juliet_vulnerability_analyzer/environment/workspace/scoring/score.py b/edgebench/tasks/juliet_vulnerability_analyzer/environment/workspace/scoring/score.py index dedc8430..68327855 100644 --- a/edgebench/tasks/juliet_vulnerability_analyzer/environment/workspace/scoring/score.py +++ b/edgebench/tasks/juliet_vulnerability_analyzer/environment/workspace/scoring/score.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Any -FORBIDDEN_RE = re.compile(r"""\b(codeql|joern|semgrep|cppcheck|clang\s+--analyze|scan-build)\b|\binfer\s+(?:(?:run|capture|analyze)\s+)?--|[\[(]\s*["'](?:[^"']*/)?infer["']""", re.I) +FORBIDDEN_RE = re.compile(r"\b(codeql|joern|semgrep|cppcheck|clang\s+--analyze|scan-build)\b", re.I) NETWORK_RE = re.compile(r"\b(curl|wget|git\s+clone|requests\.|urllib\.|socket\.)\b", re.I) CWE_FAMILY = { 'CWE-121': 'stack_buffer_overflow', diff --git a/edgebench/tasks/juliet_vulnerability_analyzer/judge/juliet-static-analyzer-evaluator-v3/scoring/score.py b/edgebench/tasks/juliet_vulnerability_analyzer/judge/juliet-static-analyzer-evaluator-v3/scoring/score.py index dedc8430..68327855 100644 --- a/edgebench/tasks/juliet_vulnerability_analyzer/judge/juliet-static-analyzer-evaluator-v3/scoring/score.py +++ b/edgebench/tasks/juliet_vulnerability_analyzer/judge/juliet-static-analyzer-evaluator-v3/scoring/score.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Any -FORBIDDEN_RE = re.compile(r"""\b(codeql|joern|semgrep|cppcheck|clang\s+--analyze|scan-build)\b|\binfer\s+(?:(?:run|capture|analyze)\s+)?--|[\[(]\s*["'](?:[^"']*/)?infer["']""", re.I) +FORBIDDEN_RE = re.compile(r"\b(codeql|joern|semgrep|cppcheck|clang\s+--analyze|scan-build)\b", re.I) NETWORK_RE = re.compile(r"\b(curl|wget|git\s+clone|requests\.|urllib\.|socket\.)\b", re.I) CWE_FAMILY = { 'CWE-121': 'stack_buffer_overflow',