diff --git a/edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md b/edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md index f47fa48a..b69a00ad 100644 --- a/edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md +++ b/edgebench/tasks/juliet_vulnerability_analyzer/NOTES.md @@ -9,12 +9,21 @@ CWE-121/122/190/476/416/78. The judge runs it on the hidden set (736 cases, 723 0..100 (maximize). The rescale is the v1 one (`linear` 0..100), so the 0..100 score equals the raw score. +## Changes + +### 1.1.0 + +`infer` is no longer a banned word. The check also matched the English word, so one comment using +it scored 0; the README itself says "Infer external inputs". Infer is not in the image and grading is +offline, so the README's rule against calling it stays without a text check. Scores can only rise, +and only for submissions containing the word. Regression summary describes 1.0.0. Issue: https://github.com/ByteDance-Seed/EdgeBench/issues/39 + ## Source - Workspace (`juliet-static-analyzer-public-v3`: starter analyzer, public facts and labels, scorer, tools) and the hidden evaluator (`juliet-static-analyzer-evaluator-v3`, with the hidden facts and labels): the task authors' repository (not public), identical by sha256 and mode to the - published v1 images. + published v1 images, except `scoring/score.py` in both, changed in 1.1.0. - `verify_task.sh`: the version shipped in the published v1 images, which dropped the Chinese README (also absent here). - `judge/eval_hidden.py` (v1's inline eval program, one statement per line), `judge/run.sh` and @@ -60,7 +69,7 @@ Kept as in v1, so scores stay comparable. Fixing any of them changes scores and for tool and network names. 2. **The empty starter scores 10 (valid).** With no findings, trace quality is vacuously perfect and the speed points apply; a spammy submission can score below that. -3. **The keyword ban is crude.** One mention of a banned word (for example `infer` or `curl`) in a +3. **The keyword ban is crude.** One mention of a banned word (for example `codeql` or `curl`) in a comment or variable name of any submitted `.py`/`.sh` file zeroes the score, while real network use through other modules is not detected. 4. **Host-dependent 10 points.** The speed component is all or nothing at 30 s wall clock, and a diff --git a/edgebench/tasks/juliet_vulnerability_analyzer/environment/workspace/scoring/score.py b/edgebench/tasks/juliet_vulnerability_analyzer/environment/workspace/scoring/score.py index 06a73d51..68327855 100644 --- a/edgebench/tasks/juliet_vulnerability_analyzer/environment/workspace/scoring/score.py +++ b/edgebench/tasks/juliet_vulnerability_analyzer/environment/workspace/scoring/score.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Any -FORBIDDEN_RE = re.compile(r"\b(codeql|joern|semgrep|infer|cppcheck|clang\s+--analyze|scan-build)\b", re.I) +FORBIDDEN_RE = re.compile(r"\b(codeql|joern|semgrep|cppcheck|clang\s+--analyze|scan-build)\b", re.I) NETWORK_RE = re.compile(r"\b(curl|wget|git\s+clone|requests\.|urllib\.|socket\.)\b", re.I) CWE_FAMILY = { 'CWE-121': 'stack_buffer_overflow', diff --git a/edgebench/tasks/juliet_vulnerability_analyzer/judge/juliet-static-analyzer-evaluator-v3/scoring/score.py b/edgebench/tasks/juliet_vulnerability_analyzer/judge/juliet-static-analyzer-evaluator-v3/scoring/score.py index 06a73d51..68327855 100644 --- a/edgebench/tasks/juliet_vulnerability_analyzer/judge/juliet-static-analyzer-evaluator-v3/scoring/score.py +++ b/edgebench/tasks/juliet_vulnerability_analyzer/judge/juliet-static-analyzer-evaluator-v3/scoring/score.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Any -FORBIDDEN_RE = re.compile(r"\b(codeql|joern|semgrep|infer|cppcheck|clang\s+--analyze|scan-build)\b", re.I) +FORBIDDEN_RE = re.compile(r"\b(codeql|joern|semgrep|cppcheck|clang\s+--analyze|scan-build)\b", re.I) NETWORK_RE = re.compile(r"\b(curl|wget|git\s+clone|requests\.|urllib\.|socket\.)\b", re.I) CWE_FAMILY = { 'CWE-121': 'stack_buffer_overflow', diff --git a/edgebench/tasks/juliet_vulnerability_analyzer/task.toml b/edgebench/tasks/juliet_vulnerability_analyzer/task.toml index 74553d0a..567b56be 100644 --- a/edgebench/tasks/juliet_vulnerability_analyzer/task.toml +++ b/edgebench/tasks/juliet_vulnerability_analyzer/task.toml @@ -4,7 +4,7 @@ name = "Juliet Vulnerability Analyzer" category = "Systems & Software Engineering" language = "Python" description = "Implement a deterministic static analyzer that processes structured program facts to detect vulnerabilities across six CWE categories (stack/heap overflow, integer overflow, null dereference, use-after-free, command injection)." -version = "1.0.0" +version = "1.1.0" status = "migrated" [environment]