From 39c62f1b5a033db5408e8bb2ff0e3a5da3aa3a7a Mon Sep 17 00:00:00 2001 From: findalexli Date: Sun, 1 Nov 2026 19:28:20 -0800 Subject: [PATCH] graph_node_classification: accept prediction keys convertible to strings problem.md allows keys convertible to strings, but the verifier looked up only string node IDs, so a complete integer-keyed run matched no node and scored about 0 while marked valid. Convert keys with str() only for a complete answer in which no key matches as given; every other submission is unaffected. --- edgebench/tasks/graph_node_classification/NOTES.md | 13 ++++++++++++- .../judge/workspace/verifier.py | 9 +++++++++ edgebench/tasks/graph_node_classification/task.toml | 2 +- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/edgebench/tasks/graph_node_classification/NOTES.md b/edgebench/tasks/graph_node_classification/NOTES.md index ffa5d993..50c2c90e 100644 --- a/edgebench/tasks/graph_node_classification/NOTES.md +++ b/edgebench/tasks/graph_node_classification/NOTES.md @@ -9,11 +9,21 @@ writes `solution/solution.py` (class `Solution` with `solve(input_data)`, CPU on (maximize). A static check rejects GPU use (`valid=false`). The rescale is the v1 one (`linear` 0..1 -> 0..100). +## Changes + +### 1.1.0 + +`problem.md` allows prediction keys "convertible to strings", but the verifier looked up only string +node IDs, so a run keyed by integers matched no node and scored about 0 while marked valid. When no +key matches as given, the converted keys are exactly the test nodes and every value converts with +`int()`, the verifier now converts prediction and confidence keys with `str()`. Those submissions +score like the same answer keyed by strings; others keep the same metric calculation. + ## Source - Starter solution and template, and the judge's `runner.py`, `verifier.py`, `config.yaml`, `judge_eval.py`: the task authors' repository (not public), identical by sha256 to the - published v1 images. + published v1 images, except `verifier.py`, changed in 1.1.0. - `environment/workspace/README.md`, `solution/problem.md`: the English versions shipped in the published v1 images (the repository has Chinese versions). - Blob `judge/blobs/squirrel_data.tar.gz` (5 MB packed, 626 MB unpacked; sha256 in `task.toml`): the @@ -32,6 +42,7 @@ writes `solution/solution.py` (class `Solution` with `solve(input_data)`, CPU on 2. A judge timeout leaves no result (invalid); v1 recorded an unrankable score. Same ranking effect. The grader's own failure cases are `valid=false, 0` in both. 3. The workspace git commit is identical to v1's (same hash). +4. `verifier.py` converts integer keys (see Changes, 1.1.0). ## Regression summary diff --git a/edgebench/tasks/graph_node_classification/judge/workspace/verifier.py b/edgebench/tasks/graph_node_classification/judge/workspace/verifier.py index e3c6d23f..bfb39f39 100644 --- a/edgebench/tasks/graph_node_classification/judge/workspace/verifier.py +++ b/edgebench/tasks/graph_node_classification/judge/workspace/verifier.py @@ -86,6 +86,15 @@ def evaluate( ground_truth: Dict[str, int | str], confidences: Dict[str, float] | None = None, ) -> Dict[str, float]: + # problem.md allows keys "convertible to strings"; ground_truth keys are strings. Convert only a + # complete answer in which no key matches as given, so every other submission scores as before. + if isinstance(predictions, dict) and set(predictions).isdisjoint(ground_truth): + as_str = {str(k): v for k, v in predictions.items()} + if set(as_str) == set(ground_truth) and len(_to_int_labels(as_str.values())) == len(as_str): + predictions = as_str + if isinstance(confidences, dict): + confidences = {str(k): v for k, v in confidences.items()} + ordered_ids = [nid for nid in ground_truth.keys() if nid in predictions] y_true = _to_int_labels(ground_truth[nid] for nid in ordered_ids) y_pred = _to_int_labels(predictions[nid] for nid in ordered_ids) diff --git a/edgebench/tasks/graph_node_classification/task.toml b/edgebench/tasks/graph_node_classification/task.toml index eaf83dc3..9828160e 100644 --- a/edgebench/tasks/graph_node_classification/task.toml +++ b/edgebench/tasks/graph_node_classification/task.toml @@ -4,7 +4,7 @@ name = "Graph Node Classification" category = "Scientific Problems & ML" language = "Python" description = "Implement graph neural networks from scratch using only base PyTorch for semi-supervised node classification on an unseen graph under CPU-only constraints." -version = "1.0.0" +version = "1.1.0" status = "migrated" [environment]