diff --git a/edgebench/tasks/graph_node_classification/NOTES.md b/edgebench/tasks/graph_node_classification/NOTES.md index ffa5d993..ef8d4629 100644 --- a/edgebench/tasks/graph_node_classification/NOTES.md +++ b/edgebench/tasks/graph_node_classification/NOTES.md @@ -9,11 +9,22 @@ writes `solution/solution.py` (class `Solution` with `solve(input_data)`, CPU on (maximize). A static check rejects GPU use (`valid=false`). The rescale is the v1 one (`linear` 0..1 -> 0..100). +## Changes + +### 1.1.0 + +`problem.md` allows prediction keys "convertible to strings", but the verifier looked up only string +node IDs, so a run keyed by integers matched no node: overall quality 0, about 2 of 100 through the +solve-time term, and marked valid. When no +key matches as given, the converted keys are exactly the test nodes and every value converts with +`int()`, the verifier now converts prediction and confidence keys with `str()`. Those submissions +score like the same answer keyed by strings; others keep the same metric calculation. + ## Source - Starter solution and template, and the judge's `runner.py`, `verifier.py`, `config.yaml`, `judge_eval.py`: the task authors' repository (not public), identical by sha256 to the - published v1 images. + published v1 images, except `verifier.py`, changed in 1.1.0. - `environment/workspace/README.md`, `solution/problem.md`: the English versions shipped in the published v1 images (the repository has Chinese versions). - Blob `judge/blobs/squirrel_data.tar.gz` (5 MB packed, 626 MB unpacked; sha256 in `task.toml`): the diff --git a/edgebench/tasks/graph_node_classification/judge/workspace/verifier.py b/edgebench/tasks/graph_node_classification/judge/workspace/verifier.py index e3c6d23f..bfb39f39 100644 --- a/edgebench/tasks/graph_node_classification/judge/workspace/verifier.py +++ b/edgebench/tasks/graph_node_classification/judge/workspace/verifier.py @@ -86,6 +86,15 @@ def evaluate( ground_truth: Dict[str, int | str], confidences: Dict[str, float] | None = None, ) -> Dict[str, float]: + # problem.md allows keys "convertible to strings"; ground_truth keys are strings. Convert only a + # complete answer in which no key matches as given, so every other submission scores as before. + if isinstance(predictions, dict) and set(predictions).isdisjoint(ground_truth): + as_str = {str(k): v for k, v in predictions.items()} + if set(as_str) == set(ground_truth) and len(_to_int_labels(as_str.values())) == len(as_str): + predictions = as_str + if isinstance(confidences, dict): + confidences = {str(k): v for k, v in confidences.items()} + ordered_ids = [nid for nid in ground_truth.keys() if nid in predictions] y_true = _to_int_labels(ground_truth[nid] for nid in ordered_ids) y_pred = _to_int_labels(predictions[nid] for nid in ordered_ids) diff --git a/edgebench/tasks/graph_node_classification/task.toml b/edgebench/tasks/graph_node_classification/task.toml index eaf83dc3..9828160e 100644 --- a/edgebench/tasks/graph_node_classification/task.toml +++ b/edgebench/tasks/graph_node_classification/task.toml @@ -4,7 +4,7 @@ name = "Graph Node Classification" category = "Scientific Problems & ML" language = "Python" description = "Implement graph neural networks from scratch using only base PyTorch for semi-supervised node classification on an unseen graph under CPU-only constraints." -version = "1.0.0" +version = "1.1.0" status = "migrated" [environment]