From 8856d58c9910c3c61df32ac6df175f37f115f660 Mon Sep 17 00:00:00 2001 From: Matheus de Almeida <69125506+matheusmra@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:32:02 -0700 Subject: [PATCH 1/8] feat: introduce flat path-keyed score_vector map for result tree Add iter_test_results() and to_score_vector() to ResultTree to project nested result nodes into a flat map keyed by stable path strings (category/subject/.../test_name -> raw score float). Store score_vector as a JSON column in SubmissionResult and expose it across API schemas and endpoints (SubmissionResponse, SubmissionDetailResponse, ExternalResultCreate). Populate score_vector in grading_service.py during _persist_success for successful pipeline runs, while remaining null for failed or interrupted runs. Add Alembic migration 003_add_score_vector_column.py, 16 unit tests in test_score_vector.py, and comprehensive documentation in docs/features/score_vector.md and docs/API.md. Design decision: FocusService tree traversal was deliberately kept separate because Focus relies on cumulative weight multiplier propagation for impact sorting, whereas score_vector is a raw score projection across all nodes. --- README.md | 1 + autograder/models/result_tree.py | 49 ++- docs/API.md | 20 +- docs/features/score_vector.md | 138 ++++++ tests/unit/models/test_score_vector.py | 399 ++++++++++++++++++ web/api/v1/submissions.py | 3 + web/database/models/submission_result.py | 3 +- .../versions/003_add_score_vector_column.py | 26 ++ web/schemas/submission.py | 2 + web/service/grading_service.py | 2 + 10 files changed, 639 insertions(+), 4 deletions(-) create mode 100644 docs/features/score_vector.md create mode 100644 tests/unit/models/test_score_vector.py create mode 100644 web/migrations/versions/003_add_score_vector_column.py diff --git a/README.md b/README.md index c5a13433..603e95ce 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ Each step is designed to maintain educational standards while providing maximum - **Actionable Feedback**: Get specific guidance on what to improve - **Iterative Learning**: Use feedback to improve and resubmit - **Transparent Grading**: See the breakdown of scores across all criteria +- **Progress Tracking**: Score vector provides a flat, queryable map of every test score for trend analysis across submissions ### For Developers diff --git a/autograder/models/result_tree.py b/autograder/models/result_tree.py index 94e48abe..4e2dac7f 100644 --- a/autograder/models/result_tree.py +++ b/autograder/models/result_tree.py @@ -21,7 +21,7 @@ """ from dataclasses import dataclass, field -from typing import List, Dict, Optional, Any +from typing import Dict, Iterator, List, Optional, Any, Tuple from autograder.models.criteria_tree import TestNode @@ -357,6 +357,53 @@ def get_passed_tests(self) -> List[TestResultNode]: """Get all test nodes with score of 100.""" return [test for test in self.get_all_test_results() if test.score >= 100] + def iter_test_results(self) -> Iterator[Tuple[str, TestResultNode]]: + """ + Yield (path, node) for every test in the tree. + + Path format: "category/subject/.../test_name" + Stable across executions of the same criteria config version. + Used by to_score_vector() and ResultComparator. + + Yields: + Tuples of (path_string, TestResultNode) for every leaf test + in the result tree, ordered by category → subject → test. + """ + + def _iter_subject( + subject: SubjectResultNode, prefix: str + ) -> Iterator[Tuple[str, TestResultNode]]: + """Recursively walk a subject node, yielding tests with paths.""" + current_prefix = f"{prefix}/{subject.name}" + for child_subject in subject.subjects: + yield from _iter_subject(child_subject, current_prefix) + for test in subject.tests: + yield (f"{current_prefix}/{test.name}", test) + + def _iter_category( + category: CategoryResultNode, + ) -> Iterator[Tuple[str, TestResultNode]]: + """Walk a category node, yielding tests with paths.""" + for subject in category.subjects: + yield from _iter_subject(subject, category.name) + for test in category.tests: + yield (f"{category.name}/{test.name}", test) + + for category in self.root.get_all_categories(): + yield from _iter_category(category) + + def to_score_vector(self) -> Dict[str, float]: + """ + Flatten the result tree into a path-keyed score map. + + Keys are stable across executions of the same criteria config version. + Suitable for storage, diffing, and longitudinal SQL queries. + + Returns: + Dict mapping path strings to raw test scores (0-100). + """ + return {path: node.score for path, node in self.iter_test_results()} + def to_dict(self) -> dict: """Convert entire result tree to dictionary.""" all_tests = self.get_all_test_results() diff --git a/docs/API.md b/docs/API.md index 328f6e50..615eda9e 100644 --- a/docs/API.md +++ b/docs/API.md @@ -412,7 +412,8 @@ Content-Type: application/json "final_score": null, "feedback": null, "result_tree": null, - "focus": null + "focus": null, + "score_vector": null } ``` @@ -453,6 +454,13 @@ GET /api/v1/submissions/{submission_id} "medium_impact": [ ... ], "low_impact": [ ... ] }, + "score_vector": { + "base/functionality/correct_output": 100.0, + "base/functionality/edge_cases": 71.0, + "base/code_quality/proper_syntax": 100.0, + "base/code_quality/good_practices": 85.0, + "bonus/extra_features": 100.0 + }, "submission_files": { "main.py": "print('Hello World')" }, @@ -513,6 +521,7 @@ GET /api/v1/submissions/{submission_id} "feedback": "## Preflight Check Failed\n\n### Setup Command Failed: Compile Calculator.java\n...", "result_tree": null, "focus": null, + "score_vector": null, "submission_files": { ... }, "submission_metadata": null, "pipeline_execution": { @@ -557,6 +566,7 @@ GET /api/v1/submissions/{submission_id} | `feedback` | string\|null | Human-readable feedback report | | `result_tree` | object\|null | Detailed grading results (null if grading didn't run) | | `focus` | object\|null | Focus analysis grouping failed tests by impact | +| `score_vector` | object\|null | Flat path-keyed score map (e.g. `{"base/subject/test": 85.0}`) for longitudinal queries. `null` for failed/interrupted executions | | `submission_files` | object | Submitted files as `{filename: content}` map | | `submission_metadata` | object\|null | Optional metadata attached at submission time | | `pipeline_execution` | object\|null | Pipeline execution details with step-by-step status | @@ -590,7 +600,11 @@ GET /api/v1/submissions/user/{external_user_id}?limit=100&offset=0 "final_score": 85.5, "feedback": "...", "result_tree": { ... }, - "focus": null + "focus": null, + "score_vector": { + "base/functionality/correct_output": 100.0, + "base/functionality/edge_cases": 71.0 + } } ] ``` @@ -619,6 +633,7 @@ Persist grading results computed outside the cloud instance (e.g. GitHub Action "feedback": "## Grade: 85.5/100\n...", "result_tree": { ... }, "focus": { ... }, + "score_vector": { "base/subject/test": 85.0 }, "pipeline_execution": { ... }, "execution_time_ms": 4521, "error_message": null, @@ -642,6 +657,7 @@ Persist grading results computed outside the cloud instance (e.g. GitHub Action | `feedback` | string | ✗ | Generated feedback text | | `result_tree` | object | ✗ | Scored result tree | | `focus` | object | ✗ | Failed tests sorted by impact | +| `score_vector` | object | ✗ | Flat path-keyed score map for longitudinal queries | | `pipeline_execution` | object | ✗ | Pipeline step execution details | | `execution_time_ms` | int | ✓ | Total execution time in milliseconds (≥ 0) | | `error_message` | string | ✗ | Error message for failed runs | diff --git a/docs/features/score_vector.md b/docs/features/score_vector.md new file mode 100644 index 00000000..550f5509 --- /dev/null +++ b/docs/features/score_vector.md @@ -0,0 +1,138 @@ +# Score Vector + +**Status:** Implemented +**Related:** [Focus Feature](focus_feature.md), [Result Tree](../architecture/result_tree.md) + +--- + +## Overview + +The **score vector** is a flat, path-keyed map of every test score produced by a grading execution. It is a denormalised projection of data already computed in the `ResultTree`, designed for longitudinal queries, progress tracking, and regression detection. + +```json +{ + "base/code_quality/complexity/cyclomatic_complexity": 72.0, + "base/code_quality/documentation/docstring_coverage": 88.0, + "base/test_coverage/test_inclusion": 60.0, + "bonus/architecture/design_pattern_usage": 95.0, + "penalty/hygiene/code_duplication": 45.0 +} +``` + +Keys are stable path strings in `category/subject/.../test_name` format. Values are raw test scores (0–100). The vector is consistent across all executions of the same criteria config version. + +--- + +## Motivation + +The `result_tree` is a deeply nested JSON structure. To extract a single test score from it, a consumer must recursively traverse the tree, find the right node, and extract the score. This is expensive and fragile, especially when repeated across dozens of submissions for trend analysis. + +The score vector flattens this into a single-level dict, enabling: + +- **Direct SQL queries** against individual test scores without application-side parsing +- **Cross-submission comparison** by diffing two score vectors +- **Regression detection** by comparing the same key across submissions +- **Group-level analytics** (averages, distributions) via standard SQL aggregations + +--- + +## How It Works + +### Tree Traversal: `ResultTree.iter_test_results()` + +The `iter_test_results()` method on `ResultTree` walks the full hierarchy: + +``` +Root → Categories (base, bonus, penalty) → Subjects (recursive) → Tests +``` + +For each test leaf node, it yields a `(path, TestResultNode)` tuple where the path encodes the full ancestry: + +```python +for path, node in result_tree.iter_test_results(): + print(f"{path}: {node.score}") +# base/code_quality/complexity/cyclomatic_complexity: 72.0 +# base/code_quality/documentation/docstring_coverage: 88.0 +# ... +``` + +### Score Vector: `ResultTree.to_score_vector()` + +A thin wrapper that calls `iter_test_results()` and returns the flat dict: + +```python +vector = result_tree.to_score_vector() +# {"base/code_quality/complexity/cyclomatic_complexity": 72.0, ...} +``` + +### Storage + +The score vector is stored as a nullable JSON column (`score_vector`) on the `submission_results` table. It is populated automatically during `_persist_success` in the grading service. Failed or interrupted pipeline executions have `score_vector = NULL`. + +### API + +The `score_vector` field is included in: +- `SubmissionResponse` (list view) +- `SubmissionDetailResponse` (detail view) +- `ExternalResultCreate` (external result ingestion) + +--- + +## Score Vector vs Focus + +Both are derived from `ResultTree`, but they serve different purposes: + +| | Focus | Score Vector | +|---|---|---| +| **Purpose** | Guide immediate feedback | Enable comparison and trend queries | +| **Includes passing tests** | No | Yes | +| **Sorted** | Yes, by impact | No | +| **Key** | None (ordered list) | Stable path string | +| **Weight-aware** | Yes | No — raw test score only | +| **Consumer** | Feedback renderer | History/trend/regression queries | + +--- + +## Version Safety + +Score vectors are only comparable across submissions graded under the **same `grading_config_id` and config `version`**. If the criteria config changes (tests added, subjects renamed), the path keys change, making cross-version vectors incompatible. + +Consumers performing longitudinal queries must filter by both `grading_config_id` and `version`. + +--- + +## SQL Query Examples + +**Track one metric over time:** +```sql +SELECT + submitted_at, + (score_vector->>'base/code_quality/complexity/cyclomatic_complexity')::float AS score +FROM submission_results +JOIN submissions ON submissions.id = submission_results.submission_id +WHERE submissions.external_user_id = 'user-123' + AND submissions.grading_config_id = 42 +ORDER BY submitted_at; +``` + +**Detect regressions between two submissions:** +```sql +SELECT key, prev.value::float AS before, curr.value::float AS after +FROM jsonb_each_text( + (SELECT score_vector FROM submission_results WHERE id = :prev_id) +) prev(key, value) +JOIN jsonb_each_text( + (SELECT score_vector FROM submission_results WHERE id = :curr_id) +) curr(key, value) USING (key) +WHERE curr.value::float < prev.value::float; +``` + +**Compute class average for a specific test:** +```sql +SELECT + AVG((score_vector->>'base/code_quality/complexity/cyclomatic_complexity')::float) +FROM submission_results sr +JOIN submissions s ON s.id = sr.submission_id +WHERE s.grading_config_id = 42 + AND s.submitted_at > NOW() - INTERVAL '30 days'; +``` diff --git a/tests/unit/models/test_score_vector.py b/tests/unit/models/test_score_vector.py new file mode 100644 index 00000000..303c8179 --- /dev/null +++ b/tests/unit/models/test_score_vector.py @@ -0,0 +1,399 @@ +""" +Unit tests for ResultTree.iter_test_results() and ResultTree.to_score_vector(). + +Tests cover: +1. Simple tree with base category only +2. Full tree with base, bonus, and penalty categories +3. Deeply nested subjects (3+ levels) +4. Tests directly under categories (flat structure, no subjects) +5. Mixed structure — subjects and direct tests on the same category +6. Empty tree with no tests +7. Path format correctness +8. Score vector output matches expected dict +9. Score vector is empty for empty trees +""" + +from unittest.mock import MagicMock + +from autograder.models.criteria_tree import TestNode +from autograder.models.result_tree import ( + CategoryResultNode, + ResultTree, + RootResultNode, + SubjectResultNode, + TestResultNode, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_test_node(name: str) -> TestNode: + """Create a minimal TestNode stub for use in TestResultNode.""" + node = MagicMock(spec=TestNode) + node.name = name + node.file_target = None + return node + + +def _make_test_result(name: str, score: float, weight: float = 100.0) -> TestResultNode: + """Create a TestResultNode with the given name and score.""" + return TestResultNode( + name=name, + test_node=_make_test_node(name), + score=score, + report=f"Report for {name}", + weight=weight, + ) + + +def _make_subject( + name: str, + weight: float = 100.0, + subjects: list = None, + tests: list = None, + subjects_weight: float = None, +) -> SubjectResultNode: + """Create a SubjectResultNode.""" + return SubjectResultNode( + name=name, + weight=weight, + subjects_weight=subjects_weight, + subjects=subjects or [], + tests=tests or [], + ) + + +def _make_category( + name: str, + weight: float = 100.0, + subjects: list = None, + tests: list = None, + subjects_weight: float = None, +) -> CategoryResultNode: + """Create a CategoryResultNode.""" + return CategoryResultNode( + name=name, + weight=weight, + subjects_weight=subjects_weight, + subjects=subjects or [], + tests=tests or [], + ) + + +def _make_tree( + base: CategoryResultNode, + bonus: CategoryResultNode = None, + penalty: CategoryResultNode = None, +) -> ResultTree: + """Create a ResultTree with the given categories.""" + root = RootResultNode(base=base, bonus=bonus, penalty=penalty) + return ResultTree(root=root) + + +# --------------------------------------------------------------------------- +# iter_test_results() tests +# --------------------------------------------------------------------------- + +class TestIterTestResults: + """Tests for ResultTree.iter_test_results().""" + + def test_simple_base_only(self): + """Base category with one subject and two tests.""" + tree = _make_tree( + base=_make_category("base", subjects=[ + _make_subject("code_quality", tests=[ + _make_test_result("cyclomatic_complexity", 72.0), + _make_test_result("line_length", 95.0), + ]), + ]), + ) + + results = list(tree.iter_test_results()) + + assert len(results) == 2 + assert results[0][0] == "base/code_quality/cyclomatic_complexity" + assert results[0][1].score == 72.0 + assert results[1][0] == "base/code_quality/line_length" + assert results[1][1].score == 95.0 + + def test_all_three_categories(self): + """Tree with base, bonus, and penalty categories.""" + tree = _make_tree( + base=_make_category("base", subjects=[ + _make_subject("tests", tests=[ + _make_test_result("test_basic", 80.0), + ]), + ]), + bonus=_make_category("bonus", weight=10, subjects=[ + _make_subject("extras", tests=[ + _make_test_result("extra_credit", 100.0), + ]), + ]), + penalty=_make_category("penalty", weight=20, subjects=[ + _make_subject("hygiene", tests=[ + _make_test_result("code_duplication", 45.0), + ]), + ]), + ) + + results = list(tree.iter_test_results()) + paths = [path for path, _ in results] + + assert len(results) == 3 + assert "base/tests/test_basic" in paths + assert "bonus/extras/extra_credit" in paths + assert "penalty/hygiene/code_duplication" in paths + + def test_deeply_nested_subjects(self): + """Three levels of subject nesting: category/s1/s2/s3/test.""" + tree = _make_tree( + base=_make_category("base", subjects=[ + _make_subject("level1", subjects=[ + _make_subject("level2", subjects=[ + _make_subject("level3", tests=[ + _make_test_result("deep_test", 55.0), + ]), + ]), + ]), + ]), + ) + + results = list(tree.iter_test_results()) + + assert len(results) == 1 + assert results[0][0] == "base/level1/level2/level3/deep_test" + assert results[0][1].score == 55.0 + + def test_tests_directly_under_category(self): + """Tests attached directly to a category (no subjects).""" + tree = _make_tree( + base=_make_category("base", tests=[ + _make_test_result("flat_test_a", 90.0), + _make_test_result("flat_test_b", 60.0), + ]), + ) + + results = list(tree.iter_test_results()) + paths = [path for path, _ in results] + + assert len(results) == 2 + assert paths == ["base/flat_test_a", "base/flat_test_b"] + + def test_mixed_subjects_and_direct_tests_on_category(self): + """Category has both subjects and direct tests — both should appear.""" + tree = _make_tree( + base=_make_category("base", + subjects=[ + _make_subject("group_a", tests=[ + _make_test_result("grouped_test", 70.0), + ]), + ], + tests=[ + _make_test_result("ungrouped_test", 85.0), + ], + ), + ) + + results = list(tree.iter_test_results()) + paths = [path for path, _ in results] + + assert len(results) == 2 + # Subjects come before direct tests (iteration order) + assert paths[0] == "base/group_a/grouped_test" + assert paths[1] == "base/ungrouped_test" + + def test_mixed_subjects_and_tests_on_subject(self): + """Subject has both nested subjects and direct tests.""" + tree = _make_tree( + base=_make_category("base", subjects=[ + _make_subject("parent", + subjects=[ + _make_subject("child", tests=[ + _make_test_result("child_test", 40.0), + ]), + ], + tests=[ + _make_test_result("parent_direct_test", 75.0), + ], + ), + ]), + ) + + results = list(tree.iter_test_results()) + paths = [path for path, _ in results] + + assert len(results) == 2 + assert paths[0] == "base/parent/child/child_test" + assert paths[1] == "base/parent/parent_direct_test" + + def test_empty_tree_no_tests(self): + """Tree with categories but no tests yields nothing.""" + tree = _make_tree( + base=_make_category("base", subjects=[ + _make_subject("empty_subject"), + ]), + ) + + results = list(tree.iter_test_results()) + assert not results + + def test_empty_base_no_subjects_no_tests(self): + """Completely empty base category.""" + tree = _make_tree(base=_make_category("base")) + + results = list(tree.iter_test_results()) + assert not results + + def test_yields_actual_test_result_nodes(self): + """Verify that yielded nodes are the original TestResultNode objects.""" + test = _make_test_result("identity_check", 88.0) + tree = _make_tree( + base=_make_category("base", subjects=[ + _make_subject("s", tests=[test]), + ]), + ) + + results = list(tree.iter_test_results()) + assert results[0][1] is test # Same object, not a copy + + def test_ordering_matches_tree_traversal(self): + """Results iterate in tree order: categories in base→bonus→penalty, + then subjects in declaration order, then tests in declaration order.""" + tree = _make_tree( + base=_make_category("base", subjects=[ + _make_subject("alpha", tests=[ + _make_test_result("a1", 10.0), + _make_test_result("a2", 20.0), + ]), + _make_subject("beta", tests=[ + _make_test_result("b1", 30.0), + ]), + ]), + bonus=_make_category("bonus", weight=10, tests=[ + _make_test_result("bonus_t", 100.0), + ]), + ) + + paths = [path for path, _ in tree.iter_test_results()] + assert paths == [ + "base/alpha/a1", + "base/alpha/a2", + "base/beta/b1", + "bonus/bonus_t", + ] + + +# --------------------------------------------------------------------------- +# to_score_vector() tests +# --------------------------------------------------------------------------- + +class TestToScoreVector: + """Tests for ResultTree.to_score_vector().""" + + def test_returns_dict_of_path_to_score(self): + """Basic score vector output.""" + tree = _make_tree( + base=_make_category("base", subjects=[ + _make_subject("code_quality", subjects=[ + _make_subject("complexity", tests=[ + _make_test_result("cyclomatic_complexity", 72.0), + ]), + _make_subject("documentation", tests=[ + _make_test_result("docstring_coverage", 88.0), + ]), + ]), + _make_subject("test_coverage", tests=[ + _make_test_result("test_inclusion", 60.0), + ]), + ]), + bonus=_make_category("bonus", weight=10, subjects=[ + _make_subject("architecture", tests=[ + _make_test_result("design_pattern_usage", 95.0), + ]), + ]), + penalty=_make_category("penalty", weight=20, subjects=[ + _make_subject("hygiene", tests=[ + _make_test_result("code_duplication", 45.0), + ]), + ]), + ) + + vector = tree.to_score_vector() + + assert vector == { + "base/code_quality/complexity/cyclomatic_complexity": 72.0, + "base/code_quality/documentation/docstring_coverage": 88.0, + "base/test_coverage/test_inclusion": 60.0, + "bonus/architecture/design_pattern_usage": 95.0, + "penalty/hygiene/code_duplication": 45.0, + } + + def test_empty_tree_returns_empty_dict(self): + """Empty tree produces empty score vector.""" + tree = _make_tree(base=_make_category("base")) + assert tree.to_score_vector() == {} + + def test_includes_passing_tests(self): + """Score vector retains tests with score 100 (unlike Focus).""" + tree = _make_tree( + base=_make_category("base", subjects=[ + _make_subject("s", tests=[ + _make_test_result("passing_test", 100.0), + _make_test_result("failing_test", 30.0), + ]), + ]), + ) + + vector = tree.to_score_vector() + + assert "base/s/passing_test" in vector + assert vector["base/s/passing_test"] == 100.0 + assert vector["base/s/failing_test"] == 30.0 + + def test_preserves_raw_scores(self): + """Scores are raw floats, not weighted or rounded.""" + tree = _make_tree( + base=_make_category("base", subjects=[ + _make_subject("s", tests=[ + _make_test_result("precise_score", 72.333), + ]), + ]), + ) + + vector = tree.to_score_vector() + assert vector["base/s/precise_score"] == 72.333 + + def test_score_vector_keys_are_strings(self): + """All keys are strings, all values are floats.""" + tree = _make_tree( + base=_make_category("base", tests=[ + _make_test_result("t1", 50.0), + _make_test_result("t2", 75.5), + ]), + ) + + vector = tree.to_score_vector() + for key, value in vector.items(): + assert isinstance(key, str), f"Key {key!r} is not a string" + assert isinstance(value, float), f"Value {value!r} is not a float" + + def test_score_vector_matches_iter_test_results(self): + """to_score_vector() is consistent with iter_test_results().""" + tree = _make_tree( + base=_make_category("base", subjects=[ + _make_subject("a", tests=[ + _make_test_result("t1", 10.0), + _make_test_result("t2", 20.0), + ]), + ]), + bonus=_make_category("bonus", weight=5, tests=[ + _make_test_result("bt", 100.0), + ]), + ) + + vector = tree.to_score_vector() + iter_dict = {path: node.score for path, node in tree.iter_test_results()} + + assert vector == iter_dict diff --git a/web/api/v1/submissions.py b/web/api/v1/submissions.py index 797494bd..4071244d 100644 --- a/web/api/v1/submissions.py +++ b/web/api/v1/submissions.py @@ -187,6 +187,7 @@ async def get_submission( "feedback": None, "result_tree": None, "focus": None, + "score_vector": None, "pipeline_execution": None, } @@ -197,6 +198,7 @@ async def get_submission( "feedback": submission.result.feedback, "result_tree": submission.result.result_tree, "focus": submission.result.focus, + "score_vector": submission.result.score_vector, "pipeline_execution": submission.result.pipeline_execution, }) @@ -316,6 +318,7 @@ async def ingest_external_result( result_tree=payload.result_tree, feedback=payload.feedback, focus=payload.focus, + score_vector=payload.score_vector, pipeline_execution=payload.pipeline_execution, execution_time_ms=payload.execution_time_ms, pipeline_status=pipeline_status, diff --git a/web/database/models/submission_result.py b/web/database/models/submission_result.py index ba7655cd..d476cd8c 100644 --- a/web/database/models/submission_result.py +++ b/web/database/models/submission_result.py @@ -1,6 +1,6 @@ """SubmissionResult database model.""" -from datetime import datetime, timezone +from datetime import datetime from enum import Enum from typing import Optional @@ -32,6 +32,7 @@ class SubmissionResult(Base): result_tree: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True) feedback: Mapped[Optional[str]] = mapped_column(Text, nullable=True) focus: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True) # Focus object with test impacts + score_vector: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True) # Flat path-keyed score map pipeline_execution: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True) # NEW: Pipeline step details execution_time_ms: Mapped[int] = mapped_column(Integer, nullable=False) pipeline_status: Mapped[PipelineStatus] = mapped_column( diff --git a/web/migrations/versions/003_add_score_vector_column.py b/web/migrations/versions/003_add_score_vector_column.py new file mode 100644 index 00000000..ac565df1 --- /dev/null +++ b/web/migrations/versions/003_add_score_vector_column.py @@ -0,0 +1,26 @@ +"""add score_vector column to submission_results + +Revision ID: 003 +Revises: 002 +Create Date: 2026-07-27 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '003' +down_revision = '002' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Add score_vector column to submission_results table.""" + op.add_column('submission_results', + sa.Column('score_vector', sa.JSON(), nullable=True)) + + +def downgrade() -> None: + """Remove score_vector column from submission_results table.""" + op.drop_column('submission_results', 'score_vector') diff --git a/web/schemas/submission.py b/web/schemas/submission.py index 5b437e11..5b2af8bd 100644 --- a/web/schemas/submission.py +++ b/web/schemas/submission.py @@ -72,6 +72,7 @@ class SubmissionResponse(BaseModel): feedback: Optional[str] = None result_tree: Optional[Dict[str, Any]] = None focus: Optional[Dict[str, Any]] = None + score_vector: Optional[Dict[str, float]] = None class SubmissionDetailResponse(SubmissionResponse): @@ -99,6 +100,7 @@ class ExternalResultCreate(BaseModel): result_tree: Optional[Dict[str, Any]] = Field(None, description="Scored result tree") focus: Optional[Dict[str, Any]] = Field(None, description="Sorted failed tests by impact") pipeline_execution: Optional[Dict[str, Any]] = Field(None, description="Pipeline step execution details") + score_vector: Optional[Dict[str, float]] = Field(None, description="Flat path-keyed score map for longitudinal queries") execution_time_ms: int = Field(..., description="Total execution time in milliseconds", ge=0) error_message: Optional[str] = Field(None, description="Error message for failed runs") submission_metadata: Optional[Dict[str, Any]] = Field(None, description="Repository/run metadata") diff --git a/web/service/grading_service.py b/web/service/grading_service.py index b72f6a50..9ec61132 100644 --- a/web/service/grading_service.py +++ b/web/service/grading_service.py @@ -126,6 +126,7 @@ async def _persist_success(result_repo, submission_repo, request: GradingRequest focus_dict = result.focus.to_dict() if result.focus else None pipeline_summary = PipelineExecutionSerializer.serialize(pipeline_execution) + score_vector = result.result_tree.to_score_vector() if result.result_tree else None await result_repo.create( submission_id=request.submission_id, @@ -133,6 +134,7 @@ async def _persist_success(result_repo, submission_repo, request: GradingRequest result_tree=result_tree_dict, feedback=result.feedback, focus=focus_dict, + score_vector=score_vector, pipeline_execution=pipeline_summary, execution_time_ms=execution_time_ms, pipeline_status=PipelineStatus.SUCCESS, From 43bab0d0b827fdd49623539b752e5c9ee7a38551 Mon Sep 17 00:00:00 2001 From: Matheus de Almeida <69125506+matheusmra@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:42:30 -0700 Subject: [PATCH 2/8] docs: fix broken result_tree link and add score_vector to mkdocs nav --- docs/features/score_vector.md | 2 +- mkdocs.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/features/score_vector.md b/docs/features/score_vector.md index 550f5509..aeb35f49 100644 --- a/docs/features/score_vector.md +++ b/docs/features/score_vector.md @@ -1,7 +1,7 @@ # Score Vector **Status:** Implemented -**Related:** [Focus Feature](focus_feature.md), [Result Tree](../architecture/result_tree.md) +**Related:** [Focus Feature](focus_feature.md), [Core Structures](../architecture/core_structures.md) --- diff --git a/mkdocs.yml b/mkdocs.yml index f5183be5..da69512a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -78,6 +78,7 @@ nav: - Features: - Grading Engine: features/grading_engine.md - Focus Feature: features/focus_feature.md + - Score Vector: features/score_vector.md - Deliberate Code Execution: features/deliberate_code_execution.md - Setup Config: features/setup_config_feature.md - Command Resolver: features/command_resolver.md From ee2b5bfee04b910eec8cda9a55e6eefa2da35de9 Mon Sep 17 00:00:00 2001 From: Matheus de Almeida <69125506+matheusmra@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:50:43 -0700 Subject: [PATCH 3/8] refactor(focus): use category.iter_test_results() for FocusService traversal --- autograder/models/result_tree.py | 74 ++++++++++++++++++++-------- autograder/services/focus_service.py | 63 +---------------------- 2 files changed, 55 insertions(+), 82 deletions(-) diff --git a/autograder/models/result_tree.py b/autograder/models/result_tree.py index 4e2dac7f..0d6a7604 100644 --- a/autograder/models/result_tree.py +++ b/autograder/models/result_tree.py @@ -143,6 +143,32 @@ def get_all_test_results(self) -> List[TestResultNode]: results.extend(subject.get_all_test_results()) return results + def iter_test_results( + self, prefix: str, parent_multiplier: float = 1.0 + ) -> Iterator[Tuple[str, TestResultNode, float]]: + """ + Recursively yield (path, node, multiplier) for tests under this subject. + """ + current_prefix = f"{prefix}/{self.name}" + current_subject_multiplier = parent_multiplier + current_test_multiplier = parent_multiplier + + if self.subjects_weight is not None: + subj_group_w = self.subjects_weight / 100.0 + test_group_w = (100.0 - self.subjects_weight) / 100.0 + current_subject_multiplier *= subj_group_w + current_test_multiplier *= test_group_w + + for child_subject in self.subjects: + child_weight_factor = child_subject.weight / 100.0 + yield from child_subject.iter_test_results( + prefix=current_prefix, + parent_multiplier=current_subject_multiplier * child_weight_factor, + ) + + for test in self.tests: + yield (f"{current_prefix}/{test.name}", test, current_test_multiplier) + @dataclass class CategoryResultNode: @@ -217,6 +243,31 @@ def get_all_test_results(self) -> List[TestResultNode]: results.extend(subject.get_all_test_results()) return results + def iter_test_results( + self, prefix: Optional[str] = None, parent_multiplier: float = 1.0 + ) -> Iterator[Tuple[str, TestResultNode, float]]: + """ + Yield (path, node, multiplier) for tests under this category. + """ + cat_prefix = prefix or self.name + initial_mult = parent_multiplier * (self.weight / 100.0) + subj_mult = initial_mult + test_mult = initial_mult + + if self.subjects_weight is not None: + subj_mult *= self.subjects_weight / 100.0 + test_mult *= (100.0 - self.subjects_weight) / 100.0 + + for subject in self.subjects: + child_weight_factor = subject.weight / 100.0 + yield from subject.iter_test_results( + prefix=cat_prefix, + parent_multiplier=subj_mult * child_weight_factor, + ) + + for test in self.tests: + yield (f"{cat_prefix}/{test.name}", test, test_mult) + @dataclass class RootResultNode: @@ -369,28 +420,9 @@ def iter_test_results(self) -> Iterator[Tuple[str, TestResultNode]]: Tuples of (path_string, TestResultNode) for every leaf test in the result tree, ordered by category → subject → test. """ - - def _iter_subject( - subject: SubjectResultNode, prefix: str - ) -> Iterator[Tuple[str, TestResultNode]]: - """Recursively walk a subject node, yielding tests with paths.""" - current_prefix = f"{prefix}/{subject.name}" - for child_subject in subject.subjects: - yield from _iter_subject(child_subject, current_prefix) - for test in subject.tests: - yield (f"{current_prefix}/{test.name}", test) - - def _iter_category( - category: CategoryResultNode, - ) -> Iterator[Tuple[str, TestResultNode]]: - """Walk a category node, yielding tests with paths.""" - for subject in category.subjects: - yield from _iter_subject(subject, category.name) - for test in category.tests: - yield (f"{category.name}/{test.name}", test) - for category in self.root.get_all_categories(): - yield from _iter_category(category) + for path, test_node, _multiplier in category.iter_test_results(): + yield (path, test_node) def to_score_vector(self) -> Dict[str, float]: """ diff --git a/autograder/services/focus_service.py b/autograder/services/focus_service.py index ea3b13bf..0ecec060 100644 --- a/autograder/services/focus_service.py +++ b/autograder/services/focus_service.py @@ -3,7 +3,6 @@ from autograder.models.result_tree import ( CategoryResultNode, ResultTree, - SubjectResultNode, TestResultNode, ) @@ -25,72 +24,14 @@ def __calculate_impact( points_missed = 100 - test.score return points_missed * (test.weight / 100) * cumulative_multiplier - def __process_subject( - self, subject: SubjectResultNode, parent_multiplier: float - ) -> List[FocusedTest]: - focused_tests = [] - - # Determine the multiplier for children of this subject - # If this subject has sub-subjects and tests, we might need to split the weight - current_subject_multiplier = parent_multiplier - current_test_multiplier = parent_multiplier - - if subject.subjects_weight is not None: - # If subjects_weight is defined, it splits the pie between - # Sub-Subjects group and Tests group. - - # The 'weight' of the sub-subjects group - subj_group_w = subject.subjects_weight / 100 - # The 'weight' of the tests group - test_group_w = (100 - subject.subjects_weight) / 100 - - current_subject_multiplier *= subj_group_w - current_test_multiplier *= test_group_w - - for child_subject in subject.subjects: - # The child subject's weight contributes to the 'Subjects Group' - child_weight_factor = child_subject.weight / 100 - focused_tests.extend( - self.__process_subject( - child_subject, current_subject_multiplier * child_weight_factor - ) - ) - - for test in subject.tests: - # The test's weight contributes to the 'Tests Group' - focused_tests.append( - FocusedTest( - test_result=test, - diff_score=self.__calculate_impact(test, current_test_multiplier), - ) - ) - - return focused_tests - def __process_category(self, category: CategoryResultNode) -> List[FocusedTest]: focused_tests: List[FocusedTest] = [] - # Initial Multiplier for a Category Root is 1.0 (100%) - # Logic follows the same split as Subject if subjects_weight exists - initial_mult = category.weight / 100.0 - subj_mult = initial_mult - test_mult = initial_mult - - if category.subjects_weight is not None: - subj_mult *= category.subjects_weight / 100.0 - test_mult *= (100.0 - category.subjects_weight) / 100.0 - - for subject in category.subjects: - child_weight_factor = subject.weight / 100 - focused_tests.extend( - self.__process_subject(subject, subj_mult * child_weight_factor) - ) - - for test in category.tests: + for _path, test, multiplier in category.iter_test_results(): focused_tests.append( FocusedTest( test_result=test, - diff_score=self.__calculate_impact(test, test_mult), + diff_score=self.__calculate_impact(test, multiplier), ) ) From a77262a37e4b75776bd790e160ee3114c582273c Mon Sep 17 00:00:00 2001 From: Matheus de Almeida <69125506+matheusmra@users.noreply.github.com> Date: Sun, 26 Jul 2026 21:05:35 -0700 Subject: [PATCH 4/8] feat: introduce post-pipeline baseline result comparison Add ResultComparator and ComparisonResult to compare a head grading execution against a baseline result_tree from a previous submission without secondary pipeline runs or code execution. Add ResultTree.from_dict() classmethods across all result tree node models to reconstruct a ResultTree from a serialized dictionary. Compute score_delta and test_deltas with status classifications (improved, regressed, unchanged, introduced, removed) for all test paths. Persist comparison JSON column in SubmissionResult table via Alembic migration 004_add_comparison_column_to_submission_results.py. Expose baseline_result_tree in SubmissionCreate input schema and surface comparison in SubmissionResponse, SubmissionDetailResponse, and ExternalResultCreate schemas and endpoints. Add 9 unit tests in test_result_comparator.py and documentation in docs/features/baseline_comparison.md, docs/API.md, and mkdocs.yml. --- .../models/dataclass/comparison_result.py | 57 +++ autograder/models/dataclass/grading_result.py | 2 + autograder/models/result_tree.py | 127 ++++++- autograder/services/result_comparator.py | 94 +++++ docs/API.md | 2 + docs/features/baseline_comparison.md | 111 ++++++ mkdocs.yml | 1 + tests/unit/services/test_result_comparator.py | 334 ++++++++++++++++++ web/api/v1/submissions.py | 4 + web/database/models/submission_result.py | 1 + ...comparison_column_to_submission_results.py | 26 ++ web/schemas/submission.py | 26 ++ web/service/grading_service.py | 20 ++ 13 files changed, 799 insertions(+), 6 deletions(-) create mode 100644 autograder/models/dataclass/comparison_result.py create mode 100644 autograder/services/result_comparator.py create mode 100644 docs/features/baseline_comparison.md create mode 100644 tests/unit/services/test_result_comparator.py create mode 100644 web/migrations/versions/004_add_comparison_column_to_submission_results.py diff --git a/autograder/models/dataclass/comparison_result.py b/autograder/models/dataclass/comparison_result.py new file mode 100644 index 00000000..b762c375 --- /dev/null +++ b/autograder/models/dataclass/comparison_result.py @@ -0,0 +1,57 @@ +from dataclasses import dataclass, field +from typing import List, Optional, Any, Dict + + +@dataclass +class TestDelta: + """ + Represents the score difference and status for a single test path + between a baseline and a head grading execution. + + Attributes: + path: Stable path string in "category/subject/.../test_name" format. + status: One of "improved", "regressed", "unchanged", "introduced", "removed". + baseline_score: Score in the baseline run, or None if introduced. + head_score: Score in the head run, or None if removed. + delta: head_score - baseline_score, or None if introduced/removed. + """ + + path: str + status: str + baseline_score: Optional[float] = None + head_score: Optional[float] = None + delta: Optional[float] = None + + def to_dict(self) -> Dict[str, Any]: + """Convert TestDelta to a serializable dictionary.""" + return { + "path": self.path, + "status": self.status, + "baseline_score": self.baseline_score, + "head_score": self.head_score, + "delta": self.delta, + } + + +@dataclass +class ComparisonResult: + """ + Structured outcome of comparing two ResultTree objects (baseline vs head). + + Attributes: + score_delta: Change in final score (head.final_score - baseline.final_score). + improved: True if score_delta > 0. + test_deltas: List of per-test comparisons across all test paths. + """ + + score_delta: float + improved: bool + test_deltas: List[TestDelta] = field(default_factory=list) + + def to_dict(self) -> Dict[str, Any]: + """Convert ComparisonResult to a serializable dictionary.""" + return { + "score_delta": self.score_delta, + "improved": self.improved, + "test_deltas": [delta.to_dict() for delta in self.test_deltas], + } diff --git a/autograder/models/dataclass/grading_result.py b/autograder/models/dataclass/grading_result.py index 00db2685..350a9542 100644 --- a/autograder/models/dataclass/grading_result.py +++ b/autograder/models/dataclass/grading_result.py @@ -2,6 +2,7 @@ from typing import Optional from autograder.models.result_tree import ResultTree from autograder.models.dataclass.focus import Focus +from autograder.models.dataclass.comparison_result import ComparisonResult @dataclass @@ -12,6 +13,7 @@ class GradingResult: feedback: Optional[str] = None result_tree: Optional['ResultTree'] = None focus: Optional['Focus'] = None # Focus object organizing tests by impact + comparison: Optional['ComparisonResult'] = None # Structured comparison against a baseline # In case of error error: Optional[str] = None diff --git a/autograder/models/result_tree.py b/autograder/models/result_tree.py index 0d6a7604..df9a15bf 100644 --- a/autograder/models/result_tree.py +++ b/autograder/models/result_tree.py @@ -45,9 +45,9 @@ class TestResultNode: """ name: str - test_node: TestNode score: float report: str + test_node: Optional[TestNode] = None weight: float = 100.0 parameters: Optional[Dict[str, Any]] = field(default_factory=dict) metadata: Dict[str, Any] = field(default_factory=dict) @@ -64,11 +64,24 @@ def to_dict(self) -> dict: "score": round(self.score, 2), "weight": self.weight, "report": self.report, - "file_target": self.test_node.file_target, + "file_target": self.test_node.file_target if self.test_node else None, "parameters": self.parameters, "metadata": self.metadata, } + @classmethod + def from_dict(cls, data: dict) -> "TestResultNode": + """Reconstruct a TestResultNode from a dictionary representation.""" + return cls( + name=data.get("name", ""), + score=float(data.get("score", 0.0)), + report=data.get("report", ""), + test_node=None, + weight=float(data.get("weight", 100.0)), + parameters=data.get("parameters") or {}, + metadata=data.get("metadata") or {}, + ) + @dataclass class SubjectResultNode: @@ -88,8 +101,8 @@ class SubjectResultNode: """ name: str - weight: float - subjects_weight: Optional[float] + weight: float = 100.0 + subjects_weight: Optional[float] = None score: float = 0.0 subjects: List["SubjectResultNode"] = field(default_factory=list) tests: List[TestResultNode] = field(default_factory=list) @@ -136,6 +149,27 @@ def to_dict(self) -> dict: "metadata": self.metadata, } + @classmethod + def from_dict(cls, data: dict) -> "SubjectResultNode": + """Reconstruct a SubjectResultNode from a dictionary representation.""" + subjects = [ + SubjectResultNode.from_dict(s) + for s in data.get("subjects") or [] + ] + tests = [ + TestResultNode.from_dict(t) + for t in data.get("tests") or [] + ] + return cls( + name=data.get("name", ""), + weight=float(data.get("weight", 100.0)), + score=float(data.get("score", 0.0)), + subjects_weight=data.get("subjects_weight"), + subjects=subjects, + tests=tests, + metadata=data.get("metadata") or {}, + ) + def get_all_test_results(self) -> List[TestResultNode]: """Recursively collect all test results under this subject.""" results = list(self.tests) @@ -236,6 +270,27 @@ def to_dict(self) -> dict: "metadata": self.metadata, } + @classmethod + def from_dict(cls, data: dict) -> "CategoryResultNode": + """Reconstruct a CategoryResultNode from a dictionary representation.""" + subjects = [ + SubjectResultNode.from_dict(s) + for s in data.get("subjects") or [] + ] + tests = [ + TestResultNode.from_dict(t) + for t in data.get("tests") or [] + ] + return cls( + name=data.get("name", ""), + weight=float(data.get("weight", 100.0)), + score=float(data.get("score", 0.0)), + subjects_weight=data.get("subjects_weight"), + subjects=subjects, + tests=tests, + metadata=data.get("metadata") or {}, + ) + def get_all_test_results(self) -> List[TestResultNode]: """Recursively collect all test results under this category.""" results = list(self.tests) @@ -343,6 +398,34 @@ def to_dict(self) -> dict: return result + @classmethod + def from_dict(cls, data: dict) -> "RootResultNode": + """Reconstruct a RootResultNode from a dictionary representation.""" + base_data = data.get("base") + if not base_data: + raise ValueError("Root node dictionary must contain a 'base' category.") + + base_cat = CategoryResultNode.from_dict(base_data) + bonus_cat = ( + CategoryResultNode.from_dict(data["bonus"]) + if data.get("bonus") + else None + ) + penalty_cat = ( + CategoryResultNode.from_dict(data["penalty"]) + if data.get("penalty") + else None + ) + + return cls( + name=data.get("name", "root"), + score=float(data.get("score", 0.0)), + base=base_cat, + bonus=bonus_cat, + penalty=penalty_cat, + metadata=data.get("metadata") or {}, + ) + def get_all_categories(self) -> List[CategoryResultNode]: """Get all category nodes.""" categories = [] @@ -445,11 +528,43 @@ def to_dict(self) -> dict: return { "template_name": self.template_name, "final_score": round(self.root.score, 2), - "tree": self.root.to_dict(), - "metadata": self.metadata, "summary": { "total_tests": len(all_tests), "passed_tests": len(passed_tests), "failed_tests": len(failed_tests), }, + "tree": self.root.to_dict(), + "root": self.root.to_dict(), + "metadata": self.metadata, } + + @classmethod + def from_dict(cls, data: dict) -> "ResultTree": + """ + Reconstruct a ResultTree from a dictionary representation. + + Supports: + - ResultTree.to_dict() format (with 'root' or 'tree' key) + - DB stored format (with 'children' key) + - Raw RootResultNode dict (with 'base' key directly) + """ + if not data: + raise ValueError("Cannot deserialize empty dictionary into ResultTree") + + root_dict = data + if "root" in data and isinstance(data["root"], dict): + root_dict = data["root"] + elif "tree" in data and isinstance(data["tree"], dict): + root_dict = data["tree"] + elif "children" in data and isinstance(data["children"], dict): + root_dict = data["children"] + + root = RootResultNode.from_dict(root_dict) + if "final_score" in data: + root.score = float(data["final_score"]) + + return cls( + root=root, + template_name=data.get("template_name"), + metadata=data.get("metadata") or {}, + ) diff --git a/autograder/services/result_comparator.py b/autograder/services/result_comparator.py new file mode 100644 index 00000000..9a71bdfe --- /dev/null +++ b/autograder/services/result_comparator.py @@ -0,0 +1,94 @@ +from typing import Dict, List +from autograder.models.result_tree import ResultTree, TestResultNode +from autograder.models.dataclass.comparison_result import ComparisonResult, TestDelta + + +class ResultComparator: + """ + Stateless utility service that compares two ResultTree objects (baseline vs head) + and computes a structured ComparisonResult containing overall score deltas + and per-test status transitions. + """ + + @staticmethod + def compare( + baseline: ResultTree, + head: ResultTree, + ) -> ComparisonResult: + """ + Compare a baseline ResultTree with a head ResultTree. + + Args: + baseline: The baseline ResultTree (reference run). + head: The head ResultTree (current run). + + Returns: + ComparisonResult containing score_delta, improved flag, and test_deltas. + """ + baseline_map: Dict[str, TestResultNode] = dict(baseline.iter_test_results()) + head_map: Dict[str, TestResultNode] = dict(head.iter_test_results()) + + # Calculate overall final score delta + score_delta = round(head.root.score - baseline.root.score, 2) + improved = score_delta > 0 + + # Preserve traversal order of head, then append removed paths from baseline + all_paths: List[str] = list(head_map.keys()) + for path in baseline_map.keys(): + if path not in head_map: + all_paths.append(path) + + test_deltas: List[TestDelta] = [] + for path in all_paths: + in_head = path in head_map + in_baseline = path in baseline_map + + if in_head and in_baseline: + base_score = baseline_map[path].score + head_score = head_map[path].score + delta = round(head_score - base_score, 2) + + if delta > 0: + status = "improved" + elif delta < 0: + status = "regressed" + else: + status = "unchanged" + + test_deltas.append( + TestDelta( + path=path, + status=status, + baseline_score=base_score, + head_score=head_score, + delta=delta, + ) + ) + elif in_head: + head_score = head_map[path].score + test_deltas.append( + TestDelta( + path=path, + status="introduced", + baseline_score=None, + head_score=head_score, + delta=None, + ) + ) + else: + base_score = baseline_map[path].score + test_deltas.append( + TestDelta( + path=path, + status="removed", + baseline_score=base_score, + head_score=None, + delta=None, + ) + ) + + return ComparisonResult( + score_delta=score_delta, + improved=improved, + test_deltas=test_deltas, + ) diff --git a/docs/API.md b/docs/API.md index 615eda9e..a714fbae 100644 --- a/docs/API.md +++ b/docs/API.md @@ -398,6 +398,7 @@ Content-Type: application/json | `files` | list[object] | ✓ | List of files with `filename` and `content` | | `language` | string | ✗ | Language override (defaults to first language in config) | | `metadata` | object | ✗ | Optional metadata to attach to the submission | +| `baseline_result_tree` | object | ✗ | Serialised `result_tree` from a previous submission response to calculate a `ComparisonResult` | **Response (200 OK):** ```json @@ -567,6 +568,7 @@ GET /api/v1/submissions/{submission_id} | `result_tree` | object\|null | Detailed grading results (null if grading didn't run) | | `focus` | object\|null | Focus analysis grouping failed tests by impact | | `score_vector` | object\|null | Flat path-keyed score map (e.g. `{"base/subject/test": 85.0}`) for longitudinal queries. `null` for failed/interrupted executions | +| `comparison` | object\|null | Baseline comparison result (`score_delta`, `improved`, `test_deltas`) if `baseline_result_tree` was provided | | `submission_files` | object | Submitted files as `{filename: content}` map | | `submission_metadata` | object\|null | Optional metadata attached at submission time | | `pipeline_execution` | object\|null | Pipeline execution details with step-by-step status | diff --git a/docs/features/baseline_comparison.md b/docs/features/baseline_comparison.md new file mode 100644 index 00000000..03d0ec23 --- /dev/null +++ b/docs/features/baseline_comparison.md @@ -0,0 +1,111 @@ +# Baseline Comparison + +**Status:** Implemented +**Related:** [Score Vector](score_vector.md), [Focus Feature](focus_feature.md), [Core Structures](../architecture/core_structures.md) + +--- + +## Overview + +**Baseline Comparison** allows the autograder to compare two grading results produced from the same criteria config version and return a structured delta (`ComparisonResult`). + +This is a post-pipeline feature executed in the web layer after `pipeline.run()`. It does not trigger a second pipeline execution or run code in a sandbox — it operates entirely on the stored `result_tree` structures. + +```json +{ + "score_delta": 15.0, + "improved": true, + "test_deltas": [ + { + "path": "base/code_quality/cyclomatic_complexity", + "status": "improved", + "baseline_score": 60.0, + "head_score": 90.0, + "delta": 30.0 + }, + { + "path": "base/code_quality/docstring_coverage", + "status": "regressed", + "baseline_score": 100.0, + "head_score": 75.0, + "delta": -25.0 + }, + { + "path": "base/new_module/unit_tests", + "status": "introduced", + "baseline_score": null, + "head_score": 100.0, + "delta": null + } + ] +} +``` + +--- + +## Motivation + +The pipeline produces an absolute score (0–100) and a result tree. For gamification, regression alerts, and progress tracking, knowing the absolute score alone is insufficient. Callers need to answer questions like: + +- *"Did this submission improve over the student's previous attempt?"* +- *"Which specific tests regressed since the last commit?"* +- *"How many points were gained or lost overall?"* + +The caller provides the `baseline_result_tree` from a previous submission, and the autograder computes the comparison alongside the new grading execution. + +--- + +## Data Structure: `ComparisonResult` & `TestDelta` + +### `TestDelta.status` Values + +| Status | Meaning | +|---|---| +| `improved` | `delta > 0` (score increased) | +| `regressed` | `delta < 0` (score decreased) | +| `unchanged` | `delta == 0` (score stayed the same) | +| `introduced` | Path present in head, missing in baseline (new test/file) | +| `removed` | Path present in baseline, missing in head (removed test/file) | + +--- + +## How It Works + +1. Caller submits code with optional `baseline_result_tree` in `SubmissionCreate`. +2. Pipeline grades current submission, producing `head_tree` (`ResultTree`). +3. If `baseline_result_tree` is provided: + - Deserialises baseline into a `ResultTree` using `ResultTree.from_dict()`. + - Calls `ResultComparator.compare(baseline=baseline_tree, head=head_tree)`. + - Attaches `ComparisonResult` to `GradingResult.comparison`. +4. Persists `comparison` JSON column in `submission_results` table. +5. Returns `comparison` object in `SubmissionResponse` / `SubmissionDetailResponse`. + +--- + +## API Usage + +### Creating a Submission with Baseline + +```http +POST /api/v1/submissions +Content-Type: application/json + +{ + "external_assignment_id": "assignment-01", + "external_user_id": "user-123", + "username": "student", + "files": [ + { "filename": "main.py", "content": "print('hello')" } + ], + "baseline_result_tree": { + "final_score": 75.0, + "children": { ... } + } +} +``` + +--- + +## Domain Agnosticism + +`ResultComparator` is domain-agnostic. It knows nothing about git, commits, or LMS platforms. It simply compares two `ResultTree` objects. diff --git a/mkdocs.yml b/mkdocs.yml index da69512a..aa1fea4b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -79,6 +79,7 @@ nav: - Grading Engine: features/grading_engine.md - Focus Feature: features/focus_feature.md - Score Vector: features/score_vector.md + - Baseline Comparison: features/baseline_comparison.md - Deliberate Code Execution: features/deliberate_code_execution.md - Setup Config: features/setup_config_feature.md - Command Resolver: features/command_resolver.md diff --git a/tests/unit/services/test_result_comparator.py b/tests/unit/services/test_result_comparator.py new file mode 100644 index 00000000..cf72e84d --- /dev/null +++ b/tests/unit/services/test_result_comparator.py @@ -0,0 +1,334 @@ +""" +Unit tests for ResultComparator and ResultTree.from_dict(). + +Tests cover: +1. Equal trees (score_delta = 0, improved = False, status = unchanged) +2. Score improvements (score_delta > 0, improved = True, status = improved) +3. Score regressions (score_delta < 0, improved = False, status = regressed) +4. Introduced tests (in head, not in baseline -> status = introduced) +5. Removed tests (in baseline, not in head -> status = removed) +6. Complex mixed trees with multiple categories and nested subjects +7. ResultTree.from_dict() deserialization and round-trip verification +8. ComparisonResult and TestDelta serialization +""" + +from autograder.models.dataclass.comparison_result import ComparisonResult, TestDelta +from autograder.models.result_tree import ( + CategoryResultNode, + ResultTree, + RootResultNode, + SubjectResultNode, + TestResultNode, +) +from autograder.services.result_comparator import ResultComparator + + +# --------------------------------------------------------------------------- +# Test Helpers +# --------------------------------------------------------------------------- + + +def _make_test(name: str, score: float, weight: float = 100.0) -> TestResultNode: + return TestResultNode( + name=name, + score=score, + report=f"Report for {name}", + test_node=None, + weight=weight, + ) + + +def _make_subject( + name: str, + tests=None, + subjects=None, + weight: float = 100.0, + score: float = 100.0, +) -> SubjectResultNode: + return SubjectResultNode( + name=name, + weight=weight, + score=score, + subjects=subjects or [], + tests=tests or [], + ) + + +def _make_category( + name: str, + subjects=None, + tests=None, + weight: float = 100.0, + score: float = 100.0, +) -> CategoryResultNode: + return CategoryResultNode( + name=name, + weight=weight, + score=score, + subjects=subjects or [], + tests=tests or [], + ) + + +def _make_tree( + base: CategoryResultNode, + bonus: CategoryResultNode = None, + penalty: CategoryResultNode = None, + final_score: float = 100.0, +) -> ResultTree: + root = RootResultNode(name="root", score=final_score, base=base, bonus=bonus, penalty=penalty) + return ResultTree(root=root) + + +# --------------------------------------------------------------------------- +# Test Cases +# --------------------------------------------------------------------------- + + +class TestResultComparator: + """Test suite for ResultComparator logic.""" + + def test_identical_trees_produce_unchanged_status(self): + """Verify identical trees return 0 delta and unchanged status.""" + t1 = _make_test("t1", 100.0) + base = _make_category("base", subjects=[_make_subject("s1", tests=[t1])]) + tree_a = _make_tree(base=base, final_score=100.0) + tree_b = _make_tree(base=base, final_score=100.0) + + result = ResultComparator.compare(baseline=tree_a, head=tree_b) + + assert result.score_delta == 0.0 + assert result.improved is False + assert len(result.test_deltas) == 1 + delta = result.test_deltas[0] + assert delta.path == "base/s1/t1" + assert delta.status == "unchanged" + assert delta.baseline_score == 100.0 + assert delta.head_score == 100.0 + assert delta.delta == 0.0 + + def test_score_improvement(self): + """Verify score improvement returns positive delta and improved status.""" + baseline_test = _make_test("t1", 60.0) + head_test = _make_test("t1", 90.0) + + baseline_tree = _make_tree( + base=_make_category("base", subjects=[_make_subject("s1", tests=[baseline_test])]), + final_score=60.0, + ) + head_tree = _make_tree( + base=_make_category("base", subjects=[_make_subject("s1", tests=[head_test])]), + final_score=90.0, + ) + + result = ResultComparator.compare(baseline=baseline_tree, head=head_tree) + + assert result.score_delta == 30.0 + assert result.improved is True + assert len(result.test_deltas) == 1 + delta = result.test_deltas[0] + assert delta.path == "base/s1/t1" + assert delta.status == "improved" + assert delta.baseline_score == 60.0 + assert delta.head_score == 90.0 + assert delta.delta == 30.0 + + def test_score_regression(self): + """Verify score regression returns negative delta and regressed status.""" + baseline_test = _make_test("t1", 100.0) + head_test = _make_test("t1", 40.0) + + baseline_tree = _make_tree( + base=_make_category("base", subjects=[_make_subject("s1", tests=[baseline_test])]), + final_score=100.0, + ) + head_tree = _make_tree( + base=_make_category("base", subjects=[_make_subject("s1", tests=[head_test])]), + final_score=40.0, + ) + + result = ResultComparator.compare(baseline=baseline_tree, head=head_tree) + + assert result.score_delta == -60.0 + assert result.improved is False + assert len(result.test_deltas) == 1 + delta = result.test_deltas[0] + assert delta.path == "base/s1/t1" + assert delta.status == "regressed" + assert delta.baseline_score == 100.0 + assert delta.head_score == 40.0 + assert delta.delta == -60.0 + + def test_introduced_test(self): + """Verify new test in head returns introduced status.""" + t1 = _make_test("t1", 80.0) + t2 = _make_test("t2", 100.0) + + baseline_tree = _make_tree( + base=_make_category("base", subjects=[_make_subject("s1", tests=[t1])]), + final_score=80.0, + ) + head_tree = _make_tree( + base=_make_category("base", subjects=[_make_subject("s1", tests=[t1, t2])]), + final_score=90.0, + ) + + result = ResultComparator.compare(baseline=baseline_tree, head=head_tree) + + assert len(result.test_deltas) == 2 + d1, d2 = result.test_deltas + assert d1.path == "base/s1/t1" + assert d1.status == "unchanged" + + assert d2.path == "base/s1/t2" + assert d2.status == "introduced" + assert d2.baseline_score is None + assert d2.head_score == 100.0 + assert d2.delta is None + + def test_removed_test(self): + """Verify removed test from baseline returns removed status.""" + t1 = _make_test("t1", 80.0) + t2 = _make_test("t2", 50.0) + + baseline_tree = _make_tree( + base=_make_category("base", subjects=[_make_subject("s1", tests=[t1, t2])]), + final_score=65.0, + ) + head_tree = _make_tree( + base=_make_category("base", subjects=[_make_subject("s1", tests=[t1])]), + final_score=80.0, + ) + + result = ResultComparator.compare(baseline=baseline_tree, head=head_tree) + + assert len(result.test_deltas) == 2 + d1, d2 = result.test_deltas + assert d1.path == "base/s1/t1" + assert d1.status == "unchanged" + + assert d2.path == "base/s1/t2" + assert d2.status == "removed" + assert d2.baseline_score == 50.0 + assert d2.head_score is None + assert d2.delta is None + + def test_complex_tree_mixed_statuses(self): + """Verify complex tree with mixed transitions.""" + # Baseline tests + b_t1 = _make_test("t1", 50.0) + b_t2 = _make_test("t2", 100.0) + b_t3 = _make_test("t3", 80.0) + + # Head tests + h_t1 = _make_test("t1", 80.0) # improved + h_t2 = _make_test("t2", 40.0) # regressed + h_t4 = _make_test("t4", 100.0) # introduced + # b_t3 is removed + + baseline_tree = _make_tree( + base=_make_category("base", subjects=[ + _make_subject("s1", tests=[b_t1, b_t2]), + _make_subject("s2", tests=[b_t3]), + ]), + final_score=76.67, + ) + head_tree = _make_tree( + base=_make_category("base", subjects=[ + _make_subject("s1", tests=[h_t1, h_t2]), + _make_subject("s2", tests=[h_t4]), + ]), + final_score=73.33, + ) + + result = ResultComparator.compare(baseline=baseline_tree, head=head_tree) + + assert result.score_delta == -3.34 + assert result.improved is False + + deltas = {d.path: d for d in result.test_deltas} + + assert deltas["base/s1/t1"].status == "improved" + assert deltas["base/s1/t1"].delta == 30.0 + + assert deltas["base/s1/t2"].status == "regressed" + assert deltas["base/s1/t2"].delta == -60.0 + + assert deltas["base/s2/t3"].status == "removed" + assert deltas["base/s2/t3"].baseline_score == 80.0 + + assert deltas["base/s2/t4"].status == "introduced" + assert deltas["base/s2/t4"].head_score == 100.0 + + +class TestResultTreeFromDict: + """Test suite for ResultTree.from_dict deserialization.""" + + def test_deserialization_round_trip(self): + """Verify round trip serialization and deserialization.""" + t1 = _make_test("t1", 85.0) + t2 = _make_test("t2", 95.0) + cat_base = _make_category("base", subjects=[_make_subject("sub1", tests=[t1, t2])]) + original_tree = _make_tree(base=cat_base, final_score=90.0) + + tree_dict = original_tree.to_dict() + reconstructed = ResultTree.from_dict(tree_dict) + + assert reconstructed.root.score == 90.0 + assert reconstructed.root.base.name == "base" + assert len(reconstructed.root.base.subjects) == 1 + + vector = reconstructed.to_score_vector() + assert vector["base/sub1/t1"] == 85.0 + assert vector["base/sub1/t2"] == 95.0 + + def test_deserialization_db_children_wrapper_format(self): + """Verify deserialization from DB children wrapper format.""" + db_dict = { + "final_score": 85.5, + "children": { + "name": "root", + "score": 85.5, + "base": { + "name": "base", + "weight": 100.0, + "score": 85.5, + "tests": [ + {"name": "check_syntax", "score": 100.0, "weight": 50.0}, + {"name": "check_logic", "score": 71.0, "weight": 50.0}, + ], + }, + }, + } + + reconstructed = ResultTree.from_dict(db_dict) + assert reconstructed.root.score == 85.5 + vector = reconstructed.to_score_vector() + assert vector["base/check_syntax"] == 100.0 + assert vector["base/check_logic"] == 71.0 + + +class TestComparisonResultSerialization: + """Test suite for ComparisonResult dataclass serialization.""" + + def test_to_dict_format(self): + """Verify to_dict output format for ComparisonResult.""" + delta = TestDelta( + path="base/s1/t1", + status="improved", + baseline_score=50.0, + head_score=80.0, + delta=30.0, + ) + res = ComparisonResult(score_delta=30.0, improved=True, test_deltas=[delta]) + + d = res.to_dict() + assert d["score_delta"] == 30.0 + assert d["improved"] is True + assert len(d["test_deltas"]) == 1 + td = d["test_deltas"][0] + assert td["path"] == "base/s1/t1" + assert td["status"] == "improved" + assert td["baseline_score"] == 50.0 + assert td["head_score"] == 80.0 + assert td["delta"] == 30.0 diff --git a/web/api/v1/submissions.py b/web/api/v1/submissions.py index 4071244d..d1cfcc64 100644 --- a/web/api/v1/submissions.py +++ b/web/api/v1/submissions.py @@ -125,6 +125,7 @@ async def create_submission( external_user_id=db_submission.external_user_id, submission_files=db_submission.submission_files, locale=submission.locale, + baseline_result_tree=submission.baseline_result_tree, ) task = asyncio.create_task(grade_submission(grading_request)) @@ -188,6 +189,7 @@ async def get_submission( "result_tree": None, "focus": None, "score_vector": None, + "comparison": None, "pipeline_execution": None, } @@ -199,6 +201,7 @@ async def get_submission( "result_tree": submission.result.result_tree, "focus": submission.result.focus, "score_vector": submission.result.score_vector, + "comparison": submission.result.comparison, "pipeline_execution": submission.result.pipeline_execution, }) @@ -319,6 +322,7 @@ async def ingest_external_result( feedback=payload.feedback, focus=payload.focus, score_vector=payload.score_vector, + comparison=payload.comparison, pipeline_execution=payload.pipeline_execution, execution_time_ms=payload.execution_time_ms, pipeline_status=pipeline_status, diff --git a/web/database/models/submission_result.py b/web/database/models/submission_result.py index d476cd8c..57d72584 100644 --- a/web/database/models/submission_result.py +++ b/web/database/models/submission_result.py @@ -33,6 +33,7 @@ class SubmissionResult(Base): feedback: Mapped[Optional[str]] = mapped_column(Text, nullable=True) focus: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True) # Focus object with test impacts score_vector: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True) # Flat path-keyed score map + comparison: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True) # Baseline comparison output pipeline_execution: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True) # NEW: Pipeline step details execution_time_ms: Mapped[int] = mapped_column(Integer, nullable=False) pipeline_status: Mapped[PipelineStatus] = mapped_column( diff --git a/web/migrations/versions/004_add_comparison_column_to_submission_results.py b/web/migrations/versions/004_add_comparison_column_to_submission_results.py new file mode 100644 index 00000000..3b6c15d5 --- /dev/null +++ b/web/migrations/versions/004_add_comparison_column_to_submission_results.py @@ -0,0 +1,26 @@ +"""add comparison column to submission_results + +Revision ID: 004 +Revises: 003 +Create Date: 2026-07-27 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '004' +down_revision = '003' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Add comparison column to submission_results table.""" + op.add_column('submission_results', + sa.Column('comparison', sa.JSON(), nullable=True)) + + +def downgrade() -> None: + """Remove comparison column from submission_results table.""" + op.drop_column('submission_results', 'comparison') diff --git a/web/schemas/submission.py b/web/schemas/submission.py index 5b2af8bd..a44a9e64 100644 --- a/web/schemas/submission.py +++ b/web/schemas/submission.py @@ -23,6 +23,22 @@ class SubmissionFileData(BaseModel): content: str = Field(..., description="Content of the file") +class TestDeltaResponse(BaseModel): + """Schema for a single test delta in a baseline comparison.""" + path: str = Field(..., description="Stable test path string (category/subject/.../test_name)") + status: str = Field(..., description="Status transition: improved, regressed, unchanged, introduced, or removed") + baseline_score: Optional[float] = Field(None, description="Score in baseline run") + head_score: Optional[float] = Field(None, description="Score in head run") + delta: Optional[float] = Field(None, description="Score change (head - baseline)") + + +class ComparisonResultResponse(BaseModel): + """Schema for baseline comparison results.""" + score_delta: float = Field(..., description="Overall final score change") + improved: bool = Field(..., description="True if score_delta > 0") + test_deltas: List[TestDeltaResponse] = Field(default_factory=list, description="Per-test deltas") + + class SubmissionCreate(BaseModel): """Schema for creating a new submission.""" external_assignment_id: str = Field(..., description="External assignment ID") @@ -32,6 +48,14 @@ class SubmissionCreate(BaseModel): language: Optional[str] = Field(None, description="Optional language override") locale: Optional[str] = Field("en", description="Optional locale for feedback (e.g., 'en', 'pt_br')") metadata: Optional[Dict[str, Any]] = Field(None, description="Optional submission metadata") + baseline_result_tree: Optional[Dict[str, Any]] = Field( + None, + description=( + "Serialised result_tree from a previous submission response. " + "When provided, the autograder computes a ComparisonResult and " + "attaches it to the grading result." + ), + ) @field_validator('language') @classmethod @@ -73,6 +97,7 @@ class SubmissionResponse(BaseModel): result_tree: Optional[Dict[str, Any]] = None focus: Optional[Dict[str, Any]] = None score_vector: Optional[Dict[str, float]] = None + comparison: Optional[ComparisonResultResponse] = None class SubmissionDetailResponse(SubmissionResponse): @@ -101,6 +126,7 @@ class ExternalResultCreate(BaseModel): focus: Optional[Dict[str, Any]] = Field(None, description="Sorted failed tests by impact") pipeline_execution: Optional[Dict[str, Any]] = Field(None, description="Pipeline step execution details") score_vector: Optional[Dict[str, float]] = Field(None, description="Flat path-keyed score map for longitudinal queries") + comparison: Optional[Dict[str, Any]] = Field(None, description="Baseline comparison output") execution_time_ms: int = Field(..., description="Total execution time in milliseconds", ge=0) error_message: Optional[str] = Field(None, description="Error message for failed runs") submission_metadata: Optional[Dict[str, Any]] = Field(None, description="Repository/run metadata") diff --git a/web/service/grading_service.py b/web/service/grading_service.py index 9ec61132..fc5178da 100644 --- a/web/service/grading_service.py +++ b/web/service/grading_service.py @@ -4,9 +4,12 @@ import time from dataclasses import dataclass from datetime import datetime, timezone +from typing import Optional from autograder.autograder import build_pipeline from autograder.models.dataclass.submission import Submission as AutograderSubmission, SubmissionFile +from autograder.models.result_tree import ResultTree +from autograder.services.result_comparator import ResultComparator from autograder.utils.feedback_generator import generate_preflight_feedback from sandbox_manager.models.sandbox_models import Language from web.config.logging import get_logger @@ -35,6 +38,7 @@ class GradingRequest: external_user_id: str submission_files: dict locale: str = "en" + baseline_result_tree: Optional[dict] = None async def grade_submission(request: GradingRequest) -> None: @@ -62,6 +66,20 @@ async def grade_submission(request: GradingRequest) -> None: execution_time_ms = int((time.time() - start_time) * 1000) if pipeline_execution.result: + if request.baseline_result_tree and pipeline_execution.result.result_tree: + try: + baseline_tree = ResultTree.from_dict(request.baseline_result_tree) + head_tree = pipeline_execution.result.result_tree + pipeline_execution.result.comparison = ResultComparator.compare( + baseline=baseline_tree, + head=head_tree, + ) + except Exception as exc: # pylint: disable=broad-exception-caught + logger.warning( + "Failed to perform baseline comparison for submission %d: %s", + request.submission_id, str(exc) + ) + await _persist_success(result_repo, submission_repo, request, pipeline_execution, execution_time_ms) else: await _persist_failure(result_repo, submission_repo, request, pipeline_execution, execution_time_ms) @@ -125,6 +143,7 @@ async def _persist_success(result_repo, submission_repo, request: GradingRequest } focus_dict = result.focus.to_dict() if result.focus else None + comparison_dict = result.comparison.to_dict() if result.comparison else None pipeline_summary = PipelineExecutionSerializer.serialize(pipeline_execution) score_vector = result.result_tree.to_score_vector() if result.result_tree else None @@ -135,6 +154,7 @@ async def _persist_success(result_repo, submission_repo, request: GradingRequest feedback=result.feedback, focus=focus_dict, score_vector=score_vector, + comparison=comparison_dict, pipeline_execution=pipeline_summary, execution_time_ms=execution_time_ms, pipeline_status=PipelineStatus.SUCCESS, From c00477b5bba99721f1aeffcb115c5e150ab9623f Mon Sep 17 00:00:00 2001 From: Matheus de Almeida <69125506+matheusmra@users.noreply.github.com> Date: Sun, 26 Jul 2026 21:34:46 -0700 Subject: [PATCH 5/8] feat: add contribution-aware evaluation context --- .../dataclass/structural_analysis_result.py | 6 +++-- autograder/models/dataclass/submission.py | 27 +++++++++++++++++-- autograder/models/pipeline_execution.py | 6 +++++ autograder/services/grader/criteria_grader.py | 13 ++++++++- autograder/services/grader/grader_service.py | 4 ++- autograder/steps/grade_step.py | 1 + autograder/steps/structural_analysis_step.py | 21 +++++++++++++-- web/api/v1/submissions.py | 12 ++++++++- web/schemas/__init__.py | 2 ++ web/schemas/submission.py | 21 +++++++++++++++ web/service/grading_service.py | 24 +++++++++++++++-- 11 files changed, 126 insertions(+), 11 deletions(-) diff --git a/autograder/models/dataclass/structural_analysis_result.py b/autograder/models/dataclass/structural_analysis_result.py index c725e488..3ed10ace 100644 --- a/autograder/models/dataclass/structural_analysis_result.py +++ b/autograder/models/dataclass/structural_analysis_result.py @@ -1,5 +1,5 @@ -from dataclasses import dataclass -from typing import Dict, Optional, TYPE_CHECKING +from dataclasses import dataclass, field +from typing import Dict, Optional, Set, TYPE_CHECKING if TYPE_CHECKING: from ast_grep_py import SgRoot @@ -12,9 +12,11 @@ class StructuralAnalysisResult: Attributes: roots: A dictionary mapping filenames to their corresponding ast-grep root nodes. If a file could not be parsed, the value is None. + changed_lines: Changed line numbers supplied for each parsed file. available: Whether structural analysis infrastructure was available and attempted. reason: Optional reason explaining why analysis was unavailable/skipped. """ roots: Dict[str, Optional['SgRoot']] available: bool = True reason: Optional[str] = None + changed_lines: Dict[str, Set[int]] = field(default_factory=dict) diff --git a/autograder/models/dataclass/submission.py b/autograder/models/dataclass/submission.py index cc328dde..3465c042 100644 --- a/autograder/models/dataclass/submission.py +++ b/autograder/models/dataclass/submission.py @@ -1,12 +1,34 @@ -from typing import Dict, Optional from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Set + from sandbox_manager.models.sandbox_models import Language + @dataclass class SubmissionFile: """Represents a single file in a submission.""" filename: str content: str + changed_lines: Optional[Set[int]] = None + metadata: Optional[Dict[str, Any]] = None + + @property + def is_contribution_aware(self) -> bool: + """Return whether changed-line information was supplied for this file.""" + return self.changed_lines is not None + + +@dataclass +class EvaluationScope: + """ + Defines which files are the primary subject of an evaluation. + + When present, pipeline analysis steps restrict their work to these files. + When absent, all submission files are treated equally. + """ + + scoped_files: List[str] + @dataclass class Submission: @@ -14,6 +36,7 @@ class Submission: username: str user_id: int assignment_id: int - submission_files: Dict[str,SubmissionFile] + submission_files: Dict[str, SubmissionFile] language: Optional[Language] = None locale: str = "en" + evaluation_scope: Optional[EvaluationScope] = None diff --git a/autograder/models/pipeline_execution.py b/autograder/models/pipeline_execution.py index 35bb2eeb..d0a5eb4c 100644 --- a/autograder/models/pipeline_execution.py +++ b/autograder/models/pipeline_execution.py @@ -11,6 +11,7 @@ if TYPE_CHECKING: from autograder.models.abstract.template import Template from autograder.models.criteria_tree import CriteriaTree + from autograder.models.dataclass.submission import EvaluationScope from autograder.models.dataclass.focus import Focus from autograder.models.dataclass.grade_step_result import GradeStepResult from autograder.models.result_tree import ResultTree @@ -50,6 +51,11 @@ def locale(self) -> str: """Returns the locale for this pipeline execution, sourced from the submission.""" return self.submission.locale + @property + def evaluation_scope(self) -> Optional["EvaluationScope"]: + """Return the optional evaluation scope supplied with the submission.""" + return self.submission.evaluation_scope + def add_step_result(self, step_result: StepResult) -> 'PipelineExecution': """ Adds a single StepResult to the list of execution results. diff --git a/autograder/services/grader/criteria_grader.py b/autograder/services/grader/criteria_grader.py index ccdb69eb..c7216fff 100644 --- a/autograder/services/grader/criteria_grader.py +++ b/autograder/services/grader/criteria_grader.py @@ -7,7 +7,7 @@ SubjectNode, TestNode, ) -from autograder.models.dataclass.submission import SubmissionFile +from autograder.models.dataclass.submission import EvaluationScope, SubmissionFile from autograder.models.dataclass.test_result import TestResult from autograder.models.result_tree import ( CategoryResultNode, @@ -32,6 +32,7 @@ def __init__( locale: str = "en", pre_computed_results: Optional[Dict[str, TestResult]] = None, structural_analysis=None, + evaluation_scope: Optional[EvaluationScope] = None, ): self.logger = logging.getLogger("SubmissionGrader") self.submission_files = submission_files @@ -41,6 +42,7 @@ def __init__( self.locale = locale self.pre_computed_results = pre_computed_results self.structural_analysis = structural_analysis + self.evaluation_scope = evaluation_scope def __balance_nodes( self, @@ -147,6 +149,13 @@ def process_test(self, test: TestNode) -> TestResultNode: if self.submission_language is not None else config_submission_language ) + test_params.pop("evaluation_scope", None) + test_params.pop("file_metadata", None) + + file_metadata = { + sub_file.filename: sub_file.metadata + for sub_file in file_target or [] + } test_result = test.test_function.execute( files=file_target, @@ -155,6 +164,8 @@ def process_test(self, test: TestNode) -> TestResultNode: pre_computed_results=self.pre_computed_results, structural_analysis=self.structural_analysis, submission_language=effective_submission_language, + evaluation_scope=self.evaluation_scope, + file_metadata=file_metadata, **test_params, ) return TestResultNode( diff --git a/autograder/services/grader/grader_service.py b/autograder/services/grader/grader_service.py index 3577ba90..7b68a576 100644 --- a/autograder/services/grader/grader_service.py +++ b/autograder/services/grader/grader_service.py @@ -2,7 +2,7 @@ from typing import Dict, Optional from autograder.models.criteria_tree import CriteriaTree -from autograder.models.dataclass.submission import SubmissionFile +from autograder.models.dataclass.submission import EvaluationScope, SubmissionFile from autograder.models.dataclass.test_result import TestResult from autograder.models.result_tree import ( ResultTree, @@ -28,6 +28,7 @@ def grade_from_tree( locale: str = "en", pre_computed_results: Optional[Dict[str, TestResult]] = None, structural_analysis=None, + evaluation_scope: Optional[EvaluationScope] = None, ) -> ResultTree: """Traverse the generic built criteria tree to resolve inputs, grades and report to ResultTree.""" grader = SubmissionGrader( @@ -38,6 +39,7 @@ def grade_from_tree( locale=locale, pre_computed_results=pre_computed_results, structural_analysis=structural_analysis, + evaluation_scope=evaluation_scope, ) base_result = grader.process_category(criteria_tree.base) diff --git a/autograder/steps/grade_step.py b/autograder/steps/grade_step.py index e6adf5d2..df9a80b2 100644 --- a/autograder/steps/grade_step.py +++ b/autograder/steps/grade_step.py @@ -67,6 +67,7 @@ def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: locale=pipeline_exec.locale, pre_computed_results=pre_computed_results, structural_analysis=structural_analysis, + evaluation_scope=pipeline_exec.evaluation_scope, ) # Create grading result diff --git a/autograder/steps/structural_analysis_step.py b/autograder/steps/structural_analysis_step.py index 0d64413e..8bc530a2 100644 --- a/autograder/steps/structural_analysis_step.py +++ b/autograder/steps/structural_analysis_step.py @@ -1,5 +1,5 @@ import logging -from typing import Dict, Optional +from typing import Dict, Optional, Set from autograder.models.abstract.step import Step from autograder.models.pipeline_execution import PipelineExecution @@ -69,7 +69,17 @@ def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: ) roots: Dict[str, Optional[SgRoot]] = {} + changed_lines: Dict[str, Set[int]] = {} + scoped_files = ( + set(pipeline_exec.evaluation_scope.scoped_files) + if pipeline_exec.evaluation_scope is not None + else None + ) + for filename, sub_file in submission.submission_files.items(): + if scoped_files is not None and filename not in scoped_files: + continue + # Only parse files that likely contain code if not self._is_code_file(filename): continue @@ -80,7 +90,14 @@ def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: logger.warning("Failed to parse %s with ast-grep: %s", filename, str(e)) roots[filename] = None - result = StructuralAnalysisResult(roots=roots, available=True) + if sub_file.changed_lines is not None: + changed_lines[filename] = set(sub_file.changed_lines) + + result = StructuralAnalysisResult( + roots=roots, + changed_lines=changed_lines, + available=True, + ) return pipeline_exec.add_step_result(StepResult.success(self.step_name, result)) def _map_language(self, language: Language) -> Optional[str]: diff --git a/web/api/v1/submissions.py b/web/api/v1/submissions.py index d1cfcc64..fb7c5290 100644 --- a/web/api/v1/submissions.py +++ b/web/api/v1/submissions.py @@ -90,7 +90,12 @@ async def create_submission( # Convert list of SubmissionFileData to dict format for storage and quick access # This indexing by filename allows O(1) file lookups during grading submission_files_dict = { - file_data.filename: {"filename": file_data.filename, "content": file_data.content} + file_data.filename: { + "filename": file_data.filename, + "content": file_data.content, + "changed_lines": file_data.changed_lines, + "file_metadata": file_data.file_metadata, + } for file_data in submission.files } @@ -126,6 +131,11 @@ async def create_submission( submission_files=db_submission.submission_files, locale=submission.locale, baseline_result_tree=submission.baseline_result_tree, + evaluation_scope=( + submission.evaluation_scope.model_dump() + if submission.evaluation_scope is not None + else None + ), ) task = asyncio.create_task(grade_submission(grading_request)) diff --git a/web/schemas/__init__.py b/web/schemas/__init__.py index 3385ee03..987fd2e5 100644 --- a/web/schemas/__init__.py +++ b/web/schemas/__init__.py @@ -11,6 +11,7 @@ SubmissionDetailResponse, SubmissionStatus, SubmissionFileData, + EvaluationScopeData, ExternalResultCreate, ExternalResultResponse, ) @@ -28,6 +29,7 @@ "SubmissionDetailResponse", "SubmissionStatus", "SubmissionFileData", + "EvaluationScopeData", "ExternalResultCreate", "ExternalResultResponse", "DeliberateCodeExecutionRequest", diff --git a/web/schemas/submission.py b/web/schemas/submission.py index a44a9e64..7c3bafe4 100644 --- a/web/schemas/submission.py +++ b/web/schemas/submission.py @@ -21,6 +21,23 @@ class SubmissionFileData(BaseModel): """Schema for a submission file.""" filename: str = Field(..., description="Name of the file") content: str = Field(..., description="Content of the file") + changed_lines: Optional[List[int]] = Field( + None, + description="One-indexed line numbers added or modified in this file", + ) + file_metadata: Optional[Dict[str, Any]] = Field( + None, + description="Optional opaque metadata for this file", + ) + + +class EvaluationScopeData(BaseModel): + """Schema defining the files that are the primary evaluation subject.""" + + scoped_files: List[str] = Field( + ..., + description="Filenames to include in scope-aware pipeline analysis", + ) class TestDeltaResponse(BaseModel): @@ -48,6 +65,10 @@ class SubmissionCreate(BaseModel): language: Optional[str] = Field(None, description="Optional language override") locale: Optional[str] = Field("en", description="Optional locale for feedback (e.g., 'en', 'pt_br')") metadata: Optional[Dict[str, Any]] = Field(None, description="Optional submission metadata") + evaluation_scope: Optional[EvaluationScopeData] = Field( + None, + description="Optional file scope for pipeline analysis", + ) baseline_result_tree: Optional[Dict[str, Any]] = Field( None, description=( diff --git a/web/service/grading_service.py b/web/service/grading_service.py index fc5178da..55913f8b 100644 --- a/web/service/grading_service.py +++ b/web/service/grading_service.py @@ -7,7 +7,11 @@ from typing import Optional from autograder.autograder import build_pipeline -from autograder.models.dataclass.submission import Submission as AutograderSubmission, SubmissionFile +from autograder.models.dataclass.submission import ( + EvaluationScope, + Submission as AutograderSubmission, + SubmissionFile, +) from autograder.models.result_tree import ResultTree from autograder.services.result_comparator import ResultComparator from autograder.utils.feedback_generator import generate_preflight_feedback @@ -39,6 +43,7 @@ class GradingRequest: submission_files: dict locale: str = "en" baseline_result_tree: Optional[dict] = None + evaluation_scope: Optional[dict] = None async def grade_submission(request: GradingRequest) -> None: @@ -116,9 +121,23 @@ async def _run_pipeline(request: GradingRequest): ) files_to_grade = { - name: SubmissionFile(filename=f["filename"], content=f["content"]) + name: SubmissionFile( + filename=f["filename"], + content=f["content"], + changed_lines=( + set(f["changed_lines"]) + if f.get("changed_lines") is not None + else None + ), + metadata=f.get("file_metadata"), + ) for name, f in request.submission_files.items() } + evaluation_scope = ( + EvaluationScope(**request.evaluation_scope) + if request.evaluation_scope is not None + else None + ) autograder_submission = AutograderSubmission( username=request.username, @@ -127,6 +146,7 @@ async def _run_pipeline(request: GradingRequest): submission_files=files_to_grade, language=Language[request.language.upper()] if request.language else None, locale=request.locale, + evaluation_scope=evaluation_scope, ) return await asyncio.to_thread(pipeline.run, autograder_submission) From 227ea5f2f756360ad5c209e1dbc2de4b8b6b1ee3 Mon Sep 17 00:00:00 2001 From: Matheus de Almeida <69125506+matheusmra@users.noreply.github.com> Date: Sun, 26 Jul 2026 21:34:53 -0700 Subject: [PATCH 6/8] test: cover contribution-aware metadata flow --- tests/unit/models/test_submission.py | 44 +++++ tests/unit/pipeline/test_ai_batch_step.py | 9 +- .../test_pipeline_execution_accessors.py | 10 +- .../pipeline/test_structural_analysis_step.py | 46 +++++- .../services/grader/test_submission_grader.py | 49 +++++- tests/unit/test_file_metadata_passthrough.py | 152 ++++++++++++++++++ tests/unit/test_language_validation.py | 26 ++- tests/web/test_grading_service.py | 48 +++++- tests/web/test_routes.py | 49 +++++- 9 files changed, 425 insertions(+), 8 deletions(-) create mode 100644 tests/unit/models/test_submission.py create mode 100644 tests/unit/test_file_metadata_passthrough.py diff --git a/tests/unit/models/test_submission.py b/tests/unit/models/test_submission.py new file mode 100644 index 00000000..d4f39bce --- /dev/null +++ b/tests/unit/models/test_submission.py @@ -0,0 +1,44 @@ +"""Tests for submission evaluation context models.""" + +from autograder.models.dataclass.submission import ( + EvaluationScope, + Submission, + SubmissionFile, +) + + +def test_submission_file_context_is_optional(): + """Snapshot submissions retain their context-free defaults.""" + sub_file = SubmissionFile(filename="main.py", content="print('hello')") + + assert sub_file.changed_lines is None + assert sub_file.metadata is None + assert sub_file.is_contribution_aware is False + + +def test_submission_file_reports_changed_line_context(): + """Supplying even an empty changed-line set makes the file context-aware.""" + sub_file = SubmissionFile( + filename="main.py", + content="print('hello')", + changed_lines=set(), + metadata={"change_status": "modified"}, + ) + + assert sub_file.is_contribution_aware is True + assert sub_file.metadata == {"change_status": "modified"} + + +def test_submission_evaluation_scope_is_optional(): + """Evaluation scope is additive and available through the submission.""" + submission = Submission( + username="student", + user_id=1, + assignment_id=2, + submission_files={}, + ) + assert submission.evaluation_scope is None + + scope = EvaluationScope(scoped_files=["main.py"]) + submission.evaluation_scope = scope + assert submission.evaluation_scope is scope diff --git a/tests/unit/pipeline/test_ai_batch_step.py b/tests/unit/pipeline/test_ai_batch_step.py index b6c70717..299ecd36 100644 --- a/tests/unit/pipeline/test_ai_batch_step.py +++ b/tests/unit/pipeline/test_ai_batch_step.py @@ -23,7 +23,11 @@ ) from autograder.models.dataclass.param_description import ParamDescription from autograder.models.dataclass.step_result import StepName, StepResult, StepStatus -from autograder.models.dataclass.submission import Submission, SubmissionFile +from autograder.models.dataclass.submission import ( + EvaluationScope, + Submission, + SubmissionFile, +) from autograder.models.dataclass.test_result import TestResult from autograder.models.pipeline_execution import PipelineExecution from autograder.models.result_tree import CategoryResultNode, ResultTree, RootResultNode @@ -408,6 +412,8 @@ def test_pre_computed_results_passed_to_grader_service(self): """GradeStep forwards AI_BATCH pre_computed_results to GraderService.grade_from_tree.""" pre_computed = {"ai_code_review": TestResult("ai_code_review", 77, "good", "")} pipeline_exec = self._make_pipeline_with_tree_and_ai_batch(pre_computed) + scope = EvaluationScope(scoped_files=["main.py"]) + pipeline_exec.submission.evaluation_scope = scope captured = {} @@ -422,6 +428,7 @@ def _fake_grade_from_tree(**kwargs): GradeStep().execute(pipeline_exec) assert captured.get("pre_computed_results") is pre_computed + assert captured.get("evaluation_scope") is scope def test_pre_computed_results_is_none_when_no_ai_batch_step(self): """When AI_BATCH step was not in the pipeline, None must be passed.""" diff --git a/tests/unit/pipeline/test_pipeline_execution_accessors.py b/tests/unit/pipeline/test_pipeline_execution_accessors.py index ea437ef8..a220103d 100644 --- a/tests/unit/pipeline/test_pipeline_execution_accessors.py +++ b/tests/unit/pipeline/test_pipeline_execution_accessors.py @@ -2,7 +2,7 @@ from autograder.models.dataclass.focus import Focus from autograder.models.dataclass.grade_step_result import GradeStepResult from autograder.models.dataclass.step_result import StepName, StepResult, StepStatus -from autograder.models.dataclass.submission import Submission, SubmissionFile +from autograder.models.dataclass.submission import EvaluationScope, Submission, SubmissionFile from autograder.models.pipeline_execution import PipelineExecution from autograder.models.result_tree import CategoryResultNode, ResultTree, RootResultNode from autograder.template_library.input_output import InputOutputTemplate @@ -50,6 +50,14 @@ def test_ai_batch_results_accessor(): assert pipeline_exec.get_ai_batch_results() == {"some": "data"} +def test_evaluation_scope_accessor(): + scope = EvaluationScope(scoped_files=["main.py"]) + pipeline_exec = _build_pipeline_execution() + pipeline_exec.submission.evaluation_scope = scope + + assert pipeline_exec.evaluation_scope is scope + + def test_typed_accessors_raise_on_missing_required_artifacts(): pipeline_exec = _build_pipeline_execution() diff --git a/tests/unit/pipeline/test_structural_analysis_step.py b/tests/unit/pipeline/test_structural_analysis_step.py index 033271ee..023a301a 100644 --- a/tests/unit/pipeline/test_structural_analysis_step.py +++ b/tests/unit/pipeline/test_structural_analysis_step.py @@ -2,7 +2,7 @@ from unittest.mock import MagicMock, patch from autograder.steps.structural_analysis_step import StructuralAnalysisStep from autograder.models.pipeline_execution import PipelineExecution -from autograder.models.dataclass.submission import Submission, SubmissionFile +from autograder.models.dataclass.submission import EvaluationScope, Submission, SubmissionFile from autograder.models.dataclass.step_result import StepName, StepStatus from sandbox_manager.models.sandbox_models import Language @@ -41,10 +41,54 @@ def test_structural_analysis_step_execution_success(mock_sg_root, mock_pipeline_ assert "main.py" in step_result.data.roots assert "data.txt" not in step_result.data.roots # Heuristic should skip .txt assert step_result.data.roots["main.py"] == mock_root_instance + assert step_result.data.changed_lines == {} # Verify SgRoot called correctly mock_sg_root.assert_called_once_with("print('hello')", "python") + +@patch("autograder.steps.structural_analysis_step.SgRoot") +def test_structural_analysis_restricts_parsing_to_evaluation_scope( + mock_sg_root, + mock_pipeline_exec, +): + mock_pipeline_exec.submission.submission_files["helper.py"] = SubmissionFile( + filename="helper.py", + content="def helper(): return True", + changed_lines={1}, + ) + mock_pipeline_exec.submission.submission_files["main.py"].changed_lines = {1, 3} + mock_pipeline_exec.submission.evaluation_scope = EvaluationScope( + scoped_files=["helper.py"], + ) + + result_exec = StructuralAnalysisStep().execute(mock_pipeline_exec) + analysis = result_exec.get_step_result(StepName.STRUCTURAL_ANALYSIS).data + + assert set(analysis.roots) == {"helper.py"} + assert analysis.changed_lines == {"helper.py": {1}} + mock_sg_root.assert_called_once_with("def helper(): return True", "python") + + +@patch("autograder.steps.structural_analysis_step.SgRoot") +def test_structural_analysis_parses_all_code_files_without_scope( + mock_sg_root, + mock_pipeline_exec, +): + mock_pipeline_exec.submission.submission_files["helper.py"] = SubmissionFile( + filename="helper.py", + content="def helper(): return True", + changed_lines={1}, + ) + mock_pipeline_exec.submission.submission_files["main.py"].changed_lines = {1} + + result_exec = StructuralAnalysisStep().execute(mock_pipeline_exec) + analysis = result_exec.get_step_result(StepName.STRUCTURAL_ANALYSIS).data + + assert set(analysis.roots) == {"main.py", "helper.py"} + assert analysis.changed_lines == {"main.py": {1}, "helper.py": {1}} + assert mock_sg_root.call_count == 2 + @patch("autograder.steps.structural_analysis_step.SgRoot") def test_structural_analysis_step_parsing_failure(mock_sg_root, mock_pipeline_exec): # Setup mock to raise error for parsing diff --git a/tests/unit/services/grader/test_submission_grader.py b/tests/unit/services/grader/test_submission_grader.py index 31812dc7..98e64eff 100644 --- a/tests/unit/services/grader/test_submission_grader.py +++ b/tests/unit/services/grader/test_submission_grader.py @@ -4,6 +4,7 @@ from autograder.services.grader.criteria_grader import SubmissionGrader from autograder.models.criteria_tree import CategoryNode, SubjectNode, TestNode from autograder.models.abstract.test_function import TestFunction +from autograder.models.dataclass.submission import EvaluationScope, SubmissionFile from autograder.models.dataclass.test_result import TestResult class MockTestFunction(TestFunction): @@ -19,6 +20,19 @@ def parameter_description(self) -> list: def execute(self, files=None, sandbox=None, **kwargs): return TestResult(test_name="mock_test", score=kwargs.get('score', 100.0), report="OK") + +class CapturingTestFunction(MockTestFunction): + """Test function that records the pipeline context passed to execute.""" + + def __init__(self): + self.files = None + self.kwargs = {} + + def execute(self, files=None, sandbox=None, **kwargs): + self.files = files + self.kwargs = kwargs + return super().execute(files=files, sandbox=sandbox, **kwargs) + @pytest.fixture def grader(): command_resolver = MagicMock() @@ -96,8 +110,6 @@ def test_balance_nodes_zero_weights(grader): assert result.subjects[1].weight == 50.0 assert result.calculate_score() == 100.0 -from autograder.models.dataclass.submission import SubmissionFile - def test_balance_nodes_subjects_and_tests_missing_subjects_weight(grader): tf = MockTestFunction() s1 = SubjectNode(name="S1", weight=100, tests=[TestNode(name="T1", test_function=tf)]) @@ -163,3 +175,36 @@ def test_get_file_target_specific(): target_files = grader.get_file_target(t1) assert len(target_files) == 1 assert target_files[0] is file1 + + +def test_process_test_passes_scope_and_target_file_metadata(): + scope = EvaluationScope(scoped_files=["file1.py"]) + file1 = SubmissionFile( + filename="file1.py", + content="", + metadata={"change_status": "modified"}, + ) + file2 = SubmissionFile( + filename="file2.py", + content="", + metadata={"change_status": "added"}, + ) + grader = SubmissionGrader( + submission_files={"file1.py": file1, "file2.py": file2}, + command_resolver=MagicMock(), + evaluation_scope=scope, + ) + test_function = CapturingTestFunction() + test_node = TestNode( + name="T1", + test_function=test_function, + file_target=["file1.py"], + ) + + grader.process_test(test_node) + + assert test_function.files == [file1] + assert test_function.kwargs["evaluation_scope"] is scope + assert test_function.kwargs["file_metadata"] == { + "file1.py": {"change_status": "modified"}, + } diff --git a/tests/unit/test_file_metadata_passthrough.py b/tests/unit/test_file_metadata_passthrough.py new file mode 100644 index 00000000..71268de2 --- /dev/null +++ b/tests/unit/test_file_metadata_passthrough.py @@ -0,0 +1,152 @@ +"""Regression coverage for rich per-file metadata passthrough.""" + +from copy import deepcopy +from unittest.mock import Mock, patch + +import pytest + +from autograder.models.abstract.test_function import TestFunction +from autograder.models.criteria_tree import CategoryNode, CriteriaTree, TestNode +from autograder.models.dataclass.test_result import TestResult +from autograder.services.grader.grader_service import GraderService +from web.service.grading_service import GradingRequest, _run_pipeline + + +class MetadataCapturingTest(TestFunction): + """Test function that records the files and context received by the grader.""" + + def __init__(self): + self.files = None + self.kwargs = None + + @property + def name(self) -> str: + return "metadata_capture" + + @property + def description(self) -> str: + return "Capture opaque per-file metadata." + + @property + def parameter_description(self) -> list: + return [] + + def execute(self, files, sandbox, *args, **kwargs) -> TestResult: + self.files = files + self.kwargs = kwargs + return TestResult( + test_name=self.name, + score=100.0, + report="Metadata received.", + ) + + +@pytest.mark.asyncio +async def test_rich_repository_metadata_survives_hydration_and_grader_passthrough(): + """Nested repository context remains opaque, unchanged, and correctly targeted.""" + repository_file_metadata = { + "provider": "github", + "change_status": "modified", + "patch": "@@ -88,6 +88,12 @@ class PaymentService:", + "blob": { + "sha": "f00ba4", + "url": "https://example.invalid/blob/f00ba4", + }, + "stats": { + "additions": 12, + "deletions": 3, + "changes": 15, + }, + "review": { + "labels": ["backend", "security"], + "requested_reviewers": ["alice", "bob"], + "approved": False, + }, + "annotations": [ + {"line": 91, "kind": "security-sensitive"}, + {"line": 95, "kind": "new-branch"}, + ], + "optional_context": None, + } + other_file_metadata = { + "provider": "github", + "change_status": "added", + } + stored_submission_files = { + "service/payment.py": { + "filename": "service/payment.py", + "content": "class PaymentService:\n pass\n", + "changed_lines": [1, 2], + "file_metadata": repository_file_metadata, + }, + "README.md": { + "filename": "README.md", + "content": "# Payment service\n", + "changed_lines": [1], + "file_metadata": other_file_metadata, + }, + } + original_stored_data = deepcopy(stored_submission_files) + request = GradingRequest( + submission_id=10, + grading_config_id=20, + template_name="static_analysis", + criteria_config={"base": {}}, + setup_config={}, + feedback_config={}, + include_feedback=False, + language="python", + username="repository-user", + external_user_id="external-user", + submission_files=stored_submission_files, + evaluation_scope={"scoped_files": ["service/payment.py"]}, + ) + pipeline = Mock() + pipeline.run.side_effect = lambda submission: submission + + with patch( + "web.service.grading_service.build_pipeline", + return_value=pipeline, + ): + hydrated_submission = await _run_pipeline(request) + + capturing_test = MetadataCapturingTest() + criteria_tree = CriteriaTree( + base=CategoryNode( + name="base", + weight=100, + tests=[ + TestNode( + name="metadata_capture", + test_function=capturing_test, + file_target=["service/payment.py"], + ) + ], + ) + ) + + GraderService().grade_from_tree( + criteria_tree=criteria_tree, + submission_files=hydrated_submission.submission_files, + evaluation_scope=hydrated_submission.evaluation_scope, + ) + + assert stored_submission_files == original_stored_data + assert hydrated_submission.evaluation_scope.scoped_files == [ + "service/payment.py" + ] + + target_file = hydrated_submission.submission_files["service/payment.py"] + assert target_file.changed_lines == {1, 2} + assert target_file.metadata == repository_file_metadata + assert target_file.metadata is repository_file_metadata + + assert capturing_test.files == [target_file] + assert capturing_test.kwargs["evaluation_scope"] is hydrated_submission.evaluation_scope + assert capturing_test.kwargs["file_metadata"] == { + "service/payment.py": repository_file_metadata, + } + assert capturing_test.kwargs["file_metadata"]["service/payment.py"] is ( + repository_file_metadata + ) + assert "README.md" not in capturing_test.kwargs["file_metadata"] diff --git a/tests/unit/test_language_validation.py b/tests/unit/test_language_validation.py index 8c20a6cc..c074e9f8 100644 --- a/tests/unit/test_language_validation.py +++ b/tests/unit/test_language_validation.py @@ -4,7 +4,11 @@ from pydantic import ValidationError from web.schemas.assignment import GradingConfigCreate, GradingConfigUpdate -from web.schemas.submission import SubmissionCreate, SubmissionFileData +from web.schemas.submission import ( + EvaluationScopeData, + SubmissionCreate, + SubmissionFileData, +) class TestLanguageValidation: @@ -140,3 +144,23 @@ def test_submission_create_case_insensitive(self): ) assert submission.language == "python" + def test_submission_create_accepts_evaluation_context(self): + """Submission schemas accept typed scope and per-file context.""" + submission = SubmissionCreate( + external_assignment_id="test-001", + external_user_id="user-001", + username="testuser", + files=[ + SubmissionFileData( + filename="test.py", + content="print('hello')", + changed_lines=[1], + file_metadata={"change_status": "modified"}, + ) + ], + evaluation_scope=EvaluationScopeData(scoped_files=["test.py"]), + ) + + assert submission.files[0].changed_lines == [1] + assert submission.files[0].file_metadata == {"change_status": "modified"} + assert submission.evaluation_scope.scoped_files == ["test.py"] diff --git a/tests/web/test_grading_service.py b/tests/web/test_grading_service.py index b9eff52f..8e06be16 100644 --- a/tests/web/test_grading_service.py +++ b/tests/web/test_grading_service.py @@ -4,7 +4,12 @@ import time from unittest.mock import Mock, AsyncMock, patch -from web.service.grading_service import grade_submission, _node_to_dict +from web.service.grading_service import ( + GradingRequest, + _node_to_dict, + _run_pipeline, + grade_submission, +) from web.database.models.submission import SubmissionStatus from web.database.models.submission_result import PipelineStatus @@ -169,3 +174,44 @@ def test_node_to_dict(): result = _node_to_dict(mock_nodes) assert result == [{"id": 1}, {"id": 2}] + +@pytest.mark.asyncio +async def test_run_pipeline_hydrates_evaluation_context(): + """Stored API data is reconstructed as typed core evaluation context.""" + mock_pipeline = Mock() + mock_pipeline.run.return_value = Mock() + request = GradingRequest( + submission_id=3, + grading_config_id=5, + template_name="static_analysis", + criteria_config={"base": {}}, + setup_config={}, + feedback_config={}, + include_feedback=False, + language="python", + username="student", + external_user_id="user-003", + submission_files={ + "main.py": { + "filename": "main.py", + "content": "print('hello')", + "changed_lines": [1, 3], + "file_metadata": {"change_status": "modified"}, + } + }, + evaluation_scope={"scoped_files": ["main.py"]}, + ) + + with patch( + "web.service.grading_service.build_pipeline", + return_value=mock_pipeline, + ): + result = await _run_pipeline(request) + + assert result is mock_pipeline.run.return_value + core_submission = mock_pipeline.run.call_args.args[0] + assert core_submission.evaluation_scope.scoped_files == ["main.py"] + assert core_submission.submission_files["main.py"].changed_lines == {1, 3} + assert core_submission.submission_files["main.py"].metadata == { + "change_status": "modified", + } diff --git a/tests/web/test_routes.py b/tests/web/test_routes.py index c72d46b9..34ea0cb9 100644 --- a/tests/web/test_routes.py +++ b/tests/web/test_routes.py @@ -295,6 +295,54 @@ async def test_create_submission(self, client): assert data["username"] == "student1" assert data["status"] == "pending" + @pytest.mark.asyncio + async def test_create_submission_persists_evaluation_context(self, client): + """Scope, changed lines, and file metadata reach the grading request.""" + config_data = { + "external_assignment_id": "submit-scope-test", + "template_name": "input_output", + "languages": ["python"], + "criteria_config": {"base": {}}, + } + await client.post("/api/v1/configs", json=config_data) + + mock_grading_tasks = set() + with patch( + "web.api.v1.submissions.grade_submission", + new_callable=AsyncMock, + ) as mock_grade, patch( + "web.api.v1.submissions.get_grading_tasks", + return_value=mock_grading_tasks, + ): + response = await client.post( + "/api/v1/submissions", + json={ + "external_assignment_id": "submit-scope-test", + "external_user_id": "user_scope", + "username": "student_scope", + "files": [ + { + "filename": "main.py", + "content": "print('hello')", + "changed_lines": [1], + "file_metadata": {"change_status": "modified"}, + } + ], + "evaluation_scope": {"scoped_files": ["main.py"]}, + }, + ) + + assert response.status_code == 200 + mock_grade.assert_called_once() + grading_request = mock_grade.call_args.args[0] + assert grading_request.submission_files["main.py"]["changed_lines"] == [1] + assert grading_request.submission_files["main.py"]["file_metadata"] == { + "change_status": "modified", + } + assert grading_request.evaluation_scope == { + "scoped_files": ["main.py"], + } + @pytest.mark.asyncio async def test_create_submission_with_language(self, client): @@ -421,4 +469,3 @@ async def test_get_user_submissions(self, client): assert isinstance(data, list) assert len(data) >= 3 assert all(s["external_user_id"] == "user_multi" for s in data) - From 34e3a77b22acf845004f502a96e4bf3a54b859c5 Mon Sep 17 00:00:00 2001 From: Matheus de Almeida <69125506+matheusmra@users.noreply.github.com> Date: Sun, 26 Jul 2026 21:35:01 -0700 Subject: [PATCH 7/8] docs: explain evaluation scope and metadata ownership --- docs/API.md | 26 +++++++++++--- docs/architecture/core_structures.md | 43 ++++++++++++++++++++--- docs/pipeline/04.8-structural-analysis.md | 18 ++++++---- 3 files changed, 73 insertions(+), 14 deletions(-) diff --git a/docs/API.md b/docs/API.md index a714fbae..c114f81d 100644 --- a/docs/API.md +++ b/docs/API.md @@ -378,12 +378,20 @@ Content-Type: application/json "files": [ { "filename": "main.py", - "content": "print('Hello World')" + "content": "print('Hello World')", + "changed_lines": [1], + "file_metadata": { + "change_status": "modified" + } } ], + "evaluation_scope": { + "scoped_files": ["main.py"] + }, "language": "python", "metadata": { - "attempt": 1 + "attempt": 1, + "source_revision": "example-revision" } } ``` @@ -395,11 +403,21 @@ Content-Type: application/json | `external_assignment_id` | string | ✓ | External assignment ID (must match an existing grading config) | | `external_user_id` | string | ✓ | External user ID from your platform | | `username` | string | ✓ | Username of the submitter | -| `files` | list[object] | ✓ | List of files with `filename` and `content` | +| `files` | list[object] | ✓ | Files with `filename`, full `content`, and optional evaluation context | +| `files[].changed_lines` | list[integer] | ✗ | One-indexed line numbers added or modified in the file | +| `files[].file_metadata` | object | ✗ | Opaque per-file context passed through to test functions | +| `evaluation_scope` | object | ✗ | Optional file scope for pipeline analysis | +| `evaluation_scope.scoped_files` | list[string] | ✓ when scope is present | Filenames that are the primary evaluation subject | | `language` | string | ✗ | Language override (defaults to first language in config) | -| `metadata` | object | ✗ | Optional metadata to attach to the submission | +| `metadata` | object | ✗ | Passive submission-level provenance stored for audit/correlation; the pipeline never reads it | | `baseline_result_tree` | object | ✗ | Serialised `result_tree` from a previous submission response to calculate a `ComparisonResult` | +`evaluation_scope` and `changed_lines` are typed pipeline inputs. +`file_metadata` is stored as `SubmissionFile.metadata` and remains opaque to +the pipeline, while top-level `metadata` is stored as `submission_metadata` +and is not passed into core grading. If `evaluation_scope` is omitted, +structural analysis continues to consider all eligible submitted files. + **Response (200 OK):** ```json { diff --git a/docs/architecture/core_structures.md b/docs/architecture/core_structures.md index 5c1c3714..771dddfb 100644 --- a/docs/architecture/core_structures.md +++ b/docs/architecture/core_structures.md @@ -371,18 +371,54 @@ class Submission: username: str # Student identifier user_id: int # Student ID assignment_id: int # Assignment identifier - submission_files: Dict[str, SubmissionFile] # Uploaded files keyed by filename + submission_files: Dict[str, SubmissionFile] # Uploaded files keyed by filename language: Optional[Language] = None # Programming language + locale: str = "en" # Feedback locale + evaluation_scope: Optional[EvaluationScope] = None ``` ### Submission File ```python class SubmissionFile: - filename: str # Name of file - content: str # File contents (text) + filename: str # Name of file + content: str # Full file contents (text) + changed_lines: Optional[Set[int]] = None # One-indexed added/modified lines + metadata: Optional[Dict[str, Any]] = None # Opaque per-file context ``` +`SubmissionFile.is_contribution_aware` is `True` when `changed_lines` was +provided, including when the caller explicitly provides an empty set. + +### Evaluation Scope + +```python +class EvaluationScope: + scoped_files: List[str] # Files that are the primary evaluation subject +``` + +When a scope is present, scope-aware pipeline steps restrict their work to the +listed filenames. Currently, structural analysis uses the scope to avoid +parsing unrelated files. Omitting the scope preserves snapshot behavior and +analyzes all eligible submission files. + +### Metadata Ownership + +Evaluation context is deliberately split by responsibility: + +| Field | Pipeline behavior | Intended use | +|-------|-------------------|--------------| +| `EvaluationScope.scoped_files` | Read by scope-aware steps | Select files that analysis should focus on | +| `SubmissionFile.changed_lines` | Read by structural analysis and available to tests | Identify added or modified lines | +| `SubmissionFile.metadata` | Never interpreted; forwarded to test functions | Carry arbitrary per-file context | +| API `metadata` / database `submission_metadata` | Never read by the core pipeline | Preserve submission-level provenance and audit context | + +Platform- or domain-specific data belongs in the open-ended metadata fields, +not in new typed core model fields. Test functions receive `evaluation_scope` +and a filename-keyed `file_metadata` mapping through `execute(**kwargs)`. They +also receive the original `SubmissionFile` objects, including `changed_lines` +and `metadata`, through the `files` argument. + ## Template Template provides test functions for a specific assignment type. @@ -412,4 +448,3 @@ class TestFunction: """Execute the test and return result""" pass ``` - diff --git a/docs/pipeline/04.8-structural-analysis.md b/docs/pipeline/04.8-structural-analysis.md index feb71c77..215f6a02 100644 --- a/docs/pipeline/04.8-structural-analysis.md +++ b/docs/pipeline/04.8-structural-analysis.md @@ -7,9 +7,10 @@ The Structural Analysis step parses student submission files into Abstract Synta ## How It Works 1. **Detect Language** — The step identifies the submission language (Python, Java, Node.js, C++, or C). -2. **Heuristic File Filtering** — It scans the submission files and identifies those likely to contain source code, skipping binary files, images, and non-code configurations (e.g., `.png`, `.json`, `.yaml`, `.md`). -3. **Parse ASTs** — For each identified code file, it uses `ast-grep-py` to parse the content into an `SgRoot` object. -4. **Store Results** — The resulting mapping of `Dict[filename, SgRoot]` is stored in `StepResult.data` under `StepName.STRUCTURAL_ANALYSIS`. +2. **Apply Evaluation Scope** — When the submission includes an `EvaluationScope`, only filenames in `scoped_files` are considered. Without a scope, all files are considered for backward compatibility. +3. **Heuristic File Filtering** — It identifies files likely to contain source code, skipping binary files, images, and non-code configurations (e.g., `.png`, `.json`, `.yaml`, `.md`). +4. **Parse ASTs** — For each identified code file, it uses `ast-grep-py` to parse the content into an `SgRoot` object. +5. **Store Results** — Parsed roots and any caller-supplied changed-line sets are stored in `StructuralAnalysisResult`. If `ast-grep-py` is not installed or the language is not supported, the step logs a warning and proceeds with an empty result set, allowing the pipeline to continue. @@ -23,24 +24,29 @@ If `ast-grep-py` is not installed or the language is not supported, the step log | Source | Data | |--------|------| -| Pipeline | `pipeline_exec.submission` → submission files and language | +| Pipeline | `pipeline_exec.submission` → submission files, language, and optional evaluation scope | ## Output | Field | Type | Description | |-------|------|-------------| -| `data` | `StructuralAnalysisResult` | Contains a mapping of filenames to `ast-grep` `SgRoot` objects | +| `data` | `StructuralAnalysisResult` | Contains filename-keyed `roots` and `changed_lines` mappings | | `status` | `StepStatus.SUCCESS` | Usually succeeds even if parsing fails for some files (stores `None` for those files) | ## How It Integrates with Grade The `GradeStep` reads the `STRUCTURAL_ANALYSIS` step result from the pipeline and passes it as `structural_analysis` to `GraderService.grade_from_tree()`. The grader threads this object through the tree traversal into every `process_test()` call, which forwards it as a kwarg to `test_function.execute()`. -Test functions like `ForbiddenKeywordTest` can then use this pre-computed AST to perform efficient and accurate queries using `ast-grep`'s pattern matching syntax. +Test functions like `ForbiddenKeywordTest` can then use this pre-computed AST +to perform efficient queries using `ast-grep`'s pattern matching syntax. +Contribution-aware tests can use `StructuralAnalysisResult.changed_lines` to +limit findings to added or modified lines. ## Key Design Decisions - **Pre-computed ASTs** — Parsing is done once at the pipeline level rather than inside individual tests to ensure efficiency when multiple structural tests are defined. +- **Optional scope** — `EvaluationScope` selects the files to parse without introducing platform-specific concepts into the core engine. An explicit empty scope parses no files. +- **Changed-line passthrough** — Changed lines are typed because pipeline and test implementations can act on them; opaque domain context remains in `SubmissionFile.metadata`. - **Fail-safe** — If parsing fails for a specific file (e.g., due to syntax errors), the step stores `None` for that file instead of failing the entire pipeline. - **Library Choice** — `ast-grep` was chosen for its performance (Rust-based) and its ability to provide high-level, language-agnostic pattern matching queries. From 034005e4293b39467cc11fb7d61238948350e3cb Mon Sep 17 00:00:00 2001 From: Matheus de Almeida <69125506+matheusmra@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:55:42 -0700 Subject: [PATCH 8/8] feat: integration tests and e2e tests --- tests/e2e/test_contribution_aware_e2e.py | 391 ++++++++++ .../test_contribution_aware_integration.py | 725 ++++++++++++++++++ 2 files changed, 1116 insertions(+) create mode 100644 tests/e2e/test_contribution_aware_e2e.py create mode 100644 tests/integration/test_contribution_aware_integration.py diff --git a/tests/e2e/test_contribution_aware_e2e.py b/tests/e2e/test_contribution_aware_e2e.py new file mode 100644 index 00000000..a64448ce --- /dev/null +++ b/tests/e2e/test_contribution_aware_e2e.py @@ -0,0 +1,391 @@ +""" +End-to-end tests for contribution-aware evaluation via the HTTP API. + +These tests exercise the full flow: + 1. Create a grading config + 2. Submit code with contribution-aware fields (evaluation_scope, changed_lines, file_metadata) + 3. Poll until completed → verify grading succeeds + +Tests cover: +- Submission with evaluation_scope is accepted and graded +- Submission without evaluation_scope is accepted and graded +- Submission with changed_lines and file_metadata is accepted +- Combined contribution-aware submission with baseline_result_tree +""" + +import json +import time +import requests +import pytest + + +def poll_submission(api_base_url, submission_id, auth_headers, timeout=60): + """Poll until a submission reaches a terminal status.""" + start_time = time.time() + while time.time() - start_time < timeout: + response = requests.get( + f"{api_base_url}/submissions/{submission_id}", + headers=auth_headers, + ) + assert response.status_code == 200 + data = response.json() + if data["status"] in ["completed", "failed"]: + return data + time.sleep(2) + + response = requests.get( + f"{api_base_url}/submissions/{submission_id}", + headers=auth_headers, + ) + print(f"DEBUG: Timeout reached. State: {json.dumps(response.json(), indent=2)}") + pytest.fail(f"Submission {submission_id} timed out") + + +@pytest.fixture +def run_id(): + return int(time.time()) + + +def _create_static_config(api_base_url, auth_headers, config_id, forbidden_imports): + """Create a static_analysis grading config.""" + config_payload = { + "external_assignment_id": config_id, + "template_name": "static_analysis", + "languages": ["python"], + "criteria_config": { + "base": { + "weight": 100.0, + "tests": [ + { + "name": f"no_{imp}", + "type": "forbidden_import", + "forbidden_imports": [imp], + "submission_language": "python", + } + for imp in forbidden_imports + ], + } + }, + } + response = requests.post( + f"{api_base_url}/configs", + json=config_payload, + headers=auth_headers, + ) + assert response.status_code in [200, 201] + return response.json() + + +def _submit( + api_base_url, + auth_headers, + config_id, + user_id, + files, + evaluation_scope=None, + baseline_result_tree=None, +): + """Submit code for grading with optional contribution-aware fields.""" + payload = { + "external_assignment_id": config_id, + "external_user_id": user_id, + "username": f"student-{user_id}", + "files": files, + } + if evaluation_scope is not None: + payload["evaluation_scope"] = evaluation_scope + if baseline_result_tree is not None: + payload["baseline_result_tree"] = baseline_result_tree + + response = requests.post( + f"{api_base_url}/submissions", + json=payload, + headers=auth_headers, + ) + assert response.status_code in [200, 201], ( + f"Submission failed: {response.status_code} — {response.text}" + ) + return response.json() + + +# ============================================================================== +# EVALUATION SCOPE E2E TESTS +# ============================================================================== + + +class TestEvaluationScopeE2E: + """E2E tests for evaluation_scope via the HTTP API.""" + + def test_submission_with_evaluation_scope_accepted( + self, api_base_url, auth_headers, run_id + ): + """Submit with evaluation_scope → 200, grading completes successfully.""" + config_id = f"scope-accept-{run_id}" + _create_static_config(api_base_url, auth_headers, config_id, ["os"]) + + files = [{"filename": "main.py", "content": "x = 42\nprint(x)"}] + sub = _submit( + api_base_url, + auth_headers, + config_id, + f"u-scope-{run_id}", + files, + evaluation_scope={"scoped_files": ["main.py"]}, + ) + result = poll_submission(api_base_url, sub["id"], auth_headers) + + assert result["status"] == "completed" + assert result["final_score"] == 100.0 + + def test_submission_without_evaluation_scope_accepted( + self, api_base_url, auth_headers, run_id + ): + """Submit without evaluation_scope → 200, grading completes successfully.""" + config_id = f"scope-none-{run_id}" + _create_static_config(api_base_url, auth_headers, config_id, ["os"]) + + files = [{"filename": "main.py", "content": "x = 42\nprint(x)"}] + sub = _submit( + api_base_url, + auth_headers, + config_id, + f"u-noscope-{run_id}", + files, + ) + result = poll_submission(api_base_url, sub["id"], auth_headers) + + assert result["status"] == "completed" + assert result["final_score"] == 100.0 + + def test_evaluation_scope_does_not_affect_scoring_for_static_analysis( + self, api_base_url, auth_headers, run_id + ): + """ + Same code with and without evaluation_scope → same score. + static_analysis tests don't filter by scope, so scores should be identical. + """ + config_id = f"scope-score-{run_id}" + _create_static_config(api_base_url, auth_headers, config_id, ["os"]) + + files = [{"filename": "main.py", "content": "x = 42\nprint(x)"}] + + # Without scope + sub1 = _submit( + api_base_url, + auth_headers, + config_id, + f"u-score1-{run_id}", + files, + ) + result1 = poll_submission(api_base_url, sub1["id"], auth_headers) + + # With scope + sub2 = _submit( + api_base_url, + auth_headers, + config_id, + f"u-score2-{run_id}", + files, + evaluation_scope={"scoped_files": ["main.py"]}, + ) + result2 = poll_submission(api_base_url, sub2["id"], auth_headers) + + assert result1["status"] == "completed" + assert result2["status"] == "completed" + assert result1["final_score"] == result2["final_score"] + + +# ============================================================================== +# CHANGED LINES E2E TESTS +# ============================================================================== + + +class TestChangedLinesE2E: + """E2E tests for changed_lines and file_metadata via the HTTP API.""" + + def test_submission_with_changed_lines_accepted( + self, api_base_url, auth_headers, run_id + ): + """Submit with changed_lines on files → 200, grading completes.""" + config_id = f"cl-accept-{run_id}" + _create_static_config(api_base_url, auth_headers, config_id, ["os"]) + + files = [ + { + "filename": "main.py", + "content": "x = 42\nprint(x)", + "changed_lines": [1, 2], + } + ] + sub = _submit( + api_base_url, + auth_headers, + config_id, + f"u-cl-{run_id}", + files, + ) + result = poll_submission(api_base_url, sub["id"], auth_headers) + + assert result["status"] == "completed" + assert result["final_score"] == 100.0 + + def test_changed_lines_and_file_metadata_preserved( + self, api_base_url, auth_headers, run_id + ): + """Submit with changed_lines and file_metadata → grading succeeds without errors.""" + config_id = f"cl-meta-{run_id}" + _create_static_config(api_base_url, auth_headers, config_id, ["os"]) + + files = [ + { + "filename": "main.py", + "content": "x = 42\nprint(x)", + "changed_lines": [1], + "file_metadata": { + "change_status": "modified", + "provider": "github", + "stats": {"additions": 1, "deletions": 0}, + }, + } + ] + sub = _submit( + api_base_url, + auth_headers, + config_id, + f"u-clm-{run_id}", + files, + ) + result = poll_submission(api_base_url, sub["id"], auth_headers) + + assert result["status"] == "completed" + assert result["final_score"] == 100.0 + + def test_submission_without_changed_lines_accepted( + self, api_base_url, auth_headers, run_id + ): + """Submit without changed_lines → 200, grading completes (backward compatible).""" + config_id = f"cl-none-{run_id}" + _create_static_config(api_base_url, auth_headers, config_id, ["os"]) + + files = [{"filename": "main.py", "content": "x = 42"}] + sub = _submit( + api_base_url, + auth_headers, + config_id, + f"u-clnone-{run_id}", + files, + ) + result = poll_submission(api_base_url, sub["id"], auth_headers) + + assert result["status"] == "completed" + + +# ============================================================================== +# COMBINED CONTRIBUTION-AWARE E2E TESTS +# ============================================================================== + + +class TestCombinedContributionAwareE2E: + """E2E tests combining all contribution-aware features.""" + + def test_full_contribution_aware_submission( + self, api_base_url, auth_headers, run_id + ): + """ + Submit with evaluation_scope + changed_lines + file_metadata + baseline_result_tree + → everything works together, comparison present in response. + """ + config_id = f"combined-{run_id}" + _create_static_config(api_base_url, auth_headers, config_id, ["os"]) + + # First submission (baseline) — violation code + baseline_files = [ + { + "filename": "main.py", + "content": "import os\nprint(os.getcwd())", + "changed_lines": [1, 2], + "file_metadata": {"change_status": "added"}, + } + ] + sub1 = _submit( + api_base_url, + auth_headers, + config_id, + f"u-comb-base-{run_id}", + baseline_files, + evaluation_scope={"scoped_files": ["main.py"]}, + ) + result1 = poll_submission(api_base_url, sub1["id"], auth_headers) + assert result1["status"] == "completed" + assert result1["final_score"] < 100.0 + assert result1["comparison"] is None # No baseline → no comparison + baseline_tree = result1["result_tree"] + + # Second submission (head) — clean code with baseline + head_files = [ + { + "filename": "main.py", + "content": "x = 42\nprint(x)", + "changed_lines": [1, 2], + "file_metadata": {"change_status": "modified"}, + } + ] + sub2 = _submit( + api_base_url, + auth_headers, + config_id, + f"u-comb-head-{run_id}", + head_files, + evaluation_scope={"scoped_files": ["main.py"]}, + baseline_result_tree=baseline_tree, + ) + result2 = poll_submission(api_base_url, sub2["id"], auth_headers) + + assert result2["status"] == "completed" + assert result2["final_score"] == 100.0 + + # Comparison should show improvement + comparison = result2["comparison"] + assert comparison is not None + assert comparison["score_delta"] > 0 + assert comparison["improved"] is True + + # score_vector should also be present + assert result2["score_vector"] is not None + + def test_multi_file_with_scope_and_metadata( + self, api_base_url, auth_headers, run_id + ): + """ + Multi-file submission with evaluation_scope scoping to a subset. + Verify grading completes successfully. + """ + config_id = f"multi-scope-{run_id}" + _create_static_config(api_base_url, auth_headers, config_id, ["os"]) + + files = [ + { + "filename": "main.py", + "content": "x = 42", + "changed_lines": [1], + "file_metadata": {"change_status": "modified"}, + }, + { + "filename": "utils.py", + "content": "def helper(): return 1", + "changed_lines": None, + "file_metadata": {"change_status": "unchanged"}, + }, + ] + sub = _submit( + api_base_url, + auth_headers, + config_id, + f"u-multi-{run_id}", + files, + evaluation_scope={"scoped_files": ["main.py"]}, + ) + result = poll_submission(api_base_url, sub["id"], auth_headers) + + assert result["status"] == "completed" + assert result["final_score"] == 100.0 diff --git a/tests/integration/test_contribution_aware_integration.py b/tests/integration/test_contribution_aware_integration.py new file mode 100644 index 00000000..6c12707d --- /dev/null +++ b/tests/integration/test_contribution_aware_integration.py @@ -0,0 +1,725 @@ +""" +Integration tests for contribution-aware evaluation features. + +Tests cover: +1. EvaluationScope propagation through the pipeline +2. changed_lines propagation through the pipeline +3. file_metadata passthrough to test functions +4. GradingService hydration of contribution-aware fields from HTTP payload format + +These tests use the static_analysis template (no sandbox) so they can run +without Docker infrastructure. +""" + +import time + +import pytest +from unittest.mock import AsyncMock, Mock, patch + +from autograder.autograder import build_pipeline +from autograder.models.abstract.test_function import TestFunction +from autograder.models.criteria_tree import CategoryNode, CriteriaTree, TestNode +from autograder.models.dataclass.submission import ( + EvaluationScope, + Submission, + SubmissionFile, +) +from autograder.models.dataclass.test_result import TestResult +from autograder.services.grader.grader_service import GraderService +from web.service.grading_service import GradingRequest, grade_submission + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_submission( + files: dict[str, SubmissionFile], + evaluation_scope: EvaluationScope | None = None, +) -> Submission: + """Build a Submission with optional contribution-aware fields.""" + return Submission( + username="integration-student", + user_id="integration-user", + assignment_id=1, + submission_files=files, + language=None, # static_analysis doesn't require language + locale="en", + evaluation_scope=evaluation_scope, + ) + + +def _static_analysis_criteria(forbidden_imports: list) -> dict: + """Build a criteria config for static_analysis with forbidden imports.""" + return { + "base": { + "weight": 100.0, + "tests": [ + { + "name": f"no_{imp}", + "type": "forbidden_import", + "forbidden_imports": [imp], + "submission_language": "python", + } + for imp in forbidden_imports + ], + } + } + + +class ContextCapturingTest(TestFunction): + """Test function that captures files, evaluation_scope, and file_metadata.""" + + def __init__(self): + self.files = None + self.kwargs = None + + @property + def name(self) -> str: + return "context_capture" + + @property + def description(self) -> str: + return "Capture contribution-aware context." + + @property + def parameter_description(self) -> list: + return [] + + def execute(self, files, sandbox, *args, **kwargs) -> TestResult: + self.files = files + self.kwargs = kwargs + return TestResult( + test_name=self.name, + score=100.0, + report="Context captured.", + ) + + +# --------------------------------------------------------------------------- +# Integration: EvaluationScope through the pipeline +# --------------------------------------------------------------------------- + + +class TestEvaluationScopeIntegration: + """Verify evaluation_scope restricts structural analysis and propagates to test functions.""" + + def test_structural_analysis_respects_evaluation_scope(self): + """Pipeline with 2 files but scope limited to 1 → structural analysis parses only scoped file.""" + from sandbox_manager.models.sandbox_models import Language + + criteria = _static_analysis_criteria(["os"]) + + files = { + "main.py": SubmissionFile(filename="main.py", content="import os\nx = 1"), + "helper.py": SubmissionFile(filename="helper.py", content="import os\ny = 2"), + } + scope = EvaluationScope(scoped_files=["main.py"]) + + submission = Submission( + username="scope-student", + user_id="scope-user", + assignment_id=1, + submission_files=files, + language=Language.PYTHON, + locale="en", + evaluation_scope=scope, + ) + + pipeline = build_pipeline( + template_name="static_analysis", + include_feedback=False, + grading_criteria=criteria, + feedback_config={}, + ) + execution = pipeline.run(submission) + + # Pipeline should succeed + assert execution.result is not None + + # Structural analysis should have only parsed main.py (the scoped file) + sa_result = execution.get_structural_analysis_result() + assert sa_result is not None + assert sa_result.available is True + assert "main.py" in sa_result.roots + assert "helper.py" not in sa_result.roots + + def test_grading_with_evaluation_scope_passes_to_test_functions(self): + """Verify evaluation_scope is propagated as kwarg to test functions via GraderService.""" + capturing_test = ContextCapturingTest() + criteria_tree = CriteriaTree( + base=CategoryNode( + name="base", + weight=100, + tests=[ + TestNode( + name="context_capture", + test_function=capturing_test, + file_target=["main.py"], + ) + ], + ) + ) + + files = { + "main.py": SubmissionFile(filename="main.py", content="x = 1"), + } + scope = EvaluationScope(scoped_files=["main.py"]) + + GraderService().grade_from_tree( + criteria_tree=criteria_tree, + submission_files=files, + evaluation_scope=scope, + ) + + assert capturing_test.kwargs is not None + assert capturing_test.kwargs["evaluation_scope"] is scope + + def test_grading_without_evaluation_scope_processes_all_files(self): + """Without scope, all submission files are processed normally.""" + from sandbox_manager.models.sandbox_models import Language + + criteria = _static_analysis_criteria(["os"]) + files = { + "main.py": SubmissionFile(filename="main.py", content="x = 1"), + "helper.py": SubmissionFile(filename="helper.py", content="y = 2"), + } + + submission = Submission( + username="noscope-student", + user_id="noscope-user", + assignment_id=1, + submission_files=files, + language=Language.PYTHON, + locale="en", + evaluation_scope=None, + ) + + pipeline = build_pipeline( + template_name="static_analysis", + include_feedback=False, + grading_criteria=criteria, + feedback_config={}, + ) + execution = pipeline.run(submission) + + assert execution.result is not None + + # Without scope, structural analysis should parse both files + sa_result = execution.get_structural_analysis_result() + assert sa_result is not None + assert "main.py" in sa_result.roots + assert "helper.py" in sa_result.roots + + def test_evaluation_scope_none_passed_to_test_functions(self): + """When evaluation_scope is None, test functions receive None.""" + capturing_test = ContextCapturingTest() + criteria_tree = CriteriaTree( + base=CategoryNode( + name="base", + weight=100, + tests=[ + TestNode( + name="context_capture", + test_function=capturing_test, + ) + ], + ) + ) + + files = {"main.py": SubmissionFile(filename="main.py", content="x = 1")} + + GraderService().grade_from_tree( + criteria_tree=criteria_tree, + submission_files=files, + evaluation_scope=None, + ) + + assert capturing_test.kwargs["evaluation_scope"] is None + + +# --------------------------------------------------------------------------- +# Integration: changed_lines through the pipeline +# --------------------------------------------------------------------------- + + +class TestChangedLinesIntegration: + """Verify changed_lines propagation through the pipeline.""" + + def test_changed_lines_propagated_through_pipeline(self): + """SubmissionFile with changed_lines → structural analysis captures them.""" + from sandbox_manager.models.sandbox_models import Language + + criteria = _static_analysis_criteria(["os"]) + files = { + "main.py": SubmissionFile( + filename="main.py", + content="import os\nx = 1", + changed_lines={1, 2}, + ), + } + + submission = Submission( + username="cl-student", + user_id="cl-user", + assignment_id=1, + submission_files=files, + language=Language.PYTHON, + locale="en", + ) + + pipeline = build_pipeline( + template_name="static_analysis", + include_feedback=False, + grading_criteria=criteria, + feedback_config={}, + ) + execution = pipeline.run(submission) + + assert execution.result is not None + + # Structural analysis should carry the changed_lines + sa_result = execution.get_structural_analysis_result() + assert sa_result is not None + assert "main.py" in sa_result.changed_lines + assert sa_result.changed_lines["main.py"] == {1, 2} + + def test_no_changed_lines_still_works(self): + """Submission without changed_lines → pipeline works normally, is_contribution_aware=False.""" + from sandbox_manager.models.sandbox_models import Language + + criteria = _static_analysis_criteria(["os"]) + file_obj = SubmissionFile(filename="main.py", content="x = 1") + assert file_obj.is_contribution_aware is False + + files = {"main.py": file_obj} + + submission = Submission( + username="no-cl-student", + user_id="no-cl-user", + assignment_id=1, + submission_files=files, + language=Language.PYTHON, + locale="en", + ) + + pipeline = build_pipeline( + template_name="static_analysis", + include_feedback=False, + grading_criteria=criteria, + feedback_config={}, + ) + execution = pipeline.run(submission) + + assert execution.result is not None + assert execution.result.final_score == 100.0 + + # Structural analysis should have empty changed_lines + sa_result = execution.get_structural_analysis_result() + assert sa_result.changed_lines == {} + + def test_changed_lines_set_means_contribution_aware(self): + """SubmissionFile with changed_lines set → is_contribution_aware is True.""" + file_obj = SubmissionFile( + filename="main.py", + content="x = 1", + changed_lines={1}, + ) + assert file_obj.is_contribution_aware is True + + +# --------------------------------------------------------------------------- +# Integration: file_metadata passthrough +# --------------------------------------------------------------------------- + + +class TestFileMetadataIntegration: + """Verify file_metadata is available in test functions via file_metadata kwarg.""" + + def test_file_metadata_available_in_test_function(self): + """Test function receives file_metadata dict keyed by filename.""" + capturing_test = ContextCapturingTest() + criteria_tree = CriteriaTree( + base=CategoryNode( + name="base", + weight=100, + tests=[ + TestNode( + name="context_capture", + test_function=capturing_test, + file_target=["main.py"], + ) + ], + ) + ) + + metadata = {"change_status": "modified", "provider": "github"} + files = { + "main.py": SubmissionFile( + filename="main.py", + content="x = 1", + metadata=metadata, + ), + } + + GraderService().grade_from_tree( + criteria_tree=criteria_tree, + submission_files=files, + ) + + assert capturing_test.kwargs is not None + assert capturing_test.kwargs["file_metadata"] == {"main.py": metadata} + assert capturing_test.kwargs["file_metadata"]["main.py"] is metadata + + def test_file_metadata_none_when_not_provided(self): + """Test function receives file_metadata with None values when metadata is absent.""" + capturing_test = ContextCapturingTest() + criteria_tree = CriteriaTree( + base=CategoryNode( + name="base", + weight=100, + tests=[ + TestNode( + name="context_capture", + test_function=capturing_test, + file_target=["main.py"], + ) + ], + ) + ) + + files = { + "main.py": SubmissionFile(filename="main.py", content="x = 1"), + } + + GraderService().grade_from_tree( + criteria_tree=criteria_tree, + submission_files=files, + ) + + assert capturing_test.kwargs["file_metadata"] == {"main.py": None} + + def test_file_metadata_multi_file_only_targeted_files(self): + """file_metadata contains only the files targeted by file_target.""" + capturing_test = ContextCapturingTest() + criteria_tree = CriteriaTree( + base=CategoryNode( + name="base", + weight=100, + tests=[ + TestNode( + name="context_capture", + test_function=capturing_test, + file_target=["main.py"], + ) + ], + ) + ) + + files = { + "main.py": SubmissionFile( + filename="main.py", + content="x = 1", + metadata={"change_status": "modified"}, + ), + "helper.py": SubmissionFile( + filename="helper.py", + content="y = 2", + metadata={"change_status": "added"}, + ), + } + + GraderService().grade_from_tree( + criteria_tree=criteria_tree, + submission_files=files, + ) + + # Only main.py should be in file_metadata (file_target = ["main.py"]) + assert "main.py" in capturing_test.kwargs["file_metadata"] + assert "helper.py" not in capturing_test.kwargs["file_metadata"] + + +# --------------------------------------------------------------------------- +# Integration: GradingService hydration of contribution-aware fields +# --------------------------------------------------------------------------- + + +class TestGradingServiceContributionAware: + """Verify the grading service correctly hydrates contribution-aware fields.""" + + @pytest.mark.asyncio + async def test_grading_service_hydrates_evaluation_scope(self): + """GradingRequest with evaluation_scope dict → AutograderSubmission has EvaluationScope object.""" + mock_result = Mock() + mock_result.final_score = 100.0 + mock_result.feedback = None + mock_result.result_tree = None + mock_result.focus = Mock() + mock_result.focus.to_dict = Mock(return_value={"base": []}) + mock_result.comparison = None + + mock_execution = Mock() + mock_execution.result = mock_result + mock_execution.start_time = time.time() + mock_execution.step_results = [] + + mock_submission_repo = Mock() + mock_submission_repo.update_status = AsyncMock() + mock_submission_repo.update = AsyncMock() + + mock_result_repo = Mock() + mock_result_repo.create = AsyncMock() + + mock_session = AsyncMock() + mock_session.commit = AsyncMock() + + captured_submission = {} + + def capture_pipeline_run(submission): + captured_submission["obj"] = submission + return mock_execution + + with patch("web.service.grading_service.build_pipeline") as mock_build, \ + patch("web.service.grading_service.get_session") as mock_get_session, \ + patch("asyncio.to_thread", side_effect=lambda fn, sub: capture_pipeline_run(sub)), \ + patch("web.service.grading_service.SubmissionRepository", return_value=mock_submission_repo), \ + patch("web.service.grading_service.ResultRepository", return_value=mock_result_repo), \ + patch("web.service.grading_service.PipelineExecutionSerializer") as mock_serializer: + + mock_get_session.return_value.__aenter__.return_value = mock_session + mock_serializer.serialize.return_value = {"status": "success"} + + request = GradingRequest( + submission_id=1, + grading_config_id=1, + template_name="static_analysis", + criteria_config={}, + setup_config={}, + feedback_config={}, + include_feedback=False, + language="python", + username="student", + external_user_id="u1", + submission_files={ + "main.py": { + "filename": "main.py", + "content": "x = 1", + "changed_lines": None, + "file_metadata": None, + } + }, + evaluation_scope={"scoped_files": ["main.py"]}, + ) + await grade_submission(request) + + sub = captured_submission["obj"] + assert sub.evaluation_scope is not None + assert sub.evaluation_scope.scoped_files == ["main.py"] + + @pytest.mark.asyncio + async def test_grading_service_hydrates_changed_lines(self): + """GradingRequest with changed_lines in files → SubmissionFile has set(changed_lines).""" + mock_result = Mock() + mock_result.final_score = 100.0 + mock_result.feedback = None + mock_result.result_tree = None + mock_result.focus = Mock() + mock_result.focus.to_dict = Mock(return_value={"base": []}) + mock_result.comparison = None + + mock_execution = Mock() + mock_execution.result = mock_result + mock_execution.start_time = time.time() + mock_execution.step_results = [] + + mock_submission_repo = Mock() + mock_submission_repo.update_status = AsyncMock() + mock_submission_repo.update = AsyncMock() + + mock_result_repo = Mock() + mock_result_repo.create = AsyncMock() + + mock_session = AsyncMock() + mock_session.commit = AsyncMock() + + captured_submission = {} + + def capture_pipeline_run(submission): + captured_submission["obj"] = submission + return mock_execution + + with patch("web.service.grading_service.build_pipeline") as mock_build, \ + patch("web.service.grading_service.get_session") as mock_get_session, \ + patch("asyncio.to_thread", side_effect=lambda fn, sub: capture_pipeline_run(sub)), \ + patch("web.service.grading_service.SubmissionRepository", return_value=mock_submission_repo), \ + patch("web.service.grading_service.ResultRepository", return_value=mock_result_repo), \ + patch("web.service.grading_service.PipelineExecutionSerializer") as mock_serializer: + + mock_get_session.return_value.__aenter__.return_value = mock_session + mock_serializer.serialize.return_value = {"status": "success"} + + request = GradingRequest( + submission_id=2, + grading_config_id=1, + template_name="static_analysis", + criteria_config={}, + setup_config={}, + feedback_config={}, + include_feedback=False, + language="python", + username="student", + external_user_id="u2", + submission_files={ + "main.py": { + "filename": "main.py", + "content": "x = 1", + "changed_lines": [1, 3, 5], + "file_metadata": {"change_status": "modified"}, + } + }, + ) + await grade_submission(request) + + sub = captured_submission["obj"] + main_file = sub.submission_files["main.py"] + assert main_file.changed_lines == {1, 3, 5} + assert main_file.metadata == {"change_status": "modified"} + assert main_file.is_contribution_aware is True + + @pytest.mark.asyncio + async def test_grading_service_evaluation_scope_none_when_absent(self): + """GradingRequest without evaluation_scope → submission.evaluation_scope is None.""" + mock_result = Mock() + mock_result.final_score = 100.0 + mock_result.feedback = None + mock_result.result_tree = None + mock_result.focus = Mock() + mock_result.focus.to_dict = Mock(return_value={"base": []}) + mock_result.comparison = None + + mock_execution = Mock() + mock_execution.result = mock_result + mock_execution.start_time = time.time() + mock_execution.step_results = [] + + mock_submission_repo = Mock() + mock_submission_repo.update_status = AsyncMock() + mock_submission_repo.update = AsyncMock() + + mock_result_repo = Mock() + mock_result_repo.create = AsyncMock() + + mock_session = AsyncMock() + mock_session.commit = AsyncMock() + + captured_submission = {} + + def capture_pipeline_run(submission): + captured_submission["obj"] = submission + return mock_execution + + with patch("web.service.grading_service.build_pipeline") as mock_build, \ + patch("web.service.grading_service.get_session") as mock_get_session, \ + patch("asyncio.to_thread", side_effect=lambda fn, sub: capture_pipeline_run(sub)), \ + patch("web.service.grading_service.SubmissionRepository", return_value=mock_submission_repo), \ + patch("web.service.grading_service.ResultRepository", return_value=mock_result_repo), \ + patch("web.service.grading_service.PipelineExecutionSerializer") as mock_serializer: + + mock_get_session.return_value.__aenter__.return_value = mock_session + mock_serializer.serialize.return_value = {"status": "success"} + + request = GradingRequest( + submission_id=3, + grading_config_id=1, + template_name="static_analysis", + criteria_config={}, + setup_config={}, + feedback_config={}, + include_feedback=False, + language="python", + username="student", + external_user_id="u3", + submission_files={ + "main.py": { + "filename": "main.py", + "content": "x = 1", + } + }, + evaluation_scope=None, + ) + await grade_submission(request) + + sub = captured_submission["obj"] + assert sub.evaluation_scope is None + main_file = sub.submission_files["main.py"] + assert main_file.changed_lines is None + assert main_file.is_contribution_aware is False + + @pytest.mark.asyncio + async def test_grading_service_changed_lines_none_when_absent(self): + """GradingRequest with file missing changed_lines key → SubmissionFile.changed_lines is None.""" + mock_result = Mock() + mock_result.final_score = 100.0 + mock_result.feedback = None + mock_result.result_tree = None + mock_result.focus = Mock() + mock_result.focus.to_dict = Mock(return_value={"base": []}) + mock_result.comparison = None + + mock_execution = Mock() + mock_execution.result = mock_result + mock_execution.start_time = time.time() + mock_execution.step_results = [] + + mock_submission_repo = Mock() + mock_submission_repo.update_status = AsyncMock() + mock_submission_repo.update = AsyncMock() + + mock_result_repo = Mock() + mock_result_repo.create = AsyncMock() + + mock_session = AsyncMock() + mock_session.commit = AsyncMock() + + captured_submission = {} + + def capture_pipeline_run(submission): + captured_submission["obj"] = submission + return mock_execution + + with patch("web.service.grading_service.build_pipeline") as mock_build, \ + patch("web.service.grading_service.get_session") as mock_get_session, \ + patch("asyncio.to_thread", side_effect=lambda fn, sub: capture_pipeline_run(sub)), \ + patch("web.service.grading_service.SubmissionRepository", return_value=mock_submission_repo), \ + patch("web.service.grading_service.ResultRepository", return_value=mock_result_repo), \ + patch("web.service.grading_service.PipelineExecutionSerializer") as mock_serializer: + + mock_get_session.return_value.__aenter__.return_value = mock_session + mock_serializer.serialize.return_value = {"status": "success"} + + # Simulate the minimal storage format (no changed_lines or file_metadata keys) + request = GradingRequest( + submission_id=4, + grading_config_id=1, + template_name="static_analysis", + criteria_config={}, + setup_config={}, + feedback_config={}, + include_feedback=False, + language="python", + username="student", + external_user_id="u4", + submission_files={ + "main.py": { + "filename": "main.py", + "content": "x = 1", + } + }, + ) + await grade_submission(request) + + sub = captured_submission["obj"] + main_file = sub.submission_files["main.py"] + assert main_file.changed_lines is None + assert main_file.metadata is None