diff --git a/autograder/models/dataclass/structural_analysis_result.py b/autograder/models/dataclass/structural_analysis_result.py index c725e488..3ed10ace 100644 --- a/autograder/models/dataclass/structural_analysis_result.py +++ b/autograder/models/dataclass/structural_analysis_result.py @@ -1,5 +1,5 @@ -from dataclasses import dataclass -from typing import Dict, Optional, TYPE_CHECKING +from dataclasses import dataclass, field +from typing import Dict, Optional, Set, TYPE_CHECKING if TYPE_CHECKING: from ast_grep_py import SgRoot @@ -12,9 +12,11 @@ class StructuralAnalysisResult: Attributes: roots: A dictionary mapping filenames to their corresponding ast-grep root nodes. If a file could not be parsed, the value is None. + changed_lines: Changed line numbers supplied for each parsed file. available: Whether structural analysis infrastructure was available and attempted. reason: Optional reason explaining why analysis was unavailable/skipped. """ roots: Dict[str, Optional['SgRoot']] available: bool = True reason: Optional[str] = None + changed_lines: Dict[str, Set[int]] = field(default_factory=dict) diff --git a/autograder/models/dataclass/submission.py b/autograder/models/dataclass/submission.py index cc328dde..3465c042 100644 --- a/autograder/models/dataclass/submission.py +++ b/autograder/models/dataclass/submission.py @@ -1,12 +1,34 @@ -from typing import Dict, Optional from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Set + from sandbox_manager.models.sandbox_models import Language + @dataclass class SubmissionFile: """Represents a single file in a submission.""" filename: str content: str + changed_lines: Optional[Set[int]] = None + metadata: Optional[Dict[str, Any]] = None + + @property + def is_contribution_aware(self) -> bool: + """Return whether changed-line information was supplied for this file.""" + return self.changed_lines is not None + + +@dataclass +class EvaluationScope: + """ + Defines which files are the primary subject of an evaluation. + + When present, pipeline analysis steps restrict their work to these files. + When absent, all submission files are treated equally. + """ + + scoped_files: List[str] + @dataclass class Submission: @@ -14,6 +36,7 @@ class Submission: username: str user_id: int assignment_id: int - submission_files: Dict[str,SubmissionFile] + submission_files: Dict[str, SubmissionFile] language: Optional[Language] = None locale: str = "en" + evaluation_scope: Optional[EvaluationScope] = None diff --git a/autograder/models/pipeline_execution.py b/autograder/models/pipeline_execution.py index 35bb2eeb..d0a5eb4c 100644 --- a/autograder/models/pipeline_execution.py +++ b/autograder/models/pipeline_execution.py @@ -11,6 +11,7 @@ if TYPE_CHECKING: from autograder.models.abstract.template import Template from autograder.models.criteria_tree import CriteriaTree + from autograder.models.dataclass.submission import EvaluationScope from autograder.models.dataclass.focus import Focus from autograder.models.dataclass.grade_step_result import GradeStepResult from autograder.models.result_tree import ResultTree @@ -50,6 +51,11 @@ def locale(self) -> str: """Returns the locale for this pipeline execution, sourced from the submission.""" return self.submission.locale + @property + def evaluation_scope(self) -> Optional["EvaluationScope"]: + """Return the optional evaluation scope supplied with the submission.""" + return self.submission.evaluation_scope + def add_step_result(self, step_result: StepResult) -> 'PipelineExecution': """ Adds a single StepResult to the list of execution results. diff --git a/autograder/services/grader/criteria_grader.py b/autograder/services/grader/criteria_grader.py index ccdb69eb..c7216fff 100644 --- a/autograder/services/grader/criteria_grader.py +++ b/autograder/services/grader/criteria_grader.py @@ -7,7 +7,7 @@ SubjectNode, TestNode, ) -from autograder.models.dataclass.submission import SubmissionFile +from autograder.models.dataclass.submission import EvaluationScope, SubmissionFile from autograder.models.dataclass.test_result import TestResult from autograder.models.result_tree import ( CategoryResultNode, @@ -32,6 +32,7 @@ def __init__( locale: str = "en", pre_computed_results: Optional[Dict[str, TestResult]] = None, structural_analysis=None, + evaluation_scope: Optional[EvaluationScope] = None, ): self.logger = logging.getLogger("SubmissionGrader") self.submission_files = submission_files @@ -41,6 +42,7 @@ def __init__( self.locale = locale self.pre_computed_results = pre_computed_results self.structural_analysis = structural_analysis + self.evaluation_scope = evaluation_scope def __balance_nodes( self, @@ -147,6 +149,13 @@ def process_test(self, test: TestNode) -> TestResultNode: if self.submission_language is not None else config_submission_language ) + test_params.pop("evaluation_scope", None) + test_params.pop("file_metadata", None) + + file_metadata = { + sub_file.filename: sub_file.metadata + for sub_file in file_target or [] + } test_result = test.test_function.execute( files=file_target, @@ -155,6 +164,8 @@ def process_test(self, test: TestNode) -> TestResultNode: pre_computed_results=self.pre_computed_results, structural_analysis=self.structural_analysis, submission_language=effective_submission_language, + evaluation_scope=self.evaluation_scope, + file_metadata=file_metadata, **test_params, ) return TestResultNode( diff --git a/autograder/services/grader/grader_service.py b/autograder/services/grader/grader_service.py index 3577ba90..7b68a576 100644 --- a/autograder/services/grader/grader_service.py +++ b/autograder/services/grader/grader_service.py @@ -2,7 +2,7 @@ from typing import Dict, Optional from autograder.models.criteria_tree import CriteriaTree -from autograder.models.dataclass.submission import SubmissionFile +from autograder.models.dataclass.submission import EvaluationScope, SubmissionFile from autograder.models.dataclass.test_result import TestResult from autograder.models.result_tree import ( ResultTree, @@ -28,6 +28,7 @@ def grade_from_tree( locale: str = "en", pre_computed_results: Optional[Dict[str, TestResult]] = None, structural_analysis=None, + evaluation_scope: Optional[EvaluationScope] = None, ) -> ResultTree: """Traverse the generic built criteria tree to resolve inputs, grades and report to ResultTree.""" grader = SubmissionGrader( @@ -38,6 +39,7 @@ def grade_from_tree( locale=locale, pre_computed_results=pre_computed_results, structural_analysis=structural_analysis, + evaluation_scope=evaluation_scope, ) base_result = grader.process_category(criteria_tree.base) diff --git a/autograder/steps/grade_step.py b/autograder/steps/grade_step.py index e6adf5d2..df9a80b2 100644 --- a/autograder/steps/grade_step.py +++ b/autograder/steps/grade_step.py @@ -67,6 +67,7 @@ def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: locale=pipeline_exec.locale, pre_computed_results=pre_computed_results, structural_analysis=structural_analysis, + evaluation_scope=pipeline_exec.evaluation_scope, ) # Create grading result diff --git a/autograder/steps/structural_analysis_step.py b/autograder/steps/structural_analysis_step.py index 0d64413e..8bc530a2 100644 --- a/autograder/steps/structural_analysis_step.py +++ b/autograder/steps/structural_analysis_step.py @@ -1,5 +1,5 @@ import logging -from typing import Dict, Optional +from typing import Dict, Optional, Set from autograder.models.abstract.step import Step from autograder.models.pipeline_execution import PipelineExecution @@ -69,7 +69,17 @@ def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: ) roots: Dict[str, Optional[SgRoot]] = {} + changed_lines: Dict[str, Set[int]] = {} + scoped_files = ( + set(pipeline_exec.evaluation_scope.scoped_files) + if pipeline_exec.evaluation_scope is not None + else None + ) + for filename, sub_file in submission.submission_files.items(): + if scoped_files is not None and filename not in scoped_files: + continue + # Only parse files that likely contain code if not self._is_code_file(filename): continue @@ -80,7 +90,14 @@ def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: logger.warning("Failed to parse %s with ast-grep: %s", filename, str(e)) roots[filename] = None - result = StructuralAnalysisResult(roots=roots, available=True) + if sub_file.changed_lines is not None: + changed_lines[filename] = set(sub_file.changed_lines) + + result = StructuralAnalysisResult( + roots=roots, + changed_lines=changed_lines, + available=True, + ) return pipeline_exec.add_step_result(StepResult.success(self.step_name, result)) def _map_language(self, language: Language) -> Optional[str]: diff --git a/docs/API.md b/docs/API.md index a714fbae..c114f81d 100644 --- a/docs/API.md +++ b/docs/API.md @@ -378,12 +378,20 @@ Content-Type: application/json "files": [ { "filename": "main.py", - "content": "print('Hello World')" + "content": "print('Hello World')", + "changed_lines": [1], + "file_metadata": { + "change_status": "modified" + } } ], + "evaluation_scope": { + "scoped_files": ["main.py"] + }, "language": "python", "metadata": { - "attempt": 1 + "attempt": 1, + "source_revision": "example-revision" } } ``` @@ -395,11 +403,21 @@ Content-Type: application/json | `external_assignment_id` | string | ✓ | External assignment ID (must match an existing grading config) | | `external_user_id` | string | ✓ | External user ID from your platform | | `username` | string | ✓ | Username of the submitter | -| `files` | list[object] | ✓ | List of files with `filename` and `content` | +| `files` | list[object] | ✓ | Files with `filename`, full `content`, and optional evaluation context | +| `files[].changed_lines` | list[integer] | ✗ | One-indexed line numbers added or modified in the file | +| `files[].file_metadata` | object | ✗ | Opaque per-file context passed through to test functions | +| `evaluation_scope` | object | ✗ | Optional file scope for pipeline analysis | +| `evaluation_scope.scoped_files` | list[string] | ✓ when scope is present | Filenames that are the primary evaluation subject | | `language` | string | ✗ | Language override (defaults to first language in config) | -| `metadata` | object | ✗ | Optional metadata to attach to the submission | +| `metadata` | object | ✗ | Passive submission-level provenance stored for audit/correlation; the pipeline never reads it | | `baseline_result_tree` | object | ✗ | Serialised `result_tree` from a previous submission response to calculate a `ComparisonResult` | +`evaluation_scope` and `changed_lines` are typed pipeline inputs. +`file_metadata` is stored as `SubmissionFile.metadata` and remains opaque to +the pipeline, while top-level `metadata` is stored as `submission_metadata` +and is not passed into core grading. If `evaluation_scope` is omitted, +structural analysis continues to consider all eligible submitted files. + **Response (200 OK):** ```json { diff --git a/docs/architecture/core_structures.md b/docs/architecture/core_structures.md index 5c1c3714..771dddfb 100644 --- a/docs/architecture/core_structures.md +++ b/docs/architecture/core_structures.md @@ -371,18 +371,54 @@ class Submission: username: str # Student identifier user_id: int # Student ID assignment_id: int # Assignment identifier - submission_files: Dict[str, SubmissionFile] # Uploaded files keyed by filename + submission_files: Dict[str, SubmissionFile] # Uploaded files keyed by filename language: Optional[Language] = None # Programming language + locale: str = "en" # Feedback locale + evaluation_scope: Optional[EvaluationScope] = None ``` ### Submission File ```python class SubmissionFile: - filename: str # Name of file - content: str # File contents (text) + filename: str # Name of file + content: str # Full file contents (text) + changed_lines: Optional[Set[int]] = None # One-indexed added/modified lines + metadata: Optional[Dict[str, Any]] = None # Opaque per-file context ``` +`SubmissionFile.is_contribution_aware` is `True` when `changed_lines` was +provided, including when the caller explicitly provides an empty set. + +### Evaluation Scope + +```python +class EvaluationScope: + scoped_files: List[str] # Files that are the primary evaluation subject +``` + +When a scope is present, scope-aware pipeline steps restrict their work to the +listed filenames. Currently, structural analysis uses the scope to avoid +parsing unrelated files. Omitting the scope preserves snapshot behavior and +analyzes all eligible submission files. + +### Metadata Ownership + +Evaluation context is deliberately split by responsibility: + +| Field | Pipeline behavior | Intended use | +|-------|-------------------|--------------| +| `EvaluationScope.scoped_files` | Read by scope-aware steps | Select files that analysis should focus on | +| `SubmissionFile.changed_lines` | Read by structural analysis and available to tests | Identify added or modified lines | +| `SubmissionFile.metadata` | Never interpreted; forwarded to test functions | Carry arbitrary per-file context | +| API `metadata` / database `submission_metadata` | Never read by the core pipeline | Preserve submission-level provenance and audit context | + +Platform- or domain-specific data belongs in the open-ended metadata fields, +not in new typed core model fields. Test functions receive `evaluation_scope` +and a filename-keyed `file_metadata` mapping through `execute(**kwargs)`. They +also receive the original `SubmissionFile` objects, including `changed_lines` +and `metadata`, through the `files` argument. + ## Template Template provides test functions for a specific assignment type. @@ -412,4 +448,3 @@ class TestFunction: """Execute the test and return result""" pass ``` - diff --git a/docs/pipeline/04.8-structural-analysis.md b/docs/pipeline/04.8-structural-analysis.md index feb71c77..215f6a02 100644 --- a/docs/pipeline/04.8-structural-analysis.md +++ b/docs/pipeline/04.8-structural-analysis.md @@ -7,9 +7,10 @@ The Structural Analysis step parses student submission files into Abstract Synta ## How It Works 1. **Detect Language** — The step identifies the submission language (Python, Java, Node.js, C++, or C). -2. **Heuristic File Filtering** — It scans the submission files and identifies those likely to contain source code, skipping binary files, images, and non-code configurations (e.g., `.png`, `.json`, `.yaml`, `.md`). -3. **Parse ASTs** — For each identified code file, it uses `ast-grep-py` to parse the content into an `SgRoot` object. -4. **Store Results** — The resulting mapping of `Dict[filename, SgRoot]` is stored in `StepResult.data` under `StepName.STRUCTURAL_ANALYSIS`. +2. **Apply Evaluation Scope** — When the submission includes an `EvaluationScope`, only filenames in `scoped_files` are considered. Without a scope, all files are considered for backward compatibility. +3. **Heuristic File Filtering** — It identifies files likely to contain source code, skipping binary files, images, and non-code configurations (e.g., `.png`, `.json`, `.yaml`, `.md`). +4. **Parse ASTs** — For each identified code file, it uses `ast-grep-py` to parse the content into an `SgRoot` object. +5. **Store Results** — Parsed roots and any caller-supplied changed-line sets are stored in `StructuralAnalysisResult`. If `ast-grep-py` is not installed or the language is not supported, the step logs a warning and proceeds with an empty result set, allowing the pipeline to continue. @@ -23,24 +24,29 @@ If `ast-grep-py` is not installed or the language is not supported, the step log | Source | Data | |--------|------| -| Pipeline | `pipeline_exec.submission` → submission files and language | +| Pipeline | `pipeline_exec.submission` → submission files, language, and optional evaluation scope | ## Output | Field | Type | Description | |-------|------|-------------| -| `data` | `StructuralAnalysisResult` | Contains a mapping of filenames to `ast-grep` `SgRoot` objects | +| `data` | `StructuralAnalysisResult` | Contains filename-keyed `roots` and `changed_lines` mappings | | `status` | `StepStatus.SUCCESS` | Usually succeeds even if parsing fails for some files (stores `None` for those files) | ## How It Integrates with Grade The `GradeStep` reads the `STRUCTURAL_ANALYSIS` step result from the pipeline and passes it as `structural_analysis` to `GraderService.grade_from_tree()`. The grader threads this object through the tree traversal into every `process_test()` call, which forwards it as a kwarg to `test_function.execute()`. -Test functions like `ForbiddenKeywordTest` can then use this pre-computed AST to perform efficient and accurate queries using `ast-grep`'s pattern matching syntax. +Test functions like `ForbiddenKeywordTest` can then use this pre-computed AST +to perform efficient queries using `ast-grep`'s pattern matching syntax. +Contribution-aware tests can use `StructuralAnalysisResult.changed_lines` to +limit findings to added or modified lines. ## Key Design Decisions - **Pre-computed ASTs** — Parsing is done once at the pipeline level rather than inside individual tests to ensure efficiency when multiple structural tests are defined. +- **Optional scope** — `EvaluationScope` selects the files to parse without introducing platform-specific concepts into the core engine. An explicit empty scope parses no files. +- **Changed-line passthrough** — Changed lines are typed because pipeline and test implementations can act on them; opaque domain context remains in `SubmissionFile.metadata`. - **Fail-safe** — If parsing fails for a specific file (e.g., due to syntax errors), the step stores `None` for that file instead of failing the entire pipeline. - **Library Choice** — `ast-grep` was chosen for its performance (Rust-based) and its ability to provide high-level, language-agnostic pattern matching queries. diff --git a/tests/e2e/test_contribution_aware_e2e.py b/tests/e2e/test_contribution_aware_e2e.py new file mode 100644 index 00000000..a64448ce --- /dev/null +++ b/tests/e2e/test_contribution_aware_e2e.py @@ -0,0 +1,391 @@ +""" +End-to-end tests for contribution-aware evaluation via the HTTP API. + +These tests exercise the full flow: + 1. Create a grading config + 2. Submit code with contribution-aware fields (evaluation_scope, changed_lines, file_metadata) + 3. Poll until completed → verify grading succeeds + +Tests cover: +- Submission with evaluation_scope is accepted and graded +- Submission without evaluation_scope is accepted and graded +- Submission with changed_lines and file_metadata is accepted +- Combined contribution-aware submission with baseline_result_tree +""" + +import json +import time +import requests +import pytest + + +def poll_submission(api_base_url, submission_id, auth_headers, timeout=60): + """Poll until a submission reaches a terminal status.""" + start_time = time.time() + while time.time() - start_time < timeout: + response = requests.get( + f"{api_base_url}/submissions/{submission_id}", + headers=auth_headers, + ) + assert response.status_code == 200 + data = response.json() + if data["status"] in ["completed", "failed"]: + return data + time.sleep(2) + + response = requests.get( + f"{api_base_url}/submissions/{submission_id}", + headers=auth_headers, + ) + print(f"DEBUG: Timeout reached. State: {json.dumps(response.json(), indent=2)}") + pytest.fail(f"Submission {submission_id} timed out") + + +@pytest.fixture +def run_id(): + return int(time.time()) + + +def _create_static_config(api_base_url, auth_headers, config_id, forbidden_imports): + """Create a static_analysis grading config.""" + config_payload = { + "external_assignment_id": config_id, + "template_name": "static_analysis", + "languages": ["python"], + "criteria_config": { + "base": { + "weight": 100.0, + "tests": [ + { + "name": f"no_{imp}", + "type": "forbidden_import", + "forbidden_imports": [imp], + "submission_language": "python", + } + for imp in forbidden_imports + ], + } + }, + } + response = requests.post( + f"{api_base_url}/configs", + json=config_payload, + headers=auth_headers, + ) + assert response.status_code in [200, 201] + return response.json() + + +def _submit( + api_base_url, + auth_headers, + config_id, + user_id, + files, + evaluation_scope=None, + baseline_result_tree=None, +): + """Submit code for grading with optional contribution-aware fields.""" + payload = { + "external_assignment_id": config_id, + "external_user_id": user_id, + "username": f"student-{user_id}", + "files": files, + } + if evaluation_scope is not None: + payload["evaluation_scope"] = evaluation_scope + if baseline_result_tree is not None: + payload["baseline_result_tree"] = baseline_result_tree + + response = requests.post( + f"{api_base_url}/submissions", + json=payload, + headers=auth_headers, + ) + assert response.status_code in [200, 201], ( + f"Submission failed: {response.status_code} — {response.text}" + ) + return response.json() + + +# ============================================================================== +# EVALUATION SCOPE E2E TESTS +# ============================================================================== + + +class TestEvaluationScopeE2E: + """E2E tests for evaluation_scope via the HTTP API.""" + + def test_submission_with_evaluation_scope_accepted( + self, api_base_url, auth_headers, run_id + ): + """Submit with evaluation_scope → 200, grading completes successfully.""" + config_id = f"scope-accept-{run_id}" + _create_static_config(api_base_url, auth_headers, config_id, ["os"]) + + files = [{"filename": "main.py", "content": "x = 42\nprint(x)"}] + sub = _submit( + api_base_url, + auth_headers, + config_id, + f"u-scope-{run_id}", + files, + evaluation_scope={"scoped_files": ["main.py"]}, + ) + result = poll_submission(api_base_url, sub["id"], auth_headers) + + assert result["status"] == "completed" + assert result["final_score"] == 100.0 + + def test_submission_without_evaluation_scope_accepted( + self, api_base_url, auth_headers, run_id + ): + """Submit without evaluation_scope → 200, grading completes successfully.""" + config_id = f"scope-none-{run_id}" + _create_static_config(api_base_url, auth_headers, config_id, ["os"]) + + files = [{"filename": "main.py", "content": "x = 42\nprint(x)"}] + sub = _submit( + api_base_url, + auth_headers, + config_id, + f"u-noscope-{run_id}", + files, + ) + result = poll_submission(api_base_url, sub["id"], auth_headers) + + assert result["status"] == "completed" + assert result["final_score"] == 100.0 + + def test_evaluation_scope_does_not_affect_scoring_for_static_analysis( + self, api_base_url, auth_headers, run_id + ): + """ + Same code with and without evaluation_scope → same score. + static_analysis tests don't filter by scope, so scores should be identical. + """ + config_id = f"scope-score-{run_id}" + _create_static_config(api_base_url, auth_headers, config_id, ["os"]) + + files = [{"filename": "main.py", "content": "x = 42\nprint(x)"}] + + # Without scope + sub1 = _submit( + api_base_url, + auth_headers, + config_id, + f"u-score1-{run_id}", + files, + ) + result1 = poll_submission(api_base_url, sub1["id"], auth_headers) + + # With scope + sub2 = _submit( + api_base_url, + auth_headers, + config_id, + f"u-score2-{run_id}", + files, + evaluation_scope={"scoped_files": ["main.py"]}, + ) + result2 = poll_submission(api_base_url, sub2["id"], auth_headers) + + assert result1["status"] == "completed" + assert result2["status"] == "completed" + assert result1["final_score"] == result2["final_score"] + + +# ============================================================================== +# CHANGED LINES E2E TESTS +# ============================================================================== + + +class TestChangedLinesE2E: + """E2E tests for changed_lines and file_metadata via the HTTP API.""" + + def test_submission_with_changed_lines_accepted( + self, api_base_url, auth_headers, run_id + ): + """Submit with changed_lines on files → 200, grading completes.""" + config_id = f"cl-accept-{run_id}" + _create_static_config(api_base_url, auth_headers, config_id, ["os"]) + + files = [ + { + "filename": "main.py", + "content": "x = 42\nprint(x)", + "changed_lines": [1, 2], + } + ] + sub = _submit( + api_base_url, + auth_headers, + config_id, + f"u-cl-{run_id}", + files, + ) + result = poll_submission(api_base_url, sub["id"], auth_headers) + + assert result["status"] == "completed" + assert result["final_score"] == 100.0 + + def test_changed_lines_and_file_metadata_preserved( + self, api_base_url, auth_headers, run_id + ): + """Submit with changed_lines and file_metadata → grading succeeds without errors.""" + config_id = f"cl-meta-{run_id}" + _create_static_config(api_base_url, auth_headers, config_id, ["os"]) + + files = [ + { + "filename": "main.py", + "content": "x = 42\nprint(x)", + "changed_lines": [1], + "file_metadata": { + "change_status": "modified", + "provider": "github", + "stats": {"additions": 1, "deletions": 0}, + }, + } + ] + sub = _submit( + api_base_url, + auth_headers, + config_id, + f"u-clm-{run_id}", + files, + ) + result = poll_submission(api_base_url, sub["id"], auth_headers) + + assert result["status"] == "completed" + assert result["final_score"] == 100.0 + + def test_submission_without_changed_lines_accepted( + self, api_base_url, auth_headers, run_id + ): + """Submit without changed_lines → 200, grading completes (backward compatible).""" + config_id = f"cl-none-{run_id}" + _create_static_config(api_base_url, auth_headers, config_id, ["os"]) + + files = [{"filename": "main.py", "content": "x = 42"}] + sub = _submit( + api_base_url, + auth_headers, + config_id, + f"u-clnone-{run_id}", + files, + ) + result = poll_submission(api_base_url, sub["id"], auth_headers) + + assert result["status"] == "completed" + + +# ============================================================================== +# COMBINED CONTRIBUTION-AWARE E2E TESTS +# ============================================================================== + + +class TestCombinedContributionAwareE2E: + """E2E tests combining all contribution-aware features.""" + + def test_full_contribution_aware_submission( + self, api_base_url, auth_headers, run_id + ): + """ + Submit with evaluation_scope + changed_lines + file_metadata + baseline_result_tree + → everything works together, comparison present in response. + """ + config_id = f"combined-{run_id}" + _create_static_config(api_base_url, auth_headers, config_id, ["os"]) + + # First submission (baseline) — violation code + baseline_files = [ + { + "filename": "main.py", + "content": "import os\nprint(os.getcwd())", + "changed_lines": [1, 2], + "file_metadata": {"change_status": "added"}, + } + ] + sub1 = _submit( + api_base_url, + auth_headers, + config_id, + f"u-comb-base-{run_id}", + baseline_files, + evaluation_scope={"scoped_files": ["main.py"]}, + ) + result1 = poll_submission(api_base_url, sub1["id"], auth_headers) + assert result1["status"] == "completed" + assert result1["final_score"] < 100.0 + assert result1["comparison"] is None # No baseline → no comparison + baseline_tree = result1["result_tree"] + + # Second submission (head) — clean code with baseline + head_files = [ + { + "filename": "main.py", + "content": "x = 42\nprint(x)", + "changed_lines": [1, 2], + "file_metadata": {"change_status": "modified"}, + } + ] + sub2 = _submit( + api_base_url, + auth_headers, + config_id, + f"u-comb-head-{run_id}", + head_files, + evaluation_scope={"scoped_files": ["main.py"]}, + baseline_result_tree=baseline_tree, + ) + result2 = poll_submission(api_base_url, sub2["id"], auth_headers) + + assert result2["status"] == "completed" + assert result2["final_score"] == 100.0 + + # Comparison should show improvement + comparison = result2["comparison"] + assert comparison is not None + assert comparison["score_delta"] > 0 + assert comparison["improved"] is True + + # score_vector should also be present + assert result2["score_vector"] is not None + + def test_multi_file_with_scope_and_metadata( + self, api_base_url, auth_headers, run_id + ): + """ + Multi-file submission with evaluation_scope scoping to a subset. + Verify grading completes successfully. + """ + config_id = f"multi-scope-{run_id}" + _create_static_config(api_base_url, auth_headers, config_id, ["os"]) + + files = [ + { + "filename": "main.py", + "content": "x = 42", + "changed_lines": [1], + "file_metadata": {"change_status": "modified"}, + }, + { + "filename": "utils.py", + "content": "def helper(): return 1", + "changed_lines": None, + "file_metadata": {"change_status": "unchanged"}, + }, + ] + sub = _submit( + api_base_url, + auth_headers, + config_id, + f"u-multi-{run_id}", + files, + evaluation_scope={"scoped_files": ["main.py"]}, + ) + result = poll_submission(api_base_url, sub["id"], auth_headers) + + assert result["status"] == "completed" + assert result["final_score"] == 100.0 diff --git a/tests/integration/test_contribution_aware_integration.py b/tests/integration/test_contribution_aware_integration.py new file mode 100644 index 00000000..6c12707d --- /dev/null +++ b/tests/integration/test_contribution_aware_integration.py @@ -0,0 +1,725 @@ +""" +Integration tests for contribution-aware evaluation features. + +Tests cover: +1. EvaluationScope propagation through the pipeline +2. changed_lines propagation through the pipeline +3. file_metadata passthrough to test functions +4. GradingService hydration of contribution-aware fields from HTTP payload format + +These tests use the static_analysis template (no sandbox) so they can run +without Docker infrastructure. +""" + +import time + +import pytest +from unittest.mock import AsyncMock, Mock, patch + +from autograder.autograder import build_pipeline +from autograder.models.abstract.test_function import TestFunction +from autograder.models.criteria_tree import CategoryNode, CriteriaTree, TestNode +from autograder.models.dataclass.submission import ( + EvaluationScope, + Submission, + SubmissionFile, +) +from autograder.models.dataclass.test_result import TestResult +from autograder.services.grader.grader_service import GraderService +from web.service.grading_service import GradingRequest, grade_submission + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_submission( + files: dict[str, SubmissionFile], + evaluation_scope: EvaluationScope | None = None, +) -> Submission: + """Build a Submission with optional contribution-aware fields.""" + return Submission( + username="integration-student", + user_id="integration-user", + assignment_id=1, + submission_files=files, + language=None, # static_analysis doesn't require language + locale="en", + evaluation_scope=evaluation_scope, + ) + + +def _static_analysis_criteria(forbidden_imports: list) -> dict: + """Build a criteria config for static_analysis with forbidden imports.""" + return { + "base": { + "weight": 100.0, + "tests": [ + { + "name": f"no_{imp}", + "type": "forbidden_import", + "forbidden_imports": [imp], + "submission_language": "python", + } + for imp in forbidden_imports + ], + } + } + + +class ContextCapturingTest(TestFunction): + """Test function that captures files, evaluation_scope, and file_metadata.""" + + def __init__(self): + self.files = None + self.kwargs = None + + @property + def name(self) -> str: + return "context_capture" + + @property + def description(self) -> str: + return "Capture contribution-aware context." + + @property + def parameter_description(self) -> list: + return [] + + def execute(self, files, sandbox, *args, **kwargs) -> TestResult: + self.files = files + self.kwargs = kwargs + return TestResult( + test_name=self.name, + score=100.0, + report="Context captured.", + ) + + +# --------------------------------------------------------------------------- +# Integration: EvaluationScope through the pipeline +# --------------------------------------------------------------------------- + + +class TestEvaluationScopeIntegration: + """Verify evaluation_scope restricts structural analysis and propagates to test functions.""" + + def test_structural_analysis_respects_evaluation_scope(self): + """Pipeline with 2 files but scope limited to 1 → structural analysis parses only scoped file.""" + from sandbox_manager.models.sandbox_models import Language + + criteria = _static_analysis_criteria(["os"]) + + files = { + "main.py": SubmissionFile(filename="main.py", content="import os\nx = 1"), + "helper.py": SubmissionFile(filename="helper.py", content="import os\ny = 2"), + } + scope = EvaluationScope(scoped_files=["main.py"]) + + submission = Submission( + username="scope-student", + user_id="scope-user", + assignment_id=1, + submission_files=files, + language=Language.PYTHON, + locale="en", + evaluation_scope=scope, + ) + + pipeline = build_pipeline( + template_name="static_analysis", + include_feedback=False, + grading_criteria=criteria, + feedback_config={}, + ) + execution = pipeline.run(submission) + + # Pipeline should succeed + assert execution.result is not None + + # Structural analysis should have only parsed main.py (the scoped file) + sa_result = execution.get_structural_analysis_result() + assert sa_result is not None + assert sa_result.available is True + assert "main.py" in sa_result.roots + assert "helper.py" not in sa_result.roots + + def test_grading_with_evaluation_scope_passes_to_test_functions(self): + """Verify evaluation_scope is propagated as kwarg to test functions via GraderService.""" + capturing_test = ContextCapturingTest() + criteria_tree = CriteriaTree( + base=CategoryNode( + name="base", + weight=100, + tests=[ + TestNode( + name="context_capture", + test_function=capturing_test, + file_target=["main.py"], + ) + ], + ) + ) + + files = { + "main.py": SubmissionFile(filename="main.py", content="x = 1"), + } + scope = EvaluationScope(scoped_files=["main.py"]) + + GraderService().grade_from_tree( + criteria_tree=criteria_tree, + submission_files=files, + evaluation_scope=scope, + ) + + assert capturing_test.kwargs is not None + assert capturing_test.kwargs["evaluation_scope"] is scope + + def test_grading_without_evaluation_scope_processes_all_files(self): + """Without scope, all submission files are processed normally.""" + from sandbox_manager.models.sandbox_models import Language + + criteria = _static_analysis_criteria(["os"]) + files = { + "main.py": SubmissionFile(filename="main.py", content="x = 1"), + "helper.py": SubmissionFile(filename="helper.py", content="y = 2"), + } + + submission = Submission( + username="noscope-student", + user_id="noscope-user", + assignment_id=1, + submission_files=files, + language=Language.PYTHON, + locale="en", + evaluation_scope=None, + ) + + pipeline = build_pipeline( + template_name="static_analysis", + include_feedback=False, + grading_criteria=criteria, + feedback_config={}, + ) + execution = pipeline.run(submission) + + assert execution.result is not None + + # Without scope, structural analysis should parse both files + sa_result = execution.get_structural_analysis_result() + assert sa_result is not None + assert "main.py" in sa_result.roots + assert "helper.py" in sa_result.roots + + def test_evaluation_scope_none_passed_to_test_functions(self): + """When evaluation_scope is None, test functions receive None.""" + capturing_test = ContextCapturingTest() + criteria_tree = CriteriaTree( + base=CategoryNode( + name="base", + weight=100, + tests=[ + TestNode( + name="context_capture", + test_function=capturing_test, + ) + ], + ) + ) + + files = {"main.py": SubmissionFile(filename="main.py", content="x = 1")} + + GraderService().grade_from_tree( + criteria_tree=criteria_tree, + submission_files=files, + evaluation_scope=None, + ) + + assert capturing_test.kwargs["evaluation_scope"] is None + + +# --------------------------------------------------------------------------- +# Integration: changed_lines through the pipeline +# --------------------------------------------------------------------------- + + +class TestChangedLinesIntegration: + """Verify changed_lines propagation through the pipeline.""" + + def test_changed_lines_propagated_through_pipeline(self): + """SubmissionFile with changed_lines → structural analysis captures them.""" + from sandbox_manager.models.sandbox_models import Language + + criteria = _static_analysis_criteria(["os"]) + files = { + "main.py": SubmissionFile( + filename="main.py", + content="import os\nx = 1", + changed_lines={1, 2}, + ), + } + + submission = Submission( + username="cl-student", + user_id="cl-user", + assignment_id=1, + submission_files=files, + language=Language.PYTHON, + locale="en", + ) + + pipeline = build_pipeline( + template_name="static_analysis", + include_feedback=False, + grading_criteria=criteria, + feedback_config={}, + ) + execution = pipeline.run(submission) + + assert execution.result is not None + + # Structural analysis should carry the changed_lines + sa_result = execution.get_structural_analysis_result() + assert sa_result is not None + assert "main.py" in sa_result.changed_lines + assert sa_result.changed_lines["main.py"] == {1, 2} + + def test_no_changed_lines_still_works(self): + """Submission without changed_lines → pipeline works normally, is_contribution_aware=False.""" + from sandbox_manager.models.sandbox_models import Language + + criteria = _static_analysis_criteria(["os"]) + file_obj = SubmissionFile(filename="main.py", content="x = 1") + assert file_obj.is_contribution_aware is False + + files = {"main.py": file_obj} + + submission = Submission( + username="no-cl-student", + user_id="no-cl-user", + assignment_id=1, + submission_files=files, + language=Language.PYTHON, + locale="en", + ) + + pipeline = build_pipeline( + template_name="static_analysis", + include_feedback=False, + grading_criteria=criteria, + feedback_config={}, + ) + execution = pipeline.run(submission) + + assert execution.result is not None + assert execution.result.final_score == 100.0 + + # Structural analysis should have empty changed_lines + sa_result = execution.get_structural_analysis_result() + assert sa_result.changed_lines == {} + + def test_changed_lines_set_means_contribution_aware(self): + """SubmissionFile with changed_lines set → is_contribution_aware is True.""" + file_obj = SubmissionFile( + filename="main.py", + content="x = 1", + changed_lines={1}, + ) + assert file_obj.is_contribution_aware is True + + +# --------------------------------------------------------------------------- +# Integration: file_metadata passthrough +# --------------------------------------------------------------------------- + + +class TestFileMetadataIntegration: + """Verify file_metadata is available in test functions via file_metadata kwarg.""" + + def test_file_metadata_available_in_test_function(self): + """Test function receives file_metadata dict keyed by filename.""" + capturing_test = ContextCapturingTest() + criteria_tree = CriteriaTree( + base=CategoryNode( + name="base", + weight=100, + tests=[ + TestNode( + name="context_capture", + test_function=capturing_test, + file_target=["main.py"], + ) + ], + ) + ) + + metadata = {"change_status": "modified", "provider": "github"} + files = { + "main.py": SubmissionFile( + filename="main.py", + content="x = 1", + metadata=metadata, + ), + } + + GraderService().grade_from_tree( + criteria_tree=criteria_tree, + submission_files=files, + ) + + assert capturing_test.kwargs is not None + assert capturing_test.kwargs["file_metadata"] == {"main.py": metadata} + assert capturing_test.kwargs["file_metadata"]["main.py"] is metadata + + def test_file_metadata_none_when_not_provided(self): + """Test function receives file_metadata with None values when metadata is absent.""" + capturing_test = ContextCapturingTest() + criteria_tree = CriteriaTree( + base=CategoryNode( + name="base", + weight=100, + tests=[ + TestNode( + name="context_capture", + test_function=capturing_test, + file_target=["main.py"], + ) + ], + ) + ) + + files = { + "main.py": SubmissionFile(filename="main.py", content="x = 1"), + } + + GraderService().grade_from_tree( + criteria_tree=criteria_tree, + submission_files=files, + ) + + assert capturing_test.kwargs["file_metadata"] == {"main.py": None} + + def test_file_metadata_multi_file_only_targeted_files(self): + """file_metadata contains only the files targeted by file_target.""" + capturing_test = ContextCapturingTest() + criteria_tree = CriteriaTree( + base=CategoryNode( + name="base", + weight=100, + tests=[ + TestNode( + name="context_capture", + test_function=capturing_test, + file_target=["main.py"], + ) + ], + ) + ) + + files = { + "main.py": SubmissionFile( + filename="main.py", + content="x = 1", + metadata={"change_status": "modified"}, + ), + "helper.py": SubmissionFile( + filename="helper.py", + content="y = 2", + metadata={"change_status": "added"}, + ), + } + + GraderService().grade_from_tree( + criteria_tree=criteria_tree, + submission_files=files, + ) + + # Only main.py should be in file_metadata (file_target = ["main.py"]) + assert "main.py" in capturing_test.kwargs["file_metadata"] + assert "helper.py" not in capturing_test.kwargs["file_metadata"] + + +# --------------------------------------------------------------------------- +# Integration: GradingService hydration of contribution-aware fields +# --------------------------------------------------------------------------- + + +class TestGradingServiceContributionAware: + """Verify the grading service correctly hydrates contribution-aware fields.""" + + @pytest.mark.asyncio + async def test_grading_service_hydrates_evaluation_scope(self): + """GradingRequest with evaluation_scope dict → AutograderSubmission has EvaluationScope object.""" + mock_result = Mock() + mock_result.final_score = 100.0 + mock_result.feedback = None + mock_result.result_tree = None + mock_result.focus = Mock() + mock_result.focus.to_dict = Mock(return_value={"base": []}) + mock_result.comparison = None + + mock_execution = Mock() + mock_execution.result = mock_result + mock_execution.start_time = time.time() + mock_execution.step_results = [] + + mock_submission_repo = Mock() + mock_submission_repo.update_status = AsyncMock() + mock_submission_repo.update = AsyncMock() + + mock_result_repo = Mock() + mock_result_repo.create = AsyncMock() + + mock_session = AsyncMock() + mock_session.commit = AsyncMock() + + captured_submission = {} + + def capture_pipeline_run(submission): + captured_submission["obj"] = submission + return mock_execution + + with patch("web.service.grading_service.build_pipeline") as mock_build, \ + patch("web.service.grading_service.get_session") as mock_get_session, \ + patch("asyncio.to_thread", side_effect=lambda fn, sub: capture_pipeline_run(sub)), \ + patch("web.service.grading_service.SubmissionRepository", return_value=mock_submission_repo), \ + patch("web.service.grading_service.ResultRepository", return_value=mock_result_repo), \ + patch("web.service.grading_service.PipelineExecutionSerializer") as mock_serializer: + + mock_get_session.return_value.__aenter__.return_value = mock_session + mock_serializer.serialize.return_value = {"status": "success"} + + request = GradingRequest( + submission_id=1, + grading_config_id=1, + template_name="static_analysis", + criteria_config={}, + setup_config={}, + feedback_config={}, + include_feedback=False, + language="python", + username="student", + external_user_id="u1", + submission_files={ + "main.py": { + "filename": "main.py", + "content": "x = 1", + "changed_lines": None, + "file_metadata": None, + } + }, + evaluation_scope={"scoped_files": ["main.py"]}, + ) + await grade_submission(request) + + sub = captured_submission["obj"] + assert sub.evaluation_scope is not None + assert sub.evaluation_scope.scoped_files == ["main.py"] + + @pytest.mark.asyncio + async def test_grading_service_hydrates_changed_lines(self): + """GradingRequest with changed_lines in files → SubmissionFile has set(changed_lines).""" + mock_result = Mock() + mock_result.final_score = 100.0 + mock_result.feedback = None + mock_result.result_tree = None + mock_result.focus = Mock() + mock_result.focus.to_dict = Mock(return_value={"base": []}) + mock_result.comparison = None + + mock_execution = Mock() + mock_execution.result = mock_result + mock_execution.start_time = time.time() + mock_execution.step_results = [] + + mock_submission_repo = Mock() + mock_submission_repo.update_status = AsyncMock() + mock_submission_repo.update = AsyncMock() + + mock_result_repo = Mock() + mock_result_repo.create = AsyncMock() + + mock_session = AsyncMock() + mock_session.commit = AsyncMock() + + captured_submission = {} + + def capture_pipeline_run(submission): + captured_submission["obj"] = submission + return mock_execution + + with patch("web.service.grading_service.build_pipeline") as mock_build, \ + patch("web.service.grading_service.get_session") as mock_get_session, \ + patch("asyncio.to_thread", side_effect=lambda fn, sub: capture_pipeline_run(sub)), \ + patch("web.service.grading_service.SubmissionRepository", return_value=mock_submission_repo), \ + patch("web.service.grading_service.ResultRepository", return_value=mock_result_repo), \ + patch("web.service.grading_service.PipelineExecutionSerializer") as mock_serializer: + + mock_get_session.return_value.__aenter__.return_value = mock_session + mock_serializer.serialize.return_value = {"status": "success"} + + request = GradingRequest( + submission_id=2, + grading_config_id=1, + template_name="static_analysis", + criteria_config={}, + setup_config={}, + feedback_config={}, + include_feedback=False, + language="python", + username="student", + external_user_id="u2", + submission_files={ + "main.py": { + "filename": "main.py", + "content": "x = 1", + "changed_lines": [1, 3, 5], + "file_metadata": {"change_status": "modified"}, + } + }, + ) + await grade_submission(request) + + sub = captured_submission["obj"] + main_file = sub.submission_files["main.py"] + assert main_file.changed_lines == {1, 3, 5} + assert main_file.metadata == {"change_status": "modified"} + assert main_file.is_contribution_aware is True + + @pytest.mark.asyncio + async def test_grading_service_evaluation_scope_none_when_absent(self): + """GradingRequest without evaluation_scope → submission.evaluation_scope is None.""" + mock_result = Mock() + mock_result.final_score = 100.0 + mock_result.feedback = None + mock_result.result_tree = None + mock_result.focus = Mock() + mock_result.focus.to_dict = Mock(return_value={"base": []}) + mock_result.comparison = None + + mock_execution = Mock() + mock_execution.result = mock_result + mock_execution.start_time = time.time() + mock_execution.step_results = [] + + mock_submission_repo = Mock() + mock_submission_repo.update_status = AsyncMock() + mock_submission_repo.update = AsyncMock() + + mock_result_repo = Mock() + mock_result_repo.create = AsyncMock() + + mock_session = AsyncMock() + mock_session.commit = AsyncMock() + + captured_submission = {} + + def capture_pipeline_run(submission): + captured_submission["obj"] = submission + return mock_execution + + with patch("web.service.grading_service.build_pipeline") as mock_build, \ + patch("web.service.grading_service.get_session") as mock_get_session, \ + patch("asyncio.to_thread", side_effect=lambda fn, sub: capture_pipeline_run(sub)), \ + patch("web.service.grading_service.SubmissionRepository", return_value=mock_submission_repo), \ + patch("web.service.grading_service.ResultRepository", return_value=mock_result_repo), \ + patch("web.service.grading_service.PipelineExecutionSerializer") as mock_serializer: + + mock_get_session.return_value.__aenter__.return_value = mock_session + mock_serializer.serialize.return_value = {"status": "success"} + + request = GradingRequest( + submission_id=3, + grading_config_id=1, + template_name="static_analysis", + criteria_config={}, + setup_config={}, + feedback_config={}, + include_feedback=False, + language="python", + username="student", + external_user_id="u3", + submission_files={ + "main.py": { + "filename": "main.py", + "content": "x = 1", + } + }, + evaluation_scope=None, + ) + await grade_submission(request) + + sub = captured_submission["obj"] + assert sub.evaluation_scope is None + main_file = sub.submission_files["main.py"] + assert main_file.changed_lines is None + assert main_file.is_contribution_aware is False + + @pytest.mark.asyncio + async def test_grading_service_changed_lines_none_when_absent(self): + """GradingRequest with file missing changed_lines key → SubmissionFile.changed_lines is None.""" + mock_result = Mock() + mock_result.final_score = 100.0 + mock_result.feedback = None + mock_result.result_tree = None + mock_result.focus = Mock() + mock_result.focus.to_dict = Mock(return_value={"base": []}) + mock_result.comparison = None + + mock_execution = Mock() + mock_execution.result = mock_result + mock_execution.start_time = time.time() + mock_execution.step_results = [] + + mock_submission_repo = Mock() + mock_submission_repo.update_status = AsyncMock() + mock_submission_repo.update = AsyncMock() + + mock_result_repo = Mock() + mock_result_repo.create = AsyncMock() + + mock_session = AsyncMock() + mock_session.commit = AsyncMock() + + captured_submission = {} + + def capture_pipeline_run(submission): + captured_submission["obj"] = submission + return mock_execution + + with patch("web.service.grading_service.build_pipeline") as mock_build, \ + patch("web.service.grading_service.get_session") as mock_get_session, \ + patch("asyncio.to_thread", side_effect=lambda fn, sub: capture_pipeline_run(sub)), \ + patch("web.service.grading_service.SubmissionRepository", return_value=mock_submission_repo), \ + patch("web.service.grading_service.ResultRepository", return_value=mock_result_repo), \ + patch("web.service.grading_service.PipelineExecutionSerializer") as mock_serializer: + + mock_get_session.return_value.__aenter__.return_value = mock_session + mock_serializer.serialize.return_value = {"status": "success"} + + # Simulate the minimal storage format (no changed_lines or file_metadata keys) + request = GradingRequest( + submission_id=4, + grading_config_id=1, + template_name="static_analysis", + criteria_config={}, + setup_config={}, + feedback_config={}, + include_feedback=False, + language="python", + username="student", + external_user_id="u4", + submission_files={ + "main.py": { + "filename": "main.py", + "content": "x = 1", + } + }, + ) + await grade_submission(request) + + sub = captured_submission["obj"] + main_file = sub.submission_files["main.py"] + assert main_file.changed_lines is None + assert main_file.metadata is None diff --git a/tests/unit/models/test_submission.py b/tests/unit/models/test_submission.py new file mode 100644 index 00000000..d4f39bce --- /dev/null +++ b/tests/unit/models/test_submission.py @@ -0,0 +1,44 @@ +"""Tests for submission evaluation context models.""" + +from autograder.models.dataclass.submission import ( + EvaluationScope, + Submission, + SubmissionFile, +) + + +def test_submission_file_context_is_optional(): + """Snapshot submissions retain their context-free defaults.""" + sub_file = SubmissionFile(filename="main.py", content="print('hello')") + + assert sub_file.changed_lines is None + assert sub_file.metadata is None + assert sub_file.is_contribution_aware is False + + +def test_submission_file_reports_changed_line_context(): + """Supplying even an empty changed-line set makes the file context-aware.""" + sub_file = SubmissionFile( + filename="main.py", + content="print('hello')", + changed_lines=set(), + metadata={"change_status": "modified"}, + ) + + assert sub_file.is_contribution_aware is True + assert sub_file.metadata == {"change_status": "modified"} + + +def test_submission_evaluation_scope_is_optional(): + """Evaluation scope is additive and available through the submission.""" + submission = Submission( + username="student", + user_id=1, + assignment_id=2, + submission_files={}, + ) + assert submission.evaluation_scope is None + + scope = EvaluationScope(scoped_files=["main.py"]) + submission.evaluation_scope = scope + assert submission.evaluation_scope is scope diff --git a/tests/unit/pipeline/test_ai_batch_step.py b/tests/unit/pipeline/test_ai_batch_step.py index b6c70717..299ecd36 100644 --- a/tests/unit/pipeline/test_ai_batch_step.py +++ b/tests/unit/pipeline/test_ai_batch_step.py @@ -23,7 +23,11 @@ ) from autograder.models.dataclass.param_description import ParamDescription from autograder.models.dataclass.step_result import StepName, StepResult, StepStatus -from autograder.models.dataclass.submission import Submission, SubmissionFile +from autograder.models.dataclass.submission import ( + EvaluationScope, + Submission, + SubmissionFile, +) from autograder.models.dataclass.test_result import TestResult from autograder.models.pipeline_execution import PipelineExecution from autograder.models.result_tree import CategoryResultNode, ResultTree, RootResultNode @@ -408,6 +412,8 @@ def test_pre_computed_results_passed_to_grader_service(self): """GradeStep forwards AI_BATCH pre_computed_results to GraderService.grade_from_tree.""" pre_computed = {"ai_code_review": TestResult("ai_code_review", 77, "good", "")} pipeline_exec = self._make_pipeline_with_tree_and_ai_batch(pre_computed) + scope = EvaluationScope(scoped_files=["main.py"]) + pipeline_exec.submission.evaluation_scope = scope captured = {} @@ -422,6 +428,7 @@ def _fake_grade_from_tree(**kwargs): GradeStep().execute(pipeline_exec) assert captured.get("pre_computed_results") is pre_computed + assert captured.get("evaluation_scope") is scope def test_pre_computed_results_is_none_when_no_ai_batch_step(self): """When AI_BATCH step was not in the pipeline, None must be passed.""" diff --git a/tests/unit/pipeline/test_pipeline_execution_accessors.py b/tests/unit/pipeline/test_pipeline_execution_accessors.py index ea437ef8..a220103d 100644 --- a/tests/unit/pipeline/test_pipeline_execution_accessors.py +++ b/tests/unit/pipeline/test_pipeline_execution_accessors.py @@ -2,7 +2,7 @@ from autograder.models.dataclass.focus import Focus from autograder.models.dataclass.grade_step_result import GradeStepResult from autograder.models.dataclass.step_result import StepName, StepResult, StepStatus -from autograder.models.dataclass.submission import Submission, SubmissionFile +from autograder.models.dataclass.submission import EvaluationScope, Submission, SubmissionFile from autograder.models.pipeline_execution import PipelineExecution from autograder.models.result_tree import CategoryResultNode, ResultTree, RootResultNode from autograder.template_library.input_output import InputOutputTemplate @@ -50,6 +50,14 @@ def test_ai_batch_results_accessor(): assert pipeline_exec.get_ai_batch_results() == {"some": "data"} +def test_evaluation_scope_accessor(): + scope = EvaluationScope(scoped_files=["main.py"]) + pipeline_exec = _build_pipeline_execution() + pipeline_exec.submission.evaluation_scope = scope + + assert pipeline_exec.evaluation_scope is scope + + def test_typed_accessors_raise_on_missing_required_artifacts(): pipeline_exec = _build_pipeline_execution() diff --git a/tests/unit/pipeline/test_structural_analysis_step.py b/tests/unit/pipeline/test_structural_analysis_step.py index 033271ee..023a301a 100644 --- a/tests/unit/pipeline/test_structural_analysis_step.py +++ b/tests/unit/pipeline/test_structural_analysis_step.py @@ -2,7 +2,7 @@ from unittest.mock import MagicMock, patch from autograder.steps.structural_analysis_step import StructuralAnalysisStep from autograder.models.pipeline_execution import PipelineExecution -from autograder.models.dataclass.submission import Submission, SubmissionFile +from autograder.models.dataclass.submission import EvaluationScope, Submission, SubmissionFile from autograder.models.dataclass.step_result import StepName, StepStatus from sandbox_manager.models.sandbox_models import Language @@ -41,10 +41,54 @@ def test_structural_analysis_step_execution_success(mock_sg_root, mock_pipeline_ assert "main.py" in step_result.data.roots assert "data.txt" not in step_result.data.roots # Heuristic should skip .txt assert step_result.data.roots["main.py"] == mock_root_instance + assert step_result.data.changed_lines == {} # Verify SgRoot called correctly mock_sg_root.assert_called_once_with("print('hello')", "python") + +@patch("autograder.steps.structural_analysis_step.SgRoot") +def test_structural_analysis_restricts_parsing_to_evaluation_scope( + mock_sg_root, + mock_pipeline_exec, +): + mock_pipeline_exec.submission.submission_files["helper.py"] = SubmissionFile( + filename="helper.py", + content="def helper(): return True", + changed_lines={1}, + ) + mock_pipeline_exec.submission.submission_files["main.py"].changed_lines = {1, 3} + mock_pipeline_exec.submission.evaluation_scope = EvaluationScope( + scoped_files=["helper.py"], + ) + + result_exec = StructuralAnalysisStep().execute(mock_pipeline_exec) + analysis = result_exec.get_step_result(StepName.STRUCTURAL_ANALYSIS).data + + assert set(analysis.roots) == {"helper.py"} + assert analysis.changed_lines == {"helper.py": {1}} + mock_sg_root.assert_called_once_with("def helper(): return True", "python") + + +@patch("autograder.steps.structural_analysis_step.SgRoot") +def test_structural_analysis_parses_all_code_files_without_scope( + mock_sg_root, + mock_pipeline_exec, +): + mock_pipeline_exec.submission.submission_files["helper.py"] = SubmissionFile( + filename="helper.py", + content="def helper(): return True", + changed_lines={1}, + ) + mock_pipeline_exec.submission.submission_files["main.py"].changed_lines = {1} + + result_exec = StructuralAnalysisStep().execute(mock_pipeline_exec) + analysis = result_exec.get_step_result(StepName.STRUCTURAL_ANALYSIS).data + + assert set(analysis.roots) == {"main.py", "helper.py"} + assert analysis.changed_lines == {"main.py": {1}, "helper.py": {1}} + assert mock_sg_root.call_count == 2 + @patch("autograder.steps.structural_analysis_step.SgRoot") def test_structural_analysis_step_parsing_failure(mock_sg_root, mock_pipeline_exec): # Setup mock to raise error for parsing diff --git a/tests/unit/services/grader/test_submission_grader.py b/tests/unit/services/grader/test_submission_grader.py index 31812dc7..98e64eff 100644 --- a/tests/unit/services/grader/test_submission_grader.py +++ b/tests/unit/services/grader/test_submission_grader.py @@ -4,6 +4,7 @@ from autograder.services.grader.criteria_grader import SubmissionGrader from autograder.models.criteria_tree import CategoryNode, SubjectNode, TestNode from autograder.models.abstract.test_function import TestFunction +from autograder.models.dataclass.submission import EvaluationScope, SubmissionFile from autograder.models.dataclass.test_result import TestResult class MockTestFunction(TestFunction): @@ -19,6 +20,19 @@ def parameter_description(self) -> list: def execute(self, files=None, sandbox=None, **kwargs): return TestResult(test_name="mock_test", score=kwargs.get('score', 100.0), report="OK") + +class CapturingTestFunction(MockTestFunction): + """Test function that records the pipeline context passed to execute.""" + + def __init__(self): + self.files = None + self.kwargs = {} + + def execute(self, files=None, sandbox=None, **kwargs): + self.files = files + self.kwargs = kwargs + return super().execute(files=files, sandbox=sandbox, **kwargs) + @pytest.fixture def grader(): command_resolver = MagicMock() @@ -96,8 +110,6 @@ def test_balance_nodes_zero_weights(grader): assert result.subjects[1].weight == 50.0 assert result.calculate_score() == 100.0 -from autograder.models.dataclass.submission import SubmissionFile - def test_balance_nodes_subjects_and_tests_missing_subjects_weight(grader): tf = MockTestFunction() s1 = SubjectNode(name="S1", weight=100, tests=[TestNode(name="T1", test_function=tf)]) @@ -163,3 +175,36 @@ def test_get_file_target_specific(): target_files = grader.get_file_target(t1) assert len(target_files) == 1 assert target_files[0] is file1 + + +def test_process_test_passes_scope_and_target_file_metadata(): + scope = EvaluationScope(scoped_files=["file1.py"]) + file1 = SubmissionFile( + filename="file1.py", + content="", + metadata={"change_status": "modified"}, + ) + file2 = SubmissionFile( + filename="file2.py", + content="", + metadata={"change_status": "added"}, + ) + grader = SubmissionGrader( + submission_files={"file1.py": file1, "file2.py": file2}, + command_resolver=MagicMock(), + evaluation_scope=scope, + ) + test_function = CapturingTestFunction() + test_node = TestNode( + name="T1", + test_function=test_function, + file_target=["file1.py"], + ) + + grader.process_test(test_node) + + assert test_function.files == [file1] + assert test_function.kwargs["evaluation_scope"] is scope + assert test_function.kwargs["file_metadata"] == { + "file1.py": {"change_status": "modified"}, + } diff --git a/tests/unit/test_file_metadata_passthrough.py b/tests/unit/test_file_metadata_passthrough.py new file mode 100644 index 00000000..71268de2 --- /dev/null +++ b/tests/unit/test_file_metadata_passthrough.py @@ -0,0 +1,152 @@ +"""Regression coverage for rich per-file metadata passthrough.""" + +from copy import deepcopy +from unittest.mock import Mock, patch + +import pytest + +from autograder.models.abstract.test_function import TestFunction +from autograder.models.criteria_tree import CategoryNode, CriteriaTree, TestNode +from autograder.models.dataclass.test_result import TestResult +from autograder.services.grader.grader_service import GraderService +from web.service.grading_service import GradingRequest, _run_pipeline + + +class MetadataCapturingTest(TestFunction): + """Test function that records the files and context received by the grader.""" + + def __init__(self): + self.files = None + self.kwargs = None + + @property + def name(self) -> str: + return "metadata_capture" + + @property + def description(self) -> str: + return "Capture opaque per-file metadata." + + @property + def parameter_description(self) -> list: + return [] + + def execute(self, files, sandbox, *args, **kwargs) -> TestResult: + self.files = files + self.kwargs = kwargs + return TestResult( + test_name=self.name, + score=100.0, + report="Metadata received.", + ) + + +@pytest.mark.asyncio +async def test_rich_repository_metadata_survives_hydration_and_grader_passthrough(): + """Nested repository context remains opaque, unchanged, and correctly targeted.""" + repository_file_metadata = { + "provider": "github", + "change_status": "modified", + "patch": "@@ -88,6 +88,12 @@ class PaymentService:", + "blob": { + "sha": "f00ba4", + "url": "https://example.invalid/blob/f00ba4", + }, + "stats": { + "additions": 12, + "deletions": 3, + "changes": 15, + }, + "review": { + "labels": ["backend", "security"], + "requested_reviewers": ["alice", "bob"], + "approved": False, + }, + "annotations": [ + {"line": 91, "kind": "security-sensitive"}, + {"line": 95, "kind": "new-branch"}, + ], + "optional_context": None, + } + other_file_metadata = { + "provider": "github", + "change_status": "added", + } + stored_submission_files = { + "service/payment.py": { + "filename": "service/payment.py", + "content": "class PaymentService:\n pass\n", + "changed_lines": [1, 2], + "file_metadata": repository_file_metadata, + }, + "README.md": { + "filename": "README.md", + "content": "# Payment service\n", + "changed_lines": [1], + "file_metadata": other_file_metadata, + }, + } + original_stored_data = deepcopy(stored_submission_files) + request = GradingRequest( + submission_id=10, + grading_config_id=20, + template_name="static_analysis", + criteria_config={"base": {}}, + setup_config={}, + feedback_config={}, + include_feedback=False, + language="python", + username="repository-user", + external_user_id="external-user", + submission_files=stored_submission_files, + evaluation_scope={"scoped_files": ["service/payment.py"]}, + ) + pipeline = Mock() + pipeline.run.side_effect = lambda submission: submission + + with patch( + "web.service.grading_service.build_pipeline", + return_value=pipeline, + ): + hydrated_submission = await _run_pipeline(request) + + capturing_test = MetadataCapturingTest() + criteria_tree = CriteriaTree( + base=CategoryNode( + name="base", + weight=100, + tests=[ + TestNode( + name="metadata_capture", + test_function=capturing_test, + file_target=["service/payment.py"], + ) + ], + ) + ) + + GraderService().grade_from_tree( + criteria_tree=criteria_tree, + submission_files=hydrated_submission.submission_files, + evaluation_scope=hydrated_submission.evaluation_scope, + ) + + assert stored_submission_files == original_stored_data + assert hydrated_submission.evaluation_scope.scoped_files == [ + "service/payment.py" + ] + + target_file = hydrated_submission.submission_files["service/payment.py"] + assert target_file.changed_lines == {1, 2} + assert target_file.metadata == repository_file_metadata + assert target_file.metadata is repository_file_metadata + + assert capturing_test.files == [target_file] + assert capturing_test.kwargs["evaluation_scope"] is hydrated_submission.evaluation_scope + assert capturing_test.kwargs["file_metadata"] == { + "service/payment.py": repository_file_metadata, + } + assert capturing_test.kwargs["file_metadata"]["service/payment.py"] is ( + repository_file_metadata + ) + assert "README.md" not in capturing_test.kwargs["file_metadata"] diff --git a/tests/unit/test_language_validation.py b/tests/unit/test_language_validation.py index 8c20a6cc..c074e9f8 100644 --- a/tests/unit/test_language_validation.py +++ b/tests/unit/test_language_validation.py @@ -4,7 +4,11 @@ from pydantic import ValidationError from web.schemas.assignment import GradingConfigCreate, GradingConfigUpdate -from web.schemas.submission import SubmissionCreate, SubmissionFileData +from web.schemas.submission import ( + EvaluationScopeData, + SubmissionCreate, + SubmissionFileData, +) class TestLanguageValidation: @@ -140,3 +144,23 @@ def test_submission_create_case_insensitive(self): ) assert submission.language == "python" + def test_submission_create_accepts_evaluation_context(self): + """Submission schemas accept typed scope and per-file context.""" + submission = SubmissionCreate( + external_assignment_id="test-001", + external_user_id="user-001", + username="testuser", + files=[ + SubmissionFileData( + filename="test.py", + content="print('hello')", + changed_lines=[1], + file_metadata={"change_status": "modified"}, + ) + ], + evaluation_scope=EvaluationScopeData(scoped_files=["test.py"]), + ) + + assert submission.files[0].changed_lines == [1] + assert submission.files[0].file_metadata == {"change_status": "modified"} + assert submission.evaluation_scope.scoped_files == ["test.py"] diff --git a/tests/web/test_grading_service.py b/tests/web/test_grading_service.py index b9eff52f..8e06be16 100644 --- a/tests/web/test_grading_service.py +++ b/tests/web/test_grading_service.py @@ -4,7 +4,12 @@ import time from unittest.mock import Mock, AsyncMock, patch -from web.service.grading_service import grade_submission, _node_to_dict +from web.service.grading_service import ( + GradingRequest, + _node_to_dict, + _run_pipeline, + grade_submission, +) from web.database.models.submission import SubmissionStatus from web.database.models.submission_result import PipelineStatus @@ -169,3 +174,44 @@ def test_node_to_dict(): result = _node_to_dict(mock_nodes) assert result == [{"id": 1}, {"id": 2}] + +@pytest.mark.asyncio +async def test_run_pipeline_hydrates_evaluation_context(): + """Stored API data is reconstructed as typed core evaluation context.""" + mock_pipeline = Mock() + mock_pipeline.run.return_value = Mock() + request = GradingRequest( + submission_id=3, + grading_config_id=5, + template_name="static_analysis", + criteria_config={"base": {}}, + setup_config={}, + feedback_config={}, + include_feedback=False, + language="python", + username="student", + external_user_id="user-003", + submission_files={ + "main.py": { + "filename": "main.py", + "content": "print('hello')", + "changed_lines": [1, 3], + "file_metadata": {"change_status": "modified"}, + } + }, + evaluation_scope={"scoped_files": ["main.py"]}, + ) + + with patch( + "web.service.grading_service.build_pipeline", + return_value=mock_pipeline, + ): + result = await _run_pipeline(request) + + assert result is mock_pipeline.run.return_value + core_submission = mock_pipeline.run.call_args.args[0] + assert core_submission.evaluation_scope.scoped_files == ["main.py"] + assert core_submission.submission_files["main.py"].changed_lines == {1, 3} + assert core_submission.submission_files["main.py"].metadata == { + "change_status": "modified", + } diff --git a/tests/web/test_routes.py b/tests/web/test_routes.py index c72d46b9..34ea0cb9 100644 --- a/tests/web/test_routes.py +++ b/tests/web/test_routes.py @@ -295,6 +295,54 @@ async def test_create_submission(self, client): assert data["username"] == "student1" assert data["status"] == "pending" + @pytest.mark.asyncio + async def test_create_submission_persists_evaluation_context(self, client): + """Scope, changed lines, and file metadata reach the grading request.""" + config_data = { + "external_assignment_id": "submit-scope-test", + "template_name": "input_output", + "languages": ["python"], + "criteria_config": {"base": {}}, + } + await client.post("/api/v1/configs", json=config_data) + + mock_grading_tasks = set() + with patch( + "web.api.v1.submissions.grade_submission", + new_callable=AsyncMock, + ) as mock_grade, patch( + "web.api.v1.submissions.get_grading_tasks", + return_value=mock_grading_tasks, + ): + response = await client.post( + "/api/v1/submissions", + json={ + "external_assignment_id": "submit-scope-test", + "external_user_id": "user_scope", + "username": "student_scope", + "files": [ + { + "filename": "main.py", + "content": "print('hello')", + "changed_lines": [1], + "file_metadata": {"change_status": "modified"}, + } + ], + "evaluation_scope": {"scoped_files": ["main.py"]}, + }, + ) + + assert response.status_code == 200 + mock_grade.assert_called_once() + grading_request = mock_grade.call_args.args[0] + assert grading_request.submission_files["main.py"]["changed_lines"] == [1] + assert grading_request.submission_files["main.py"]["file_metadata"] == { + "change_status": "modified", + } + assert grading_request.evaluation_scope == { + "scoped_files": ["main.py"], + } + @pytest.mark.asyncio async def test_create_submission_with_language(self, client): @@ -421,4 +469,3 @@ async def test_get_user_submissions(self, client): assert isinstance(data, list) assert len(data) >= 3 assert all(s["external_user_id"] == "user_multi" for s in data) - diff --git a/web/api/v1/submissions.py b/web/api/v1/submissions.py index d1cfcc64..fb7c5290 100644 --- a/web/api/v1/submissions.py +++ b/web/api/v1/submissions.py @@ -90,7 +90,12 @@ async def create_submission( # Convert list of SubmissionFileData to dict format for storage and quick access # This indexing by filename allows O(1) file lookups during grading submission_files_dict = { - file_data.filename: {"filename": file_data.filename, "content": file_data.content} + file_data.filename: { + "filename": file_data.filename, + "content": file_data.content, + "changed_lines": file_data.changed_lines, + "file_metadata": file_data.file_metadata, + } for file_data in submission.files } @@ -126,6 +131,11 @@ async def create_submission( submission_files=db_submission.submission_files, locale=submission.locale, baseline_result_tree=submission.baseline_result_tree, + evaluation_scope=( + submission.evaluation_scope.model_dump() + if submission.evaluation_scope is not None + else None + ), ) task = asyncio.create_task(grade_submission(grading_request)) diff --git a/web/schemas/__init__.py b/web/schemas/__init__.py index 3385ee03..987fd2e5 100644 --- a/web/schemas/__init__.py +++ b/web/schemas/__init__.py @@ -11,6 +11,7 @@ SubmissionDetailResponse, SubmissionStatus, SubmissionFileData, + EvaluationScopeData, ExternalResultCreate, ExternalResultResponse, ) @@ -28,6 +29,7 @@ "SubmissionDetailResponse", "SubmissionStatus", "SubmissionFileData", + "EvaluationScopeData", "ExternalResultCreate", "ExternalResultResponse", "DeliberateCodeExecutionRequest", diff --git a/web/schemas/submission.py b/web/schemas/submission.py index a44a9e64..7c3bafe4 100644 --- a/web/schemas/submission.py +++ b/web/schemas/submission.py @@ -21,6 +21,23 @@ class SubmissionFileData(BaseModel): """Schema for a submission file.""" filename: str = Field(..., description="Name of the file") content: str = Field(..., description="Content of the file") + changed_lines: Optional[List[int]] = Field( + None, + description="One-indexed line numbers added or modified in this file", + ) + file_metadata: Optional[Dict[str, Any]] = Field( + None, + description="Optional opaque metadata for this file", + ) + + +class EvaluationScopeData(BaseModel): + """Schema defining the files that are the primary evaluation subject.""" + + scoped_files: List[str] = Field( + ..., + description="Filenames to include in scope-aware pipeline analysis", + ) class TestDeltaResponse(BaseModel): @@ -48,6 +65,10 @@ class SubmissionCreate(BaseModel): language: Optional[str] = Field(None, description="Optional language override") locale: Optional[str] = Field("en", description="Optional locale for feedback (e.g., 'en', 'pt_br')") metadata: Optional[Dict[str, Any]] = Field(None, description="Optional submission metadata") + evaluation_scope: Optional[EvaluationScopeData] = Field( + None, + description="Optional file scope for pipeline analysis", + ) baseline_result_tree: Optional[Dict[str, Any]] = Field( None, description=( diff --git a/web/service/grading_service.py b/web/service/grading_service.py index fc5178da..55913f8b 100644 --- a/web/service/grading_service.py +++ b/web/service/grading_service.py @@ -7,7 +7,11 @@ from typing import Optional from autograder.autograder import build_pipeline -from autograder.models.dataclass.submission import Submission as AutograderSubmission, SubmissionFile +from autograder.models.dataclass.submission import ( + EvaluationScope, + Submission as AutograderSubmission, + SubmissionFile, +) from autograder.models.result_tree import ResultTree from autograder.services.result_comparator import ResultComparator from autograder.utils.feedback_generator import generate_preflight_feedback @@ -39,6 +43,7 @@ class GradingRequest: submission_files: dict locale: str = "en" baseline_result_tree: Optional[dict] = None + evaluation_scope: Optional[dict] = None async def grade_submission(request: GradingRequest) -> None: @@ -116,9 +121,23 @@ async def _run_pipeline(request: GradingRequest): ) files_to_grade = { - name: SubmissionFile(filename=f["filename"], content=f["content"]) + name: SubmissionFile( + filename=f["filename"], + content=f["content"], + changed_lines=( + set(f["changed_lines"]) + if f.get("changed_lines") is not None + else None + ), + metadata=f.get("file_metadata"), + ) for name, f in request.submission_files.items() } + evaluation_scope = ( + EvaluationScope(**request.evaluation_scope) + if request.evaluation_scope is not None + else None + ) autograder_submission = AutograderSubmission( username=request.username, @@ -127,6 +146,7 @@ async def _run_pipeline(request: GradingRequest): submission_files=files_to_grade, language=Language[request.language.upper()] if request.language else None, locale=request.locale, + evaluation_scope=evaluation_scope, ) return await asyncio.to_thread(pipeline.run, autograder_submission)