From 3f64db36dac291956215672792efaeea4b51718d Mon Sep 17 00:00:00 2001 From: appleweiping Date: Sun, 20 Sep 2026 01:54:46 -0500 Subject: [PATCH] Add bounded capacity and conflict what-if reports Signed-off-by: appleweiping --- .github/workflows/ci.yml | 6 + CHANGELOG.md | 7 + CITATION.cff | 4 +- README.md | 8 +- docs/what-if.md | 42 +++++ examples/what-if-scenarios.json | 23 +++ pyproject.toml | 2 +- src/peermatchlab/__init__.py | 6 +- src/peermatchlab/cli.py | 66 ++++++++ src/peermatchlab/io.py | 14 +- src/peermatchlab/what_if.py | 284 ++++++++++++++++++++++++++++++++ tests/test_what_if.py | 247 +++++++++++++++++++++++++++ uv.lock | 2 +- 13 files changed, 703 insertions(+), 8 deletions(-) create mode 100644 docs/what-if.md create mode 100644 examples/what-if-scenarios.json create mode 100644 src/peermatchlab/what_if.py create mode 100644 tests/test_what_if.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f72f7e2..737b0a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,6 +68,11 @@ jobs: --experts examples/experts.json --conflicts examples/conflicts.json --affinities examples/affinities.csv --config examples/fair-local-config.json --output fair-local-smoke.json + - run: >- + uv run --locked peermatch what-if --documents examples/documents.json + --experts examples/experts.json --conflicts examples/conflicts.json + --config examples/config.json --scenarios examples/what-if-scenarios.json + --output what-if-smoke.json - run: uv run --locked python -m build --no-isolation - name: Verify the wheel in a clean environment shell: bash @@ -76,6 +81,7 @@ jobs: .ci-wheel-check/bin/python -m pip install --disable-pip-version-check --no-deps dist/*.whl .ci-wheel-check/bin/python -c "import peermatchlab; print(peermatchlab.__version__)" .ci-wheel-check/bin/python -c "from peermatchlab.keyphrases import KeyphraseConfig, rank_keyphrases; assert rank_keyphrases('alpha beta', config=KeyphraseConfig())" + .ci-wheel-check/bin/python -c "from peermatchlab.what_if import WhatIfPlan, compare_what_if; assert WhatIfPlan and compare_what_if" .ci-wheel-check/bin/peermatch --help tests: diff --git a/CHANGELOG.md b/CHANGELOG.md index 5415f53..979f8cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ All notable changes are documented here. The project follows semantic versioning _No changes yet._ +## 0.15.0 - 2026-09-20 + +- Added bounded, deterministic local capacity/conflict what-if comparisons with + non-cumulative scenarios, independent audits, signed assignment/workload + deltas, strict plan/reference validation, a CLI example, and no input mutation. + This is not a causal constraint diagnosis or an interactive web UI. + ## 0.14.0 - 2026-09-20 - Added opt-in `fair-local` assignment: a maximum-cardinality flow baseline diff --git a/CITATION.cff b/CITATION.cff index 51795d8..45c16bc 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -6,8 +6,8 @@ authors: - family-names: "Yan" given-names: "Weiping" alias: "appleweiping" -version: 0.14.0 -date-released: 2026-09-19 +version: 0.15.0 +date-released: 2026-09-20 license: MIT repository-code: "https://github.com/appleweiping/PeerMatchLab" url: "https://github.com/appleweiping/PeerMatchLab" diff --git a/README.md b/README.md index 244cc65..30c074c 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ Most matching prototypes stop after computing pairwise similarity. Real allocati - Per-document demand overrides and minimum acceptable score thresholds. - Machine-readable unmet-demand diagnostics for conflicts, zero capacity, score filtering, sparse affinities, senior reservations, institution gates, and global capacity coupling. +- Bounded [capacity/conflict what-if comparisons](docs/what-if.md) that rerun and audit + independent scenarios without changing baseline inputs. - Independent checks for conflict, capacity, demand, references, duplicate assignments, coverage, workload inequality, and institution duplication. - Strict JSON and JSONL adapters with unknown-field and duplicate-ID rejection. @@ -42,7 +44,7 @@ Most matching prototypes stop after computing pairwise similarity. Real allocati `Retry-After` handling, proactive request pacing, injectable transport, and strict response schemas. - Opt-in OpenReview reviewer profile/publication acquisition with exact author-ID joins, invitation/date/content filters, privacy-minimized snapshots, and replayable file hashes. -- `validate`, `score`, `match`, `match-affinity`, `audit`, `import-openreview`, `fetch-openreview`, +- `validate`, `score`, `match`, `match-affinity`, `what-if`, `audit`, `import-openreview`, `fetch-openreview`, and `expertise` CLI commands. - Stable JSON output suitable for review, version control, and downstream systems. @@ -134,6 +136,7 @@ The public modules have intentionally narrow responsibilities: | `openreview` | Loss-aware conversion of local OpenReview exports | | `openreview_api` | Bounded OpenReview API v2 synchronization and evidence manifests | | `pipeline` | One-call score → assign → audit orchestration | +| `what_if` | Bounded independent capacity/conflict scenario comparison | | `cli` | Reproducible command-line workflows | | `report` | Portable, script-free HTML review dashboard | @@ -484,7 +487,8 @@ checksum, and build-provenance guarantees. - Optional calibrated embedding adapters without making a hosted service mandatory. - Group constraints beyond institution diversity and senior coverage, where they can be expressed exactly rather than approximated. -- Interactive what-if reports for capacity and conflict changes. +- An interactive what-if UI and broader constraint scenarios; the bounded local + [capacity/conflict comparison](docs/what-if.md) is available from the CLI. - Import/export adapters for common review-management schemas. ## License diff --git a/docs/what-if.md b/docs/what-if.md new file mode 100644 index 0000000..7f89510 --- /dev/null +++ b/docs/what-if.md @@ -0,0 +1,42 @@ +# Bounded capacity and conflict what-if comparisons + +`peermatch what-if` compares a baseline match with 1–8 independent local +scenarios. A scenario can change expert capacities or explicitly add/remove +declared document–expert conflicts. Every scenario starts from the same +unchanged baseline, runs the existing matcher, and undergoes the independent +assignment audit. The JSON report includes assigned and unmet demand, status, +scores, assignments, per-expert workload, audit and unmet reason codes, plus +signed deltas and added/removed assignment pairs. + +```sh +peermatch what-if \ + --documents examples/documents.json \ + --experts examples/experts.json \ + --conflicts examples/conflicts.json \ + --config examples/config.json \ + --scenarios examples/what-if-scenarios.json \ + --output what-if-report.json +``` + +The schema-1 plan has a `scenarios` array. Each scenario requires a unique, +short lowercase `id` and all three change fields: `capacity_overrides` maps +existing expert IDs to integer capacities from 0 to 128; `add_conflicts` and +`remove_conflicts` are arrays of `[document_id, expert_id]` pairs. Adding an +existing conflict or removing a nonexistent one is an error. A scenario must +have at least one change. Scenario IDs are evaluated and reported in sorted +order, regardless of plan order. Conflict changes affect *only* the named +scenario. The source files and Python domain objects are never modified. + +The local comparison is limited to 32 documents, 64 experts, 128 requested +slots, 2,048 conflicts, 8 scenarios, 1 MiB per document/expert/conflict CLI +input, and 64 KiB per plan/config CLI input. The output path cannot alias an +input path, including through symlinks and hard links. It is an ordinary +replaceable local JSON output, not a signed or create-only registry record. + +This is a deterministic sensitivity *observation*, not a causal attribution +or proof that a particular constraint is the unique cause of unmet demand. +Changing capacity or conflict declarations can alter assignments globally. +The tool neither infers real-world conflicts nor recommends removing them; +human operators must verify the declarations and approve any actual change. +It does not implement an interactive web UI, OpenReview write-back, or the +frozen Matcher solver families still listed as gaps in the capability map. diff --git a/examples/what-if-scenarios.json b/examples/what-if-scenarios.json new file mode 100644 index 0000000..930dd5a --- /dev/null +++ b/examples/what-if-scenarios.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "scenarios": [ + { + "id": "capacity-drop", + "capacity_overrides": {"expert-a": 0}, + "add_conflicts": [], + "remove_conflicts": [] + }, + { + "id": "new-conflict", + "capacity_overrides": {}, + "add_conflicts": [["paper-search", "expert-a"]], + "remove_conflicts": [] + }, + { + "id": "remove-declared-conflict", + "capacity_overrides": {}, + "add_conflicts": [], + "remove_conflicts": [["paper-search", "expert-d"]] + } + ] +} diff --git a/pyproject.toml b/pyproject.toml index 34b597e..7145c20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "peermatchlab" -version = "0.14.0" +version = "0.15.0" description = "Transparent, constraint-aware expert matching for offline workflows" readme = "README.md" requires-python = ">=3.11" diff --git a/src/peermatchlab/__init__.py b/src/peermatchlab/__init__.py index 67cbe7e..abfdfae 100644 --- a/src/peermatchlab/__init__.py +++ b/src/peermatchlab/__init__.py @@ -87,6 +87,7 @@ write_publish_artifact, ) from peermatchlab.scoring import MatchScorer, ScoreWeights +from peermatchlab.what_if import WhatIfPlan, WhatIfScenario, compare_what_if __all__ = [ "Affinity", @@ -138,7 +139,10 @@ "ScoreWeights", "TermContribution", "UnmetReason", + "WhatIfPlan", + "WhatIfScenario", "build_expertise_corpus", + "compare_what_if", "embedding_requests", "extract_keyphrases", "fetch_openreview_snapshot", @@ -166,4 +170,4 @@ "write_publish_artifact", ] -__version__ = "0.14.0" +__version__ = "0.15.0" diff --git a/src/peermatchlab/cli.py b/src/peermatchlab/cli.py index f8a20f5..2467515 100644 --- a/src/peermatchlab/cli.py +++ b/src/peermatchlab/cli.py @@ -28,8 +28,11 @@ from peermatchlab.gold_evaluation import GoldEvaluationConfig, evaluate_gold_files from peermatchlab.io import ( load_conflicts, + load_conflicts_text, load_documents, + load_documents_text, load_experts, + load_experts_text, load_json_text, plan_from_dict, plan_to_dict, @@ -51,6 +54,7 @@ ) from peermatchlab.pipeline import MatchRun, run_affinity_matching, run_matching from peermatchlab.report import write_html +from peermatchlab.what_if import WhatIfPlan, compare_what_if def _inputs(parser: argparse.ArgumentParser) -> None: @@ -84,6 +88,14 @@ def build_parser() -> argparse.ArgumentParser: match.add_argument("--scores", help="optional full score-matrix JSON path") match.add_argument("--html", help="optional self-contained HTML report path") + what_if = commands.add_parser( + "what-if", help="compare bounded non-cumulative capacity/conflict scenarios" + ) + _inputs(what_if) + what_if.add_argument("--config", help="optional JSON run configuration") + what_if.add_argument("--scenarios", required=True, help="schema-1 scenario plan JSON") + what_if.add_argument("--output", required=True, help="comparison JSON path") + affinity = commands.add_parser("match-affinity", help="assign a sparse external affinity CSV") _inputs(affinity) affinity.add_argument( @@ -249,6 +261,58 @@ def _config(path: str | None) -> MatchConfig: return MatchConfig.from_json(path) if path else MatchConfig() +def _bounded_text(path: str, maximum: int, label: str) -> str: + with Path(path).open("rb") as stream: + source = stream.read(maximum + 1) + if len(source) > maximum: + raise DataValidationError(f"{label} exceeds {maximum} bytes") + try: + return source.decode("utf-8") + except UnicodeDecodeError as error: + raise DataValidationError(f"{label} must be UTF-8") from error + + +def _what_if(args: argparse.Namespace) -> int: + inputs = [args.documents, args.experts, args.scenarios] + if args.conflicts: + inputs.append(args.conflicts) + if args.config: + inputs.append(args.config) + if any(_same_file(args.output, source) for source in inputs): + raise DataValidationError("output path must not refer to an input") + documents = load_documents_text( + _bounded_text(args.documents, 1024 * 1024, "documents"), + source=args.documents, + max_records=32, + ) + experts = load_experts_text( + _bounded_text(args.experts, 1024 * 1024, "experts"), + source=args.experts, + max_records=64, + ) + conflicts = ( + load_conflicts_text( + _bounded_text(args.conflicts, 1024 * 1024, "conflicts"), + source=args.conflicts, + max_records=2048, + ) + if args.conflicts + else () + ) + config = MatchConfig() + if args.config: + raw_config = load_json_text(_bounded_text(args.config, 64 * 1024, "config")) + if not isinstance(raw_config, dict): + raise DataValidationError("what-if config must be a JSON object") + config = MatchConfig.from_mapping(raw_config) + with Path(args.scenarios).open("rb") as stream: + plan = WhatIfPlan.from_bytes(stream.read(64 * 1024 + 1)) + result = compare_what_if(documents, experts, plan, conflicts=conflicts, config=config) + write_json(args.output, result) + print(f"compared {len(plan.scenarios)} scenarios to {args.output}") + return 0 + + def _import_openreview(args: argparse.Namespace) -> int: imported_documents = load_openreview_submissions( args.submissions, @@ -521,6 +585,8 @@ def main(argv: Sequence[str] | None = None) -> int: args = build_parser().parse_args(argv) try: + if args.command == "what-if": + return _what_if(args) if args.command == "import-openreview": return _import_openreview(args) if args.command == "fetch-openreview": diff --git a/src/peermatchlab/io.py b/src/peermatchlab/io.py index 337ca2a..70bc6b6 100644 --- a/src/peermatchlab/io.py +++ b/src/peermatchlab/io.py @@ -381,8 +381,20 @@ def load_conflicts(path: str | Path | None) -> tuple[Conflict, ...]: if path is None: return () + return _conflicts_from_records(_records(path)) + + +def load_conflicts_text( + text: str, *, source: str = "conflict input", max_records: int | None = None +) -> tuple[Conflict, ...]: + """Validate conflicts from an already bounded, decoded byte snapshot.""" + + return _conflicts_from_records(_records_from_text(text, source, max_records=max_records)) + + +def _conflicts_from_records(rows: Iterable[Mapping[str, Any]]) -> tuple[Conflict, ...]: result: list[Conflict] = [] - for row in _records(path): + for row in rows: unknown = set(row) - {"document_id", "expert_id", "reason"} if unknown: raise DataValidationError(f"unknown conflict fields: {sorted(unknown)}") diff --git a/src/peermatchlab/what_if.py b/src/peermatchlab/what_if.py new file mode 100644 index 0000000..60beb65 --- /dev/null +++ b/src/peermatchlab/what_if.py @@ -0,0 +1,284 @@ +"""Bounded, independent capacity/conflict scenario comparisons for local matching.""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, replace +from typing import Any + +from peermatchlab.config import MatchConfig +from peermatchlab.io import load_json_text +from peermatchlab.models import Conflict, DataValidationError, Document, Expert +from peermatchlab.pipeline import MatchRun, run_matching + +MAX_PLAN_BYTES = 64 * 1024 +MAX_DOCUMENTS = 32 +MAX_EXPERTS = 64 +MAX_SCENARIOS = 8 +MAX_REQUESTED = 128 +_SCENARIO_ID = re.compile(r"[a-z][a-z0-9_-]{0,31}\Z") +Pair = tuple[str, str] + + +def _pairs(value: object, label: str) -> tuple[Pair, ...]: + if not isinstance(value, list) or len(value) > MAX_DOCUMENTS * MAX_EXPERTS: + raise DataValidationError(f"{label} must be a bounded array") + result: list[Pair] = [] + for row in value: + if ( + not isinstance(row, list) + or len(row) != 2 + or any(not isinstance(item, str) or not item or item != item.strip() for item in row) + ): + raise DataValidationError(f"{label} must contain [document_id, expert_id] pairs") + result.append((row[0], row[1])) + if len(set(result)) != len(result): + raise DataValidationError(f"{label} contains duplicate pairs") + return tuple(sorted(result)) + + +@dataclass(frozen=True, slots=True) +class WhatIfScenario: + id: str + capacity_overrides: tuple[tuple[str, int], ...] + add_conflicts: tuple[Pair, ...] + remove_conflicts: tuple[Pair, ...] + + def __post_init__(self) -> None: + if type(self.id) is not str or _SCENARIO_ID.fullmatch(self.id) is None: + raise DataValidationError("scenario id must be short lowercase ASCII") + if ( + type(self.capacity_overrides) is not tuple + or len(self.capacity_overrides) > MAX_EXPERTS + or any( + not isinstance(row, tuple) + or len(row) != 2 + or not isinstance(row[0], str) + or not row[0] + or row[0] != row[0].strip() + or type(row[1]) is not int + or not 0 <= row[1] <= MAX_REQUESTED + for row in self.capacity_overrides + ) + or len({row[0] for row in self.capacity_overrides}) != len(self.capacity_overrides) + ): + raise DataValidationError("scenario capacity overrides are invalid") + for label, pairs in ( + ("add_conflicts", self.add_conflicts), + ("remove_conflicts", self.remove_conflicts), + ): + if ( + type(pairs) is not tuple + or len(pairs) > MAX_DOCUMENTS * MAX_EXPERTS + or any( + not isinstance(pair, tuple) + or len(pair) != 2 + or any( + not isinstance(item, str) or not item or item != item.strip() + for item in pair + ) + for pair in pairs + ) + or len(set(pairs)) != len(pairs) + ): + raise DataValidationError(f"scenario {label} pairs are invalid") + if set(self.add_conflicts) & set(self.remove_conflicts): + raise DataValidationError("the same conflict cannot be added and removed") + if not self.capacity_overrides and not self.add_conflicts and not self.remove_conflicts: + raise DataValidationError("scenario must change capacity or conflicts") + object.__setattr__(self, "capacity_overrides", tuple(sorted(self.capacity_overrides))) + object.__setattr__(self, "add_conflicts", tuple(sorted(self.add_conflicts))) + object.__setattr__(self, "remove_conflicts", tuple(sorted(self.remove_conflicts))) + + @classmethod + def from_mapping(cls, value: object) -> WhatIfScenario: + if not isinstance(value, dict) or set(value) != { + "id", + "capacity_overrides", + "add_conflicts", + "remove_conflicts", + }: + raise DataValidationError("scenario has missing or unknown fields") + ident = value["id"] + if type(ident) is not str or _SCENARIO_ID.fullmatch(ident) is None: + raise DataValidationError("scenario id must be short lowercase ASCII") + overrides = value["capacity_overrides"] + if not isinstance(overrides, dict) or len(overrides) > MAX_EXPERTS: + raise DataValidationError("capacity_overrides must be a bounded object") + if any( + not isinstance(key, str) + or not key + or key != key.strip() + or type(count) is not int + or not 0 <= count <= MAX_REQUESTED + for key, count in overrides.items() + ): + raise DataValidationError("capacity override must map expert ids to bounded integers") + added = _pairs(value["add_conflicts"], "add_conflicts") + removed = _pairs(value["remove_conflicts"], "remove_conflicts") + if set(added) & set(removed): + raise DataValidationError("the same conflict cannot be added and removed") + if not overrides and not added and not removed: + raise DataValidationError("scenario must change capacity or conflicts") + return cls(ident, tuple(sorted(overrides.items())), added, removed) + + def to_dict(self) -> dict[str, object]: + return { + "id": self.id, + "capacity_overrides": dict(self.capacity_overrides), + "add_conflicts": [list(pair) for pair in self.add_conflicts], + "remove_conflicts": [list(pair) for pair in self.remove_conflicts], + } + + +@dataclass(frozen=True, slots=True) +class WhatIfPlan: + scenarios: tuple[WhatIfScenario, ...] + + def __post_init__(self) -> None: + if ( + type(self.scenarios) is not tuple + or not 1 <= len(self.scenarios) <= MAX_SCENARIOS + or any(not isinstance(row, WhatIfScenario) for row in self.scenarios) + or len({row.id for row in self.scenarios}) != len(self.scenarios) + ): + raise DataValidationError("what-if plan requires 1 to 8 unique scenarios") + object.__setattr__(self, "scenarios", tuple(sorted(self.scenarios, key=lambda row: row.id))) + + @classmethod + def from_bytes(cls, source: bytes) -> WhatIfPlan: + if type(source) is not bytes or len(source) > MAX_PLAN_BYTES: + raise DataValidationError("scenario plan must be a bounded immutable byte snapshot") + try: + raw = load_json_text(source.decode("utf-8")) + except UnicodeDecodeError as error: + raise DataValidationError("scenario plan must be UTF-8") from error + except json.JSONDecodeError as error: + raise DataValidationError("scenario plan must be valid JSON") from error + if not isinstance(raw, dict) or set(raw) != {"schema_version", "scenarios"}: + raise DataValidationError("scenario plan has missing or unknown fields") + if type(raw["schema_version"]) is not int or raw["schema_version"] != 1: + raise DataValidationError("scenario plan requires schema_version 1") + rows = raw["scenarios"] + if not isinstance(rows, list) or not 1 <= len(rows) <= MAX_SCENARIOS: + raise DataValidationError("scenario plan requires 1 to 8 scenarios") + scenarios = tuple(WhatIfScenario.from_mapping(row) for row in rows) + if len({item.id for item in scenarios}) != len(scenarios): + raise DataValidationError("scenario ids must be unique") + return cls(tuple(sorted(scenarios, key=lambda item: item.id))) + + +def _summary(run: MatchRun) -> dict[str, Any]: + diagnostics = run.plan.diagnostics + if diagnostics is None: + raise DataValidationError("generated plan lacks demand diagnostics") + return { + "assigned": diagnostics.assigned, + "unmet": diagnostics.unmet, + "status": diagnostics.status.value, + "certified": diagnostics.certified, + "total_score": run.plan.total_score, + "assignments": sorted([[row.document_id, row.expert_id] for row in run.plan.assignments]), + "workload": dict(sorted(run.audit.workload.items())), + "audit": run.audit.as_dict(), + "safe": run.audit.safe, + "reasons": { + item.document_id: [code.value for code in item.reason_codes] + for item in diagnostics.documents + if item.unmet + }, + } + + +def _delta(base: dict[str, Any], result: dict[str, Any]) -> dict[str, object]: + original = {tuple(pair) for pair in base["assignments"]} + changed = {tuple(pair) for pair in result["assignments"]} + return { + "assigned": result["assigned"] - base["assigned"], + "unmet": result["unmet"] - base["unmet"], + "total_score": result["total_score"] - base["total_score"], + "added_assignments": [list(pair) for pair in sorted(changed - original)], + "removed_assignments": [list(pair) for pair in sorted(original - changed)], + "workload": { + ident: result["workload"][ident] - count for ident, count in base["workload"].items() + }, + } + + +def compare_what_if( + documents: tuple[Document, ...], + experts: tuple[Expert, ...], + plan: WhatIfPlan, + *, + conflicts: tuple[Conflict, ...] = (), + config: MatchConfig | None = None, +) -> dict[str, object]: + """Compare non-cumulative scenarios against one independently audited baseline.""" + if not isinstance(plan, WhatIfPlan) or not isinstance(config, (MatchConfig, type(None))): + raise DataValidationError("invalid what-if plan or match configuration") + if ( + not 1 <= len(documents) <= MAX_DOCUMENTS + or not 1 <= len(experts) <= MAX_EXPERTS + or len(documents) * len(experts) > MAX_DOCUMENTS * MAX_EXPERTS + or len(conflicts) > MAX_DOCUMENTS * MAX_EXPERTS + ): + raise DataValidationError("what-if domain exceeds document/expert/conflict limits") + if ( + any(not isinstance(item, Document) for item in documents) + or any(not isinstance(item, Expert) for item in experts) + or any(not isinstance(item, Conflict) for item in conflicts) + ): + raise DataValidationError("what-if inputs have invalid domain objects") + document_ids = {item.id for item in documents} + expert_ids = {item.id for item in experts} + if len(document_ids) != len(documents) or len(expert_ids) != len(experts): + raise DataValidationError("what-if document and expert ids must be unique") + selected = config or MatchConfig() + demand = sum(item.required_experts or selected.reviewers_per_document for item in documents) + if demand > MAX_REQUESTED: + raise DataValidationError("what-if requested assignment count exceeds limit") + baseline_pairs = {(item.document_id, item.expert_id) for item in conflicts} + if any(d not in document_ids or e not in expert_ids for d, e in baseline_pairs): + raise DataValidationError("baseline conflict references unknown document or expert") + for scenario in plan.scenarios: + if any(ident not in expert_ids for ident, _ in scenario.capacity_overrides): + raise DataValidationError("capacity override references unknown expert") + if any( + d not in document_ids or e not in expert_ids + for d, e in (*scenario.add_conflicts, *scenario.remove_conflicts) + ): + raise DataValidationError("scenario conflict references unknown document or expert") + if set(scenario.add_conflicts) & baseline_pairs: + raise DataValidationError("added conflict already exists in baseline") + if set(scenario.remove_conflicts) - baseline_pairs: + raise DataValidationError("removed conflict does not exist in baseline") + baseline_run = run_matching(documents, experts, conflicts=conflicts, config=selected) + baseline = _summary(baseline_run) + if not baseline["safe"]: + raise DataValidationError("baseline matching audit failed") + scenarios: list[dict[str, object]] = [] + for scenario in plan.scenarios: + capacities = dict(scenario.capacity_overrides) + changed_experts = tuple( + replace(item, capacity=capacities[item.id]) if item.id in capacities else item + for item in experts + ) + removed = set(scenario.remove_conflicts) + changed_conflicts = tuple( + item for item in conflicts if (item.document_id, item.expert_id) not in removed + ) + tuple(Conflict(d, e, "what-if scenario") for d, e in scenario.add_conflicts) + result = _summary( + run_matching(documents, changed_experts, conflicts=changed_conflicts, config=selected) + ) + if not result["safe"]: + raise DataValidationError(f"scenario {scenario.id} matching audit failed") + scenarios.append( + { + "id": scenario.id, + "changes": scenario.to_dict(), + "result": result, + "delta": _delta(baseline, result), + } + ) + return {"schema_version": 1, "baseline": baseline, "scenarios": scenarios} diff --git a/tests/test_what_if.py b/tests/test_what_if.py new file mode 100644 index 0000000..1bd33c9 --- /dev/null +++ b/tests/test_what_if.py @@ -0,0 +1,247 @@ +"""Independent capacity/conflict what-if oracles and CLI boundaries.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from peermatchlab.cli import main +from peermatchlab.config import MatchConfig +from peermatchlab.models import Conflict, DataValidationError, Document, Expert +from peermatchlab.what_if import WhatIfPlan, WhatIfScenario, compare_what_if + + +def _inputs() -> tuple[tuple[Document, ...], tuple[Expert, ...]]: + documents = (Document("d", "Graph neural networks", required_experts=1),) + experts = ( + Expert("a", "Graph specialist", topics=("graph",), capacity=1), + Expert("b", "General reviewer", topics=("review",), capacity=1), + ) + return documents, experts + + +def _plan() -> bytes: + return json.dumps( + { + "schema_version": 1, + "scenarios": [ + { + "id": "disable-a", + "capacity_overrides": {"a": 0}, + "add_conflicts": [], + "remove_conflicts": [], + }, + { + "id": "conflict-a", + "capacity_overrides": {}, + "add_conflicts": [["d", "a"]], + "remove_conflicts": [], + }, + ], + }, + sort_keys=True, + ).encode() + + +def test_capacity_and_conflict_scenarios_are_independent_and_audited() -> None: + documents, experts = _inputs() + before = (experts[0].capacity, experts[1].capacity) + report = compare_what_if( + documents, + experts, + WhatIfPlan.from_bytes(_plan()), + config=MatchConfig(reviewers_per_document=1), + ) + assert report == compare_what_if( + documents, + experts, + WhatIfPlan.from_bytes(_plan()), + config=MatchConfig(reviewers_per_document=1), + ) + assert report["baseline"]["assigned"] == 1 + assert report["baseline"]["unmet"] == 0 + assert report["baseline"]["safe"] is True + baseline = report["baseline"]["assignments"] + assert len(baseline) == 1 + assert baseline[0] == ["d", "a"] + assert [row["id"] for row in report["scenarios"]] == ["conflict-a", "disable-a"] + for scenario in report["scenarios"]: + assert scenario["result"]["assignments"] == [["d", "b"]] + assert scenario["result"]["safe"] is True + assert scenario["delta"]["added_assignments"] == [["d", "b"]] + assert scenario["delta"]["removed_assignments"] == [["d", "a"]] + assert (experts[0].capacity, experts[1].capacity) == before + + +def test_removing_declared_conflict_is_explicit_not_inferred() -> None: + documents, experts = _inputs() + raw = json.loads(_plan()) + raw["scenarios"] = [ + { + "id": "remove", + "capacity_overrides": {}, + "add_conflicts": [], + "remove_conflicts": [["d", "a"]], + } + ] + plan = WhatIfPlan.from_bytes(json.dumps(raw).encode()) + with pytest.raises(DataValidationError, match="does not exist"): + compare_what_if(documents, experts, plan) + report = compare_what_if( + documents, + experts, + plan, + conflicts=(Conflict("d", "a"),), + config=MatchConfig(reviewers_per_document=1), + ) + assert report["baseline"]["assignments"] == [["d", "b"]] + assert report["scenarios"][0]["result"]["assignments"] == [["d", "a"]] + + +def test_complete_capacity_removal_creates_one_unmet_slot_and_negative_delta() -> None: + documents, experts = _inputs() + raw = json.loads(_plan()) + raw["scenarios"] = [ + { + "id": "none-available", + "capacity_overrides": {"a": 0, "b": 0}, + "add_conflicts": [], + "remove_conflicts": [], + } + ] + report = compare_what_if( + documents, + experts, + WhatIfPlan.from_bytes(json.dumps(raw).encode()), + config=MatchConfig(reviewers_per_document=1), + ) + result = report["scenarios"][0] + assert result["result"]["assigned"] == 0 + assert result["result"]["unmet"] == 1 + assert result["result"]["status"] == "infeasible" + assert result["delta"]["assigned"] == -1 + assert result["delta"]["unmet"] == 1 + assert result["delta"]["removed_assignments"] == [["d", "a"]] + assert result["delta"]["workload"] == {"a": -1, "b": 0} + + +def test_zero_explicit_document_demand_is_invalid_domain_input() -> None: + with pytest.raises(DataValidationError, match="required_experts must be positive"): + Document("d", "Graph neural networks", required_experts=0) + + +def test_plan_duplicate_json_fields_and_resource_limits_are_rejected() -> None: + with pytest.raises(DataValidationError, match="valid JSON"): + WhatIfPlan.from_bytes(b"{") + with pytest.raises(DataValidationError, match="UTF-8"): + WhatIfPlan.from_bytes(b"\xff") + with pytest.raises(DataValidationError, match="duplicate JSON field"): + WhatIfPlan.from_bytes(b'{"schema_version":1,"schema_version":1,"scenarios":[]}') + with pytest.raises(DataValidationError, match="bounded immutable"): + WhatIfPlan.from_bytes(b" " * (64 * 1024 + 1)) + raw = json.loads(_plan()) + raw["scenarios"] = raw["scenarios"] * 5 + with pytest.raises(DataValidationError, match="1 to 8"): + WhatIfPlan.from_bytes(json.dumps(raw).encode()) + _, experts = _inputs() + with pytest.raises(DataValidationError, match="requested"): + compare_what_if( + (Document("d", "Graph neural networks", required_experts=129),), + experts, + WhatIfPlan.from_bytes(_plan()), + ) + with pytest.raises(DataValidationError, match="1 to 8 unique"): + WhatIfPlan(()) + with pytest.raises(DataValidationError, match="capacity overrides"): + WhatIfScenario("bad", (("a", True),), (), ()) + with pytest.raises(DataValidationError, match="pairs are invalid"): + WhatIfScenario("bad", (), (("d", "a"), ("d", "a")), ()) + + +@pytest.mark.parametrize( + "mutate", + [ + lambda p: p.update(schema_version=2), + lambda p: p["scenarios"][1].update(id="disable-a"), + lambda p: p["scenarios"][0]["capacity_overrides"].update(a=True), + lambda p: p["scenarios"][0]["capacity_overrides"].update(unknown=1), + lambda p: p["scenarios"][0]["add_conflicts"].append(["d", "unknown"]), + lambda p: p["scenarios"][0].update(other=1), + ], +) +def test_bad_plan_or_reference_is_rejected(mutate: object) -> None: + documents, experts = _inputs() + raw = json.loads(_plan()) + mutate(raw) + with pytest.raises(DataValidationError): + compare_what_if(documents, experts, WhatIfPlan.from_bytes(json.dumps(raw).encode())) + + +def test_cli_smoke_and_input_output_alias_guard( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + documents = tmp_path / "documents.json" + experts = tmp_path / "experts.json" + plan = tmp_path / "scenarios.json" + output = tmp_path / "report.json" + documents.write_text( + json.dumps([{"id": "d", "title": "Graph neural networks", "required_experts": 1}]) + ) + experts.write_text( + json.dumps( + [ + {"id": "a", "name": "Graph specialist", "topics": ["graph"], "capacity": 1}, + {"id": "b", "name": "General reviewer", "topics": ["review"], "capacity": 1}, + ] + ) + ) + plan.write_bytes(_plan()) + args = [ + "what-if", + "--documents", + str(documents), + "--experts", + str(experts), + "--scenarios", + str(plan), + "--output", + str(output), + ] + assert main(args) == 0 + saved = json.loads(output.read_text()) + assert saved["baseline"]["safe"] is True + assert len(saved["scenarios"]) == 2 + assert main([*args[:-1], str(plan)]) == 2 + assert "must not refer" in capsys.readouterr().err + + +def test_cli_rejects_oversize_source_before_output( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + documents = tmp_path / "documents.json" + experts = tmp_path / "experts.json" + plan = tmp_path / "scenarios.json" + output = tmp_path / "report.json" + documents.write_bytes(b" " * (1024 * 1024 + 1)) + experts.write_text('[{"id":"a","name":"A"}]', encoding="utf-8") + plan.write_bytes(_plan()) + assert ( + main( + [ + "what-if", + "--documents", + str(documents), + "--experts", + str(experts), + "--scenarios", + str(plan), + "--output", + str(output), + ] + ) + == 2 + ) + assert "exceeds" in capsys.readouterr().err + assert not output.exists() diff --git a/uv.lock b/uv.lock index 2cd743c..f0c3b02 100644 --- a/uv.lock +++ b/uv.lock @@ -478,7 +478,7 @@ wheels = [ [[package]] name = "peermatchlab" -version = "0.14.0" +version = "0.15.0" source = { editable = "." } [package.optional-dependencies]