From e55541991331d578d88660dcbb34af60506a534d Mon Sep 17 00:00:00 2001 From: Nick Petalas Date: Tue, 3 Nov 2026 14:16:02 +0000 Subject: [PATCH 1/2] feat(research): filter source overlap by author and creation date --- docs/research/agent-procedure.md | 18 +++ research/source-reference-intake/README.md | 104 +++++++++++++ research/source-reference-intake/SOURCES.json | 18 +++ research/source-reference-intake/TASK.json | 87 +++++++++++ .../source-reference-intake/VALIDATION.json | 34 +++++ research/source-reference-intake/VERIFY.json | 43 ++++++ research/source-reference-intake/verify.py | 48 ++++++ scripts/find_overlap.py | 101 ++++++++++++- tests/test_research_overlap_intake.py | 143 ++++++++++++++++++ 9 files changed, 591 insertions(+), 5 deletions(-) create mode 100644 research/source-reference-intake/README.md create mode 100644 research/source-reference-intake/SOURCES.json create mode 100644 research/source-reference-intake/TASK.json create mode 100644 research/source-reference-intake/VALIDATION.json create mode 100644 research/source-reference-intake/VERIFY.json create mode 100644 research/source-reference-intake/verify.py create mode 100644 tests/test_research_overlap_intake.py diff --git a/docs/research/agent-procedure.md b/docs/research/agent-procedure.md index 06b35b9805..580305ef40 100644 --- a/docs/research/agent-procedure.md +++ b/docs/research/agent-procedure.md @@ -121,6 +121,24 @@ that a task is open. A deterministic seed is a reproducibility aid, not novelty. Each card supplies a first-step command, expected evidence, a next experiment, its connection to the bottleneck, allowed variations and scope-change triggers. +Before taking a source's next question, query its issue/PR identifier directly +in the overlap body corpus. Author and creation-date filters can isolate an +account's recent contributions without relying on GitHub's indexed body search: + +```sh +python3 -B scripts/find_overlap.py --reference 6537 --pr-author petalas \ + --created-since 2026-10-27 --include-closed --cache /tmp/source-overlap.json +``` + +Use the actual source number, login and UTC cutoff. `--reference` is repeatable +and excludes decimal/hash substring hits; ordinary positional terms retain +literal substring semantics. Author/date filters search only PR titles and +bodies. `--include-closed` adds a separately paginated closed/merged corpus. +Default searches are unchanged. Online queries refresh legacy PR caches that +lack author/creation metadata; offline queries requiring missing metadata fail +explicitly rather than reporting no matches. Filtered results are leads, not +evidence of semantic novelty, and do not replace live claim checks. + **Resolve the generic orientation command before running a historical card.** If its `first_step.command` is `python3 -B scripts/research_baseline.py`, use the current orientation command in [current priorities](current-priorities.md). diff --git a/research/source-reference-intake/README.md b/research/source-reference-intake/README.md new file mode 100644 index 0000000000..030175c426 --- /dev/null +++ b/research/source-reference-intake/README.md @@ -0,0 +1,104 @@ +# Source-reference intake for research overlap + +The overlap helper now accepts identifier-aware reference queries and filters +PR bodies by author and creation date. This makes an account's recent work +on a proposed source visible directly in the fetched body corpus. Missing +filter metadata produces an explicit error, so an old cache cannot silently +hide a possible lead. + +Claim type: workflow tooling with behavioral regression evidence. This +contribution was produced by Nick Petalas. No mathematical bound, kappa +improvement or predecessor-result verification is claimed. + +## Use + +From the repository root: + +```sh +~/git/integer-mult-bounds/.venv/bin/python3.12 -B scripts/find_overlap.py \ + --reference 6537 --pr-author petalas --created-since 2026-10-27 \ + --include-closed --cache /tmp/source-overlap.json +``` + +Use the actual source number, login and UTC cutoff. `--reference` is +repeatable and applies identifier boundaries to numeric terms. Hashes and +decimal substrings cannot displace a real reference. Positional terms retain +their literal search semantics. The date filter uses creation, not update, +and includes the cutoff day's midnight UTC. + +Author/date filters search PR titles and bodies only. Closed/merged bodies +require `--include-closed`, with separate coverage and timestamps. They do +not search local notes or Discussions. The ordinary unfiltered search keeps +its existing scope. A filtered online query refreshes legacy PR metadata; +an offline cache missing the requested fields fails with an explanation. +`--pr-author` takes a login, rather than `@me` or a search expression. + +## Evidence and reproduction + +Nine new regression groups cover identifier noise and spelling, literal +queries, inclusive dates and author case, optional closed bodies, legacy +offline errors, invalid metadata, creation/update separation, selective +online refresh and invalid CLI input. The existing 37 overlap groups also +pass. The registered replay runs all 46 groups in normal and optimized +Python, without network access. +The separate local run of all 18 test modules that mention the overlap helper +also passed, with 132 tests. That broader run is recorded as local evidence; +the registered package runs the focused 46-test suite. + +```sh +~/git/integer-mult-bounds/.venv/bin/python3.12 -B scripts/research_package.py --package research/source-reference-intake +~/git/integer-mult-bounds/.venv/bin/python3.12 -I -W error -B research/source-reference-intake/verify.py +~/git/integer-mult-bounds/.venv/bin/python3.12 -B scripts/check_research_commit.py --package research/source-reference-intake --check-source-history +``` + +Expected: `Ran 46 tests`, `OK`, `PASS_SOURCE_REFERENCE_INTAKE` and the +registered package PASS. [VALIDATION.json](VALIDATION.json) is generated +evidence bound to the tested script/test hashes. Default replay writes +nothing persistent. `verify.py --write NEW_PATH` records a fresh execution +and refuses to overwrite an existing path. + +A live author/date query over the 587-open-PR snapshot fetched at +2026-11-03T14:10:56.715135+00:00 returned PR6571 as a source-6537 lead. +This only establishes the tool's retrieval behavior on that snapshot. +Neither its mathematics nor the source's result was evaluated. The nine +synthetic regression groups make the behavior reproducible offline. + +## Limits + +Reference text is an overlap lead, not evidence of semantic novelty or +duplication. Work can share a model without citing its source. Current +claims, package questions and merged work still need inspection. Queries +do not provide an atomic server snapshot or a seven-day policy decision. +No selected frontier or frozen research package changes. + +## Open next questions + +Could a separately scoped provenance index also surface uncited package +reuse? First compare exact package-path references in PR bodies with a +complete changed-path inventory. Missing citations, renamed packages and +semantic equivalence remain blockers; they must not become automatic +novelty claims. + +## Implementation and credit + +The new filters traverse finite cached PR lists and validate dates before +comparing them. Unknown author/date values fail explicitly. New helpers +are at most 33 physical lines. The existing CLI dispatcher is shortened +to 170 lines by extracting argument parsing; its broader refactoring is +deferred to avoid changing unrelated path/coverage reporting. Existing +network pagination remains outside this local behavior change. + +Power of Ten rule 3 deviation: the existing Python cache/search command +allocates lists and dictionaries at runtime. The new pass adds linear +storage in the existing validated PR corpus size. A fixed-capacity service +port would require a separately declared corpus-size contract. Rule 9 +deviation: unittest dispatch and mocks use their fixed test callbacks. +The replay runs one fixed subprocess with a 90-second timeout. Warning-strict +normal/optimized execution, AST compilation and the full affected suite +provide diagnostics; Ruff is unavailable in the prescribed environment. + +Douglas Colkitt's overlap helper and its credited repository contributors +retain their attribution. Existing identifier matching and cache/pagination +machinery are reused. The new explicit query and metadata filters, regression +fixtures and integration notes were produced by Nick Petalas. Apache-2.0. +Independent human review is unclaimed. diff --git a/research/source-reference-intake/SOURCES.json b/research/source-reference-intake/SOURCES.json new file mode 100644 index 0000000000..594089b9c7 --- /dev/null +++ b/research/source-reference-intake/SOURCES.json @@ -0,0 +1,18 @@ +{ + "schema_version": 1, + "base_commit": "7c49fe32c55de02c2c873655c804b16e4c3fef0a", + "files": [ + { + "path": "certificates/selected-result.json", + "usage": "provenance", + "git_blob": "d59b5035e7b4cbaa9d688a1d32dd0d6eb79c466a", + "sha256": "9b8d0eea95b3bf0dfcb7938a732a6439b87a2685dec2a5ab9e525b3b34debbfa" + }, + { + "path": "docs/research/agent-task-catalog.json", + "usage": "provenance", + "git_blob": "3404fbdd5e7529a0674eef26114b5071e4586cec", + "sha256": "ead6cdc9d0af6d58e8a49744d8ec8c7c15e13d2ba93c276c434bcf9b51b93a81" + } + ] +} diff --git a/research/source-reference-intake/TASK.json b/research/source-reference-intake/TASK.json new file mode 100644 index 0000000000..80c0a0833e --- /dev/null +++ b/research/source-reference-intake/TASK.json @@ -0,0 +1,87 @@ +{ + "schema_version": 1, + "task_id": "replay-runtime", + "priority_revision": 8, + "base_commit": "7c49fe32c55de02c2c873655c804b16e4c3fef0a", + "original_card": { + "id": "replay-runtime", + "question": "Can one measured replay bottleneck be sped up without dropping coverage?", + "starting_points": [ + "scripts/run_isolated_tests.py", + "scripts/verify_replay_coverage.py", + "certificates/selected-result.json" + ], + "methods": [ + "Profile one isolated replay and optimize the measured bottleneck.", + "Design source-bound reuse with explicit dependency invalidation and negative controls." + ], + "acceptance_gate": "Report before/after wall time on the same fixture, retained checks and invalidation behavior.", + "first_step": { + "command": [ + "python3", + "-B", + "scripts/research_selected_baseline.py" + ], + "expected": "PASS_RESEARCH_BASELINE_BINDINGS_ONLY", + "inspect": "Resolve the current selected review, separate complex/ordinary BIT/admitted BIT/outer budgets, and compare the card against live handoffs before starting. Historical ceilings require exact source/model equivalence.", + "scope": "Source-binding orientation only; not a scientific replay or all-size proof. Materialize selected inputs if this checkout is sparse." + }, + "first_experiment": "Measure one unchanged fixture, preserve check coverage, and test one optimization with invalidation controls.", + "bottleneck_link": "Faster trustworthy replays increase review throughput and let mechanism candidates be checked sooner.", + "allowed_variations": [ + "Profile one stage or design one source-bound cache boundary." + ], + "scope_change_triggers": [ + "Skipping coverage or comparing different fixtures to claim speedup." + ], + "overlap_terms": [ + "replay runtime", + "cache invalidation", + "isolated tests" + ] + }, + "scope_status": "changed", + "actual_scope": "Add explicit issue/PR reference queries and author/creation-date filters to the complete cached PR body search, with explicit unknown-metadata failure and legacy refresh.", + "scope_change_reason": "A source-reuse collision was already represented in the complete body cache but was missed by a targeted indexed search and a broad task query. This is an original coordination-tooling improvement, not another predecessor verification or a scientific replay speedup.", + "bottleneck_link": "Coordination side result: expose recent source reuse before spending work on a target. It changes no multiplication bound or supplier obligation.", + "budget_minutes": 60, + "stop_rule": "Stop after one working source-specific query with passing regression coverage and explicit cache-coverage limits. Do not extend to semantic novelty detection.", + "outcome": "completed", + "evidence": "Nine new behavioral regression groups and the 37 existing overlap groups pass. The actual source-reference/author/date query retrieves PR6571 from the fetched open-PR corpus. VALIDATION.json records a generated hash-bound local execution.", + "author": "Nick Petalas", + "ai_assistance": "Independent human and formal proof-kernel review are unclaimed.", + "open_questions": [ + { + "question": "Can a separately scoped package-provenance index surface uncited reuse without treating missing references as novelty?", + "why_it_matters": "Reference matching cannot detect a shared model whose source is not cited.", + "first_step": "Compare exact package-path references in PR bodies with a complete changed-file inventory and report coverage explicitly.", + "blocked_by": "Missing citations, renames and semantic equivalence require separate evidence; no automatic novelty claim is justified." + } + ], + "claim_type": "workflow tooling with behavioral regression evidence", + "claim_url": "https://github.com/CrocSwap/integer-mult-bounds/discussions/1166#discussioncomment-19375429", + "scope_update_url": "https://github.com/CrocSwap/integer-mult-bounds/discussions/1166#discussioncomment-19375643", + "originality": "The CLI integration, author/creation metadata handling and regression fixtures are new. Existing numeric-reference matching, cache paging and source-history infrastructure retain their predecessor credit.", + "unrun_obligations": [ + "Independent human review", + "Semantic novelty or duplication decisions", + "All mathematical predecessor replays and selected cold replay" + ], + "orientation": { + "command": "/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -B scripts/research_selected_baseline.py", + "output": "PASS_RESEARCH_BASELINE_BINDINGS_ONLY", + "canonical_main": "7c49fe32c55de02c2c873655c804b16e4c3fef0a", + "priority_revision": 8, + "selected_pr": 6030 + }, + "overlap": { + "coverage": "Live handoffs, active Discussions, open/account PR lists, complete cached open-PR/Discussion pagination, main-tree distinctive and general-form searches, and merged author/filter searches.", + "limits": "Arbitrary diffs, uncited semantic counterparts and unpublished work are not fully covered." + }, + "additional_local_check": { + "domain": "All 18 test modules mentioning find_overlap", + "tests_run": 132, + "outcome": "OK", + "command": "Construct unittest.TestSuite from unittest discovery of each matching file and run with warning-strict isolated CPython 3.12." + } +} diff --git a/research/source-reference-intake/VALIDATION.json b/research/source-reference-intake/VALIDATION.json new file mode 100644 index 0000000000..b9c954ece2 --- /dev/null +++ b/research/source-reference-intake/VALIDATION.json @@ -0,0 +1,34 @@ +{ + "claim_type": "workflow tooling with behavioral regression evidence", + "command": [ + "/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12", + "-I", + "-W", + "error", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_research_overlap_*.py", + "-q" + ], + "created_utc": "2026-11-03T14:13:04.601961+00:00", + "exit_code": 0, + "output": "----------------------------------------------------------------------\nRan 46 tests in 1.805s\n\nOK\n", + "platform": "macOS-27.0.1-arm64-arm-64bit", + "python": "3.12.15 (main, Oct 14 2026, 00:55:37) [Clang 23.1.3 ]", + "seconds": 1.8843645830638707, + "sha256": { + "scripts/find_overlap.py": "de10463786ae31392b320b6ed69394bdd0fba5f6bd5df1db648ac891cb566691", + "scripts/research_package.py": "e1256016e7f51827d04fa2179255989f639ae1d97afaa9c16265f85a0c6abd70", + "tests/test_research_overlap_cache.py": "e49cf1c1707a16c0181f642ab8d969eb8e8689262277b42dba343d0a13295bad", + "tests/test_research_overlap_closed.py": "2a5d28e13fe57191702d86630763fdfd33842696077837d225c5677aea819b03", + "tests/test_research_overlap_intake.py": "118c1cb1ff90df7b9c2e691be0e6ee4a09eb8fbc150b5ecfc9d180e5296065e9", + "tests/test_research_overlap_local.py": "e7deb16c3f8d14523d31b568a7ea58f683cc56489af2009b2f5cada58a94c2da", + "tests/test_research_overlap_pagination.py": "b83e490c45f76b70eb394dd9706ff0b2f627211efe975909f9e1b40d8995f99c", + "tests/test_research_overlap_terms.py": "7dd29a1b44ab0c80b8b58a6a1eb7cff810dda942019754029b3bc38c9ff3e601" + } +} diff --git a/research/source-reference-intake/VERIFY.json b/research/source-reference-intake/VERIFY.json new file mode 100644 index 0000000000..60289f7844 --- /dev/null +++ b/research/source-reference-intake/VERIFY.json @@ -0,0 +1,43 @@ +{ + "schema_version": 1, + "status": "ready", + "python": "3.12", + "requirements": [], + "inputs": [ + "scripts/find_overlap.py", + "scripts/research_package.py", + "tests/test_research_overlap_cache.py", + "tests/test_research_overlap_closed.py", + "tests/test_research_overlap_intake.py", + "tests/test_research_overlap_local.py", + "tests/test_research_overlap_pagination.py", + "tests/test_research_overlap_terms.py" + ], + "commands": [ + { + "argv": [ + "{python}", + "-I", + "-W", + "error", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 120, + "checks": "The focused overlap-named suite: nine new intake groups and all 37 existing groups; offline behavior, metadata boundaries, cache refresh and default coverage." + }, + { + "argv": [ + "{python}", + "-I", + "-O", + "-W", + "error", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 120, + "checks": "The same complete suite with optimized Python, propagating optimization into the test subprocess." + } + ] +} diff --git a/research/source-reference-intake/verify.py b/research/source-reference-intake/verify.py new file mode 100644 index 0000000000..044d159e1c --- /dev/null +++ b/research/source-reference-intake/verify.py @@ -0,0 +1,48 @@ +"""Replay source-intake behavior and the focused overlap regression suite. + +Produced by Nick Petalas. Apache-2.0. +""" + +import argparse +from datetime import datetime, timezone +from hashlib import sha256 +import json +from pathlib import Path +import platform +import subprocess +import sys +import time + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--write", type=Path, help="Write fresh evidence; refuse overwrite") + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + command = [sys.executable, "-I", "-W", "error", "-B"] + if sys.flags.optimize: + command.append("-O") + command.extend(["-m", "unittest", "discover", "-s", "tests", + "-p", "test_research_overlap_*.py", "-q"]) + started = time.perf_counter() + result = subprocess.run(command, cwd=root, text=True, capture_output=True, + timeout=90, check=True) + elapsed = time.perf_counter() - started + print(result.stdout + result.stderr, end="") + if args.write is not None: + inputs = ["scripts/find_overlap.py", "scripts/research_package.py", + *sorted(str(path.relative_to(root)) for path in + (root / "tests").glob("test_research_overlap_*.py"))] + evidence = {"claim_type": "workflow tooling with behavioral regression evidence", + "created_utc": datetime.now(timezone.utc).isoformat(), + "python": sys.version, "platform": platform.platform(), + "command": command, "seconds": elapsed, "exit_code": result.returncode, + "output": result.stdout + result.stderr, + "sha256": {name: sha256((root / name).read_bytes()).hexdigest() for name in inputs}} + with args.write.open("x", encoding="utf-8") as stream: + stream.write(json.dumps(evidence, indent=2, sort_keys=True) + "\n") + print("PASS_SOURCE_REFERENCE_INTAKE") + + +if __name__ == "__main__": + main() diff --git a/scripts/find_overlap.py b/scripts/find_overlap.py index d20f85824e..e4d2c73560 100644 --- a/scripts/find_overlap.py +++ b/scripts/find_overlap.py @@ -63,6 +63,8 @@ def pull_records(pages): return [{"kind": "pr", "number": pr["number"], "title": pr["title"], "body": pr.get("body") or "", "url": pr["html_url"], "head": pr["head"]["sha"], "base": pr.get("base", {}).get("sha"), + "author": (pr.get("user") or {}).get("login"), + "created_at": pr.get("created_at"), "updated_at": pr["updated_at"], "draft": pr.get("draft", False), "state": pr["state"]} for page in pages for pr in page] @@ -279,8 +281,13 @@ def validate_cached_source(source, name): return stamp +def has_pr_metadata(source): + """Schema-1 legacy rows lack the author/creation fields required by filters.""" + return all(row.get("author") and row.get("created_at") for row in source["records"]) + + def remote_snapshot(cache, offline=False, refresh=False, max_age=300, need_files=False, - include_closed=False): + include_closed=False, need_pr_metadata=False): warnings = [] try: snapshot = json.loads(cache.read_text(encoding="utf-8")) @@ -316,6 +323,8 @@ def remote_snapshot(cache, offline=False, refresh=False, max_age=300, need_files sources.pop(name, None) cached = None fresh = cached is not None and age is not None and 0 <= age <= max_age + if need_pr_metadata and name in ("open_prs", "closed_prs") and cached: + fresh = fresh and has_pr_metadata(cached) if need_files and name in ("pr_files", "open_prs") and cached: # Old schema-1 caches omitted endpoints. Refresh only affected sources; # offline/failed refreshes retain the evidence with explicit warnings. @@ -467,10 +476,57 @@ def search(records, terms, all_terms=False, reference_terms=()): return sorted(matches, key=lambda r: (-r["score"], r["kind"], r["title"], r.get("url", ""))) -def main(): +def reference_number(value): + if not re.fullmatch(r"[1-9][0-9]{0,9}", value): + raise argparse.ArgumentTypeError("reference must be a positive ASCII issue/PR number") + return value + + +def creation_cutoff(value): + try: + if not re.fullmatch(r"[0-9]{4}-[0-9]{2}-[0-9]{2}", value): + raise ValueError("expected YYYY-MM-DD") + return datetime.strptime(value, "%Y-%m-%d").replace(tzinfo=timezone.utc) + except ValueError as exc: + raise argparse.ArgumentTypeError("creation cutoff must be a valid YYYY-MM-DD UTC date") from exc + + +def filter_pulls(records, author=None, created_since=None): + """Reject unknown filter metadata rather than silently omit a possible lead.""" + selected, unknown = [], [] + for row in records: + login, stamp = row.get("author"), row.get("created_at") + created = None + if created_since is not None and isinstance(stamp, str): + try: + created = datetime.fromisoformat(stamp) + except ValueError: + pass + if ((author is not None and (not isinstance(login, str) or not login)) + or (created_since is not None and (created is None or created.tzinfo is None))): + unknown.append(row["number"]) + continue + if author is not None and login.lower() != author.lower(): + continue + if created_since is not None and created < created_since: + continue + selected.append(row) + if unknown: + examples = ", ".join(str(number) for number in sorted(unknown)[:20]) + raise ValueError(f"PR filter metadata unavailable for {len(unknown)} records ({examples}); " + "refresh the cache or inspect an unfiltered report; unknown is not no overlap") + return selected + + +def parse_arguments(): ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("terms", nargs="*", help="literal case-insensitive terms; quote multiword phrases") ap.add_argument("--task", help="add the named catalogue card's overlap terms") + ap.add_argument("--reference", action="append", type=reference_number, default=[], + metavar="NUMBER", help="match an issue/PR identifier, not decimal or hash substrings; repeatable") + ap.add_argument("--pr-author", metavar="LOGIN", help="search PR bodies only, restricted to this GitHub login") + ap.add_argument("--created-since", type=creation_cutoff, metavar="YYYY-MM-DD", + help="search PR bodies only, created on/after this UTC date; not their update date") ap.add_argument("--path", action="append", default=[], metavar="PATTERN", help="match open PRs by case-sensitive Git path glob or substring; spaces are literal; repeatable") ap.add_argument("--offline", action="store_true") @@ -487,8 +543,19 @@ def main(): ap.error("use nonnegative max-age, positive limit, and either offline or refresh") if args.include_closed and args.path: ap.error("--include-closed searches titles/bodies only; omit --path (its index covers open PRs)") + pr_filters = args.pr_author is not None or args.created_since is not None + if args.pr_author is not None and not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9-]{0,38}", args.pr_author): + ap.error("--pr-author requires a GitHub login, not @me or a search expression") + if pr_filters and args.path: + ap.error("PR author/creation filters search titles/bodies only; omit --path") if args.cache.resolve().is_relative_to(ROOT.resolve()): ap.error("keep the cache outside the repository") + return ap, args + + +def main(): + ap, args = parse_arguments() + pr_filters = args.pr_author is not None or args.created_since is not None terms = list(args.terms) explicit_terms = {term.strip().lower() for term in args.terms} reference_terms = [] @@ -510,15 +577,18 @@ def main(): parts = PurePosixPath(point).parts if len(parts) >= 3 and parts[0] == "research": terms.append(parts[1]) + terms.extend(args.reference) + reference_terms = list(dict.fromkeys([*reference_terms, *args.reference])) if not any(t.strip() for t in terms) and not args.path: - ap.error("provide terms, --task or --path") + ap.error("provide terms, --task, --reference or --path") patterns = list(args.path) if any(not p for p in patterns): ap.error("--path must not be empty") local = local_records(ROOT) local_status, local_warnings = local_coverage(ROOT, local) snapshot, warnings = remote_snapshot(args.cache, args.offline, args.refresh, args.max_age, - need_files=bool(patterns), include_closed=args.include_closed) + need_files=bool(patterns), include_closed=args.include_closed, + need_pr_metadata=pr_filters) warnings = local_warnings + warnings records = list(local) + pull_corpus(snapshot, args.include_closed) coverage = {"local": local_status} @@ -530,6 +600,12 @@ def main(): if name == "discussions": records.extend(source["records"]) coverage[name]["records"] = len(source["records"]) + if pr_filters: + try: + records = filter_pulls(pull_corpus(snapshot, args.include_closed), + args.pr_author, args.created_since) + except ValueError as exc: + ap.error(str(exc)) path_matches = [] if patterns: source = snapshot["sources"].get("pr_files") @@ -580,6 +656,18 @@ def main(): "it does not certify HEAD contents or historical coverage. " "No code diffs or arbitrary package files are searched. Recheck relevant heads before submission."), "matches_total": len(matches), "matches": matches[:args.limit]} + if pr_filters: + report["pr_filters"] = {"author": args.pr_author, + "created_since": args.created_since.isoformat() if args.created_since else None, + "scope": "PR titles and bodies only; local notes and Discussions excluded", + "records_after_filter": len(records)} + report["limits"] = ("Author/creation filters search the declared PR title/body corpora only. " + "Local notes and Discussions are excluded. " + + ("Closed/merged PRs are included with separate coverage timestamps. " + if args.include_closed else "Closed/merged PRs are not included; use --include-closed. ") + + "Creation dates are not update dates. Unknown filter metadata is an error. " + "Snapshots are not atomic; no diffs or semantic equivalence are checked. " + "Matches are overlap leads, not a novelty certificate.") if patterns: report["path_patterns"] = patterns report["path_limits"] = ("Path matching covers the changed file paths GitHub reports for open PRs, " @@ -606,7 +694,10 @@ def main(): local_summary["unsearched_head_paths_count"] = None if omitted is None else len(omitted) print("Coverage: " + json.dumps({**coverage, "local": local_summary})) if reference_terms: - print("Task references (identifier matching): " + ", ".join(reference_terms)) + label = "References" if args.reference else "Task references" + print(label + " (identifier matching): " + ", ".join(reference_terms)) + if pr_filters: + print("PR filters: " + json.dumps(report["pr_filters"])) print(report["limits"]) if patterns: print(report["path_limits"]) diff --git a/tests/test_research_overlap_intake.py b/tests/test_research_overlap_intake.py new file mode 100644 index 0000000000..24c7dc7e2c --- /dev/null +++ b/tests/test_research_overlap_intake.py @@ -0,0 +1,143 @@ +"""Source-reference intake must retain exact PR metadata and expose unknowns.""" + +import contextlib +from datetime import datetime, timezone +import io +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +import find_overlap as overlap +sys.path.pop(0) + + +def pull(number, body="Builds on PR6537.", author="petalas", created="2026-11-03T00:00:00Z"): + return {"kind": "pr", "number": number, "title": "A follow-up", "body": body, + "url": f"https://example.invalid/pull/{number}", "author": author, + "created_at": created, "updated_at": "2026-11-03T12:00:00Z", + "state": "open", "head": "a" * 40, "base": "b" * 40} + + +class IntakeTests(unittest.TestCase): + def run_report(self, rows, arguments=(), closed=()): + with tempfile.TemporaryDirectory() as tmp: + cache = Path(tmp) / "cache.json" + sources = {name: {"fetched_at": overlap.now(), "complete_pagination": True, + "records": records} + for name, records in (("open_prs", rows), ("discussions", []), + ("closed_prs", list(closed)))} + raw = json.dumps({"schema_version": 1, "repository": overlap.REPO, "sources": sources}) + cache.write_text(raw, encoding="utf-8") + output = io.StringIO() + with (patch.object(overlap, "ROOT", Path(tmp) / "empty-repo"), + patch.object(overlap, "local_coverage", return_value=({}, [])), + patch.object(overlap, "gh_json", side_effect=AssertionError("network")), + patch.object(sys, "argv", ["find_overlap.py", "--offline", "--json", + "--cache", str(cache), *arguments]), + contextlib.redirect_stdout(output)): + overlap.main() + self.assertEqual(cache.read_text(encoding="utf-8"), raw) + return json.loads(output.getvalue()) + + def test_reference_bypasses_hash_and_decimal_noise(self): + noise = [pull(i, "Commit a6537b, saving 0.006537 and parameter 6537.5.") for i in range(21)] + result = self.run_report([*noise, pull(99)], ["--reference", "6537"]) + self.assertEqual(result["matches_total"], 1) + self.assertEqual([row["number"] for row in result["matches"]], [99]) + self.assertEqual(result["reference_terms"], ["6537"]) + + def test_reference_forms_and_literal_query_semantics(self): + forms = ["#6537", "PR6537", "pr #6537", "PR 6537", + "https://github.com/CrocSwap/integer-mult-bounds/pull/6537"] + rows = [pull(i, body) for i, body in enumerate(forms)] + result = self.run_report(rows, ["--reference", "6537", "--reference", "6537"]) + self.assertEqual(result["matches_total"], len(forms)) + self.assertEqual(result["reference_terms"], ["6537"]) + result = self.run_report([pull(1, "a6537b")], ["6537"]) + self.assertEqual(result["matches_total"], 1) + + def test_author_and_creation_cutoff_are_inclusive_and_case_insensitive(self): + rows = [pull(1, created="2026-10-27T00:00:00Z"), + pull(2, created="2026-10-26T23:59:59Z"), pull(3, author="someone-else")] + result = self.run_report(rows, ["--reference", "6537", "--pr-author", "PeTaLaS", + "--created-since", "2026-10-27"]) + self.assertEqual([row["number"] for row in result["matches"]], [1]) + self.assertEqual(result["pr_filters"]["records_after_filter"], 1) + self.assertEqual(result["pr_filters"]["created_since"], "2026-10-27T00:00:00+00:00") + self.assertIn("Discussions excluded", result["pr_filters"]["scope"]) + + def test_closed_body_is_only_included_when_requested(self): + row = dict(pull(9), state="closed") + args = ["--reference", "6537", "--pr-author", "petalas"] + self.assertEqual(self.run_report([], args, [row])["matches_total"], 0) + result = self.run_report([], [*args, "--include-closed"], [row]) + self.assertEqual([match["number"] for match in result["matches"]], [9]) + self.assertTrue(result["coverage"]["closed_prs"]["complete_pagination"]) + + def test_legacy_offline_metadata_is_an_error_not_an_empty_result(self): + row = pull(1) + del row["author"] + with contextlib.redirect_stderr(io.StringIO()) as errors, self.assertRaises(SystemExit) as exc: + self.run_report([row], ["--reference", "6537", "--pr-author", "petalas"]) + self.assertEqual(exc.exception.code, 2) + self.assertIn("unknown is not no overlap", errors.getvalue()) + # Unfiltered offline reports remain compatible with legacy metadata. + self.assertEqual(self.run_report([row], ["--reference", "6537"])["matches_total"], 1) + + def test_missing_or_bad_creation_times_cannot_be_silently_filtered_out(self): + cutoff = datetime(2026, 10, 27, tzinfo=timezone.utc) + for stamp in (None, 123, "yesterday", "2026-11-03T00:00:00"): + with self.subTest(stamp=stamp), self.assertRaisesRegex(ValueError, "metadata unavailable"): + overlap.filter_pulls([pull(1, created=stamp)], created_since=cutoff) + + def test_normalized_rest_metadata_retains_creation_not_update(self): + raw = {"number": 7, "title": "A lead", "body": None, "html_url": "url", + "head": {"sha": "a" * 40}, "state": "closed", "user": {"login": "petalas"}, + "created_at": "2026-10-01T00:00:00Z", "updated_at": "2026-11-03T00:00:00Z"} + normalized = overlap.pull_records([[raw]])[0] + self.assertEqual(normalized["author"], "petalas") + self.assertEqual(normalized["created_at"], raw["created_at"]) + cutoff = datetime(2026, 10, 27, tzinfo=timezone.utc) + self.assertEqual(overlap.filter_pulls([normalized], "petalas", cutoff), []) + self.assertIsNone(overlap.pull_records([[dict(raw, user=None)]])[0]["author"]) + + def test_fresh_legacy_cache_refreshes_pr_metadata_without_refetching_discussions(self): + with tempfile.TemporaryDirectory() as tmp: + cache = Path(tmp) / "cache.json" + old = pull(1) + del old["author"] + section = lambda rows: {"fetched_at": overlap.now(), "complete_pagination": True, "records": rows} + cache.write_text(json.dumps({"schema_version": 1, "repository": overlap.REPO, + "sources": {"open_prs": section([old]), + "discussions": section([])}}), encoding="utf-8") + raw = {"number": 1, "title": "A lead", "html_url": "url", "head": {"sha": "a" * 40}, + "state": "open", "updated_at": overlap.now(), "created_at": overlap.now(), + "user": {"login": "petalas"}} + with (patch.object(overlap, "gh_json", return_value=[[raw]]) as fetch, + patch.object(overlap, "fetch_discussions", side_effect=AssertionError("discussion refetch"))): + snapshot, warnings = overlap.remote_snapshot(cache, need_pr_metadata=True) + self.assertEqual(fetch.call_count, 1) + self.assertEqual(warnings, []) + self.assertEqual(snapshot["sources"]["open_prs"]["records"][0]["author"], "petalas") + + def test_invalid_query_values_stop_before_network(self): + for arguments in (["--reference", "0"], ["--reference", "06537"], + ["--reference", "６５３７"], ["--reference", "-1"], + ["--reference", "6537", "--created-since", "2026-02-30"], + ["--reference", "6537", "--created-since", "2026-1-1"], + ["--reference", "6537", "--pr-author", "@me"], + ["--reference", "6537", "--pr-author", "petalas", "--path", "research/*"]): + with self.subTest(arguments=arguments), contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as exc: + self.run_report([], arguments) + self.assertEqual(exc.exception.code, 2) + + +if __name__ == "__main__": + unittest.main() From b994ea2487733dace57c67bed957bb95bf3fc745 Mon Sep 17 00:00:00 2001 From: Nick Petalas Date: Tue, 3 Nov 2026 14:40:22 +0000 Subject: [PATCH 2/2] fix(research): isolate source intake from the pinned overlap helper --- docs/research/agent-procedure.md | 10 +- research/source-reference-intake/README.md | 57 +++-- research/source-reference-intake/TASK.json | 17 +- .../source-reference-intake/VALIDATION.json | 11 +- research/source-reference-intake/VERIFY.json | 3 +- research/source-reference-intake/verify.py | 2 +- scripts/find_overlap.py | 101 +------- scripts/find_source_overlap.py | 225 ++++++++++++++++++ tests/test_research_overlap_intake.py | 37 ++- 9 files changed, 320 insertions(+), 143 deletions(-) create mode 100644 scripts/find_source_overlap.py diff --git a/docs/research/agent-procedure.md b/docs/research/agent-procedure.md index 580305ef40..f65395f408 100644 --- a/docs/research/agent-procedure.md +++ b/docs/research/agent-procedure.md @@ -122,11 +122,11 @@ Each card supplies a first-step command, expected evidence, a next experiment, its connection to the bottleneck, allowed variations and scope-change triggers. Before taking a source's next question, query its issue/PR identifier directly -in the overlap body corpus. Author and creation-date filters can isolate an -account's recent contributions without relying on GitHub's indexed body search: +in the dedicated PR body intake command. Author and creation-date filters can +isolate an account's recent contributions without GitHub's indexed body search: ```sh -python3 -B scripts/find_overlap.py --reference 6537 --pr-author petalas \ +python3 -B scripts/find_source_overlap.py --reference 6537 --pr-author petalas \ --created-since 2026-10-27 --include-closed --cache /tmp/source-overlap.json ``` @@ -134,8 +134,8 @@ Use the actual source number, login and UTC cutoff. `--reference` is repeatable and excludes decimal/hash substring hits; ordinary positional terms retain literal substring semantics. Author/date filters search only PR titles and bodies. `--include-closed` adds a separately paginated closed/merged corpus. -Default searches are unchanged. Online queries refresh legacy PR caches that -lack author/creation metadata; offline queries requiring missing metadata fail +The existing `find_overlap.py` CLI is unchanged. Online intake queries refresh +PR caches that lack author/creation metadata; offline queries with missing metadata fail explicitly rather than reporting no matches. Filtered results are leads, not evidence of semantic novelty, and do not replace live claim checks. diff --git a/research/source-reference-intake/README.md b/research/source-reference-intake/README.md index 030175c426..21696637ff 100644 --- a/research/source-reference-intake/README.md +++ b/research/source-reference-intake/README.md @@ -1,7 +1,7 @@ # Source-reference intake for research overlap -The overlap helper now accepts identifier-aware reference queries and filters -PR bodies by author and creation date. This makes an account's recent work +A dedicated `scripts/find_source_overlap.py` CLI accepts reference queries +and filters PR bodies by author and creation date. This makes an account's recent work on a proposed source visible directly in the fetched body corpus. Missing filter metadata produces an explicit error, so an old cache cannot silently hide a possible lead. @@ -15,7 +15,7 @@ improvement or predecessor-result verification is claimed. From the repository root: ```sh -~/git/integer-mult-bounds/.venv/bin/python3.12 -B scripts/find_overlap.py \ +~/git/integer-mult-bounds/.venv/bin/python3.12 -B scripts/find_source_overlap.py \ --reference 6537 --pr-author petalas --created-since 2026-10-27 \ --include-closed --cache /tmp/source-overlap.json ``` @@ -26,24 +26,22 @@ decimal substrings cannot displace a real reference. Positional terms retain their literal search semantics. The date filter uses creation, not update, and includes the cutoff day's midnight UTC. -Author/date filters search PR titles and bodies only. Closed/merged bodies -require `--include-closed`, with separate coverage and timestamps. They do -not search local notes or Discussions. The ordinary unfiltered search keeps -its existing scope. A filtered online query refreshes legacy PR metadata; -an offline cache missing the requested fields fails with an explanation. +The dedicated intake CLI searches PR titles and bodies only. Closed/merged +bodies require `--include-closed`, with separate coverage and timestamps. +Local notes and Discussions are excluded. The existing `scripts/find_overlap.py` +remains byte-identical to the PR base; its functions are imported without +mutation. A filtered online query refreshes legacy PR metadata. An offline +cache missing the requested fields fails with an explanation. `--pr-author` takes a login, rather than `@me` or a search expression. ## Evidence and reproduction -Nine new regression groups cover identifier noise and spelling, literal +Eleven new regression groups cover identifier noise and spelling, literal queries, inclusive dates and author case, optional closed bodies, legacy offline errors, invalid metadata, creation/update separation, selective online refresh and invalid CLI input. The existing 37 overlap groups also -pass. The registered replay runs all 46 groups in normal and optimized +pass. The registered replay runs all 48 groups in normal and optimized Python, without network access. -The separate local run of all 18 test modules that mention the overlap helper -also passed, with 132 tests. That broader run is recorded as local evidence; -the registered package runs the focused 46-test suite. ```sh ~/git/integer-mult-bounds/.venv/bin/python3.12 -B scripts/research_package.py --package research/source-reference-intake @@ -51,16 +49,16 @@ the registered package runs the focused 46-test suite. ~/git/integer-mult-bounds/.venv/bin/python3.12 -B scripts/check_research_commit.py --package research/source-reference-intake --check-source-history ``` -Expected: `Ran 46 tests`, `OK`, `PASS_SOURCE_REFERENCE_INTAKE` and the +Expected: `Ran 48 tests`, `OK`, `PASS_SOURCE_REFERENCE_INTAKE` and the registered package PASS. [VALIDATION.json](VALIDATION.json) is generated evidence bound to the tested script/test hashes. Default replay writes nothing persistent. `verify.py --write NEW_PATH` records a fresh execution and refuses to overwrite an existing path. -A live author/date query over the 587-open-PR snapshot fetched at -2026-11-03T14:10:56.715135+00:00 returned PR6571 as a source-6537 lead. +The dedicated CLI's offline query over the retained 587-open-PR snapshot, +fetched at 2026-11-03T14:10:56.715135+00:00, returned PR6571 as a source-6537 lead. This only establishes the tool's retrieval behavior on that snapshot. -Neither its mathematics nor the source's result was evaluated. The nine +Neither its mathematics nor the source's result was evaluated. The eleven synthetic regression groups make the behavior reproducible offline. ## Limits @@ -82,13 +80,13 @@ novelty claims. ## Implementation and credit The new filters traverse finite cached PR lists and validate dates before -comparing them. Unknown author/date values fail explicitly. New helpers -are at most 33 physical lines. The existing CLI dispatcher is shortened -to 170 lines by extracting argument parsing; its broader refactoring is -deferred to avoid changing unrelated path/coverage reporting. Existing -network pagination remains outside this local behavior change. +comparing them. Unknown author/date values fail explicitly. The dedicated +CLI reuses the existing normalizer, identifier matcher, atomic cache writer +and PR deduplication functions. Its own PR-only cache refresh retains author +and creation metadata and preserves older evidence on failed publication. +The existing CLI and every other research package remain untouched. -Power of Ten rule 3 deviation: the existing Python cache/search command +Power of Ten rule 3 deviation: the Python cache/search command allocates lists and dictionaries at runtime. The new pass adds linear storage in the existing validated PR corpus size. A fixed-capacity service port would require a separately declared corpus-size contract. Rule 9 @@ -99,6 +97,17 @@ provide diagnostics; Ruff is unavailable in the prescribed environment. Douglas Colkitt's overlap helper and its credited repository contributors retain their attribution. Existing identifier matching and cache/pagination -machinery are reused. The new explicit query and metadata filters, regression +machinery are reused. The new dedicated query and metadata filters, regression fixtures and integration notes were produced by Nick Petalas. Apache-2.0. Independent human review is unclaimed. + +## CI compatibility repair + +At head `e55541991331d578d88660dcbb34af60506a534d`, the initial hosted run +passed this package and four other jobs, but the historical overlap-path-index +replay failed with `source blob provenance mismatch: scripts/find_overlap.py`. +The shared-file edit in this PR caused that job to be selected. The correction +restores the exact PR-base helper bytes and moves the feature to the dedicated +CLI. The historical package and its source pins are unchanged. A regression +checks that the new CLI does not mutate imported legacy functions or record +shapes. New hosted confirmation remains pending. diff --git a/research/source-reference-intake/TASK.json b/research/source-reference-intake/TASK.json index 80c0a0833e..43c43f48f1 100644 --- a/research/source-reference-intake/TASK.json +++ b/research/source-reference-intake/TASK.json @@ -41,13 +41,13 @@ ] }, "scope_status": "changed", - "actual_scope": "Add explicit issue/PR reference queries and author/creation-date filters to the complete cached PR body search, with explicit unknown-metadata failure and legacy refresh.", + "actual_scope": "Add a dedicated PR-source intake CLI with reference, author and creation-date queries while leaving scripts/find_overlap.py byte-identical to the PR base.", "scope_change_reason": "A source-reuse collision was already represented in the complete body cache but was missed by a targeted indexed search and a broad task query. This is an original coordination-tooling improvement, not another predecessor verification or a scientific replay speedup.", "bottleneck_link": "Coordination side result: expose recent source reuse before spending work on a target. It changes no multiplication bound or supplier obligation.", "budget_minutes": 60, "stop_rule": "Stop after one working source-specific query with passing regression coverage and explicit cache-coverage limits. Do not extend to semantic novelty detection.", "outcome": "completed", - "evidence": "Nine new behavioral regression groups and the 37 existing overlap groups pass. The actual source-reference/author/date query retrieves PR6571 from the fetched open-PR corpus. VALIDATION.json records a generated hash-bound local execution.", + "evidence": "Eleven new behavioral regression groups and the 37 existing overlap-named groups pass. The standalone CLI retrieves PR6571 from the retained body corpus. VALIDATION.json records generated hash-bound local execution.", "author": "Nick Petalas", "ai_assistance": "Independent human and formal proof-kernel review are unclaimed.", "open_questions": [ @@ -78,10 +78,13 @@ "coverage": "Live handoffs, active Discussions, open/account PR lists, complete cached open-PR/Discussion pagination, main-tree distinctive and general-form searches, and merged author/filter searches.", "limits": "Arbitrary diffs, uncited semantic counterparts and unpublished work are not fully covered." }, - "additional_local_check": { - "domain": "All 18 test modules mentioning find_overlap", - "tests_run": 132, - "outcome": "OK", - "command": "Construct unittest.TestSuite from unittest discovery of each matching file and run with warning-strict isolated CPython 3.12." + "ci_repair": { + "failed_head": "e55541991331d578d88660dcbb34af60506a534d", + "run": "https://github.com/CrocSwap/integer-mult-bounds/actions/runs/41652301463", + "job": "replay (research/overlap-path-index)", + "error": "source blob provenance mismatch: scripts/find_overlap.py", + "cause": "The shared-file edit in this PR invalidated the historical consumer.", + "correction": "Restore the shared helper to the exact PR-base bytes and move the feature into scripts/find_source_overlap.py. No other research package or source pin is edited.", + "hosted_status": "pending" } } diff --git a/research/source-reference-intake/VALIDATION.json b/research/source-reference-intake/VALIDATION.json index b9c954ece2..462e30cc48 100644 --- a/research/source-reference-intake/VALIDATION.json +++ b/research/source-reference-intake/VALIDATION.json @@ -15,18 +15,19 @@ "test_research_overlap_*.py", "-q" ], - "created_utc": "2026-11-03T14:13:04.601961+00:00", + "created_utc": "2026-11-03T14:39:20.279188+00:00", "exit_code": 0, - "output": "----------------------------------------------------------------------\nRan 46 tests in 1.805s\n\nOK\n", + "output": "----------------------------------------------------------------------\nRan 48 tests in 1.817s\n\nOK\n", "platform": "macOS-27.0.1-arm64-arm-64bit", "python": "3.12.15 (main, Oct 14 2026, 00:55:37) [Clang 23.1.3 ]", - "seconds": 1.8843645830638707, + "seconds": 1.900019499938935, "sha256": { - "scripts/find_overlap.py": "de10463786ae31392b320b6ed69394bdd0fba5f6bd5df1db648ac891cb566691", + "scripts/find_overlap.py": "c48d121d1fe4996a4c232b0d713d9163a1e47777ea3efe01e39bc36cec9aa053", + "scripts/find_source_overlap.py": "81788b200f4e2a195420016c289cc8387d3d7b531afdb60c2cb2f8aff3d489ef", "scripts/research_package.py": "e1256016e7f51827d04fa2179255989f639ae1d97afaa9c16265f85a0c6abd70", "tests/test_research_overlap_cache.py": "e49cf1c1707a16c0181f642ab8d969eb8e8689262277b42dba343d0a13295bad", "tests/test_research_overlap_closed.py": "2a5d28e13fe57191702d86630763fdfd33842696077837d225c5677aea819b03", - "tests/test_research_overlap_intake.py": "118c1cb1ff90df7b9c2e691be0e6ee4a09eb8fbc150b5ecfc9d180e5296065e9", + "tests/test_research_overlap_intake.py": "152138d48ef155d33c4cb375a34200a98200b1ac5e1f073cf848096e1b631f7a", "tests/test_research_overlap_local.py": "e7deb16c3f8d14523d31b568a7ea58f683cc56489af2009b2f5cada58a94c2da", "tests/test_research_overlap_pagination.py": "b83e490c45f76b70eb394dd9706ff0b2f627211efe975909f9e1b40d8995f99c", "tests/test_research_overlap_terms.py": "7dd29a1b44ab0c80b8b58a6a1eb7cff810dda942019754029b3bc38c9ff3e601" diff --git a/research/source-reference-intake/VERIFY.json b/research/source-reference-intake/VERIFY.json index 60289f7844..1e4c7005aa 100644 --- a/research/source-reference-intake/VERIFY.json +++ b/research/source-reference-intake/VERIFY.json @@ -4,6 +4,7 @@ "python": "3.12", "requirements": [], "inputs": [ + "scripts/find_source_overlap.py", "scripts/find_overlap.py", "scripts/research_package.py", "tests/test_research_overlap_cache.py", @@ -24,7 +25,7 @@ "{package}/verify.py" ], "timeout_seconds": 120, - "checks": "The focused overlap-named suite: nine new intake groups and all 37 existing groups; offline behavior, metadata boundaries, cache refresh and default coverage." + "checks": "The focused suite: eleven dedicated intake groups and 37 legacy overlap groups, including legacy-module isolation and failed-refresh evidence preservation." }, { "argv": [ diff --git a/research/source-reference-intake/verify.py b/research/source-reference-intake/verify.py index 044d159e1c..f0ff0e1a4c 100644 --- a/research/source-reference-intake/verify.py +++ b/research/source-reference-intake/verify.py @@ -30,7 +30,7 @@ def main(): elapsed = time.perf_counter() - started print(result.stdout + result.stderr, end="") if args.write is not None: - inputs = ["scripts/find_overlap.py", "scripts/research_package.py", + inputs = ["scripts/find_source_overlap.py", "scripts/find_overlap.py", "scripts/research_package.py", *sorted(str(path.relative_to(root)) for path in (root / "tests").glob("test_research_overlap_*.py"))] evidence = {"claim_type": "workflow tooling with behavioral regression evidence", diff --git a/scripts/find_overlap.py b/scripts/find_overlap.py index e4d2c73560..d20f85824e 100644 --- a/scripts/find_overlap.py +++ b/scripts/find_overlap.py @@ -63,8 +63,6 @@ def pull_records(pages): return [{"kind": "pr", "number": pr["number"], "title": pr["title"], "body": pr.get("body") or "", "url": pr["html_url"], "head": pr["head"]["sha"], "base": pr.get("base", {}).get("sha"), - "author": (pr.get("user") or {}).get("login"), - "created_at": pr.get("created_at"), "updated_at": pr["updated_at"], "draft": pr.get("draft", False), "state": pr["state"]} for page in pages for pr in page] @@ -281,13 +279,8 @@ def validate_cached_source(source, name): return stamp -def has_pr_metadata(source): - """Schema-1 legacy rows lack the author/creation fields required by filters.""" - return all(row.get("author") and row.get("created_at") for row in source["records"]) - - def remote_snapshot(cache, offline=False, refresh=False, max_age=300, need_files=False, - include_closed=False, need_pr_metadata=False): + include_closed=False): warnings = [] try: snapshot = json.loads(cache.read_text(encoding="utf-8")) @@ -323,8 +316,6 @@ def remote_snapshot(cache, offline=False, refresh=False, max_age=300, need_files sources.pop(name, None) cached = None fresh = cached is not None and age is not None and 0 <= age <= max_age - if need_pr_metadata and name in ("open_prs", "closed_prs") and cached: - fresh = fresh and has_pr_metadata(cached) if need_files and name in ("pr_files", "open_prs") and cached: # Old schema-1 caches omitted endpoints. Refresh only affected sources; # offline/failed refreshes retain the evidence with explicit warnings. @@ -476,57 +467,10 @@ def search(records, terms, all_terms=False, reference_terms=()): return sorted(matches, key=lambda r: (-r["score"], r["kind"], r["title"], r.get("url", ""))) -def reference_number(value): - if not re.fullmatch(r"[1-9][0-9]{0,9}", value): - raise argparse.ArgumentTypeError("reference must be a positive ASCII issue/PR number") - return value - - -def creation_cutoff(value): - try: - if not re.fullmatch(r"[0-9]{4}-[0-9]{2}-[0-9]{2}", value): - raise ValueError("expected YYYY-MM-DD") - return datetime.strptime(value, "%Y-%m-%d").replace(tzinfo=timezone.utc) - except ValueError as exc: - raise argparse.ArgumentTypeError("creation cutoff must be a valid YYYY-MM-DD UTC date") from exc - - -def filter_pulls(records, author=None, created_since=None): - """Reject unknown filter metadata rather than silently omit a possible lead.""" - selected, unknown = [], [] - for row in records: - login, stamp = row.get("author"), row.get("created_at") - created = None - if created_since is not None and isinstance(stamp, str): - try: - created = datetime.fromisoformat(stamp) - except ValueError: - pass - if ((author is not None and (not isinstance(login, str) or not login)) - or (created_since is not None and (created is None or created.tzinfo is None))): - unknown.append(row["number"]) - continue - if author is not None and login.lower() != author.lower(): - continue - if created_since is not None and created < created_since: - continue - selected.append(row) - if unknown: - examples = ", ".join(str(number) for number in sorted(unknown)[:20]) - raise ValueError(f"PR filter metadata unavailable for {len(unknown)} records ({examples}); " - "refresh the cache or inspect an unfiltered report; unknown is not no overlap") - return selected - - -def parse_arguments(): +def main(): ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("terms", nargs="*", help="literal case-insensitive terms; quote multiword phrases") ap.add_argument("--task", help="add the named catalogue card's overlap terms") - ap.add_argument("--reference", action="append", type=reference_number, default=[], - metavar="NUMBER", help="match an issue/PR identifier, not decimal or hash substrings; repeatable") - ap.add_argument("--pr-author", metavar="LOGIN", help="search PR bodies only, restricted to this GitHub login") - ap.add_argument("--created-since", type=creation_cutoff, metavar="YYYY-MM-DD", - help="search PR bodies only, created on/after this UTC date; not their update date") ap.add_argument("--path", action="append", default=[], metavar="PATTERN", help="match open PRs by case-sensitive Git path glob or substring; spaces are literal; repeatable") ap.add_argument("--offline", action="store_true") @@ -543,19 +487,8 @@ def parse_arguments(): ap.error("use nonnegative max-age, positive limit, and either offline or refresh") if args.include_closed and args.path: ap.error("--include-closed searches titles/bodies only; omit --path (its index covers open PRs)") - pr_filters = args.pr_author is not None or args.created_since is not None - if args.pr_author is not None and not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9-]{0,38}", args.pr_author): - ap.error("--pr-author requires a GitHub login, not @me or a search expression") - if pr_filters and args.path: - ap.error("PR author/creation filters search titles/bodies only; omit --path") if args.cache.resolve().is_relative_to(ROOT.resolve()): ap.error("keep the cache outside the repository") - return ap, args - - -def main(): - ap, args = parse_arguments() - pr_filters = args.pr_author is not None or args.created_since is not None terms = list(args.terms) explicit_terms = {term.strip().lower() for term in args.terms} reference_terms = [] @@ -577,18 +510,15 @@ def main(): parts = PurePosixPath(point).parts if len(parts) >= 3 and parts[0] == "research": terms.append(parts[1]) - terms.extend(args.reference) - reference_terms = list(dict.fromkeys([*reference_terms, *args.reference])) if not any(t.strip() for t in terms) and not args.path: - ap.error("provide terms, --task, --reference or --path") + ap.error("provide terms, --task or --path") patterns = list(args.path) if any(not p for p in patterns): ap.error("--path must not be empty") local = local_records(ROOT) local_status, local_warnings = local_coverage(ROOT, local) snapshot, warnings = remote_snapshot(args.cache, args.offline, args.refresh, args.max_age, - need_files=bool(patterns), include_closed=args.include_closed, - need_pr_metadata=pr_filters) + need_files=bool(patterns), include_closed=args.include_closed) warnings = local_warnings + warnings records = list(local) + pull_corpus(snapshot, args.include_closed) coverage = {"local": local_status} @@ -600,12 +530,6 @@ def main(): if name == "discussions": records.extend(source["records"]) coverage[name]["records"] = len(source["records"]) - if pr_filters: - try: - records = filter_pulls(pull_corpus(snapshot, args.include_closed), - args.pr_author, args.created_since) - except ValueError as exc: - ap.error(str(exc)) path_matches = [] if patterns: source = snapshot["sources"].get("pr_files") @@ -656,18 +580,6 @@ def main(): "it does not certify HEAD contents or historical coverage. " "No code diffs or arbitrary package files are searched. Recheck relevant heads before submission."), "matches_total": len(matches), "matches": matches[:args.limit]} - if pr_filters: - report["pr_filters"] = {"author": args.pr_author, - "created_since": args.created_since.isoformat() if args.created_since else None, - "scope": "PR titles and bodies only; local notes and Discussions excluded", - "records_after_filter": len(records)} - report["limits"] = ("Author/creation filters search the declared PR title/body corpora only. " - "Local notes and Discussions are excluded. " - + ("Closed/merged PRs are included with separate coverage timestamps. " - if args.include_closed else "Closed/merged PRs are not included; use --include-closed. ") - + "Creation dates are not update dates. Unknown filter metadata is an error. " - "Snapshots are not atomic; no diffs or semantic equivalence are checked. " - "Matches are overlap leads, not a novelty certificate.") if patterns: report["path_patterns"] = patterns report["path_limits"] = ("Path matching covers the changed file paths GitHub reports for open PRs, " @@ -694,10 +606,7 @@ def main(): local_summary["unsearched_head_paths_count"] = None if omitted is None else len(omitted) print("Coverage: " + json.dumps({**coverage, "local": local_summary})) if reference_terms: - label = "References" if args.reference else "Task references" - print(label + " (identifier matching): " + ", ".join(reference_terms)) - if pr_filters: - print("PR filters: " + json.dumps(report["pr_filters"])) + print("Task references (identifier matching): " + ", ".join(reference_terms)) print(report["limits"]) if patterns: print(report["path_limits"]) diff --git a/scripts/find_source_overlap.py b/scripts/find_source_overlap.py new file mode 100644 index 0000000000..89a459f489 --- /dev/null +++ b/scripts/find_source_overlap.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +"""Search PR body corpora for source references, authors and creation dates. + +Produced by Nick Petalas. Apache-2.0. +Reuses Douglas Colkitt's credited overlap reader without changing its bytes +or mutating its functions. Results are leads, not a novelty certificate. +""" + +import argparse +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import re +import subprocess +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +try: + import find_overlap as legacy +finally: + sys.path.pop(0) + +ROOT = Path(__file__).resolve().parents[1] +REPO = legacy.REPO +now = legacy.now +gh_json = legacy.gh_json + + +def pull_records(pages): + rows = legacy.pull_records(pages) + originals = [pull for page in pages for pull in page] + for row, pull in zip(rows, originals, strict=True): + row["author"] = (pull.get("user") or {}).get("login") + row["created_at"] = pull.get("created_at") + return rows + + +def read_snapshot(cache): + warnings = [] + empty = {"schema_version": 1, "repository": REPO, "sources": {}} + try: + snapshot = json.loads(cache.read_text(encoding="utf-8")) + if (not isinstance(snapshot, dict) or snapshot.get("schema_version") != 1 + or snapshot.get("repository") != REPO or not isinstance(snapshot.get("sources"), dict)): + raise ValueError("cache schema or repository mismatch") + except FileNotFoundError: + snapshot = empty + except (OSError, ValueError) as exc: + warnings.append("cache: unusable snapshot: " + legacy.error_summary(exc)) + snapshot = empty + return snapshot, warnings + + +def cached_source(snapshot, name, max_age, need_metadata, warnings): + cached = snapshot["sources"].get(name) + if cached is None: + return None, False + try: + stamp = legacy.validate_cached_source(cached, name) + age = (datetime.now(timezone.utc) - stamp).total_seconds() + except (KeyError, ValueError, TypeError) as exc: + warnings.append(f"{name}: unusable cached source: {legacy.error_summary(exc)}") + snapshot["sources"].pop(name, None) + return None, False + metadata = all(row.get("author") and row.get("created_at") for row in cached["records"]) + return cached, 0 <= age <= max_age and (metadata or not need_metadata) + + +def fetch_source(snapshot, name, cache): + state = "open" if name == "open_prs" else "closed" + records = pull_records(gh_json([ + "--paginate", "--slurp", + f"repos/{REPO}/pulls?state={state}&per_page=100&sort=updated&direction=desc"])) + replacement = {"fetched_at": now(), "records": records, "complete_pagination": True} + pending = {**snapshot, "sources": {**snapshot["sources"], name: replacement}} + legacy.atomic_json(cache, pending) + snapshot["sources"][name] = replacement + return replacement + + +def remote_snapshot(cache, offline=False, refresh=False, max_age=300, + include_closed=False, need_pr_metadata=False): + snapshot, warnings = read_snapshot(cache) + names = ("open_prs", "closed_prs") if include_closed else ("open_prs",) + for name in names: + cached, fresh = cached_source(snapshot, name, max_age, need_pr_metadata, warnings) + if offline: + warnings.append(f"{name}: offline; cached data may miss current changes") + elif refresh or not fresh: + try: + cached = fetch_source(snapshot, name, cache) + except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError) as exc: + warnings.append(f"{name}: refresh failed; using old data if available: " + + legacy.error_summary(exc)) + if cached is None: + warnings.append(f"{name}: no usable remote snapshot") + snapshot["sources"].pop(name, None) + return snapshot, warnings + + +def reference_number(value): + if not re.fullmatch(r"[1-9][0-9]{0,9}", value): + raise argparse.ArgumentTypeError("reference must be a positive ASCII issue/PR number") + return value + + +def creation_cutoff(value): + try: + if not re.fullmatch(r"[0-9]{4}-[0-9]{2}-[0-9]{2}", value): + raise ValueError("expected YYYY-MM-DD") + return datetime.strptime(value, "%Y-%m-%d").replace(tzinfo=timezone.utc) + except ValueError as exc: + raise argparse.ArgumentTypeError("creation cutoff must be a valid YYYY-MM-DD UTC date") from exc + + +def filter_pulls(records, author=None, created_since=None): + """Reject unknown filter metadata rather than silently omit a possible lead.""" + selected, unknown = [], [] + for row in records: + login, stamp = row.get("author"), row.get("created_at") + created = None + if created_since is not None and isinstance(stamp, str): + try: + created = datetime.fromisoformat(stamp) + except ValueError: + pass + if ((author is not None and (not isinstance(login, str) or not login)) + or (created_since is not None and (created is None or created.tzinfo is None))): + unknown.append(row["number"]) + continue + if author is not None and login.lower() != author.lower(): + continue + if created_since is not None and created < created_since: + continue + selected.append(row) + if unknown: + examples = ", ".join(str(number) for number in sorted(unknown)[:20]) + raise ValueError(f"PR filter metadata unavailable for {len(unknown)} records ({examples}); " + "refresh the cache or inspect an unfiltered report; unknown is not no overlap") + return selected + + +def parse_arguments(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("terms", nargs="*", help="literal case-insensitive PR title/body terms") + parser.add_argument("--reference", action="append", type=reference_number, default=[], metavar="NUMBER") + parser.add_argument("--pr-author", metavar="LOGIN") + parser.add_argument("--created-since", type=creation_cutoff, metavar="YYYY-MM-DD") + parser.add_argument("--include-closed", action="store_true", help="also fetch/search closed and merged PRs") + parser.add_argument("--offline", action="store_true") + parser.add_argument("--refresh", action="store_true") + parser.add_argument("--max-age", type=int, default=300) + parser.add_argument("--limit", type=int, default=20) + parser.add_argument("--all-terms", action="store_true") + parser.add_argument("--json", action="store_true", dest="json_output") + cache_root = Path(os.environ.get("XDG_CACHE_HOME", str(Path.home() / ".cache"))) + parser.add_argument("--cache", type=Path, default=cache_root / "integer-mult-bounds/source-overlap.json") + args = parser.parse_args() + if args.max_age < 0 or args.limit < 1 or (args.offline and args.refresh): + parser.error("use nonnegative max-age, positive limit, and either offline or refresh") + if args.pr_author is not None and not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9-]{0,38}", args.pr_author): + parser.error("--pr-author requires a GitHub login, not @me or a search expression") + if args.cache.resolve().is_relative_to(ROOT.resolve()): + parser.error("keep the cache outside the repository") + if not args.reference and not any(term.strip() for term in args.terms): + parser.error("provide terms or --reference") + return parser, args + + +def write_report(report, args): + if args.json_output: + print(json.dumps(report, indent=2)) + return + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(errors="backslashreplace") + print(report["status"]) + print("Coverage: " + json.dumps(report["coverage"])) + print("PR filters: " + json.dumps(report["pr_filters"])) + print(report["limits"]) + for warning in report["warnings"]: + print("WARNING: " + warning) + print(f"{report['matches_total']} possible matches; showing {len(report['matches'])}") + for match in report["matches"]: + print(f"\n[pr {match['number']}] {match['title']}\n{match.get('url', '')}") + print(match["excerpt"]) + + +def main(): + parser, args = parse_arguments() + filtered = args.pr_author is not None or args.created_since is not None + snapshot, warnings = remote_snapshot(args.cache, args.offline, args.refresh, args.max_age, + args.include_closed, filtered) + try: + records = filter_pulls(legacy.pull_corpus(snapshot, args.include_closed), + args.pr_author, args.created_since) + except ValueError as exc: + parser.error(str(exc)) + references = list(dict.fromkeys(args.reference)) + terms = [*args.terms, *references] + matches = legacy.search(records, terms, args.all_terms, references) + names = ("open_prs", "closed_prs") if args.include_closed else ("open_prs",) + coverage = {} + for name in names: + source = snapshot["sources"].get(name) + coverage[name] = ({"fetched_at": source["fetched_at"], "records": len(source["records"]), + "complete_pagination": source["complete_pagination"]} if source else None) + report = {"status": "POSSIBLE_OVERLAP_NOT_A_NOVELTY_CERTIFICATE", "searched_at": now(), + "terms": terms, "reference_terms": references, "coverage": coverage, "warnings": warnings, + "pr_filters": {"author": args.pr_author, + "created_since": args.created_since.isoformat() if args.created_since else None, + "scope": "PR titles and bodies only; local notes and Discussions excluded", + "records_after_filter": len(records)}, + "limits": ("PR titles/bodies only; local notes and Discussions are excluded. " + + ("Closed/merged PRs are included with separate timestamps. " if args.include_closed + else "Closed/merged PRs are excluded; use --include-closed. ") + + "Creation dates are not update dates. Unknown filter metadata is an error. " + "Snapshots are not atomic; no diffs or semantic equivalence are checked. " + "Matches are overlap leads, not a novelty certificate."), + "matches_total": len(matches), "matches": matches[:args.limit]} + write_report(report, args) + + +if __name__ == "__main__": + main() diff --git a/tests/test_research_overlap_intake.py b/tests/test_research_overlap_intake.py index 24c7dc7e2c..78d16e5538 100644 --- a/tests/test_research_overlap_intake.py +++ b/tests/test_research_overlap_intake.py @@ -13,7 +13,7 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) -import find_overlap as overlap +import find_source_overlap as overlap sys.path.pop(0) @@ -36,9 +36,8 @@ def run_report(self, rows, arguments=(), closed=()): cache.write_text(raw, encoding="utf-8") output = io.StringIO() with (patch.object(overlap, "ROOT", Path(tmp) / "empty-repo"), - patch.object(overlap, "local_coverage", return_value=({}, [])), patch.object(overlap, "gh_json", side_effect=AssertionError("network")), - patch.object(sys, "argv", ["find_overlap.py", "--offline", "--json", + patch.object(sys, "argv", ["find_source_overlap.py", "--offline", "--json", "--cache", str(cache), *arguments]), contextlib.redirect_stdout(output)): overlap.main() @@ -107,6 +106,36 @@ def test_normalized_rest_metadata_retains_creation_not_update(self): self.assertEqual(overlap.filter_pulls([normalized], "petalas", cutoff), []) self.assertIsNone(overlap.pull_records([[dict(raw, user=None)]])[0]["author"]) + def test_new_cli_does_not_mutate_the_legacy_reader(self): + names = ("pull_records", "remote_snapshot", "search", "atomic_json") + functions = [getattr(overlap.legacy, name) for name in names] + self.run_report([pull(1)], ["--reference", "6537", "--pr-author", "petalas"]) + self.assertEqual(functions, [getattr(overlap.legacy, name) for name in names]) + raw = {"number": 7, "title": "A lead", "html_url": "url", "head": {"sha": "a" * 40}, + "state": "open", "updated_at": overlap.now(), "user": {"login": "petalas"}, + "created_at": overlap.now()} + self.assertNotIn("author", overlap.legacy.pull_records([[raw]])[0]) + self.assertNotIn("created_at", overlap.legacy.pull_records([[raw]])[0]) + + def test_refresh_failure_preserves_cached_bytes_and_in_memory_evidence(self): + for failure in ("fetch", "publication"): + with self.subTest(failure=failure), tempfile.TemporaryDirectory() as tmp: + cache = Path(tmp) / "cache.json" + old = {"fetched_at": overlap.now(), "complete_pagination": True, "records": [pull(1)]} + raw = json.dumps({"schema_version": 1, "repository": overlap.REPO, + "sources": {"open_prs": old}}) + cache.write_text(raw, encoding="utf-8") + new = {"number": 2, "title": "New", "html_url": "url", "head": {"sha": "c" * 40}, + "state": "open", "updated_at": overlap.now()} + fetch_error = overlap.subprocess.CalledProcessError(1, ["gh"], stderr="later page failed") + with (patch.object(overlap, "gh_json", side_effect=fetch_error if failure == "fetch" else None, + return_value=[[new]]), + patch.object(overlap.legacy, "atomic_json", side_effect=OSError("write failed"))): + snapshot, warnings = overlap.remote_snapshot(cache, refresh=True) + self.assertEqual(cache.read_text(encoding="utf-8"), raw) + self.assertEqual(snapshot["sources"]["open_prs"], old) + self.assertTrue(any("refresh failed" in warning for warning in warnings)) + def test_fresh_legacy_cache_refreshes_pr_metadata_without_refetching_discussions(self): with tempfile.TemporaryDirectory() as tmp: cache = Path(tmp) / "cache.json" @@ -120,7 +149,7 @@ def test_fresh_legacy_cache_refreshes_pr_metadata_without_refetching_discussions "state": "open", "updated_at": overlap.now(), "created_at": overlap.now(), "user": {"login": "petalas"}} with (patch.object(overlap, "gh_json", return_value=[[raw]]) as fetch, - patch.object(overlap, "fetch_discussions", side_effect=AssertionError("discussion refetch"))): + patch.object(overlap.legacy, "fetch_discussions", side_effect=AssertionError("discussion refetch"))): snapshot, warnings = overlap.remote_snapshot(cache, need_pr_metadata=True) self.assertEqual(fetch.call_count, 1) self.assertEqual(warnings, [])