diff --git a/.github/workflows/upstream-sync.yml b/.github/workflows/upstream-sync.yml index 789eb3f5..f6f83324 100644 --- a/.github/workflows/upstream-sync.yml +++ b/.github/workflows/upstream-sync.yml @@ -170,6 +170,10 @@ jobs: echo "status=clean" >> "$GITHUB_OUTPUT" exit 0 fi + # rerere cannot record a modify/delete conflict, so a file we deleted + # on purpose (.fork-deleted) that upstream edited would stall every + # sync. Keep those deleted. Any other conflict still stops the run. + scripts/resolve-fork-deleted.py # rerere may have staged resolutions but left the merge in-progress. unresolved="$(git diff --name-only --diff-filter=U)" if [ -z "$unresolved" ]; then diff --git a/docs/fork/sync-cycle.md b/docs/fork/sync-cycle.md index b035c303..ab35215c 100644 --- a/docs/fork/sync-cycle.md +++ b/docs/fork/sync-cycle.md @@ -135,6 +135,11 @@ rerere replays previously-recorded resolutions for our PERMANENT deltas. copy outright. Anything left conflicting is a three-way conflict on a catalogued file. +A file we deleted that upstream then edits is a modify/delete conflict, which +rerere cannot record, so `scripts/resolve-fork-deleted.py` runs straight after +the merge and `git rm`s each conflicted path listed in `.fork-deleted`. A text +conflict, or a modify/delete on an unlisted path, still stops the sync. + **A replayed resolution is not a correct one.** rerere reproduces the edit that resolved a conflict whose preimage matched, and it has no idea whether the result still means anything. The failure it produces is specific and worth diff --git a/docs/fork/what-we-changed.md b/docs/fork/what-we-changed.md index 8b898d53..abe442ad 100644 --- a/docs/fork/what-we-changed.md +++ b/docs/fork/what-we-changed.md @@ -722,13 +722,16 @@ expects undeclared static-filter keys to persist is skipped by name in the 16 upstream workflows we do not carry. `.fork-surface` cannot cover a deletion: it reads `--diff-filter=ACMR` against the merge base, where a file we removed is unchanged and therefore invisible, so a sync reinstates it in - silence. The check fails when a listed path exists. + silence. The check fails when a listed path exists. The sync keeps a listed + path deleted on its own (`scripts/resolve-fork-deleted.py`), so only a text + conflict, or an upstream delete of a file we edited, still stops it. - `scripts/ci/__tests__/**` in `knip.json` - upstream's ratchet test. Only upstream's `main.yml` ever ran it, and we do not carry that workflow, so knip is correct that nothing uses it. Ignored rather than deleted, on the same reasoning as the orphaned upstream pages: removing an upstream file buys a - delete/modify conflict on every sync. **`scripts/ci/ratchet.mjs` itself is - therefore not wired into our CI either** - run it by hand, or give it a home. + delete/modify conflict on every sync until its path is in `.fork-deleted`. + **`scripts/ci/ratchet.mjs` itself is therefore not wired into our CI + either** - run it by hand, or give it a home. - `scripts/ci/ratchet-baseline.json` - upstream's escape-hatch ratchet, our numbers. The baseline is the floor for `as any` and `eslint-disable` counts per package, so every hatch we remove has to be locked in here or the ratchet diff --git a/scripts/__tests__/test_resolve_fork_deleted.py b/scripts/__tests__/test_resolve_fork_deleted.py new file mode 100755 index 00000000..89cb31a6 --- /dev/null +++ b/scripts/__tests__/test_resolve_fork_deleted.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# Project: dfe-hyperdx +# File: scripts/__tests__/test_resolve_fork_deleted.py +# Purpose: Prove the sync keeps fork-deleted files deleted, and nothing else +# Language: Python +# +# License: MIT (inherits upstream hyperdxio/hyperdx) +# Copyright: (c) 2026 HYPERI PTY LIMITED +"""Tests for scripts/resolve-fork-deleted.py. + +The script is a statement about git's index after a conflicted merge, so the +test makes a real one: a base commit, a fork branch that deletes and edits, an +upstream branch that edits the same files, then `git merge`. Run:: + + python3 -m unittest discover -s scripts/__tests__ -v +""" + +from __future__ import annotations + +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +SOURCE_ROOT = Path(__file__).resolve().parent.parent +SCRIPT = "resolve-fork-deleted.py" + +# path -> (base, fork, upstream). None means the side deleted the file. +FILES: dict[str, tuple[str, str | None, str | None]] = { + # Fork deleted, upstream edited, listed by exact path. + "listed/modify-delete.yml": ("a\n", None, "a upstream\n"), + # Fork deleted, upstream edited, listed by glob. + "globbed/modify-delete.yml": ("a\n", None, "a upstream\n"), + # Fork deleted, upstream edited, NOT listed. + "unlisted/modify-delete.yml": ("a\n", None, "a upstream\n"), + # Both edited the same line, NOT listed. + "unlisted/text.txt": ("a\n", "a fork\n", "a upstream\n"), + # Both edited the same line, but the path IS listed. + "listed/text.yml": ("a\n", "a fork\n", "a upstream\n"), + # We edited, upstream deleted, and the path is listed. + "listed/upstream-deleted.yml": ("a\n", "a fork\n", None), +} + +FORK_DELETED = """\ +# comment +listed/modify-delete.yml +listed/text.yml +listed/upstream-deleted.yml + +globbed/*.yml +""" + + +def git(repo: Path, *args: str, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run( + ["git", *args], + cwd=repo, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=check, + ) + + +def write_side(repo: Path, index: int) -> None: + """Apply one column of FILES (1 = fork, 2 = upstream) to the work tree.""" + for path, versions in FILES.items(): + target = repo / path + content = versions[index] + if content is None: + git(repo, "rm", "-q", "--", path) + else: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + + +class ResolveForkDeletedTest(unittest.TestCase): + def setUp(self) -> None: + self.repo = Path(tempfile.mkdtemp(prefix="fork-deleted-")) + self.addCleanup(shutil.rmtree, self.repo, ignore_errors=True) + + (self.repo / "scripts").mkdir() + shutil.copy(SOURCE_ROOT / SCRIPT, self.repo / "scripts" / SCRIPT) + + git(self.repo, "init", "-q", "-b", "main") + git(self.repo, "config", "user.email", "test@dfe.hyperi.io") + git(self.repo, "config", "user.name", "test") + git(self.repo, "config", "rerere.enabled", "false") + + for path, versions in FILES.items(): + (self.repo / path).parent.mkdir(parents=True, exist_ok=True) + (self.repo / path).write_text(versions[0], encoding="utf-8") + git(self.repo, "add", "-A") + git(self.repo, "commit", "-qm", "base") + + git(self.repo, "checkout", "-q", "-b", "upstream") + write_side(self.repo, 2) + git(self.repo, "commit", "-qam", "upstream edits") + + git(self.repo, "checkout", "-q", "main") + write_side(self.repo, 1) + (self.repo / ".fork-deleted").write_text(FORK_DELETED, encoding="utf-8") + git(self.repo, "add", "-A") + git(self.repo, "commit", "-qm", "fork") + + merged = git(self.repo, "merge", "--no-edit", "upstream", check=False) + self.assertNotEqual(merged.returncode, 0, "the fixture must conflict") + + def unmerged(self) -> set[str]: + out = git(self.repo, "diff", "--name-only", "--diff-filter=U").stdout + return set(out.split()) + + def run_script(self) -> subprocess.CompletedProcess: + return subprocess.run( + [str(self.repo / "scripts" / SCRIPT)], + cwd=self.repo, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=False, + ) + + def test_fixture_has_every_conflict_shape(self) -> None: + self.assertEqual(self.unmerged(), set(FILES)) + + def test_resolves_only_listed_modify_delete(self) -> None: + result = self.run_script() + self.assertEqual(result.returncode, 0, result.stderr) + + resolved = { + "listed/modify-delete.yml", + "globbed/modify-delete.yml", + } + self.assertEqual(self.unmerged(), set(FILES) - resolved) + for path in resolved: + self.assertIn(path, result.stdout) + self.assertFalse((self.repo / path).exists(), path) + for path in set(FILES) - resolved: + self.assertNotIn(path, result.stdout) + + def test_leaves_the_text_conflict_markers_alone(self) -> None: + self.run_script() + for path in ("listed/text.yml", "unlisted/text.txt"): + text = (self.repo / path).read_text(encoding="utf-8") + self.assertIn("<<<<<<<", text, path) + + def test_second_run_finds_nothing(self) -> None: + self.run_script() + again = self.run_script() + self.assertEqual(again.returncode, 0) + self.assertEqual(again.stdout, "") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/resolve-fork-deleted.py b/scripts/resolve-fork-deleted.py new file mode 100755 index 00000000..4aadb252 --- /dev/null +++ b/scripts/resolve-fork-deleted.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# Project: dfe-hyperdx +# File: scripts/resolve-fork-deleted.py +# Purpose: Keep fork-deleted upstream files deleted through a merge +# Language: Python +# +# License: MIT (inherits upstream hyperdxio/hyperdx) +# Copyright: (c) 2026 HYPERI PTY LIMITED +"""Resolve modify/delete conflicts on the files `.fork-deleted` lists. + +An upstream file we deleted on purpose conflicts whenever upstream edits it: +we deleted it, they modified it. rerere only records text conflicts, so it +never replays this one and every such edit stalls the sync. The answer is +always the same - the file stays deleted - so this runs after `git merge` and +`git rm`s exactly those paths. + +A path is resolved only when both hold: + + - it is unmerged with the merge-base and upstream stages present and ours + absent (we deleted it, upstream modified it) + - it matches a `.fork-deleted` entry (a path or fnmatch glob) + +Everything else is left for the merge to report: a text conflict on a listed +path, a modify/delete on an unlisted path, and a file upstream deleted that we +modified. + +Usage:: + + git merge upstream/main || scripts/resolve-fork-deleted.py + +Prints each path it resolved. Exit 0 whether or not it resolved anything, 1 +when git itself fails. +""" + +from __future__ import annotations + +import fnmatch +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +DELETIONS = REPO_ROOT / ".fork-deleted" + + +def _git(*args: str) -> str: + """Run git in the repo and return stdout, raising when it fails.""" + return subprocess.run( + ["git", *args], + cwd=REPO_ROOT, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=True, + ).stdout + + +def load_deletions() -> list[str]: + """Read `.fork-deleted` - one path or glob per line, '#' comments.""" + if not DELETIONS.is_file(): + return [] + entries: list[str] = [] + for raw in DELETIONS.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if line and not line.startswith("#"): + entries.append(line) + return entries + + +def unmerged_stages() -> dict[str, set[int]]: + """Map each unmerged path to the index stages it still has. + + Stage 1 is the merge base, 2 is ours, 3 is upstream's. `-z` keeps a path + with a quote or newline in it intact, and upstream controls these names. + """ + stages: dict[str, set[int]] = {} + for record in _git("ls-files", "--unmerged", "-z").split("\0"): + if not record: + continue + meta, _, path = record.partition("\t") + stages.setdefault(path, set()).add(int(meta.split()[2])) + return stages + + +def deleted_by_us(stages: set[int]) -> bool: + """True for a modify/delete where WE deleted: base and theirs, no ours.""" + return stages == {1, 3} + + +def resolve() -> list[str]: + """`git rm` every unmerged fork-deleted path we deleted, returning them.""" + patterns = load_deletions() + resolved = [ + path + for path, stages in sorted(unmerged_stages().items()) + if deleted_by_us(stages) and any(fnmatch.fnmatchcase(path, p) for p in patterns) + ] + for path in resolved: + _git("rm", "--quiet", "--", path) + return resolved + + +def main() -> int: + try: + resolved = resolve() + except (subprocess.CalledProcessError, OSError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + for path in resolved: + print(f"kept deleted (.fork-deleted): {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())