From 732ccfa0da2f32528fb3030522249171fa908413 Mon Sep 17 00:00:00 2001 From: Derek Date: Tue, 17 Nov 2026 06:56:53 +1100 Subject: [PATCH 1/3] fix(sync): keep fork-deleted files deleted through the upstream merge A file deleted in the fork and listed in .fork-deleted conflicts whenever upstream edits it. rerere cannot record a modify/delete conflict. Each conflict stalled the weekly sync and triggered an issue for manual resolution. Issue 224 had 7 of its 28 conflicts of this kind. scripts/resolve-fork-deleted.py runs straight after the merge in upstream-sync.yml. It runs git rm on each unmerged path listed in .fork-deleted and deleted in the fork. Text conflicts and modify/delete on unlisted paths still stop the sync. Files deleted upstream and edited in the fork still stop the sync. --- .github/workflows/upstream-sync.yml | 4 + docs/fork/sync-cycle.md | 5 + .../__tests__/test_resolve_fork_deleted.py | 158 ++++++++++++++++++ scripts/resolve-fork-deleted.py | 116 +++++++++++++ 4 files changed, 283 insertions(+) create mode 100755 scripts/__tests__/test_resolve_fork_deleted.py create mode 100755 scripts/resolve-fork-deleted.py diff --git a/.github/workflows/upstream-sync.yml b/.github/workflows/upstream-sync.yml index 789eb3f5..f6f83324 100644 --- a/.github/workflows/upstream-sync.yml +++ b/.github/workflows/upstream-sync.yml @@ -170,6 +170,10 @@ jobs: echo "status=clean" >> "$GITHUB_OUTPUT" exit 0 fi + # rerere cannot record a modify/delete conflict, so a file we deleted + # on purpose (.fork-deleted) that upstream edited would stall every + # sync. Keep those deleted. Any other conflict still stops the run. + scripts/resolve-fork-deleted.py # rerere may have staged resolutions but left the merge in-progress. unresolved="$(git diff --name-only --diff-filter=U)" if [ -z "$unresolved" ]; then diff --git a/docs/fork/sync-cycle.md b/docs/fork/sync-cycle.md index b035c303..ab35215c 100644 --- a/docs/fork/sync-cycle.md +++ b/docs/fork/sync-cycle.md @@ -135,6 +135,11 @@ rerere replays previously-recorded resolutions for our PERMANENT deltas. copy outright. Anything left conflicting is a three-way conflict on a catalogued file. +A file we deleted that upstream then edits is a modify/delete conflict, which +rerere cannot record, so `scripts/resolve-fork-deleted.py` runs straight after +the merge and `git rm`s each conflicted path listed in `.fork-deleted`. A text +conflict, or a modify/delete on an unlisted path, still stops the sync. + **A replayed resolution is not a correct one.** rerere reproduces the edit that resolved a conflict whose preimage matched, and it has no idea whether the result still means anything. The failure it produces is specific and worth diff --git a/scripts/__tests__/test_resolve_fork_deleted.py b/scripts/__tests__/test_resolve_fork_deleted.py new file mode 100755 index 00000000..89cb31a6 --- /dev/null +++ b/scripts/__tests__/test_resolve_fork_deleted.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# Project: dfe-hyperdx +# File: scripts/__tests__/test_resolve_fork_deleted.py +# Purpose: Prove the sync keeps fork-deleted files deleted, and nothing else +# Language: Python +# +# License: MIT (inherits upstream hyperdxio/hyperdx) +# Copyright: (c) 2026 HYPERI PTY LIMITED +"""Tests for scripts/resolve-fork-deleted.py. + +The script is a statement about git's index after a conflicted merge, so the +test makes a real one: a base commit, a fork branch that deletes and edits, an +upstream branch that edits the same files, then `git merge`. Run:: + + python3 -m unittest discover -s scripts/__tests__ -v +""" + +from __future__ import annotations + +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +SOURCE_ROOT = Path(__file__).resolve().parent.parent +SCRIPT = "resolve-fork-deleted.py" + +# path -> (base, fork, upstream). None means the side deleted the file. +FILES: dict[str, tuple[str, str | None, str | None]] = { + # Fork deleted, upstream edited, listed by exact path. + "listed/modify-delete.yml": ("a\n", None, "a upstream\n"), + # Fork deleted, upstream edited, listed by glob. + "globbed/modify-delete.yml": ("a\n", None, "a upstream\n"), + # Fork deleted, upstream edited, NOT listed. + "unlisted/modify-delete.yml": ("a\n", None, "a upstream\n"), + # Both edited the same line, NOT listed. + "unlisted/text.txt": ("a\n", "a fork\n", "a upstream\n"), + # Both edited the same line, but the path IS listed. + "listed/text.yml": ("a\n", "a fork\n", "a upstream\n"), + # We edited, upstream deleted, and the path is listed. + "listed/upstream-deleted.yml": ("a\n", "a fork\n", None), +} + +FORK_DELETED = """\ +# comment +listed/modify-delete.yml +listed/text.yml +listed/upstream-deleted.yml + +globbed/*.yml +""" + + +def git(repo: Path, *args: str, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run( + ["git", *args], + cwd=repo, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=check, + ) + + +def write_side(repo: Path, index: int) -> None: + """Apply one column of FILES (1 = fork, 2 = upstream) to the work tree.""" + for path, versions in FILES.items(): + target = repo / path + content = versions[index] + if content is None: + git(repo, "rm", "-q", "--", path) + else: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + + +class ResolveForkDeletedTest(unittest.TestCase): + def setUp(self) -> None: + self.repo = Path(tempfile.mkdtemp(prefix="fork-deleted-")) + self.addCleanup(shutil.rmtree, self.repo, ignore_errors=True) + + (self.repo / "scripts").mkdir() + shutil.copy(SOURCE_ROOT / SCRIPT, self.repo / "scripts" / SCRIPT) + + git(self.repo, "init", "-q", "-b", "main") + git(self.repo, "config", "user.email", "test@dfe.hyperi.io") + git(self.repo, "config", "user.name", "test") + git(self.repo, "config", "rerere.enabled", "false") + + for path, versions in FILES.items(): + (self.repo / path).parent.mkdir(parents=True, exist_ok=True) + (self.repo / path).write_text(versions[0], encoding="utf-8") + git(self.repo, "add", "-A") + git(self.repo, "commit", "-qm", "base") + + git(self.repo, "checkout", "-q", "-b", "upstream") + write_side(self.repo, 2) + git(self.repo, "commit", "-qam", "upstream edits") + + git(self.repo, "checkout", "-q", "main") + write_side(self.repo, 1) + (self.repo / ".fork-deleted").write_text(FORK_DELETED, encoding="utf-8") + git(self.repo, "add", "-A") + git(self.repo, "commit", "-qm", "fork") + + merged = git(self.repo, "merge", "--no-edit", "upstream", check=False) + self.assertNotEqual(merged.returncode, 0, "the fixture must conflict") + + def unmerged(self) -> set[str]: + out = git(self.repo, "diff", "--name-only", "--diff-filter=U").stdout + return set(out.split()) + + def run_script(self) -> subprocess.CompletedProcess: + return subprocess.run( + [str(self.repo / "scripts" / SCRIPT)], + cwd=self.repo, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=False, + ) + + def test_fixture_has_every_conflict_shape(self) -> None: + self.assertEqual(self.unmerged(), set(FILES)) + + def test_resolves_only_listed_modify_delete(self) -> None: + result = self.run_script() + self.assertEqual(result.returncode, 0, result.stderr) + + resolved = { + "listed/modify-delete.yml", + "globbed/modify-delete.yml", + } + self.assertEqual(self.unmerged(), set(FILES) - resolved) + for path in resolved: + self.assertIn(path, result.stdout) + self.assertFalse((self.repo / path).exists(), path) + for path in set(FILES) - resolved: + self.assertNotIn(path, result.stdout) + + def test_leaves_the_text_conflict_markers_alone(self) -> None: + self.run_script() + for path in ("listed/text.yml", "unlisted/text.txt"): + text = (self.repo / path).read_text(encoding="utf-8") + self.assertIn("<<<<<<<", text, path) + + def test_second_run_finds_nothing(self) -> None: + self.run_script() + again = self.run_script() + self.assertEqual(again.returncode, 0) + self.assertEqual(again.stdout, "") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/resolve-fork-deleted.py b/scripts/resolve-fork-deleted.py new file mode 100755 index 00000000..4aadb252 --- /dev/null +++ b/scripts/resolve-fork-deleted.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# Project: dfe-hyperdx +# File: scripts/resolve-fork-deleted.py +# Purpose: Keep fork-deleted upstream files deleted through a merge +# Language: Python +# +# License: MIT (inherits upstream hyperdxio/hyperdx) +# Copyright: (c) 2026 HYPERI PTY LIMITED +"""Resolve modify/delete conflicts on the files `.fork-deleted` lists. + +An upstream file we deleted on purpose conflicts whenever upstream edits it: +we deleted it, they modified it. rerere only records text conflicts, so it +never replays this one and every such edit stalls the sync. The answer is +always the same - the file stays deleted - so this runs after `git merge` and +`git rm`s exactly those paths. + +A path is resolved only when both hold: + + - it is unmerged with the merge-base and upstream stages present and ours + absent (we deleted it, upstream modified it) + - it matches a `.fork-deleted` entry (a path or fnmatch glob) + +Everything else is left for the merge to report: a text conflict on a listed +path, a modify/delete on an unlisted path, and a file upstream deleted that we +modified. + +Usage:: + + git merge upstream/main || scripts/resolve-fork-deleted.py + +Prints each path it resolved. Exit 0 whether or not it resolved anything, 1 +when git itself fails. +""" + +from __future__ import annotations + +import fnmatch +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +DELETIONS = REPO_ROOT / ".fork-deleted" + + +def _git(*args: str) -> str: + """Run git in the repo and return stdout, raising when it fails.""" + return subprocess.run( + ["git", *args], + cwd=REPO_ROOT, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=True, + ).stdout + + +def load_deletions() -> list[str]: + """Read `.fork-deleted` - one path or glob per line, '#' comments.""" + if not DELETIONS.is_file(): + return [] + entries: list[str] = [] + for raw in DELETIONS.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if line and not line.startswith("#"): + entries.append(line) + return entries + + +def unmerged_stages() -> dict[str, set[int]]: + """Map each unmerged path to the index stages it still has. + + Stage 1 is the merge base, 2 is ours, 3 is upstream's. `-z` keeps a path + with a quote or newline in it intact, and upstream controls these names. + """ + stages: dict[str, set[int]] = {} + for record in _git("ls-files", "--unmerged", "-z").split("\0"): + if not record: + continue + meta, _, path = record.partition("\t") + stages.setdefault(path, set()).add(int(meta.split()[2])) + return stages + + +def deleted_by_us(stages: set[int]) -> bool: + """True for a modify/delete where WE deleted: base and theirs, no ours.""" + return stages == {1, 3} + + +def resolve() -> list[str]: + """`git rm` every unmerged fork-deleted path we deleted, returning them.""" + patterns = load_deletions() + resolved = [ + path + for path, stages in sorted(unmerged_stages().items()) + if deleted_by_us(stages) and any(fnmatch.fnmatchcase(path, p) for p in patterns) + ] + for path in resolved: + _git("rm", "--quiet", "--", path) + return resolved + + +def main() -> int: + try: + resolved = resolve() + except (subprocess.CalledProcessError, OSError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + for path in resolved: + print(f"kept deleted (.fork-deleted): {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From e19d0143be6e6245035a332493c81913bd1e2651 Mon Sep 17 00:00:00 2001 From: Derek Date: Tue, 17 Nov 2026 07:01:40 +1100 Subject: [PATCH 2/3] docs(fork): say the sync keeps fork-deleted paths deleted The .fork-deleted entry in what-we-changed.md says a listed path stays deleted through the sync. The sync still stops only for a text conflict or an upstream delete of a file we edited. The knip note says an unlisted removal still conflicts until its path is listed. --- docs/fork/what-we-changed.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/fork/what-we-changed.md b/docs/fork/what-we-changed.md index 8b898d53..7b1e3c27 100644 --- a/docs/fork/what-we-changed.md +++ b/docs/fork/what-we-changed.md @@ -722,12 +722,14 @@ expects undeclared static-filter keys to persist is skipped by name in the 16 upstream workflows we do not carry. `.fork-surface` cannot cover a deletion: it reads `--diff-filter=ACMR` against the merge base, where a file we removed is unchanged and therefore invisible, so a sync reinstates it in - silence. The check fails when a listed path exists. + silence. The check fails when a listed path exists. The sync keeps a listed + path deleted on its own (`scripts/resolve-fork-deleted.py`), so only a text + conflict, or an upstream delete of a file we edited, still stops it. - `scripts/ci/__tests__/**` in `knip.json` - upstream's ratchet test. Only upstream's `main.yml` ever ran it, and we do not carry that workflow, so knip is correct that nothing uses it. Ignored rather than deleted, on the same reasoning as the orphaned upstream pages: removing an upstream file buys a - delete/modify conflict on every sync. **`scripts/ci/ratchet.mjs` itself is + delete/modify conflict on every sync until its path is in `.fork-deleted`. **`scripts/ci/ratchet.mjs` itself is therefore not wired into our CI either** - run it by hand, or give it a home. - `scripts/ci/ratchet-baseline.json` - upstream's escape-hatch ratchet, our numbers. The baseline is the floor for `as any` and `eslint-disable` counts From d62df94818d3dc9028c351ef1f91a359231a7608 Mon Sep 17 00:00:00 2001 From: Derek Date: Tue, 17 Nov 2026 07:11:17 +1100 Subject: [PATCH 3/3] docs(fork): wrap the fork-deleted note to prettier width --- docs/fork/what-we-changed.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/fork/what-we-changed.md b/docs/fork/what-we-changed.md index 7b1e3c27..abe442ad 100644 --- a/docs/fork/what-we-changed.md +++ b/docs/fork/what-we-changed.md @@ -729,8 +729,9 @@ expects undeclared static-filter keys to persist is skipped by name in upstream's `main.yml` ever ran it, and we do not carry that workflow, so knip is correct that nothing uses it. Ignored rather than deleted, on the same reasoning as the orphaned upstream pages: removing an upstream file buys a - delete/modify conflict on every sync until its path is in `.fork-deleted`. **`scripts/ci/ratchet.mjs` itself is - therefore not wired into our CI either** - run it by hand, or give it a home. + delete/modify conflict on every sync until its path is in `.fork-deleted`. + **`scripts/ci/ratchet.mjs` itself is therefore not wired into our CI + either** - run it by hand, or give it a home. - `scripts/ci/ratchet-baseline.json` - upstream's escape-hatch ratchet, our numbers. The baseline is the floor for `as any` and `eslint-disable` counts per package, so every hatch we remove has to be locked in here or the ratchet