From 14a8637ed1de970d34a47340fa04f516ab1dd487 Mon Sep 17 00:00:00 2001 From: v0lumehi Date: Tue, 27 Oct 2026 12:02:48 +0100 Subject: [PATCH 1/2] Merge docs/reference and the symptom index with git's union driver About four in five new reference entries go at the end of their file (437 of 551 new ### entries in the last 400 commits to docs/reference), and docs/agents/symptoms.md is generated from them line for line. So two pull requests that each add an entry to the same file, rendering.md or shaders.md most often, conflict twice: in the reference file and in the index, both at the end of the same group. .gitattributes sets merge=union for docs/reference/*.md and docs/agents/symptoms.md. Git's built-in union driver keeps both sides' new lines; it needs no setup and applies to merges, rebases and cherry-picks. Checked in a clone: two branches that each append an entry to rendering.md and regenerate the index rebase onto each other without a conflict, the index is current afterwards (docs_symptoms.py --check), and without the attribute the same rebase conflicts in both files. Where both sides change the same line, the union driver keeps both versions without a conflict. test_docs now fails on an entry with a Symptom, Cause, Fix or Found on bullet more than once, or a heading twice in one file: checked with two branches that each add a game to the same Found on line, which merge into two Found on lines that the test reports. The seven entries that already have a bullet twice are listed as the baseline, and a count there may only go down. --- .gitattributes | 5 +++++ dm/tools/tests/test_docs.py | 38 ++++++++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index 8fa2fa7865..b96623cfc7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,8 @@ # The games gallery's table (dm/tools/docs_gallery.py writes README.md's gallery from it): a new game is a row at its end. README.md itself is not merged this way; after a # conflict in its gallery, run docs_gallery.py. test_docs fails on a game in the table twice docs/images/gallery.tsv merge=union +# Findings are added as new entries, mostly at the end of a reference file, and the symptom index is generated from them line for line. Two pull requests that each add an +# entry to the same file conflicted there and in docs/agents/symptoms.md every time. The union merge keeps both sides' new lines instead. Where both sides change the same +# line, it keeps both versions without a conflict: dm/tools/tests/test_docs.py fails on an entry with a bullet twice, and docs_symptoms.py --check on an index out of order. +docs/reference/*.md merge=union +docs/agents/symptoms.md merge=union diff --git a/dm/tools/tests/test_docs.py b/dm/tools/tests/test_docs.py index af73c08784..3f71719908 100644 --- a/dm/tools/tests/test_docs.py +++ b/dm/tools/tests/test_docs.py @@ -1,7 +1,7 @@ """The repository's own Markdown stays navigable: every relative link and #anchor resolves (GitHub's heading anchors), every game profile has its note (dm/games//README.md) and a row in the games table of dm/games/README.md, the symptom index (docs/agents/symptoms.md) is the one dm/tools/docs_symptoms.py writes from docs/reference, the patch index (docs/agents/game-patches.md) is the one dm/tools/docs_patches.py writes from the patch files, host tables and profiles, and the entry points stay short: AGENTS.md at most 1,500 words, each numbered guide in docs/agents at most 6,000 (a new finding goes into a game note or a -reference, not into these), and README's games gallery is the one dm/tools/docs_gallery.py writes from docs/images/gallery.tsv. Third-party trees are left out. No game files needed.""" +reference, not into these), README's games gallery is the one dm/tools/docs_gallery.py writes from docs/images/gallery.tsv, and no reference entry has a bullet twice (the guard of .gitattributes' union merge). Third-party trees are left out. No game files needed.""" from pathlib import Path import re import subprocess @@ -174,6 +174,42 @@ def test_union_merge_rejects_conflicting_game_rows(self): docs_gallery.readme() +# Entries that already have a bullet more than once (count of the bullet in that entry); .gitattributes merges docs/reference with the union driver, which keeps both versions +# of a line two branches changed, so an entry gaining a second bullet is what a bad union merge looks like. A count here may only go down +BULLETS_TWICE = { + ('frame-rate.md', "Present snapshots: the game's swapchain texture free at once", 'Fix'): 3, + ('rendering.md', 'Pulling vertex inputs', 'Cause'): 2, + ('rendering.md', 'Pulling vertex inputs', 'Fix'): 2, + ('rendering.md', 'Transform feedback captured by a vertex entry point', 'Found on'): 2, + ('services.md', 'oe calls answered without an applet proxy', 'Symptom'): 2, + ('services.md', 'oe calls answered without an applet proxy', 'Fix'): 2, + ('shaders.md', 'Passthrough geometry shaders drawn by the vertex shader', 'Fix'): 3, + ('shaders.md', 'Passthrough geometry shaders drawn by the vertex shader', 'Found on'): 3, + ('threads.md', 'A host lock held across a same-core handoff', 'Found on'): 2, +} + + +class UnionMerge(unittest.TestCase): + def test_reference_entries_have_each_bullet_once(self): + bad = [] + for path in sorted((REPO / 'docs/reference').glob('*.md')): + head, fence, heads, count = None, False, {}, {} + for line in path.read_text().split('\n'): + if line.startswith('```'): fence = not fence + if fence: continue + m = re.match(r'### (.*?)\s*$', line) + if m: + head = m.group(1); heads[head] = heads.get(head, 0) + 1 + if heads[head] > 1: bad.append(f'{path.name}: heading "{head}" twice') + continue + if re.match(r'#{1,2} ', line): head = None + m = re.match(r'- \*\*(Symptom|Cause|Fix|Found on):\*\*', line) + if m and head: + k = (path.name, head, m.group(1)); count[k] = count.get(k, 0) + 1 + bad += [f'{f}: "{h}": {b} {n} times' for (f, h, b), n in count.items() if n > BULLETS_TWICE.get((f, h, b), 1)] + self.assertEqual(bad, [], 'a reference entry has a bullet or heading more than once (a union merge kept both versions of a line? merge them by hand):\n' + '\n'.join(bad)) + + class Length(unittest.TestCase): def test_entry_points_stay_short(self): words = lambda p: len(p.read_text().split()) From bf67417dc4b1b288a06963e6cd4764cce29ff551 Mon Sep 17 00:00:00 2001 From: v0lumehi Date: Tue, 27 Oct 2026 21:42:01 +0100 Subject: [PATCH 2/2] Regenerate the symptom index in CI on main A workflow on pushes to main runs docs_symptoms.py and commits the index when it changed. A stale index no longer fails test_docs unless DM_DOCS_INDEX_STRICT=1, which the workflow sets; the links of the index to entries with a Symptom bullet are still always checked. --- .github/workflows/docs-index.yml | 59 ++++++++++++++++++++++++++++++++ AGENTS.md | 2 +- dm/tools/docs_symptoms.py | 7 ++-- dm/tools/tests/test_docs.py | 12 ++++++- docs/agents/06-dev-loop.md | 2 +- docs/agents/symptoms.md | 2 +- docs/env_switches.tsv | 1 + 7 files changed, 78 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/docs-index.yml diff --git a/.github/workflows/docs-index.yml b/.github/workflows/docs-index.yml new file mode 100644 index 0000000000..3aef8e6dac --- /dev/null +++ b/.github/workflows/docs-index.yml @@ -0,0 +1,59 @@ +# Regenerates docs/agents/symptoms.md, the symptom index, on main: dm/tools/docs_symptoms.py writes it from the Symptom bullets in docs/reference, and when the file changes the +# job commits it as github-actions[bot]. Pull requests need not touch the index (a stale one does not fail test_docs, see DM_DOCS_INDEX_STRICT there), so two pull requests that +# each add an entry no longer conflict in it. +# It runs only on pushes to main and by hand, never on a pull request, so no pull request's code runs with write access to the repository. +# Loop guard: the bot's commit changes symptoms.md, which is one of the paths below, but a push made with GITHUB_TOKEN does not start workflows, so it does not trigger itself. +# If a branch protection rule on main forbids pushes by the bot (required pull requests or status checks without a bypass for github-actions), the push step fails and the job +# turns red: that is the visible signal; allow the bot to bypass the rule, or run docs_symptoms.py by hand and commit its output. +name: docs-index + +on: + push: + branches: [main] + paths: [docs/reference/**, dm/tools/docs_symptoms.py, docs/agents/symptoms.md] + workflow_dispatch: + +permissions: + contents: write + +# Two pushes in a row queue instead of racing; the later run starts from a main that has the earlier one's index commit. +concurrency: + group: docs-index + cancel-in-progress: false + +defaults: + run: + shell: bash + +# Actions are pinned to commit SHAs (the tag in the comment), as in tests.yml. +jobs: + regenerate: + name: regenerate the symptom index + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.14' + - name: Write the index + run: python3 dm/tools/docs_symptoms.py + - name: Check the references and the new index + env: + DM_DOCS_INDEX_STRICT: '1' + run: python3 -m unittest dm/tools/tests/test_docs.py + - name: Commit and push the index if it changed + run: | + if git diff --quiet -- docs/agents/symptoms.md; then + echo "The symptom index is current." + exit 0 + fi + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add docs/agents/symptoms.md + git commit -m 'Regenerate the symptom index' + # A push that landed meanwhile makes this one non-fast-forward: rebase onto it and try once more + git pull --rebase origin main && git push origin HEAD:main \ + || { git pull --rebase origin main && git push origin HEAD:main; } diff --git a/AGENTS.md b/AGENTS.md index 309a0f20d2..6102b77847 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,7 @@ | Another Switch title | the NSP script (below), then [docs/agents](docs/agents/README.md) | | Packing, building, prewarm lists, power | [docs/development.md](docs/development.md) | | Checking a change before a pull request | [06-dev-loop.md](docs/agents/06-dev-loop.md) | -- Where a new finding goes, once: a fact about one game (its passes, shader keys, measurements, why its profile needs a key or a cut, its status) in that game's note, in the section it belongs to and marked current or resolved; a general mechanism (a host or tool change that holds for other games) as an entry in its `docs/reference/.md` section (Symptom, Cause, Fix, Found on with the game's note linked), then `python3 dm/tools/docs_symptoms.py` to update the symptom index, and a line under the game note's "Found on this game". Link to a finding instead of repeating it. Adding a game takes its note and its row in the games table. Once Mac gameplay testing passes, add it to README's regular games gallery with a screenshot (`docs/images/gallery.tsv`, then `dm/tools/docs_gallery.py`); Mac testing is sufficient for inclusion. Add caption notes only for noteworthy features or limitations (such as a 60 fps upgrade or controller requirement); otherwise show only the name. Keep testing milestones, platform and remaining validation in the game note. This file changes only when a rule does; it states no game count (the games table is the count). `dm/tools/tests/test_docs.py` checks the links and anchors, the notes and their table rows, the symptom index, and the length of this file (1,500 words) and of each guide in `docs/agents` (6,000). +- Where a new finding goes, once: a fact about one game (its passes, shader keys, measurements, why its profile needs a key or a cut, its status) in that game's note, in the section it belongs to and marked current or resolved; a general mechanism (a host or tool change that holds for other games) as an entry in its `docs/reference/.md` section (Symptom, Cause, Fix, Found on with the game's note linked; CI adds it to the symptom index), and a line under the game note's "Found on this game". Link to a finding instead of repeating it. Adding a game takes its note and its row in the games table. Once Mac gameplay testing passes, add it to README's regular games gallery with a screenshot (`docs/images/gallery.tsv`, then `dm/tools/docs_gallery.py`); Mac testing is sufficient for inclusion. Add caption notes only for noteworthy features or limitations (such as a 60 fps upgrade or controller requirement); otherwise show only the name. Keep testing milestones, platform and remaining validation in the game note. This file changes only when a rule does; it states no game count (the games table is the count). `dm/tools/tests/test_docs.py` checks the links and anchors, the notes and their table rows, the symptom index, and the length of this file (1,500 words) and of each guide in `docs/agents` (6,000). - Each game's constants and shader tables form a game profile in `dm/games//`, where `` is the base game's title ID: `game.tsv` plus the tables (and `patches.py` when the rewriter has patches for its builds). Mario's profile is `010015100B514000`. Tools pick the profile with `DM_GAME`; `make -C dm/host` and `build_ipa.sh` pick it with `GAME`. There is no default game: set it every time (the tools stop and list the profiles otherwise). Put per-game values in the profile, not in code ([dm/games/README.md](dm/games/README.md)). - Prefer general optimizations and fixes to game-specific rules. Performance work (frame pacing, power, caches, load times) counts here as much as correctness. First look for a change that holds for every game: Horizon / NVN / Vulkan semantics, or what the binary, its SDK version, its imports or its files show. Use a game-specific rule (a profile key, a rewriter cut, a build-ID gate, a per-game branch in the host) only when it is absolutely necessary, that is, when no general rule can do it. - Find a bug's root cause and fix it there; do not just work around it. Before changing code, find why it happens: the faulting instruction and its decompilation, the value that went wrong and who wrote it, the Horizon / NVN / Vulkan / Metal semantics the host or the translator gets wrong. Then fix that place. A change that only hides the symptom (a stub, a skipped call, a clamp, a retry, a sleep or a longer timeout, a cache flush, a profile key that turns a feature off) is a workaround: use one only when the cause is known and lies where it cannot be fixed (the game's own code, which a rewriter cut changes deliberately, or a platform limit), or when the user asks for a stopgap. Then say so in the code comment and in the game note or reference entry: the cause, or what is still unknown, marked open. diff --git a/dm/tools/docs_symptoms.py b/dm/tools/docs_symptoms.py index 07a6ce0f76..6ecaeb5475 100644 --- a/dm/tools/docs_symptoms.py +++ b/dm/tools/docs_symptoms.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 # Writes docs/agents/symptoms.md, the symptom index, from the subsystem references in docs/reference: one line per entry that has a "- **Symptom:**" bullet (the text of that -# bullet, links reduced to their text) with a link to the entry's heading, grouped by reference file in the order docs/reference/README.md lists them. Run it after adding or -# changing an entry; --check only reports whether the index is current (dm/tools/tests/test_docs.py runs it). +# bullet, links reduced to their text) with a link to the entry's heading, grouped by reference file in the order docs/reference/README.md lists them. Running it +# after adding or changing an entry is optional: .github/workflows/docs-index.yml regenerates the index on main, and test_docs checks it only with DM_DOCS_INDEX_STRICT=1. +# Run it to see your entry in the index; --check only reports whether the index is current. # Usage: docs_symptoms.py [--check] import re, sys, pathlib, unicodedata @@ -39,7 +40,7 @@ def index(): lines = ['# Symptom index', '', 'One line per symptom in the [subsystem references](../reference/): what you see (an assertion, a `[RED]` line, an abort or crash site, a visible result) and the ' 'entry that explains it. Search this page for the text in your log first (`grep -i` the function name, the `[RED]` text or the address form), then read the entry and ' - 'the game notes it names. Generated by `dm/tools/docs_symptoms.py` from the entries\' Symptom bullets: change the entry, then run it.', ''] + 'the game notes it names. Generated by `dm/tools/docs_symptoms.py` from the entries\' Symptom bullets: change the entry; CI regenerates this page on `main`.', ''] for p in ref_files(): e = entries(p) if not e: continue diff --git a/dm/tools/tests/test_docs.py b/dm/tools/tests/test_docs.py index 3f71719908..cda03845e6 100644 --- a/dm/tools/tests/test_docs.py +++ b/dm/tools/tests/test_docs.py @@ -1,8 +1,9 @@ """The repository's own Markdown stays navigable: every relative link and #anchor resolves (GitHub's heading anchors), every game profile has its note (dm/games//README.md) and a row in the games table of dm/games/README.md, the symptom index (docs/agents/symptoms.md) is the one dm/tools/docs_symptoms.py writes from -docs/reference, the patch index (docs/agents/game-patches.md) is the one dm/tools/docs_patches.py writes from the patch files, host tables and profiles, and the entry points stay short: AGENTS.md at most 1,500 words, each numbered guide in docs/agents at most 6,000 (a new finding goes into a game note or a +docs/reference (checked only with DM_DOCS_INDEX_STRICT=1; CI regenerates it on main), the patch index (docs/agents/game-patches.md) is the one dm/tools/docs_patches.py writes from the patch files, host tables and profiles, and the entry points stay short: AGENTS.md at most 1,500 words, each numbered guide in docs/agents at most 6,000 (a new finding goes into a game note or a reference, not into these), README's games gallery is the one dm/tools/docs_gallery.py writes from docs/images/gallery.tsv, and no reference entry has a bullet twice (the guard of .gitattributes' union merge). Third-party trees are left out. No game files needed.""" from pathlib import Path +import os import re import subprocess import sys @@ -84,6 +85,15 @@ def test_every_profile_has_a_note_and_a_row(self): class SymptomIndex(unittest.TestCase): + # CI regenerates the index on main (.github/workflows/docs-index.yml), so a pull request may leave it stale, which also keeps two pull requests from conflicting in it. + # That workflow sets DM_DOCS_INDEX_STRICT=1 to check the index it has just written; run with it locally to see whether yours is current. Every other check of the index + # (its links reach real entries with a Symptom bullet) is the test above and always runs + def test_index_lines_point_to_entries_with_a_symptom(self): + known = {(p.name, a) for p in docs_symptoms.ref_files() for _, a, _ in docs_symptoms.entries(p)} + bad = [m.group(0) for m in re.finditer(r'\.\./reference/([a-z-]+\.md)#(\S+?)\)', docs_symptoms.OUT.read_text()) if (m.group(1), m.group(2)) not in known] + self.assertEqual(bad, [], 'docs/agents/symptoms.md links to an entry that is gone or has no Symptom bullet (CI regenerates the index on main; DM_DOCS_INDEX_STRICT=1 checks all of it)') + + @unittest.skipUnless(os.environ.get('DM_DOCS_INDEX_STRICT') == '1', 'a stale index is regenerated on main; DM_DOCS_INDEX_STRICT=1 checks it') def test_index_is_current(self): self.assertEqual(docs_symptoms.OUT.read_text(), docs_symptoms.index(), 'docs/agents/symptoms.md is out of date: run dm/tools/docs_symptoms.py') diff --git a/docs/agents/06-dev-loop.md b/docs/agents/06-dev-loop.md index 28bd7bca3a..2a833636d4 100644 --- a/docs/agents/06-dev-loop.md +++ b/docs/agents/06-dev-loop.md @@ -64,7 +64,7 @@ Reading the report (`report.md` beside `report.json` in `/reports/`): one By what the change touches (the largest that applies): -- **Docs only:** `python3 -m unittest dm/tools/tests/test_docs.py`; after a change to `docs/reference`, `python3 dm/tools/docs_symptoms.py`; after editing `docs/images/gallery.tsv`, `python3 dm/tools/docs_gallery.py` (also resolves conflicts inside README's generated gallery). +- **Docs only:** `python3 -m unittest dm/tools/tests/test_docs.py`; after a change to `docs/reference`, `python3 dm/tools/docs_symptoms.py` is optional (CI regenerates the symptom index on `main` and a stale one does not fail a pull request; run it to see your entry in the index, `DM_DOCS_INDEX_STRICT=1` makes the test check it); after editing `docs/images/gallery.tsv`, `python3 dm/tools/docs_gallery.py` (also resolves conflicts inside README's generated gallery). - **A Python tool or script** (`scripts/`, the tools under `dm/`): the ratchet `lint-python.yml` runs, ruff and pyright with no new finding: `/bin/python3 dm/tools/lint_python.py`, from a venv with `.github/requirements/lint.txt` installed, kept on the drive of the NSP folders, not in the home folder ([Continuous integration](../development.md#continuous-integration)). The system Python has neither tool, and the check stops at a `[RED]` line there: that is not a pass. A generated page (`docs_*.py`) is compared in CI on a clean checkout, so it must not read untracked or git-ignored files such as `dm/host/gen`. - **One game's profile or note** (`dm/games//`): the Python tool tests; a full shader build of that game after a change to its shader inputs; the game on the Mac (or the iPhone) along its route, with `scripts/bench.py` numbers when the change is about speed; the finding in the game's note ([AGENTS.md](../../AGENTS.md)). - **The host, the chain or the tools** (`dm/host`, `dm/gfx`, `dm/tools`, `dm/gfx/tools`, `dm/ios`, `scripts`): everything above, `make -C dm/gfx/tests GAME= run`, `dmsh_chain.py --header` after a chain change, `python3 scripts/regress.py impact main..HEAD` to see which games it reaches, then `python3 scripts/regress.py check --games ` (`--quick` while iterating, the full run before the pull request), and its `report.md` pasted into the pull request with a word on every `CHANGED`, `NEW-FAIL` or `REGRESSION`. Choose the games, not every local one: the five the change is most likely to reach or break (those `impact` names, then games that cover the APIs, engines and paths it touches). Take more than five only when the change truly needs them (a broad chain or host change no five games cover), and fewer when its reach is narrow and clear (a fix on one game's path, a tool only some games use). Name the games the report covers and why you chose them: the others are unchecked. diff --git a/docs/agents/symptoms.md b/docs/agents/symptoms.md index 40339db328..fed4ca38a5 100644 --- a/docs/agents/symptoms.md +++ b/docs/agents/symptoms.md @@ -1,6 +1,6 @@ # Symptom index -One line per symptom in the [subsystem references](../reference/): what you see (an assertion, a `[RED]` line, an abort or crash site, a visible result) and the entry that explains it. Search this page for the text in your log first (`grep -i` the function name, the `[RED]` text or the address form), then read the entry and the game notes it names. Generated by `dm/tools/docs_symptoms.py` from the entries' Symptom bullets: change the entry, then run it. +One line per symptom in the [subsystem references](../reference/): what you see (an assertion, a `[RED]` line, an abort or crash site, a visible result) and the entry that explains it. Search this page for the text in your log first (`grep -i` the function name, the `[RED]` text or the address form), then read the entry and the game notes it names. Generated by `dm/tools/docs_symptoms.py` from the entries' Symptom bullets: change the entry; CI regenerates this page on `main`. ## Packing diff --git a/docs/env_switches.tsv b/docs/env_switches.tsv index fe5d90d3fb..7ca56cc18f 100644 --- a/docs/env_switches.tsv +++ b/docs/env_switches.tsv @@ -20,6 +20,7 @@ DM_DBG_AU_FRAMES diagnostic `=` asks the Mac's Core Audio output for tha DM_DETCLOCK switch `=`: from the first present on, the guest clock advances a fixed step per present instead of following the wall clock, and the random source becomes a fixed-seed stream (replay and test runs) DM_DETCLOCK_PACE switch `=1` with `DM_DETCLOCK`: each frame takes at least one clock step of wall time, so background loading keeps up with the fixed clock (used by the screenshot gate) DM_DISPLAY_NAME switch Overrides the iOS app's name in `build_ipa.sh` (default the game name in the base NSP's control data) +DM_DOCS_INDEX_STRICT test `=1`: test_docs also requires docs/agents/symptoms.md to equal a freshly generated index (set by .github/workflows/docs-index.yml on main) DM_DYNAMIC_RESOLUTION player `1` lets NieR:Automata 1.0.3's own dynamic resolution scale between 70 and 100 %, `0` (default) keeps full resolution; only on the builds whose patches.py names the DYNAMIC_RESOLUTION capability (`hle_fpslock.c` `k_nier_dr`). The launcher's Dynamic resolution DM_ENTITLEMENTS path `=` gives a prepared Xcode project another entitlements file instead of the Increased Memory Limit one, and the ad-hoc signature of the unsigned IPA's executable carries them; empty means no entitlements DM_ENV_FILE path `=` makes `build_ipa.sh` bake its `KEY=VALUE` lines into the app as `dm/env.txt`, applied at boot; `Documents/env.txt` overrides them