From 2e746847c0c46379e23ccf01f63c7d5dca13a070 Mon Sep 17 00:00:00 2001 From: Oliver Ames Date: Mon, 9 Nov 2026 11:23:18 -0500 Subject: [PATCH 1/2] A pipeline a hybrid draw stands in for is listed in pso_user.tsv when first asked for, so the next start's prepare step builds it (from HookLink) pso_user.tsv took a pipeline's row only once its build finished. While a hybrid draw drew with its shader pair's pulling pipeline, the pipeline it wanted was built in the background; on an iPhone 18 Pro Max (HookLink's measurement) that took 5-50 s with the game running, against about 1 s in the prepare step. Castle Town asked for about 400 pipelines and finished about 60 a minute, so what was unbuilt when the run ended was never listed and the next start did not build it. demand_record writes the row when the hybrid draw first asks for the pipeline, once per key. The later row from the built pipeline is the same line and pso_user_record drops it. DM_PSO_DEMAND_RECORD=0 keeps the old order. Added t_pso_demand_record, the reference entry, the switch row and the game note line. Ported from HookLink's commit 0ad4e1a9 (https://github.com/dsi-hub/nxnative/tree/hooklink, project https://github.com/dsi-hub/HookLink), without its archive-miss memory (b3471d51) and keyed demand table (b15d36f7): main has an archive-miss memory of its own (g_pbase_miss, PR #1825) and no keyed demand table, so the call goes next to main's pso_pend_demand and runs for every stand-in draw, not only when no build is pending. Co-authored-by: dsi-hub <260802842+dsi-hub@users.noreply.github.com> --- dm/games/0400486024DEC000/README.md | 1 + dm/gfx/nvn/nvn_mtl.m | 13 +++++++++++ dm/gfx/tests/Makefile | 6 +++++ dm/gfx/tests/t_pso_demand_record.m | 35 +++++++++++++++++++++++++++++ docs/agents/symptoms.md | 1 + docs/env_switches.tsv | 1 + docs/reference/shaders.md | 8 +++++++ 7 files changed, 65 insertions(+) create mode 100644 dm/gfx/tests/t_pso_demand_record.m diff --git a/dm/games/0400486024DEC000/README.md b/dm/games/0400486024DEC000/README.md index f14f410248..1d1ce22525 100644 --- a/dm/games/0400486024DEC000/README.md +++ b/dm/games/0400486024DEC000/README.md @@ -117,6 +117,7 @@ From an empty save: at about 45 s `A` (title), `A` (New Game), then the movie. F - [Native mesh pipelines compiled at their first draw](../../../docs/reference/shaders.md#native-mesh-pipelines-compiled-at-their-first-draw): resolved. The foliage, shadows and enemies drawn with task and mesh shaders stalled at their first draw on a cold Metal cache (59.5 s of builds at the Kokiri Forest entry on an M5 Max, one frame of 42.9 s), as nothing recorded or kept their pipelines; `pso_manifest.tsv` now carries 164 `M` rows (Kokiri Forest, Hyrule Field, Castle Town, Hyrule Castle, and 41 from later play, Death Mountain Trail among it), and with the prepare step they are built before the first frame. Pipelines outside the list still stall once, then go into `pso_user.tsv`. With the first 123 alone, a cold first start (empty Metal cache and pipeline archive, M4 Pro) on a Death Mountain Trail save built 9 and 10 at their first draw in two runs, 2.7 and 4.2 s, frames of 1,775 and 1,657 ms; with all 164, none, worst frame 25.6 and 18.5 ms, the prepare step 29.6 and 29.3 s → 32.1 and 31.0 s. - [Early fragment tests from the control section](../../../docs/reference/shaders.md#early-fragment-tests-from-the-control-section): 746 of its 22,352 pixel shaders set the NVN2 flag (control +0x5b), none of them writes depth; before, no NVN2 shader got the attribute. - [Archive searches repeated while the build table is full](../../../docs/reference/frame-rate.md#archive-searches-repeated-while-the-build-table-is-full): the castle town market at about 12 fps while the GPU had time to spare. +- [A pipeline a hybrid draw stood in for listed only once built](../../../docs/reference/shaders.md#a-pipeline-a-hybrid-draw-stood-in-for-listed-only-once-built): on an iPhone 18 Pro Max, found by HookLink, the pipelines Castle Town asked for last were not in `pso_user.tsv` for the next start; they are listed when first asked for (`DM_PSO_DEMAND_RECORD=0` restores the old order). - [One Pro Controller read as two players](../../../docs/reference/services.md#one-pro-controller-read-as-two-players): resolved, no music in Kokiri Forest gameplay and its cutscenes; the controller applet opened over and over and the game kept its music suspended while it had focus. - [Compute bindings prepare texture copies before the dispatch encoder](../../../docs/reference/rendering.md#compute-bindings-prepare-texture-copies-before-the-dispatch-encoder): resolved, `DM_NVN_DISPATCH_OWN=1` crashed in the Metal driver (`AGX::ComputeContext::prepareForEnqueue`) during the Kokiri Forest load ([issue #1743](https://github.com/seedds/nxnative/issues/1743)); on an M2 Pro 5 of 5 runs crashed before the fix and 3 of 3 loaded the forest after it. - [A fixed frame step plays in slow motion on a slow Mac](../../../docs/reference/frame-rate.md#a-fixed-frame-step-plays-in-slow-motion-on-a-slow-mac): the frame-ratio division. diff --git a/dm/gfx/nvn/nvn_mtl.m b/dm/gfx/nvn/nvn_mtl.m index 95a760d6f2..f0a5701445 100644 --- a/dm/gfx/nvn/nvn_mtl.m +++ b/dm/gfx/nvn/nvn_mtl.m @@ -8826,6 +8826,18 @@ static int pso_pend_kick(const pso_key *k, NvShader *vs, NvShader *fs) { // No pso_pend_start(slot, gen, pso_desc(k, vs, fs, nil, 0), dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0)); return 1; } +// A pipeline a hybrid draw had to stand in for goes into pso_user.tsv when first asked for, not only once built. During play its build took 5-50 s on an iPhone 18 Pro Max +// (2 performance cores shared with the game), about 1 s in the prepare step before the first frame; Castle Town asked for ~400 and finished ~60 a minute, and what had not +// finished by the end of the run was never listed. Listed at once, the next start's prepare step builds every one of them. The later line from the built pipeline is a +// duplicate, dropped by pso_user_record. DM_PSO_DEMAND_RECORD=0 records only built pipelines, as before (submit thread) +static nvn_hmap g_demand_rec = {.klen = sizeof(pso_key)}; +static void demand_record(const pso_key *k, NvShader *vs, NvShader *fs) { + static int on = -1; if (on < 0) { const char *ev = getenv("DM_PSO_DEMAND_RECORD"); on = !(ev && !strcmp(ev, "0")); } + if (!on || nvn_hm_get(&g_demand_rec, k) || g_demand_rec.n >= (1u << 16)) return; + nvn_hm_put(&g_demand_rec, k, (void *)1); + pso_skey sk; char shex[sizeof sk * 2 + 1]; pso_skey_from(k, vs, fs, 0, NULL, &sk); pso_skey_hex(&sk, shex); + pso_user_record("R\t%s\n", shex); +} // The build a hybrid draw asked for while it draws with its stand-in: outside the speculative limit, on pp_bg_queue static int pso_pend_demand(const pso_key *k, NvShader *vs, NvShader *fs) { if (hm_get_id(&g_pso, k)) return 0; @@ -9224,6 +9236,7 @@ static void pso_prebuild_bg(void) { // after the prewarm, in the app without a __unsafe_unretained id h = hybrid_get(&k, vs, fs); if (h) { if (st == 0) pso_pend_demand(&k, vs, fs); + demand_record(&k, vs, fs); g_pso_pull = 1; g_hy_ns += bp_now() - bt; if (g_pp_submit_mark != g_sub_serial) { g_pp_submit_mark = g_sub_serial; uint64_t t0 = bp_now(); pso_predict_rest(); g_bp_pso_ns += bp_now() - t0; } return h; diff --git a/dm/gfx/tests/Makefile b/dm/gfx/tests/Makefile index cb7d39d847..4541408e6e 100644 --- a/dm/gfx/tests/Makefile +++ b/dm/gfx/tests/Makefile @@ -344,6 +344,12 @@ t_rt_depth_ro: t_rt_depth_ro.m $(MTL_T_DEP) clang $(CFLAGS) -fobjc-arc -o $@ t_rt_depth_ro.m $(MTL_T_SRC) $(FW) t_pso_pending: t_pso_pending.m $(MTL_T_DEP) clang $(CFLAGS) -fobjc-arc -o $@ t_pso_pending.m $(MTL_T_SRC) $(FW) +t_pso_demand_record: t_pso_demand_record.m $(MTL_T_DEP) + clang $(CFLAGS) -fobjc-arc -o $@ t_pso_demand_record.m $(MTL_T_SRC) $(FW) +.PHONY: check_t_pso_demand_record +check_t_pso_demand_record: t_pso_demand_record + ./t_pso_demand_record + ./t_pso_demand_record --off t_shmap_warm: t_shmap_warm.m $(MTL_T_DEP) clang $(CFLAGS) -fobjc-arc -o $@ t_shmap_warm.m $(MTL_T_SRC) $(FW) t_pso_depth_stencil: t_pso_depth_stencil.m $(MTL_T_DEP) diff --git a/dm/gfx/tests/t_pso_demand_record.m b/dm/gfx/tests/t_pso_demand_record.m new file mode 100644 index 0000000000..837fe4363f --- /dev/null +++ b/dm/gfx/tests/t_pso_demand_record.m @@ -0,0 +1,35 @@ +// A pipeline a hybrid draw stands in for is listed in pso_user.tsv when first asked for (demand_record), once per key; DM_PSO_DEMAND_RECORD=0 lists none of them. +// Run twice: with no argument and with --off (the Makefile's check target). The hybrid call site in the draw path is not run here: it needs translated shader libraries. +#include "../nvn/nvn_mtl.m" + +void *front_metal_layer(void) { return NULL; } +module_t *module_of(const void *p) { (void)p; return NULL; } +module_t *g_mod; int g_nmod; +void svc_thread_dump(void) {} +uint64_t now_ns(void) { return bp_now(); } + +static int g_failed; +#define CHECK(name, cond) do { int ok_ = !!(cond); printf("%s %s\n", name, ok_ ? "OK" : "FAIL"); g_failed += !ok_; } while (0) + +static int rows(const char *path) { + NSString *t = [NSString stringWithContentsOfFile:@(path) encoding:NSUTF8StringEncoding error:nil]; int n = 0; + for (NSString *l in [t componentsSeparatedByString:@"\n"]) n += [l hasPrefix:@"R\t"]; + return n; +} + +int main(int argc, char **argv) { + int off = argc > 1 && !strcmp(argv[1], "--off"); + char path[] = "/tmp/t_pso_demand_record.XXXXXX"; int fd = mkstemp(path); if (fd < 0) return 2; close(fd); + setenv("DM_PSO_USER", path, 1); unsetenv("DM_SAVE"); + if (off) setenv("DM_PSO_DEMAND_RECORD", "0", 1); else unsetenv("DM_PSO_DEMAND_RECORD"); + NvShader *vs = [NvShader new], *fs = [NvShader new]; + snprintf(vs->key, sizeof vs->key, "demand-vs"); snprintf(fs->key, sizeof fs->key, "demand-fs"); + pso_key a, b; memset(&a, 0, sizeof a); memset(&b, 0, sizeof b); a.samples = 1; b.samples = 4; + demand_record(&a, vs, fs); demand_record(&a, vs, fs); + CHECK(off ? "off: nothing listed" : "listed once for one key", rows(path) == (off ? 0 : 1)); + demand_record(&b, vs, fs); + CHECK(off ? "off: still nothing" : "a second key has its own row", rows(path) == (off ? 0 : 2)); + unlink(path); + printf("%s\n", g_failed ? "FAIL" : "PASS"); + return g_failed != 0; +} diff --git a/docs/agents/symptoms.md b/docs/agents/symptoms.md index 9db465e24f..a3467452e0 100644 --- a/docs/agents/symptoms.md +++ b/docs/agents/symptoms.md @@ -1066,6 +1066,7 @@ One line per symptom in the [subsystem references](../reference/): what you see - the first sight of a scene drawn with task and mesh shaders freezes for seconds on a cold Metal cache, on every start once the system's shader cache has dropped the pipelines, although the prewarm finished: `task / mesh pipeline: task … mesh … fragment …, N ms required at frame F`. Before ea0c98df the draw was left out instead, and the foliage stayed missing or white until the pipeline was ready. → [Native mesh pipelines compiled at their first draw](../reference/shaders.md#native-mesh-pipelines-compiled-at-their-first-draw) - a frame of 200 ms or more the first time an effect appears, although the bundled pipeline archive was trusted and the prewarm finished. With `DM_PSO_WAITLOG=1` the frame shows waits on render pipelines with no pulling entry, 34-74 ms each, for vertex shaders that have no stage inputs. → [Vertex shaders with layer entry points left out of the prebuild](../reference/shaders.md#vertex-shaders-with-layer-entry-points-left-out-of-the-prebuild) - `[RED] pipeline prewarm: seed list does not exist: …/pso_manifest.tsv` at every start, no `pipeline prewarm list …/pso_user.tsv` line, and the same compile stalls every session in places already visited, while `pso_user.tsv` keeps growing; the pipeline archive gains files every start and is never compacted (Donkey Kong Bananza: 88 files, 7 s to open them). → [The device's prewarm list unread without a seed list](../reference/shaders.md#the-devices-prewarm-list-unread-without-a-seed-list) +- on an iPhone the next start still waits on, or draws with stand-ins for, the pipelines the last session asked for last. Ocarina of Time on an iPhone 18 Pro Max asked for about 400 pipelines in Castle Town and finished about 60 a minute; whatever was still unbuilt when the run ended was not in `pso_user.tsv`, so the next start's prepare step did not build it. HookLink's measurement; we have not reproduced it. → [A pipeline a hybrid draw stood in for listed only once built](../reference/shaders.md#a-pipeline-a-hybrid-draw-stood-in-for-listed-only-once-built) - the first frame that draws with new programs loads their shader libraries on the submit thread (`[longframe] ... shader libraries N, X ms`): tens of milliseconds with the files cached, seconds when the drive is slow. → [Shader libraries loaded when the program is made](../reference/shaders.md#shader-libraries-loaded-when-the-program-is-made) - at boot, `CRASH sig=5 … pc=_dispatch_once_wait.cold.1` right after the `pipeline bases so far` line, with `bt: pso_prebuild __shmap_load_block_invoke … mtl_prog_prefetch h_nvnProgramSetShaders`. → [The prepare step inside the shader map's once](../reference/shaders.md#the-prepare-step-inside-the-shader-maps-once) - after an IPA packed without `_bp.metallib` ran once with `DM_PSO_MTL4=1`, an app with those libraries still stalls on new pipelines: `pipelines up to frame` lines show `draws with a hybrid pipeline 0` and `draws that waited: pulling base not made yet`, and the log has no `pipeline prebuild:` line. The run before it logged `[RED] pulling library of … not usable` and `pipeline prebuild done: … 19838 left to the draws`. → [A prebuild stamp written without the pulling libraries](../reference/shaders.md#a-prebuild-stamp-written-without-the-pulling-libraries) diff --git a/docs/env_switches.tsv b/docs/env_switches.tsv index e47883db1a..24e0ca209f 100644 --- a/docs/env_switches.tsv +++ b/docs/env_switches.tsv @@ -278,6 +278,7 @@ DM_PROGRAMS path Table of a game's programs on the Mac (`programs.tsv`: index, h DM_PROGRAM_ID switch `=`: the program ID the host reports from `svcGetInfo(ProgramId)` (default: the profile's `program_id`) DM_PSO_ARCHIVE path The folder of the per-game Metal 4 pipeline archive of pipeline bases (default `pso_archive/` beside `pso_user.tsv`); `off` keeps no archive DM_PSO_BUNDLE path The pipeline archive made offline (`.mtl4a`, `prebuilt.tsv`), read only; the iOS app and Mac launcher set it to their `pso/` folder; `off` ignores it +DM_PSO_DEMAND_RECORD switch `0`: a pipeline a hybrid draw stands in for goes into `pso_user.tsv` only once built, not when first asked for (default: at once, so the next start's prepare step builds it) DM_PSO_GPUS switch `build_ipa.sh`: the GPU architectures, comma-separated, to build the offline pipeline archive for (`none` builds none); unset, a profile with `pso_mtl4 on` gets its `pso_gpu` list or `g18p` when absent; profiles with `pso_mtl4 off` get none DM_PSO_HYBRID switch `0`: a draw whose pipeline is not built yet waits for it instead of drawing with its shader pair's pulling pipeline meanwhile (on by default with the Metal 4 path) DM_PSO_IOS_MIN switch `build_ipa.sh`: the minimum iOS version of the offline pipeline archives (default the SDK's major version); a device of another major version cannot open them and builds the bases itself diff --git a/docs/reference/shaders.md b/docs/reference/shaders.md index e46607025d..f287b2d590 100644 --- a/docs/reference/shaders.md +++ b/docs/reference/shaders.md @@ -2046,6 +2046,14 @@ Finding, keying and translating shaders (`dm/gfx/tools/shader_split.py`, `dmsh_b - **Fix:** a missing seed list is logged and left out; the device list is still warmed, and the prewarm returns only when neither list exists. Host only. - **Found on:** [Donkey Kong Bananza](../../dm/games/040074A01BF12000/README.md): before, a 13,200-frame session had 154 frames over 100 ms (122 mostly compiling, the worst 7.1 s), 87 s of compiling on the submit thread and 95 draws with a hybrid pipeline; after, the start warmed the 2,484-line device list in 7.6 s (1,002 bases from the archive), compacted 98 archive files into one and ran the prebuild once (20,647 pairs, 116 s before the first frame), and a 17,700-frame session had 68 such frames (40 compiling, the worst 1.9 s), 12 s of compiling and 5,320 hybrid draws. The game has a seed list now. +### A pipeline a hybrid draw stood in for listed only once built + +- **Symptom:** on an iPhone the next start still waits on, or draws with stand-ins for, the pipelines the last session asked for last. Ocarina of Time on an iPhone 18 Pro Max asked for about 400 pipelines in Castle Town and finished about 60 a minute; whatever was still unbuilt when the run ended was not in `pso_user.tsv`, so the next start's prepare step did not build it. HookLink's measurement; we have not reproduced it. +- **Cause:** `pso_user.tsv` took a pipeline's row from the submit thread only once its build finished. While a hybrid draw drew with its shader pair's pulling pipeline, the pipeline it wanted was queued in the background (`pso_pend_demand`), where a build took 5-50 s with the game running on two performance cores, against about 1 s in the prepare step before the first frame. +- **Fix:** `demand_record` (`dm/gfx/nvn/nvn_mtl.m`) writes the row when the hybrid draw first asks for the pipeline, once per key. The later row from the built pipeline is the same line, which `pso_user_record` drops. `DM_PSO_DEMAND_RECORD=0` records a pipeline only once built, as before. +- **Found on:** [The Legend of Zelda: Ocarina of Time](../../dm/games/0400486024DEC000/README.md#found-on-this-game), found by [HookLink](https://github.com/dsi-hub/HookLink) (engine branch: https://github.com/dsi-hub/nxnative/tree/hooklink, commit 0ad4e1a9) on an iPhone 18 Pro Max. +- **Check:** `t_pso_demand_record` calls `demand_record` twice for one key and checks that `pso_user.tsv` gets one row, that a second key gets its own, and that with `DM_PSO_DEMAND_RECORD=0` nothing is written. It does not draw: the hybrid call site is not run by any test. + ### Shader libraries loaded when the program is made - **Symptom:** the first frame that draws with new programs loads their shader libraries on the submit thread (`[longframe] ... shader libraries N, X ms`): tens of milliseconds with the files cached, seconds when the drive is slow. From f6eac81ad27294ee393e8847002ad730998358ac Mon Sep 17 00:00:00 2001 From: Oliver Ames Date: Thu, 12 Nov 2026 11:49:19 -0500 Subject: [PATCH 2/2] Exercise demand recording through pending hybrid pipeline lookups The earlier fixture called demand_record directly and could not detect a missing or misplaced pso_get call site. Compile synthetic Metal functions, cache real stand-ins, and suspend the production background queues while real lookups encounter both new and independently prequeued demands. An exec child checks canonical rows through warm_parse before either compile can run; completion replaces the stand-ins and preserves identical deduplicated rows. Cover recording disabled and the 65536-entry admission boundary, including eventual recording beyond that early-demand limit. With API validation, both encoder configurations pass on/off/cap cases. Removing call-site recording fails the matched control, as does limiting it to newly queued demands. Six docs tests pass and focused LLVM23 analysis adds no findings. This is test/docs only: no game assets, gameplay timing, power-loss durability or fresh-process prewarm compilation claim. Co-authored-by: Adam Kratch <70729875+akratch@users.noreply.github.com> Co-authored-by: akratch <70729875+akratch@users.noreply.github.com> --- dm/games/0400486024DEC000/README.md | 2 +- dm/gfx/tests/Makefile | 5 +- dm/gfx/tests/t_pso_demand_record.m | 136 ++++++++++++++++++++++------ docs/reference/shaders.md | 2 +- 4 files changed, 114 insertions(+), 31 deletions(-) diff --git a/dm/games/0400486024DEC000/README.md b/dm/games/0400486024DEC000/README.md index 746694bd86..38d5e6cf70 100644 --- a/dm/games/0400486024DEC000/README.md +++ b/dm/games/0400486024DEC000/README.md @@ -145,7 +145,7 @@ Its 37 kernels that correct samples, NDSR's network among them, gave about 0.25 - [Archive searches repeated while the build table is full](../../../docs/reference/frame-rate.md#archive-searches-repeated-while-the-build-table-is-full): the castle town market at about 12 fps while the GPU had time to spare. - [Pulling stand-ins made for predicted keys that are never drawn](../../../docs/reference/frame-rate.md#pulling-stand-ins-made-for-predicted-keys-that-are-never-drawn): resolved, frames of 50–170 ms on entering the castle town in every run, with one new pipeline or none. - [A texture written while it is drawn read whole at every use](../../../docs/reference/frame-rate.md#a-texture-written-while-it-is-drawn-read-whole-at-every-use): resolved, frames of 31–59 ms on entering Hyrule Castle Town, a 1800x1380x8 R16 array (38.5 MB) and three block-compressed textures re-uploaded 31–52 times while the loader wrote them, the array 16–17 times whole; 16–17 re-uploads after, the array's 7–9 at 4.7–14.2 MB, one frame of 25 ms at most. -- [A pipeline a hybrid draw stood in for listed only once built](../../../docs/reference/shaders.md#a-pipeline-a-hybrid-draw-stood-in-for-listed-only-once-built): on an iPhone 18 Pro Max, found by HookLink, the pipelines Castle Town asked for last were not in `pso_user.tsv` for the next start; they are listed when first asked for (`DM_PSO_DEMAND_RECORD=0` restores the old order). +- [A pipeline a hybrid draw stood in for listed only once built](../../../docs/reference/shaders.md#a-pipeline-a-hybrid-draw-stood-in-for-listed-only-once-built): on an iPhone 18 Pro Max, found by HookLink, the pipelines Castle Town asked for last were not in `pso_user.tsv` for the next start; they are listed when first asked for (`DM_PSO_DEMAND_RECORD=0` restores the old order). Synthetic hybrid-lookup tests now cover pending compilation, canonical persistence/reload, completion deduplication and recording limits; the iPhone timing claim is not locally reproduced. - [One Pro Controller read as two players](../../../docs/reference/services.md#one-pro-controller-read-as-two-players): resolved, no music in Kokiri Forest gameplay and its cutscenes; the controller applet opened over and over and the game kept its music suspended while it had focus. - [A keyboard receiver that declares a gamepad takes player 1](../../../docs/reference/services.md#a-keyboard-receiver-that-declares-a-gamepad-takes-player-1): current until the reporter retests ([issue #1657](https://github.com/seedds/nxnative/issues/1657)). A Keychron Link receiver was player 1 and the Pro Controller player 2, so the game took no input; a raw HID pad is now a player from its first button press. - [The controller applet leaves out the players past the maximum](../../../docs/reference/services.md#the-controller-applet-leaves-out-the-players-past-the-maximum): resolved in the host's tests, not run on the game. One Bluetooth Pro Controller came back as players 1 and 2 after a long disconnect, and the game opened the controller applet 1,671 times with no input ([issue #2237](https://github.com/seedds/nxnative/issues/2237)); the second player's controller now feeds player 1. diff --git a/dm/gfx/tests/Makefile b/dm/gfx/tests/Makefile index 8caddb6bea..b315aa58f5 100644 --- a/dm/gfx/tests/Makefile +++ b/dm/gfx/tests/Makefile @@ -358,8 +358,9 @@ t_pso_demand_record: t_pso_demand_record.m $(MTL_T_DEP) clang $(CFLAGS) -fobjc-arc -o $@ t_pso_demand_record.m $(MTL_T_SRC) $(FW) .PHONY: check_t_pso_demand_record check_t_pso_demand_record: t_pso_demand_record - ./t_pso_demand_record - ./t_pso_demand_record --off + MTL_DEBUG_LAYER=1 ./t_pso_demand_record + MTL_DEBUG_LAYER=1 ./t_pso_demand_record --off + MTL_DEBUG_LAYER=1 ./t_pso_demand_record --cap t_shmap_warm: t_shmap_warm.m $(MTL_T_DEP) clang $(CFLAGS) -fobjc-arc -o $@ t_shmap_warm.m $(MTL_T_SRC) $(FW) t_pso_depth_stencil: t_pso_depth_stencil.m $(MTL_T_DEP) diff --git a/dm/gfx/tests/t_pso_demand_record.m b/dm/gfx/tests/t_pso_demand_record.m index 837fe4363f..b005f7f936 100644 --- a/dm/gfx/tests/t_pso_demand_record.m +++ b/dm/gfx/tests/t_pso_demand_record.m @@ -1,35 +1,117 @@ -// A pipeline a hybrid draw stands in for is listed in pso_user.tsv when first asked for (demand_record), once per key; DM_PSO_DEMAND_RECORD=0 lists none of them. -// Run twice: with no argument and with --off (the Makefile's check target). The hybrid call site in the draw path is not run here: it needs translated shader libraries. +// Real pso_get hybrid misses must record demand before their queued compiles run. +// Synthetic functions and cached stand-ins need no translated/game libraries. #include "../nvn/nvn_mtl.m" - +#include +#include +#include +extern char **environ; void *front_metal_layer(void) { return NULL; } module_t *module_of(const void *p) { (void)p; return NULL; } module_t *g_mod; int g_nmod; void svc_thread_dump(void) {} uint64_t now_ns(void) { return bp_now(); } - -static int g_failed; -#define CHECK(name, cond) do { int ok_ = !!(cond); printf("%s %s\n", name, ok_ ? "OK" : "FAIL"); g_failed += !ok_; } while (0) - -static int rows(const char *path) { - NSString *t = [NSString stringWithContentsOfFile:@(path) encoding:NSUTF8StringEncoding error:nil]; int n = 0; - for (NSString *l in [t componentsSeparatedByString:@"\n"]) n += [l hasPrefix:@"R\t"]; - return n; +static int checks, failed; +#define CHECK(name, cond) do { checks++; if (!(cond)) { failed++; fprintf(stderr,"FAIL: %s (%d)\n",name,__LINE__); } } while (0) +static const char *vkey="1111111111111111_b", *fkey="2222222222222222_b"; +static NSArray *rows(const char *path) { + NSString *text=[NSString stringWithContentsOfFile:@(path) encoding:NSUTF8StringEncoding error:nil]; + CHECK("manifest is independently readable",text != nil); + NSMutableArray *r=[NSMutableArray array]; + for(NSString *line in [text componentsSeparatedByString:@"\n"]) if([line hasPrefix:@"R\t"]) [r addObject:line]; + return r; } - -int main(int argc, char **argv) { - int off = argc > 1 && !strcmp(argv[1], "--off"); - char path[] = "/tmp/t_pso_demand_record.XXXXXX"; int fd = mkstemp(path); if (fd < 0) return 2; close(fd); - setenv("DM_PSO_USER", path, 1); unsetenv("DM_SAVE"); - if (off) setenv("DM_PSO_DEMAND_RECORD", "0", 1); else unsetenv("DM_PSO_DEMAND_RECORD"); - NvShader *vs = [NvShader new], *fs = [NvShader new]; - snprintf(vs->key, sizeof vs->key, "demand-vs"); snprintf(fs->key, sizeof fs->key, "demand-fs"); - pso_key a, b; memset(&a, 0, sizeof a); memset(&b, 0, sizeof b); a.samples = 1; b.samples = 4; - demand_record(&a, vs, fs); demand_record(&a, vs, fs); - CHECK(off ? "off: nothing listed" : "listed once for one key", rows(path) == (off ? 0 : 1)); - demand_record(&b, vs, fs); - CHECK(off ? "off: still nothing" : "a second key has its own row", rows(path) == (off ? 0 : 2)); - unlink(path); - printf("%s\n", g_failed ? "FAIL" : "PASS"); - return g_failed != 0; +static void check_rows(NSArray *r, unsigned n) { + CHECK("exact number of canonical render rows",r.count == n); + for(NSUInteger i=0;i *r=rows(path);check_rows(r,n); + g_shmap=[@{@"1111111111111111":@"vertex",@"2222222222222222":@"pixel"} mutableCopy]; + long lines=-1;NSArray *tasks=warm_parse(@[@(path)],&lines); + CHECK("fresh process accepts recorded rows as prewarm tasks",lines==n&&tasks.count==(n?1u:0u)); + if(tasks.count) { + WarmTask *t=tasks[0];CHECK("rows group into the correct render pair",t->kind==1&&[t->vkey isEqualToString:@(vkey)]&&[t->fkey isEqualToString:@(fkey)]&&t->rs.count==n); + for(NSUInteger i=0;irs.count&&irs[i] isEqualToData:[NSData dataWithBytes:&key length:sizeof key]]); + } + } + printf("reload: %d checks, %d failed\n",checks,failed);return failed!=0; +} +static void child_reload(const char *exe,const char *path,unsigned n) { + char count[16];snprintf(count,sizeof count,"%u",n); + char *args[]={(char *)exe,"--reload",(char *)path,count,NULL};pid_t pid;int status=0; + int error=posix_spawn(&pid,exe,NULL,NULL,args,environ);CHECK("fresh process starts",!error); + if(!error){CHECK("fresh process exits normally",waitpid(pid,&status,0)==pid&&WIFEXITED(status)&&!WEXITSTATUS(status));} +} +static int pending(const pso_key *key) { + pthread_mutex_lock(&g_pp_mu);int i=pso_pend_find(key);int yes=i>=0&&!g_pp[i].done&&!g_pp[i].run&&g_pp[i].d!=nil;pthread_mutex_unlock(&g_pp_mu);return yes; +} +int main(int argc,char **argv) { @autoreleasepool { + alarm(90); + if(argc==4&&!strcmp(argv[1],"--reload"))return reload(argv[2],(unsigned)strtoul(argv[3],NULL,10)); + int off=argc==2&&!strcmp(argv[1],"--off"),cap=argc==2&&!strcmp(argv[1],"--cap"); + if(argc>2||(argc==2&&!off&&!cap))return 2; + char path[]="/tmp/t_pso_demand_record.XXXXXX";int fd=mkstemp(path);if(fd<0)return 2;close(fd); + setenv("DM_PSO_USER",path,1);setenv("DM_PSO_ARCHIVE","off",1);setenv("DM_PSO_MANIFEST","off",1);setenv("DM_PSO_PREBUILD","0",1); + setenv("DM_PSO_DEMAND_RECORD",off?"0":"1",1);setenv("DM_PSO_HYBRID","1",1);setenv("DM_NVN_PREDICT","0",1);unsetenv("DM_SAVE"); + mtl_init();CHECK("hybrid path is available",hybrid_on());if(!hybrid_on())return 2; + NSError *error=nil; + id lib=[g_dev newLibraryWithSource:@"#include \nusing namespace metal;\nvertex float4 v(uint i[[vertex_id]]){return float4(i==1?3.0:-1.0,i==2?3.0:-1.0,0,1);}\nfragment float4 f(){return float4(0,1,0,1);}\n" options:nil error:&error]; + CHECK("synthetic library compiles",lib!=nil);if(!lib)return 2; + NvShader *vs=[NvShader new],*fs=[NvShader new];vs->stage=0;fs->stage=1;vs->has_pull=1;fs->rf.color_known=1;fs->rf.color_mask=1; + strlcpy(vs->key,vkey,sizeof vs->key);strlcpy(fs->key,fkey,sizeof fs->key); + vs->fn=fn4_note([lib newFunctionWithName:@"v"],lib,@"v",nil);fs->fn=fn4_note([lib newFunctionWithName:@"f"],lib,@"f",nil); + CHECK("synthetic function entries exist",vs->fn&&fs->fn);if(!vs->fn||!fs->fn)return 2; + NvProg *pr=[NvProg new];pr->sh[0]=vs;pr->sh[1]=fs; + state_reset();memset(&g_pass,0,sizeof g_pass);g_pass.cfmt[0]=MTLPixelFormatRGBA8Unorm;g_pass.samples=1; + pso_key keys[2];id stand[2]; + for(int i=0;i<2;i++) { + S.chmask.mask=i?7:15;pso_key_fill(&keys[i],vs,fs,g_pass.cfmt,0,0,1,0); + // Build stand-ins outside the production base cache: the real pso_get + // archive-only lookup must miss and admit its own background demand. + stand[i]=[g_dev newRenderPipelineStateWithDescriptor:pso_desc(&keys[i],vs,fs,nil,0) error:&error]; + CHECK("real cached stand-in compiles",stand[i]!=nil);if(!stand[i])return 2; + hm_put_id(&g_pso_hy,&keys[i],stand[i]); + } + if(cap) { + for(uint32_t i=0;i<(1u<<16)-1;i++){pso_key dummy={0};dummy.samples=i+2;nvn_hm_put(&g_demand_rec,&dummy,(void *)1);} + CHECK("admission map starts one entry below its bound",g_demand_rec.n==(1u<<16)-1); + } + dispatch_queue_t queues[8];int nq=0; + for(int i=0;i<9;i++){dispatch_queue_t q=pp_bg_queue();int seen=0;for(int j=0;j a=pso_get(pr,nil); + CHECK("first real lookup returns the cached stand-in",a==stand[0]&&g_pso_pull); + CHECK("new demand remains queued, not compiled",pending(&keys[0])); + unsigned early=off?0:1;check_rows(rows(path),early); + CHECK("already-pending lookup still returns stand-in",pso_get(pr,nil)==stand[0]&&g_pso_pull); + CHECK("duplicate demand does not append a row",rows(path).count==early); + // The second pipeline was queued independently: its *first* pso_get sees + // st==1, covering placement of demand_record outside the st==0 branch. + CHECK("second demand can be independently queued",pso_pend_demand(&keys[1],vs,fs)); + S.chmask.mask=7;CHECK("prequeued lookup returns its stand-in",pso_get(pr,nil)==stand[1]&&g_pso_pull); + CHECK("both compiles remain gated",pending(&keys[0])&&pending(&keys[1])); + early=off?0:cap?1:2;NSArray *before=rows(path);check_rows(before,early); + if(cap)CHECK("map cap declines the next early record",g_demand_rec.n==(1u<<16)); + child_reload(argv[0],path,early); // fflush visibility across exec; not power-loss/fsync durability. + CHECK("reload did not need compilation to complete",pending(&keys[0])&&pending(&keys[1])); + for(int i=0;i ps=pso_get(pr,nil); + CHECK("eventual completed pipeline replaces stand-in",ps&&ps!=stand[i]&&!g_pso_pull); + CHECK("completed pipeline is retained in the normal cache",hm_get_id(&g_pso,&keys[i])==ps); + } + NSArray *after=rows(path);check_rows(after,2); + for(NSUInteger i=0;i