Experiment: defer redundant pull-only pipeline archives

dm/gfx/nvn/nvn_mtl.m

@@ -5180,6 +5180,12 @@ static int pso4_on(void) {
 // g_arch4: the files that may hold bases with a vertex layout, g_arch4p: the files that hold none (p<seq>.mtl4a: the prebuild's, and any flush whose bases all read their vertices
 // themselves). A base with a layout is looked up in g_arch4 alone: each file a lookup misses in costs about 0.06 ms, and a cold start's misses were for such bases, in the 13 prebuilt files
 static NSArray<id<MTL4Archive>> *g_arch4, *g_arch4p; static NSString *g_arch_dir; static int g_arch_seq;
+// Opt-in experiment: with a usable bundle, defer only the device's p files.
+// Mixed/layout files stay eager, preserving compaction's immutable startup state.
+// The deferred arrays are published once, never appended to while readers run.
+static NSArray<NSString *> *g_arch4p_deferred;
+static NSArray<id<MTL4Archive>> *g_arch4p_late;
+static dispatch_once_t g_arch4p_once;
 // Compaction: every start writes a file or more (each flush captures only what is new), Breath of the Wild 5-7 per start, and each file a base is not in costs a lookup about 0.06 ms.
 // With more than PSO_ARCH_COMPACT files of bases with a vertex layout, the bases the prewarm takes from them are captured again from the original shader functions, so the flush
 // after the prewarm writes them into one file; the old files are then removed (open archives stay usable for the rest of the run). Bases the
@@ -5296,6 +5302,23 @@ static int arch_file_ok(NSString *path) {
     struct stat st; int ok = !fstat(fd, &st) && S_ISREG(st.st_mode) && st.st_size > 0 && arch_container_ok(fd, (uint64_t)st.st_size);
     close(fd); return ok;
 }
+static NSArray<id<MTL4Archive>> *pso_device_pull_archives(void) {
+    if (!g_arch4p_deferred.count) return nil;
+    dispatch_once(&g_arch4p_once, ^{
+        uint64_t t0 = bp_now(); unsigned long long bytes = 0;
+        NSMutableArray<id<MTL4Archive>> *opened = [NSMutableArray array];
+        for (NSString *path in g_arch4p_deferred) {
+            NSError *error = nil;
+            id<MTL4Archive> archive = arch_file_ok(path) ? [g_dev newArchiveWithURL:[NSURL fileURLWithPath:path] error:&error] : nil;
+            if (archive) { [opened addObject:archive]; bytes += [[NSFileManager.defaultManager attributesOfItemAtPath:path error:nil] fileSize]; }
+            else MLOG("[RED] pipeline archive file %s not usable (%s): its bases are built again", path.UTF8String,
+                      error ? error.localizedDescription.UTF8String : "not a pipeline archive");
+        }
+        g_arch4p_late = [opened copy];
+        MLOG("pipeline archive: bundle miss opened %lu deferred p files, %.1f MB, in %.1f ms", (unsigned long)opened.count, bytes / 1048576.0, (bp_now() - t0) / 1e6);
+    });
+    return g_arch4p_late;
+}
 static void pso_bundle_open(NSMutableArray<id<MTL4Archive>> *archp) {
     const char *e = getenv("DM_PSO_BUNDLE");
     if (!e || !*e || !strcmp(e, "off")) return;
@@ -5373,17 +5396,24 @@ static void pso4_init(void) {
             });
 #endif
             uint64_t t0 = bp_now(); unsigned long long bytes = 0;
-            NSMutableArray<NSString *> *runf = [NSMutableArray array];
+            NSMutableArray<NSString *> *runf = [NSMutableArray array], *deferred = [NSMutableArray array];
+            const char *lazy = getenv("DM_PSO_LAZY_DEVICE_ARCHIVES");
+            int defer_pull = g_arch_bundle && lazy && !strcmp(lazy, "1");
             NSArray<NSString *> *names = [[[NSFileManager.defaultManager contentsOfDirectoryAtPath:g_arch_dir error:nil] filteredArrayUsingPredicate:[NSPredicate predicateWithFormat:@"self ENDSWITH '.mtl4a'"]]
                                           sortedArrayUsingSelector:@selector(compare:)];
             for (NSString *n in names.reverseObjectEnumerator) {   // newest first
                 NSString *p = [g_arch_dir stringByAppendingPathComponent:n]; NSError *ae = nil;
-                id<MTL4Archive> a = arch_file_ok(p) ? [g_dev newArchiveWithURL:[NSURL fileURLWithPath:p] error:&ae] : nil;
                 int pf = [n hasPrefix:@"p"];
+                // Reserve every existing sequence before any capture can flush,
+                // including archives whose opening is deferred or fails later.
+                g_arch_seq = MAX(g_arch_seq, [n substringFromIndex:pf].intValue + 1);
+                if (defer_pull && pf) { [deferred addObject:p]; continue; }
+                id<MTL4Archive> a = arch_file_ok(p) ? [g_dev newArchiveWithURL:[NSURL fileURLWithPath:p] error:&ae] : nil;
                 if (a) { [pf ? archp : arch addObject:a]; if (!pf) [runf addObject:p]; bytes += [[NSFileManager.defaultManager attributesOfItemAtPath:p error:nil] fileSize]; }
                 else MLOG("[RED] pipeline archive file %s not usable (%s): its bases are built again", p.UTF8String, ae ? ae.localizedDescription.UTF8String : "not a pipeline archive");
-                g_arch_seq = MAX(g_arch_seq, [n substringFromIndex:pf].intValue + 1);
             }
+            g_arch4p_deferred = [deferred copy];
+            if (deferred.count) MLOG("pipeline archive: %lu device p files deferred until a no-layout bundle miss (DM_PSO_LAZY_DEVICE_ARCHIVES=1); mixed/layout files remain eager", (unsigned long)deferred.count);
             if (runf.count > PSO_ARCH_COMPACT && arch_compact_allowed()) { g_compact_enabled = 1; g_compact_files = runf; }
             MLOG("pipeline archive %s: %lu files (%lu of bases without a vertex layout), %.1f MB, opened in %.1f ms", g_arch_dir.UTF8String, (unsigned long)(arch.count + archp.count - (g_arch_bundle != nil)),
                  (unsigned long)archp.count - (g_arch_bundle != nil), bytes / 1048576.0, (bp_now() - t0) / 1e6);
@@ -5517,7 +5547,11 @@ static void pbase_key_of(MTLRenderPipelineDescriptor *d, pbase_key *k) {
     else {
         uint64_t t0 = bp_now();
         int vd = 0; for (int i = 0; i < 31 && !vd; i++) vd = bk.va[i][0] != 0;   // a vertex layout (MTLVertexFormatInvalid is 0)
-        if (!vd) for (id<MTL4Archive> a in g_arch4p) if ((base = [a newRenderPipelineStateWithDescriptor:b error:nil])) break;
+        NSArray<id<MTL4Archive>> *late = nil;
+        if (!vd) {
+            for (id<MTL4Archive> a in g_arch4p) if ((base = [a newRenderPipelineStateWithDescriptor:b error:nil])) break;
+            if (!base) { late = pso_device_pull_archives(); for (id<MTL4Archive> a in late) if ((base = [a newRenderPipelineStateWithDescriptor:b error:nil])) break; }
+        }
         int from_run = 0;
         if (!base) for (id<MTL4Archive> a in g_arch4) if ((base = [a newRenderPipelineStateWithDescriptor:b error:nil])) { from_run = 1; break; }
         uint64_t t1 = bp_now();
@@ -5529,7 +5563,7 @@ static void pbase_key_of(MTLRenderPipelineDescriptor *d, pbase_key *k) {
             if (cb) { base = cb; g_compacted++; }
         }
         if (!base) {
-            if (g_arch4.count + g_arch4p.count) { g_pt_miss_n++; g_pt_miss_ns += t1 - t0; }
+            if (g_arch4.count + g_arch4p.count + late.count) { g_pt_miss_n++; g_pt_miss_ns += t1 - t0; }
             if (compile) {
                 NSError *be = nil; base = arch_capture(b, &be, vd); job->error = be; job->compiled = 1;
                 g_pt_comp_ns += bp_now() - t1;
@@ -5778,6 +5812,7 @@ static void pre_pair(NSString *vk, NSString *fk, int zero) {
         MTL4RenderPipelineDescriptor *m = pre_desc(vk, fk, zero);
         if (!m) { g_pre_skipped++; return; }
         for (id<MTL4Archive> a in g_arch4p) if ([a newRenderPipelineStateWithDescriptor:m error:nil]) { g_pre_archived++; return; }
+        for (id<MTL4Archive> a in pso_device_pull_archives()) if ([a newRenderPipelineStateWithDescriptor:m error:nil]) { g_pre_archived++; return; }
         for (id<MTL4Archive> a in g_arch4) if ([a newRenderPipelineStateWithDescriptor:m error:nil]) { g_pre_archived++; return; }
         NSError *e = nil; id ps = arch_capture(m, &e, 0);
         if (ps) g_pre_built++;

dm/gfx/tests/t_pso_bundle.m

@@ -95,15 +95,20 @@ int main(void) { @autoreleasepool {
     NSString *rest = [NSString stringWithContentsOfFile:[bundle stringByAppendingPathComponent:@"rest_pairs.tsv"] encoding:NSUTF8StringEncoding error:nil];
     EXPECT("bundle: one pair left to the device", 1, [rest isEqualToString:@"v1\tfc\n"]);
     setenv("DM_PSO_BUNDLE", bundle.UTF8String, 1);
+    // A prior device p file with a nonzero sequence: covered draws must not
+    // need its mapping in lazy mode, and captures must never overwrite it.
+    [NSFileManager.defaultManager createDirectoryAtPath:arch withIntermediateDirectories:YES attributes:nil error:nil];
+    NSString *seed = [arch stringByAppendingPathComponent:@"p00007.mtl4a"];
+    EXPECT("device fixture: seeded", 1, [archiveBytes writeToFile:seed atomically:YES]);
     shmap_load(); pso4_init();
+    const char *lazy_env = getenv("DM_PSO_LAZY_DEVICE_ARCHIVES");
+    int lazy = lazy_env && !strcmp(lazy_env, "1");
+    EXPECT("device archives: existing sequence reserved before capture", 8, g_arch_seq);
+    EXPECT("device archives: default eager, opt-in deferred", lazy ? 1 : 2, g_arch4p.count);
+    EXPECT("device archives: deferred paths immutable", lazy ? 1 : 0, g_arch4p_deferred.count);
+    EXPECT("device archives: no deferred mapping at startup", 0, g_arch4p_late.count);
     EXPECT("bundle: opened", 1, g_arch_bundle != nil);
     EXPECT("bundle: first among the files without a vertex layout", 1, g_arch4p.count && g_arch4p[0] == g_arch_bundle);
-    // its stamp matches: only the pair with function constants is built here
-    reset_counts();
-    EXPECT("covered: one pair built on the device", 1, pso_prebuild(1, 1, QOS_CLASS_USER_INITIATED));
-    EXPECT("covered: its base compiled", 1, g_pre_built);
-    EXPECT("covered: nothing else", 0, g_pre_archived + g_pre_skipped + g_pre_failed);
-    EXPECT("covered: again, nothing to do", -1, pso_prebuild(1, 1, QOS_CLASS_USER_INITIATED));
     // a draw of a covered pair (its pulling pipeline, as pso_desc_pull makes it) takes the base from the bundle: nothing compiled, nothing captured for the device's own archive
     @autoreleasepool {
         id<MTLLibrary> vl = lib_from_file([msl stringByAppendingPathComponent:@"v1_bp.metallib"], NULL), fl = lib_from_file([msl stringByAppendingPathComponent:@"f1_b.metallib"], NULL);
@@ -115,7 +120,44 @@ int main(void) { @autoreleasepool {
         EXPECT("covered draw: its base from the bundle", arch0 + 1, g_pb_archived);
         EXPECT("covered draw: none compiled", built0, g_pb_built);
         EXPECT("covered draw: nothing captured for the device's archive", new0, g_arch_new);
+        EXPECT("covered draw: deferred archives remain unopened", 0, g_arch4p_late.count);
     }
+    // A real subset bundle misses f2, which remains in the real device file.
+    // This is the first uncovered draw: no parallel archive implementation.
+    {
+        NSString *partial = [root stringByAppendingPathComponent:@"partial-bundle"];
+        put([tables stringByAppendingPathComponent:@"partial.tsv"], @"v1\tf1\n");
+        EXPECT("fallback fixture: subset archive built", 0, sh([NSString stringWithFormat:
+            @"python3 %@ --platform macos --gpu %@ --msl %@ --pairs %@/partial.tsv --out %@ --work %@/partial-work > %@/partial.log 2>&1",
+            tool, gpu, msl, tables, partial, root, root]));
+        id<MTL4Archive> subset = [g_dev newArchiveWithURL:[NSURL fileURLWithPath:[partial stringByAppendingFormat:@"/%@.mtl4a", gpu]] error:nil];
+        EXPECT("fallback fixture: usable subset bundle", 1, subset != nil);
+        NSArray *saved = g_arch4p; id<MTL4Archive> saved_bundle = g_arch_bundle;
+        g_arch_bundle = subset;
+        g_arch4p = [@[subset] arrayByAddingObjectsFromArray:[saved subarrayWithRange:NSMakeRange(1, saved.count - 1)]];
+        id<MTLLibrary> vl = lib_from_file([msl stringByAppendingPathComponent:@"v1_bp.metallib"], NULL);
+        id<MTLLibrary> fl = lib_from_file([msl stringByAppendingPathComponent:@"f2_b.metallib"], NULL);
+        MTLRenderPipelineDescriptor *d = [MTLRenderPipelineDescriptor new];
+        d.vertexFunction = fn4_note([vl newFunctionWithName:@"main0"], vl, @"main0", nil);
+        d.fragmentFunction = fn4_note([fl newFunctionWithName:@"main0"], fl, @"main0", nil);
+        d.vertexDescriptor = [MTLVertexDescriptor vertexDescriptor]; d.colorAttachments[0].pixelFormat = MTLPixelFormatRGBA8Unorm;
+        uint64_t built0 = g_pb_built, archived0 = g_pb_archived;
+        EXPECT("bundle miss: pipeline from device fallback", 1, pso_build(d, NULL) != nil);
+        EXPECT("bundle miss: actual archive hit", archived0 + 1, g_pb_archived);
+        EXPECT("bundle miss: no unnecessary compile", built0, g_pb_built);
+        EXPECT("bundle miss: deferred archive opened", lazy ? 1 : 0, g_arch4p_late.count);
+        NSArray *opened = pso_device_pull_archives();
+        EXPECT("bundle miss: later readers share immutable opened snapshot", 1, pso_device_pull_archives() == opened);
+        g_arch_bundle = saved_bundle; g_arch4p = saved;
+    }
+    // its stamp matches: only the pair with function constants is built here
+    reset_counts();
+    EXPECT("covered: one pair built on the device", 1, pso_prebuild(1, 1, QOS_CLASS_USER_INITIATED));
+    EXPECT("covered: its base compiled", 1, g_pre_built);
+    EXPECT("covered: nothing else", 0, g_pre_archived + g_pre_skipped + g_pre_failed);
+    EXPECT("covered: again, nothing to do", -1, pso_prebuild(1, 1, QOS_CLASS_USER_INITIATED));
+    EXPECT("device archive: old sequence never overwritten", 1, [[NSData dataWithContentsOfFile:seed] isEqualToData:archiveBytes]);
+    EXPECT("device archive: capture uses next reserved sequence", 1, [NSFileManager.defaultManager fileExistsAtPath:[arch stringByAppendingPathComponent:@"p00008.mtl4a"]]);
     // the near = 0 depth mode: not what the archive was made for, so the device's prebuild runs, and its main0_z bases are not in the archive
     [NSFileManager.defaultManager removeItemAtPath:[arch stringByAppendingPathComponent:@"prebuilt.tsv"] error:nil];
     reset_counts(); atomic_store(&g_depth_zero, 1);
@@ -133,7 +175,7 @@ int main(void) { @autoreleasepool {
     // a matching stamp but an archive without the bases (here: one of the device's own files): the probe finds none, so it is not trusted
     @autoreleasepool {
         NSString *txt = [NSString stringWithContentsOfFile:[tables stringByAppendingPathComponent:@"shader_pairs.tsv"] encoding:NSUTF8StringEncoding error:nil];
-        NSString *own = [arch stringByAppendingPathComponent:[[[NSFileManager.defaultManager contentsOfDirectoryAtPath:arch error:nil] filteredArrayUsingPredicate:[NSPredicate predicateWithFormat:@"self BEGINSWITH 'p' AND self ENDSWITH '.mtl4a'"]] firstObject]];
+        NSString *own = [arch stringByAppendingPathComponent:@"p00008.mtl4a"];   // the device-only function-constant pair, not the seeded full bundle
         id<MTL4Archive> real = g_arch_bundle; NSError *e = nil;
         g_arch_bundle = [g_dev newArchiveWithURL:[NSURL fileURLWithPath:own] error:&e];
         EXPECT("archive guard: real serialized archive accepted", 1, arch_file_ok(own));

docs/env_switches.tsv

@@ -213,6 +213,7 @@ DM_PSO_BUNDLE	path	The pipeline archive made offline (`<GPU architecture>.mtl4a`
 DM_PSO_GPUS	switch	`build_ipa.sh`: the GPU architectures, comma-separated, to build the offline pipeline archive for (`none` builds none); unset, `g18p` for a profile with `pso_mtl4 on`, else none
 DM_PSO_HYBRID	switch	`0`: a draw whose pipeline is not built yet waits for it instead of drawing with its shader pair's pulling pipeline meanwhile (on by default with the Metal 4 path)
 DM_PSO_IOS_MIN	switch	`build_ipa.sh`: the minimum iOS version of the offline pipeline archives (default the SDK's major version); an older system cannot open them and builds the bases itself
+DM_PSO_LAZY_DEVICE_ARCHIVES	diagnostic	`1` defers opening device p<seq>.mtl4a archives when a usable bundled archive exists, until an actual no-layout pair miss; mixed/layout archives stay eager and default `0` preserves existing startup behavior (fork-only experiment)
 DM_PSO_MANIFEST	path	The prewarm manifest to use instead of the profile's `pso_manifest.tsv`; `off` turns the prewarm off
 DM_PSO_MTL4	player	`0` builds render pipelines through the `MTLDevice` with no bases and no archive; `1` uses MTL4Compiler with the archive, hybrid pipelines and prebuild (default on on the Mac, the profile's `pso_mtl4` on iOS)
 DM_PSO_PAIRS	path	The prebuild's second shader pair table (default: `pso_pairs.tsv` beside the shader map)

docs/reference/shaders.md

@@ -920,6 +920,16 @@ Finding, keying and translating shaders (`dm/gfx/tools/shader_split.py`, `dmsh_b
 - **Found on:** [The Legend of Zelda: Breath of the Wild](../../dm/games/01007EF00011E000/README.md#found-on-this-game), during review of its offline archive path with synthetic libraries and pair lists; no new gameplay or iPhone run. The empty pair list on [ANIMAL WELL](../../dm/games/010020D01AD24000/README.md#found-on-this-game).
 - **Check:** `dm/tools/tests/test_pso_offline.py` changes vertex and fragment library bytes without changing entries or stamps, and exercises the all-constant plan, the header-only pair list and pairs all left out. `t_pso_bundle` rejects bare headers, truncated archives, malformed load commands and out-of-range universal slices, accepts real offline and serialized archives, and rejects zero-probe coverage.
 
+### Fork experiment: deferred device pull-only pipeline archives
+
+`DM_PSO_LAZY_DEVICE_ARCHIVES=1` keeps device `p<seq>.mtl4a` files closed while a usable bundled archive supplies every requested base. The default remains eager loading. Device files that can contain vertex layouts stay eager because their startup list also controls compaction, and those files can contain bases without a layout too.
+
+The first no-layout draw or prebuild pair that misses the bundle opens the deferred files once. Every reader receives the same immutable array after `dispatch_once` completes. The existing fallback order, capture path and mixed-file compaction remain in place. Startup reserves every existing filename's sequence number before any capture can flush, including files it hasn't opened yet. A bundle that is missing, unusable or disabled keeps the existing eager path.
+
+The log records the deferred file count and the time and bytes opened on the first miss. These are archive-open observations; no game startup improvement or resident-memory reduction has been measured. A function-constant remainder can trigger the fallback during prebuild, so some games may open the files shortly after startup anyway.
+
+Checks: `t_pso_bundle` passes 67 checks in default and opt-in modes with real generated Metal archives, including a covered draw that leaves the files closed, a real subset-bundle miss served by the device archive, and a capture that preserves the old file and uses the next sequence. `t_pso_archive` passes 74 checks in each mode without a bundle, including vertex-layout lookup, mixed captures, concurrency and compaction. Its fresh-process seed, replay and verification matrix also passes in each mode, with and without function constants. These are Mac synthetic GPU tests; Newt game and footprint comparisons remain pending.
+
 ### Metal's build folders left by pipeline archive flushes
 
 - **Symptom:** the user cache folder (`getconf DARWIN_USER_CACHE_DIR`, then `com.apple.gpuarchiver/`) fills with `gpuarchiver-*` folders of 0.3–5 MB each, one per pipeline archive flush (456 folders, 2.5 GB after two days). Later folders are larger, up to about 127 MB (an M4 Pro, 2026-10-29: 19 GB in 423 folders after a day of Mac runs and regression checks); why they grew is **open**. A later compile can fail with `cannot open '…/gpuarchiver-…/PersistentState': No such file or directory`.