From 18250976c12d7eb65d800093b789b44c2c0fbe1d Mon Sep 17 00:00:00 2001 From: seedds Date: Tue, 10 Nov 2026 20:06:18 -0800 Subject: [PATCH 1/9] Scratch: frame generation for Ocarina of Time (0400486024DEC000) from the scene before the HUD, its depth and the game's camera (DM_FRAMEGEN, off by default) A prototype for evaluation, not for merging as it is. Between two frames the game draws, the host shows one made by MetalFX's frame interpolator (Metal 4) from the 1440x810 scene before the HUD, the scene's depth and motion vectors reprojected from the game's camera block; the generated scene goes through the game's spatial upscale a second time and is composed under the real frame's HUD, which is never interpolated. The generated frame is presented ahead of its real frame on the present thread (DM_PRESENT_LINK=0), each for half the period; the pacing follows #2241, the presented-time shift and the delayed first engagement an outside patch (framegen-stage1-v4) read as reference. Specific to Ocarina of Time: the camera block's layout and the far-plane test that picks the scene camera, read from one build and not keyed by build ID yet. Played on the Ocarina performance line (cut6) on an M2 MacBook Air: 45-53 shown frames a second from a 30 fps cap, the game's frame rising from about 30 ms to about 37 ms. On main: the host compiles for 0400486024DEC000; no game run and no regress run. With DM_FRAMEGEN unset the hooks return at once. Co-Authored-By: Claude --- dm/gfx/nvn/nvn_framegen.inc | 345 ++++++++++++++++++++++++++++++++++++ dm/gfx/nvn/nvn_mtl.m | 32 +++- 2 files changed, 374 insertions(+), 3 deletions(-) create mode 100644 dm/gfx/nvn/nvn_framegen.inc diff --git a/dm/gfx/nvn/nvn_framegen.inc b/dm/gfx/nvn/nvn_framegen.inc new file mode 100644 index 0000000000..98ad0af71a --- /dev/null +++ b/dm/gfx/nvn/nvn_framegen.inc @@ -0,0 +1,345 @@ +// ---------- scratch (2026-11-10): frame generation for Ocarina of Time, DM_FRAMEGEN=1. An evaluation build, not for main as it is. ---------- +// A game held at 30 fps shown at 60: MetalFX's frame interpolator makes the frame between the previous scene and this one, from the scene before the HUD (the upscaler's +// input, 1440x810 linear), the scene depth and motion vectors reprojected from the game's camera block (zero where the depth says a pixel is not still scenery). +// The generated scene is upscaled by the same MetalFX spatial scaler (a second run), and the HUD is not interpolated: at the present, a pixel of the canvas that changed +// after the upscale (the HUD passes) keeps the real frame's picture, every other pixel takes the generated scene. The present thread (DM_PRESENT_LINK=0) shows the +// generated frame, then the real one, each for half the present's period; generated frames pause while frames come late (as seedds/nxnative#2241's FG_STEADY). +// Scratch limits: OoT only (the camera block is vertex uniform 0 of 13,568 bytes, the scene camera the binding with far plane 25000; the canvas is presented 1:1 inside the +// window texture's crop); DM_UPSCALER=metalfx, DM_NVN_ENC_THREAD=0 and DM_PRESENT_LINK=0 only; the constants belong in a profile if this goes further. +// Pacing and accounting borrow from seedds/nxnative#2241 (pause while late, own present allocators) and from framegen-stage1-v4.patch (the held frame's presented time shifted +// back by its hold; the first engagement delayed past boot). +// Switches: DM_FRAMEGEN=0|1 (0 still logs the shown-frame statistics, for comparison); DM_FRAMEGEN_PAUSE=0 shows every generated frame, late or not; +// DM_FRAMEGEN_VIEW=1 the generated scene without the HUD split, 2 = HUD pixels in magenta; DM_FRAMEGEN_FLIP=1 flips the canvas rows against the window's; +// DM_FRAMEGEN_SHOT=:- saves the generated and the real picture of frames a..b (slow: it waits for the GPU). +#if FG_PART == 0 +static int g_fg_mode = -1; // -1 unread, 0 off, 1 on +static int fg_on(void); static void fg_cam_note(const float *c); static void fg_depth_note(MTL4RenderPassDescriptor *pd); +static void fg_shown(int gen, double tt, double tin); +static __thread double g_fg_pres_adj; // present thread: seconds added to the presented time recorded for the real frame being presented (negative: it was held behind its generated frame) +static double g_fg_tin[64]; // by ticket: when the game queued the present (present_main), for the enqueue-to-screen time of real frames +#elif FG_PART == 1 +#define FG_WIN 3 +static int g_fg_seen, g_fg_view, g_fg_flip, g_fg_pause = 1; +static int g_fg_upto = 9; // DM_FRAMEGEN_UPTO=, to split the cost in play: 0 nothing drawn, 1 the three copies, 2 + motion pass, 3 + interpolation, 4 + second upscale, 5 + window picture (not shown), 9 all +static struct { + id fi; NSUInteger iw, ih, ow, oh; MTLPixelFormat cvf; int bad; + id c[2], d[2], mv, gen, gen_up, cv0, cv, depth_last, win[FG_WIN]; + uint64_t win_serial[FG_WIN], win_ticket[FG_WIN], win_next; + id walloc[FG_WIN]; id wcb[FG_WIN]; id wpar[FG_WIN]; + id margs, wargs; id mv_pso, comp_pso; MTLPixelFormat comp_fmt; id vf, f_mv, f_comp; + int cur, warm, prev_ok, gen_ready; long last_frame, cam_frame; float cam[96]; + uint64_t made, resets, nocam, nodepth, busy, late, behind, settling; +} FG; +static const char *g_fg_shot_dir; static long g_fg_shot_a = -1, g_fg_shot_b = -1; +static int fg_on(void) { + if (g_fg_mode >= 0) return g_fg_mode; + const char *e = getenv("DM_FRAMEGEN"), *x; int v = e && atoi(e) > 0; g_fg_seen = e != NULL; + if ((x = getenv("DM_FRAMEGEN_VIEW"))) g_fg_view = atoi(x); + if ((x = getenv("DM_FRAMEGEN_FLIP"))) g_fg_flip = atoi(x); + if ((x = getenv("DM_FRAMEGEN_PAUSE"))) g_fg_pause = atoi(x); + if ((x = getenv("DM_FRAMEGEN_UPTO"))) g_fg_upto = atoi(x); + if ((x = getenv("DM_FRAMEGEN_SHOT"))) { const char *c = strrchr(x, ':'); if (c && sscanf(c + 1, "%ld-%ld", &g_fg_shot_a, &g_fg_shot_b) == 2) g_fg_shot_dir = strndup(x, (size_t)(c - x)); } + FG.last_frame = -2; FG.cam_frame = -1; + if (v && !(g_q4 && [MTLFXFrameInterpolatorDescriptor supportsMetal4FX:g_dev])) { MLOG("[RED] frame generation (DM_FRAMEGEN): this GPU has no Metal 4 MetalFX frame interpolator; frames are shown as the game draws them"); v = 0; } + if (v && rec_on()) { MLOG("[RED] frame generation (scratch build): not with DM_NVN_ENC_THREAD=1; frames are shown as the game draws them"); v = 0; } + if (v) MLOG("frame generation (DM_FRAMEGEN=1, scratch build): the scene before the HUD is interpolated from depth and the camera, upscaled a second time, and the HUD is kept from the real frame; " + "pause while late %d, view %d, flip %d, stages up to %d", g_fg_pause, g_fg_view, g_fg_flip, g_fg_upto); + else if (g_fg_seen) MLOG("frame generation off (DM_FRAMEGEN=0): the shown-frame statistics are still logged, for comparison"); + return g_fg_mode = v; +} +// The camera block as the game binds it (encode_args_): the scene camera's binding has far plane 25000; the environment probe's (far 10000, one cube face a frame) is not taken +static void fg_cam_note(const float *c) { if (!fg_on() || !(c[57] > 20000.0f)) return; memcpy(FG.cam, c, sizeof FG.cam); FG.cam_frame = (long)atomic_load(&g_cur_frame); } +// Each render pass (td_note): the last large D32F depth attachment before the upscaler is the scene depth +static void fg_depth_note(MTL4RenderPassDescriptor *pd) { + if (!g_fg_mode || !pd || !fg_on()) return; + id d = pd.depthAttachment.texture; + if (d && d.pixelFormat == MTLPixelFormatDepth32Float && d.textureType == MTLTextureType2D && d.width >= 1024) FG.depth_last = d; +} +static int fg_inv4(const double a_[16], double r[16]) { + double a[16]; memcpy(a, a_, sizeof a); for (int i = 0; i < 16; i++) r[i] = i % 5 == 0; + for (int c = 0; c < 4; c++) { int p = c; for (int k = c + 1; k < 4; k++) if (fabs(a[k * 4 + c]) > fabs(a[p * 4 + c])) p = k; + if (fabs(a[p * 4 + c]) < 1e-12) return 0; + if (p != c) for (int k = 0; k < 4; k++) { double t = a[c * 4 + k]; a[c * 4 + k] = a[p * 4 + k]; a[p * 4 + k] = t; t = r[c * 4 + k]; r[c * 4 + k] = r[p * 4 + k]; r[p * 4 + k] = t; } + double s = 1.0 / a[c * 4 + c]; for (int k = 0; k < 4; k++) { a[c * 4 + k] *= s; r[c * 4 + k] *= s; } + for (int q = 0; q < 4; q++) if (q != c) { double f = a[q * 4 + c]; if (f != 0) for (int k = 0; k < 4; k++) { a[q * 4 + k] -= f * a[c * 4 + k]; r[q * 4 + k] -= f * r[c * 4 + k]; } } } + return 1; +} +static void fg_mul4(const double a[16], const double b[16], double r[16]) { for (int i = 0; i < 4; i++) for (int j = 0; j < 4; j++) { double s = 0; for (int k = 0; k < 4; k++) s += a[i * 4 + k] * b[k * 4 + j]; r[i * 4 + j] = s; } } +static NSString *const FG_MSL = @"#include \nusing namespace metal;\n" + "struct V { float4 p [[position]]; };\n" + "vertex V fg_vs(uint id [[vertex_id]]) { float2 uv = float2((id << 1) & 2, id & 2); V o; o.p = float4(uv * float2(2, -2) + float2(-1, 1), 0, 1); return o; }\n" + "struct MP { float4x4 reproj; float2 size; float2 inv_size; };\n" + "static float vz(float d) { return 1.0 / max(1.0 - d, 1e-7); }\n" + // Motion in pixels toward the previous position: this pixel's depth reprojected through the previous view-projection; zero where the previous depth there disagrees (a moving + // object, or a disocclusion): the test of MacNativeGames' mfx_mv (MIT), as the offline test used it + "fragment float4 fg_mv(V in [[stage_in]], depth2d d [[texture(0)]], depth2d pd [[texture(1)]], constant MP *p [[buffer(0)]]) {\n" + " uint2 g = uint2(in.p.xy); float2 uv = (float2(g) + 0.5) * p->inv_size; float d0 = d.read(g); float2 m = 0.0; int still = -1;\n" + " float4 c = p->reproj * float4(uv.x * 2.0 - 1.0, 1.0 - uv.y * 2.0, d0 * 2.0 - 1.0, 1.0);\n" + " if (c.w > 1e-7) { float2 n = c.xy / c.w; float2 prev = float2(n.x * 0.5 + 0.5, 0.5 - 0.5 * n.y); m = (prev - uv) * p->size; int2 qq = int2(floor(prev * p->size));\n" + " if (d0 < 1.0 && all(qq >= int2(1)) && all(qq < int2(p->size) - int2(1))) { float ze = vz(c.z / c.w * 0.5 + 0.5), zmin = 3.4e38, zmax = 0.0; bool nr = false;\n" + " for (int y = -1; y <= 1; y++) for (int x = -1; x <= 1; x++) { float z = vz(pd.read(uint2(qq + int2(x, y)))); zmin = min(zmin, z); zmax = max(zmax, z); nr = nr || fabs(z - ze) <= ze * 0.01; }\n" + " still = (zmax <= zmin * 1.05 ? (ze >= zmin * 0.99 && ze <= zmax * 1.01) : nr) ? 1 : 0; } }\n" + " if (still == 0) m = 0.0; if (any(isnan(m)) || any(isinf(m))) m = 0.0; return float4(m, 0.0, 0.0); }\n" + "struct CP { int2 org; int2 csize; int flip; int view; int encode; int pad; };\n" + "static float3 oetf(float3 c) { c = saturate(c); return mix(12.92 * c, 1.055 * pow(c, float3(1.0 / 2.4)) - 0.055, step(float3(0.0031308), c)); }\n" + // The generated frame's window picture: win = the real frame's window texture (encoded bytes), gen = the generated scene upscaled (linear), cv = the canvas now (with the HUD), + // cv0 = the canvas right after the real frame's upscale. A canvas pixel that changed is HUD: it keeps the real frame's picture. Any other takes the real picture plus the + // difference between the generated and the real scene, both encoded, so a still scene comes out exactly as the real frame whatever the game's last pass does to the colour + "fragment float4 fg_comp(V in [[stage_in]], texture2d win [[texture(0)]], texture2d gen [[texture(1)]], texture2d cv [[texture(2)]], texture2d cv0 [[texture(3)]], constant CP *p [[buffer(0)]]) {\n" + " uint2 g = uint2(in.p.xy); float4 w = win.read(g); int2 q = int2(g) - p->org; if (p->flip != 0) q.y = p->csize.y - 1 - q.y;\n" + " if (any(q < int2(0)) || any(q >= p->csize)) return w;\n" + " float3 a = cv.read(uint2(q)).rgb; float3 b = cv0.read(uint2(q)).rgb; bool hud = any(a != b);\n" + " if (p->view == 2 && hud) return float4(1.0, 0.0, 1.0, 1.0);\n" + " if (hud && p->view != 1) return w;\n" + " float3 c = gen.read(uint2(q)).rgb;\n" + " if (p->encode == 0) return float4(c, w.a);\n" + " if (p->view == 1) return float4(oetf(c), w.a);\n" + " return float4(saturate(w.rgb + oetf(c) - oetf(b)), w.a); }\n"; +static int fg_pipes(void) { + static int st; if (st) return st > 0; + st = -1; NSError *e = nil; uint64_t t0 = now_ns(); + id lib = [g_dev newLibraryWithSource:FG_MSL options:nil error:&e]; + if (!lib) { MLOG("[RED] frame generation: its shaders do not compile: %s", e.localizedDescription.UTF8String); return 0; } + FG.vf = [lib newFunctionWithName:@"fg_vs"]; FG.f_mv = [lib newFunctionWithName:@"fg_mv"]; FG.f_comp = [lib newFunctionWithName:@"fg_comp"]; + MTLRenderPipelineDescriptor *pd = [MTLRenderPipelineDescriptor new]; pd.vertexFunction = FG.vf; pd.fragmentFunction = FG.f_mv; pd.colorAttachments[0].pixelFormat = MTLPixelFormatRG16Float; + FG.mv_pso = FG.vf && FG.f_mv ? [g_dev newRenderPipelineStateWithDescriptor:pd error:&e] : nil; + MTL4ArgumentTableDescriptor *ad = [MTL4ArgumentTableDescriptor new]; ad.maxTextureBindCount = 4; ad.maxBufferBindCount = 1; + FG.margs = rec_table([g_dev newArgumentTableWithDescriptor:ad error:&e]); FG.wargs = [g_dev newArgumentTableWithDescriptor:ad error:&e]; + if (!FG.mv_pso || !FG.f_comp || !FG.margs || !FG.wargs) { MLOG("[RED] frame generation: no pipeline or argument table: %s", e.localizedDescription.UTF8String); return 0; } + MLOG("frame generation: shaders compiled in %.1f ms", (now_ns() - t0) / 1e6); + st = 1; return 1; +} +static id fg_tex(MTLPixelFormat f, NSUInteger w, NSUInteger h, MTLTextureUsage u, NSString *label) { + MTLTextureDescriptor *td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:f width:w height:h mipmapped:NO]; td.storageMode = MTLStorageModePrivate; td.usage = u; + id t = [g_dev newTextureWithDescriptor:td]; if (t) { t.label = label; res_add(t); } return t; +} +// The interpolator and its textures for a scene of iw x ih upscaled to ow x oh (made again when a size changes: dynamic resolution) +static int fg_setup(NSUInteger iw, NSUInteger ih, NSUInteger ow, NSUInteger oh, MTLPixelFormat cvf, id sc) { + if (FG.iw == iw && FG.ih == ih && FG.ow == ow && FG.oh == oh && FG.cvf == cvf) return !FG.bad; + id old[] = {FG.c[0], FG.c[1], FG.d[0], FG.d[1], FG.mv, FG.gen, FG.gen_up, FG.cv0}; + for (unsigned i = 0; i < sizeof old / sizeof *old; i++) if (old[i]) { res_remove(old[i]); defer_obj_release(old[i]); } + FG.c[0] = FG.c[1] = FG.d[0] = FG.d[1] = FG.mv = FG.gen = FG.gen_up = FG.cv0 = nil; FG.fi = nil; + FG.iw = iw; FG.ih = ih; FG.ow = ow; FG.oh = oh; FG.cvf = cvf; FG.bad = 1; FG.warm = 0; FG.prev_ok = 0; FG.gen_ready = 0; FG.last_frame = -2; + NSError *err = nil; uint64_t t0 = now_ns(); + if (!g_fx_comp) g_fx_comp = [g_dev newCompilerWithDescriptor:[MTL4CompilerDescriptor new] error:&err]; + MTLFXFrameInterpolatorDescriptor *d = [MTLFXFrameInterpolatorDescriptor new]; + d.colorTextureFormat = MTLPixelFormatRGBA16Float; d.outputTextureFormat = MTLPixelFormatRGBA16Float; d.depthTextureFormat = MTLPixelFormatDepth32Float; d.motionTextureFormat = MTLPixelFormatRG16Float; + d.inputWidth = iw; d.inputHeight = ih; d.outputWidth = iw; d.outputHeight = ih; + id fi = g_fx_comp ? [d newFrameInterpolatorWithDevice:g_dev compiler:g_fx_comp] : nil; + if (!fi) { MLOG("[RED] frame generation: MetalFX makes no frame interpolator for %lux%lu RGBA16Float; frames are shown as the game draws them", (unsigned long)iw, (unsigned long)ih); return 0; } + MTLTextureUsage rd = MTLTextureUsageShaderRead; + for (int i = 0; i < 2; i++) { FG.c[i] = fg_tex(MTLPixelFormatRGBA16Float, iw, ih, fi.colorTextureUsage | rd, @"framegen colour"); FG.d[i] = fg_tex(MTLPixelFormatDepth32Float, iw, ih, fi.depthTextureUsage | rd, @"framegen depth"); } + FG.mv = fg_tex(MTLPixelFormatRG16Float, iw, ih, fi.motionTextureUsage | MTLTextureUsageRenderTarget | rd, @"framegen motion"); + FG.gen = fg_tex(MTLPixelFormatRGBA16Float, iw, ih, fi.outputTextureUsage | sc.colorTextureUsage | rd, @"framegen generated scene"); + FG.gen_up = fg_tex(MTLPixelFormatRGBA16Float, ow, oh, sc.outputTextureUsage | rd, @"framegen generated scene, upscaled"); + FG.cv0 = fg_tex(cvf, ow, oh, rd, @"framegen canvas before the HUD"); + res_commit(); + if (!FG.c[0] || !FG.c[1] || !FG.d[0] || !FG.d[1] || !FG.mv || !FG.gen || !FG.gen_up || !FG.cv0) { MLOG("[RED] frame generation: a texture could not be made; frames are shown as the game draws them"); return 0; } + FG.fi = fi; FG.bad = 0; + MLOG("frame generation: MetalFX frame interpolator %lux%lu RGBA16Float, generated scene upscaled to %lux%lu (canvas format %lu), made in %.1f ms; about %.0f MB of textures", + (unsigned long)iw, (unsigned long)ih, (unsigned long)ow, (unsigned long)oh, (unsigned long)cvf, (now_ns() - t0) / 1e6, (iw * ih * (8 * 3 + 4 * 3) + ow * oh * (8 + 4)) / 1048576.0); + return 1; +} +static void fg_barrier(void) { // as up_mfx_encode: MetalFX opens encoders of its own, which take no barrier + id ce = [g_cb computeCommandEncoder]; + [ce barrierAfterQueueStages:MTLStageAll beforeStages:MTLStageBlit visibilityOptions:resource_vis()]; + [ce barrierAfterStages:MTLStageBlit beforeQueueStages:MTLStageAll visibilityOptions:resource_vis()]; + [ce endEncoding]; +} +// After the game's upscale of this frame (up_mfx_encode): color = the scene, linear RGBA16F (the scaler's input), out = the canvas the upscaled scene was just drawn into +static void fg_frame(id sc, id color, id out, const NVNcopyRegion *orr, NSUInteger cw, NSUInteger ch) { + if (!fg_on()) return; + long f = (long)atomic_load(&g_cur_frame); NSUInteger ow = (NSUInteger)orr->width, oh = (NSUInteger)orr->height; + if (orr->xoffset || orr->yoffset || ow != out.width || oh != out.height || cw != color.width || ch != color.height) { + static int once; if (!once++) MLOG("[RED] frame generation: the upscale is not whole textures (%lux%lu of %lux%lu into %lux%lu of %lux%lu); frames are shown as the game draws them", + (unsigned long)cw, (unsigned long)ch, (unsigned long)color.width, (unsigned long)color.height, (unsigned long)ow, (unsigned long)oh, (unsigned long)out.width, (unsigned long)out.height); + return; } + // Not in the first 300 upscaled frames: its shaders and the interpolator are made at first use, and making them under the boot's compile load is what framegen-stage1-v4 + // blames a crash in the shader compiler's cache on (a base M5). A stopgap as that patch's: the pipelines belong in the prebuilt set if this goes further + { static int seen; if (seen < 300) { seen++; return; } } + if (!fg_pipes() || !fg_setup(cw, ch, ow, oh, out.pixelFormat, sc)) return; + if (g_fg_upto < 1) return; + id dep = FG.depth_last; + int have_d = dep && dep.width == cw && dep.height == ch, have_cam = FG.cam_frame == f, cur = FG.cur, prev = cur ^ 1; + if (!have_d) FG.nodepth++; if (!have_cam) FG.nocam++; + MTLOrigin o0 = MTLOriginMake(0, 0, 0); + id b = blit(); + enc_b_copy_texture(b, color, 0, 0, o0, MTLSizeMake(cw, ch, 1), FG.c[cur], 0, 0, o0); + if (have_d) enc_b_copy_texture(b, dep, 0, 0, o0, MTLSizeMake(cw, ch, 1), FG.d[cur], 0, 0, o0); + enc_b_copy_texture(b, out, 0, 0, o0, MTLSizeMake(ow, oh, 1), FG.cv0, 0, 0, o0); + FG.cv = out; FG.gen_ready = 0; + int ok = have_d && have_cam && FG.prev_ok && FG.last_frame == f - 1; double m[16]; + if (ok) { // reprojection = projection x previous view (floats 84-95 of this frame's block) x inverse(view-projection) + double cam[96], ic[16], pvp[16]; for (int i = 0; i < 96; i++) cam[i] = FG.cam[i]; + double pv[16] = {cam[84], cam[85], cam[86], cam[87], cam[88], cam[89], cam[90], cam[91], cam[92], cam[93], cam[94], cam[95], 0, 0, 0, 1}; + if (fg_inv4(&cam[12], ic)) { fg_mul4(&cam[28], pv, pvp); fg_mul4(pvp, ic, m); } else ok = 0; + } + if (ok && g_fg_upto >= 2) { + rp_end(); if (g_blit) { enc_b_end(g_blit); g_blit = nil; } + id pb; size_t po = arena_alloc(96, &pb); float *p = (float *)((uint8_t *)pb.contents + po); + for (int c = 0; c < 4; c++) for (int r = 0; r < 4; r++) p[c * 4 + r] = (float)m[r * 4 + c]; + p[16] = cw; p[17] = ch; p[18] = 1.0f / cw; p[19] = 1.0f / ch; + MTL4RenderPassDescriptor *rp = [MTL4RenderPassDescriptor new]; + rp.colorAttachments[0].texture = FG.mv; rp.colorAttachments[0].loadAction = MTLLoadActionDontCare; rp.colorAttachments[0].storeAction = MTLStoreActionStore; + ser_gate(); id en = enc_r_begin(rp); + [en setArgumentTable:FG.margs atStages:MTLRenderStageFragment]; + [FG.margs setTexture:FG.d[cur].gpuResourceID atIndex:0]; [FG.margs setTexture:FG.d[prev].gpuResourceID atIndex:1]; [FG.margs setAddress:pb.gpuAddress + po atIndex:0]; + [en setViewport:(MTLViewport){0, 0, cw, ch, 0, 1}]; [en setScissorRect:(MTLScissorRect){0, 0, cw, ch}]; + [en setRenderPipelineState:FG.mv_pso]; [en drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:3]; enc_r_end(en); + id fi = FG.fi; id c1 = FG.c[cur], c0 = FG.c[prev], d1 = FG.d[cur], mv = FG.mv, gen = FG.gen, gup = FG.gen_up; + int reset = FG.warm == 0; float nearp = FG.cam[56], farp = FG.cam[57], fov = FG.cam[70] * 57.29578f, aspect = FG.cam[62]; + if (g_fg_upto >= 3) { + fg_barrier(); + rec_cb_block(g_cb, ^(id cb) { + fi.colorTexture = c1; fi.prevColorTexture = c0; fi.depthTexture = d1; fi.motionTexture = mv; fi.outputTexture = gen; + fi.motionVectorScaleX = 1; fi.motionVectorScaleY = 1; fi.jitterOffsetX = 0; fi.jitterOffsetY = 0; fi.deltaTime = 1.0f / 30; + fi.nearPlane = nearp; fi.farPlane = farp; fi.fieldOfView = fov; fi.aspectRatio = aspect; fi.depthReversed = NO; fi.shouldResetHistory = reset; + [fi encodeToCommandBuffer:cb]; + }); + } + if (g_fg_upto >= 4) { + fg_barrier(); + rec_cb_block(g_cb, ^(id cb) { // the generated scene through the game's own upscale, a second time + sc.colorTexture = gen; sc.outputTexture = gup; sc.inputContentWidth = cw; sc.inputContentHeight = ch; + [sc encodeToCommandBuffer:cb]; + }); + } + g_last_end_relaxed = 0; args_dirty_all(); g_pso_dirty = g_dss_dirty = 1; + FG.made++; + if (++FG.warm >= 3 && g_fg_upto >= 5) FG.gen_ready = 1; // the first two outputs after a new history are the new frame unchanged + } else if (!ok) { if (FG.warm) FG.resets++; FG.warm = 0; } + FG.prev_ok = have_d; FG.last_frame = f; FG.cur = prev; +} +// At the present (present_enqueue, the game's thread): the generated frame's window picture into a ring texture of the host's, in a command buffer of its own committed +// behind the frame's work. Returns the texture's serial (0 = no generated frame for this present) and its ring slot +static uint64_t fg_compose(const tex_t *t, const int crop[4], int *slot) { + if (g_fg_mode <= 0 || !FG.gen_ready) return 0; + FG.gen_ready = 0; + if (g_dm_present_link) { static int once; if (!once++) MLOG("[RED] frame generation (scratch build): DM_PRESENT_LINK is not 0, generated frames are not shown (it paces on the present thread only)"); return 0; } + id src = tex_mtl(t); if (!src) return 0; + id raw = present_raw_src(src); NSUInteger r[4]; + if (!present_crop_rect(src, crop, r)) return 0; + if (r[2] != FG.ow || r[3] != FG.oh) { static int once; if (!once++) MLOG("[RED] frame generation: the present shows %lux%lu of the window texture, not the %lux%lu canvas 1:1; frames are shown as the game draws them", + (unsigned long)r[2], (unsigned long)r[3], (unsigned long)FG.ow, (unsigned long)FG.oh); return 0; } + int k = (int)(FG.win_next % FG_WIN); + pthread_mutex_lock(&g_pq_mu); int busy = FG.win_ticket[k] > g_pq_done; pthread_mutex_unlock(&g_pq_mu); + if (busy) { FG.busy++; return 0; } + NSError *e = nil; + if (!FG.comp_pso || FG.comp_fmt != raw.pixelFormat) { + MTLRenderPipelineDescriptor *pd = [MTLRenderPipelineDescriptor new]; pd.vertexFunction = FG.vf; pd.fragmentFunction = FG.f_comp; pd.colorAttachments[0].pixelFormat = raw.pixelFormat; + FG.comp_pso = [g_dev newRenderPipelineStateWithDescriptor:pd error:&e]; FG.comp_fmt = raw.pixelFormat; + if (!FG.comp_pso) { MLOG("[RED] frame generation: no pipeline for the window format %lu: %s", (unsigned long)raw.pixelFormat, e.localizedDescription.UTF8String); return 0; } + } + if (!FG.comp_pso) return 0; + id dst = FG.win[k]; + if (!dst || dst.width != src.width || dst.height != src.height || dst.pixelFormat != raw.pixelFormat) { + dst = fg_tex(raw.pixelFormat, src.width, src.height, MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead, @"framegen window picture"); + if (!dst) { MLOG("[RED] frame generation: no texture for the generated frame's window picture"); return 0; } + if (!FG.win_serial[k]) { static uint64_t n; FG.win_serial[k] = (1ull << 61) + (1ull << 40) + ++n; } + serial_put(FG.win_serial[k], dst); FG.win[k] = dst; + MLOG("frame generation: window picture ring slot %d %lux%lu format %lu", k, (unsigned long)dst.width, (unsigned long)dst.height, (unsigned long)dst.pixelFormat); + } + if (!FG.walloc[k]) { FG.walloc[k] = [g_dev newCommandAllocator]; FG.wcb[k] = [g_dev newCommandBuffer]; FG.wpar[k] = [g_dev newBufferWithLength:64 options:MTLResourceStorageModeShared]; + if (!FG.walloc[k] || !FG.wcb[k] || !FG.wpar[k]) { MLOG("[RED] frame generation: no command allocator or buffer for the window picture"); FG.walloc[k] = nil; return 0; } + res_add(FG.wpar[k]); } + res_commit(); + int32_t *p = (int32_t *)FG.wpar[k].contents; p[0] = (int32_t)r[0]; p[1] = (int32_t)r[1]; p[2] = (int32_t)FG.ow; p[3] = (int32_t)FG.oh; p[4] = g_fg_flip; p[5] = g_fg_view; + p[6] = raw.pixelFormat == MTLPixelFormatRGBA8Unorm || raw.pixelFormat == MTLPixelFormatBGRA8Unorm; p[7] = 0; + [FG.walloc[k] reset]; + id cb = FG.wcb[k]; + [cb beginCommandBufferWithAllocator:FG.walloc[k]]; + MTL4RenderPassDescriptor *rp = [MTL4RenderPassDescriptor new]; + rp.colorAttachments[0].texture = dst; rp.colorAttachments[0].loadAction = MTLLoadActionDontCare; rp.colorAttachments[0].storeAction = MTLStoreActionStore; + id en = [cb renderCommandEncoderWithDescriptor:rp]; + if (!en) { [cb endCommandBuffer]; MLOG("[RED] frame generation: no render encoder for the window picture"); return 0; } + [en barrierAfterQueueStages:MTLStageAll beforeStages:MTLStageAll visibilityOptions:resource_vis()]; + [en setArgumentTable:FG.wargs atStages:MTLRenderStageFragment]; + [FG.wargs setTexture:raw.gpuResourceID atIndex:0]; [FG.wargs setTexture:FG.gen_up.gpuResourceID atIndex:1]; [FG.wargs setTexture:FG.cv.gpuResourceID atIndex:2]; [FG.wargs setTexture:FG.cv0.gpuResourceID atIndex:3]; + [FG.wargs setAddress:FG.wpar[k].gpuAddress atIndex:0]; + [en setViewport:(MTLViewport){0, 0, dst.width, dst.height, 0, 1}]; [en setScissorRect:(MTLScissorRect){0, 0, dst.width, dst.height}]; + [en setRenderPipelineState:FG.comp_pso]; [en drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:3]; + [en endEncoding]; + [cb endCommandBuffer]; + pthread_mutex_lock(&g_res_mu); [g_q4 commit:&cb count:1]; pthread_mutex_unlock(&g_res_mu); + *slot = k; + return FG.win_serial[k]; +} +// The present thread: whether this present's generated frame is shown. Not while frames come late or the queue is behind, until three presents in a row came in time +static int fg_pace(double t_in, double period, int waiting) { + static double prev; static int steady; + double per = period > 0 ? period : 1.0 / 30; int late = prev > 0 && t_in - prev > per * 1.25; prev = t_in; + if (g_fg_upto < 9) return 0; + if (!g_fg_pause) return 1; + if (late || waiting > 2) { steady = 0; if (late) FG.late++; else FG.behind++; return 0; } + if (steady < 3) { steady++; FG.settling++; return 0; } + return 1; +} +// Each frame put on screen (the drawable's presented handler): the intervals between shown frames and, for real frames, the time from the game's enqueue to the screen +static void fg_shown(int gen, double tt, double tin) { + static pthread_mutex_t mu = PTHREAD_MUTEX_INITIALIZER; static double last, first, iv[600], lat[600]; static int n, nl, ngen; + if (!g_fg_seen || !(tt > 0)) return; + pthread_mutex_lock(&mu); + if (last > 0 && tt > last) { if (!n) first = last; iv[n++] = (tt - last) * 1e3; ngen += gen; } + if (tt > last) last = tt; + if (!gen && tin > 0 && nl < 600) lat[nl++] = (tt - tin) * 1e3; + if (n == 600) { + for (int i = 1; i < n; i++) { double x = iv[i]; int j = i - 1; while (j >= 0 && iv[j] > x) { iv[j + 1] = iv[j]; j--; } iv[j + 1] = x; } + for (int i = 1; i < nl; i++) { double x = lat[i]; int j = i - 1; while (j >= 0 && lat[j] > x) { lat[j + 1] = lat[j]; j--; } lat[j + 1] = x; } + int over = 0; for (int i = 0; i < n; i++) over += iv[i] > 25.0; + MLOG("frame generation %s: 600 shown frames at %.1f a second, %d generated; interval between shown frames p50 %.2f ms, p95 %.2f, max %.2f, over 25 ms %d; game enqueue to screen (real frames) p50 %.1f ms, p95 %.1f; " + "interpolations %llu, history restarts %llu, no camera %llu, no depth %llu, ring slot busy %llu; generated frames not shown: late %llu, queue behind %llu, settling %llu", + g_fg_mode > 0 ? "on" : "off", 600.0 / (last - first), ngen, iv[299], iv[569], iv[599], over, nl ? lat[nl / 2] : 0.0, nl ? lat[nl * 95 / 100] : 0.0, + (unsigned long long)FG.made, (unsigned long long)FG.resets, (unsigned long long)FG.nocam, (unsigned long long)FG.nodepth, (unsigned long long)FG.busy, + (unsigned long long)FG.late, (unsigned long long)FG.behind, (unsigned long long)FG.settling); + n = nl = ngen = 0; + } + pthread_mutex_unlock(&mu); +} +#elif FG_PART == 2 +// The generated frame onto the screen, ahead of its real frame (present_main; the present thread takes the drawable itself: DM_PRESENT_LINK=0). Command allocators of its +// own (two, in turn), as #2241 gives the picture after a generated frame one, so neither present waits for the other's GPU work +static id g_fg_palloc[2]; static id g_fg_pcb[2]; static id g_fg_puv[2]; static uint64_t g_fg_pfence[2], g_fg_pdone[2], g_fg_pseq; +static int present_gen(const tex_t *t, double hold, const int crop[4]) { + id src = tex_mtl(t); + if (!src || atomic_load(&g_gpu_lost)) return 0; + CAMetalLayer *layer = (__bridge CAMetalLayer *)front_metal_layer(); + if (!layer) return 0; + if (g_present_layer_resident != layer) { [g_q4 addResidencySet:layer.residencySet]; g_present_layer_resident = layer; } + present_pipeline(layer.pixelFormat); + uint64_t seq = ++g_fg_pseq; int i = (int)(seq & 1); + if (!g_fg_palloc[i]) { g_fg_palloc[i] = [g_dev newCommandAllocator]; g_fg_pcb[i] = [g_dev newCommandBuffer]; g_fg_puv[i] = [g_dev newBufferWithLength:sizeof(float) * 8 options:MTLResourceStorageModeShared]; + if (!g_fg_palloc[i] || !g_fg_pcb[i] || !g_fg_puv[i]) { g_fg_palloc[i] = nil; return 0; } + res_add(g_fg_puv[i]); } + res_commit(); + id dr = [layer nextDrawable]; + if (!dr) return 0; + if (g_fg_pfence[i]) { pthread_mutex_lock(&g_pq_mu); while (g_fg_pdone[i] < g_fg_pfence[i]) pthread_cond_wait(&g_pq_cv, &g_pq_mu); pthread_mutex_unlock(&g_pq_mu); } + [g_fg_palloc[i] reset]; + [g_q4 waitForDrawable:dr]; + id cb = g_fg_pcb[i]; + [cb beginCommandBufferWithAllocator:g_fg_palloc[i]]; + int kind = present_encode_m4(cb, src, dr.texture, crop, g_fg_puv[i], &g_fx_present); + [cb endCommandBuffer]; + if (!kind) { [g_q4 signalDrawable:dr]; return 0; } + MTL4CommitOptions *o = [MTL4CommitOptions new]; + [o addFeedbackHandler:^(id fb) { + if (fb.error) { MLOG("[RED] frame generation: display command buffer error: %s", fb.error.description.UTF8String); gpu_lost_check(fb.error, "display (generated frame)"); } + pthread_mutex_lock(&g_pq_mu); if (seq > g_fg_pdone[i]) g_fg_pdone[i] = seq; g_pq_inflight--; pthread_cond_broadcast(&g_pq_cv); pthread_mutex_unlock(&g_pq_mu); + }]; + pthread_mutex_lock(&g_pq_mu); g_pq_inflight++; pthread_mutex_unlock(&g_pq_mu); + [g_q4 commit:&cb count:1 options:o]; + [g_q4 signalDrawable:dr]; + g_fg_pfence[i] = seq; + [dr addPresentedHandler:^(id d) { fg_shown(1, d.presentedTime, 0); }]; + if (hold > 0) [dr presentAfterMinimumDuration:hold]; else [dr present]; + return 1; +} +// DM_FRAMEGEN_SHOT: the generated and the real picture of a frame as PNGs (waits for the GPU: a diagnostic) +static void fg_shots(uint64_t frame, const tex_t *gen, const tex_t *real, const int crop[4]) { + if (!g_fg_shot_dir || (long)frame < g_fg_shot_a || (long)frame > g_fg_shot_b) return; + char path[1024]; mtl_finish_gpu(); + id g = gen ? tex_mtl(gen) : nil, r = tex_mtl(real); + if (g) { snprintf(path, sizeof path, "%s/F%llu_generated.png", g_fg_shot_dir, (unsigned long long)frame); shot(g, path, 1, crop); } + if (r) { snprintf(path, sizeof path, "%s/F%llu_real.png", g_fg_shot_dir, (unsigned long long)frame); shot(r, path, 1, crop); } +} +#endif diff --git a/dm/gfx/nvn/nvn_mtl.m b/dm/gfx/nvn/nvn_mtl.m index ee0f03a316..52c87fc7d0 100644 --- a/dm/gfx/nvn/nvn_mtl.m +++ b/dm/gfx/nvn/nvn_mtl.m @@ -272,6 +272,9 @@ static void ws_report(uint64_t frame, uint64_t frames) { MLOG("[waits] up to frame %llu, per frame:%s", (unsigned long long)frame, o ? line : " none"); } #include "nvn_rec.h" // deferred encoding (DM_NVN_ENC_THREAD) +#define FG_PART 0 +#include "nvn_framegen.inc" // scratch: frame generation (DM_FRAMEGEN) +#undef FG_PART static NSHashTable *g_res_objects; static _Atomic unsigned g_image_heap_n, g_big_n; // g_big_n: large pools in chunks, whose buffers overlap (BIG_MIN) static inline MTL4VisibilityOptions resource_vis(void) { return MTL4VisibilityOptionDevice | (g_image_heap_n || g_big_n ? MTL4VisibilityOptionResourceAlias : 0); } @@ -10662,6 +10665,7 @@ static uint64_t cbuf_slot(NvShader *sh, int nvn_stage, const nvn_rbind *b, __str uint64_t g; const uint8_t *c; if (ubd_covers(u->addr, u->addr + ul)) gpu = ubd_shadow(u->addr, u->addr + ul); else if (ubp_get(u->addr, u->addr + ul, &g, &c)) gpu = g; else RED_ONCE(92, "a uniform update deferred inside the pass covers only part of the binding (stage %d cb%d): the content before the update is read", nvn_stage, reg); } + if (g_fg_mode && nvn_stage == 0 && reg == 0 && u->size == 13568) fg_cam_note((const float *)u->addr); // scratch (DM_FRAMEGEN): Ocarina of Time's camera block return gpu; } // The uniform slots' bound sizes (DMSH_CBSIZE_BANK), which the translator's reads at computed offsets compare with: one past its slot's size, rounded up to 256 bytes (ub_bound), gives 0, as on Maxwell. 0 for an unbound slot, @@ -15248,6 +15252,7 @@ static void shot(id src, const char *path, int screen, const int cro // DM_NVN_TEXDUMP=size=WxH:path instead picks the frame by content: the first frame that opens a render pass whose first color attachment level is WxH (for one-off work whose frame differs between runs) static unsigned g_td_w, g_td_h; void td_note(MTL4RenderPassDescriptor *pd, NSUInteger nc) { + fg_depth_note(pd); // scratch (DM_FRAMEGEN) if (g_td_frame == -2) { const char *e = getenv("DM_NVN_TEXDUMP"), *c = e ? strchr(e, ':') : NULL; g_td_frame = c ? atol(e) : -1; g_td_path = c ? c + 1 : NULL; g_td_list = [NSMutableArray array]; if (c && !strncmp(e, "size=", 5) && sscanf(e + 5, "%ux%u", &g_td_w, &g_td_h) == 2) g_td_frame = LONG_MAX; if (c && !strncmp(e, "layers=", 7)) { g_td_layers = atoi(e + 7); g_td_frame = (long)atomic_load(&g_cur_frame); } } @@ -15704,6 +15709,9 @@ static int up_mfx_rect(const uint32_t v[4], NSUInteger w, NSUInteger h, NVNcopyR *r = (NVNcopyRegion){(int)v[0], (int)v[1], 0, (int)rw, (int)rh, 1}; return 1; } // The dispatch of upscaler cs as MetalFX's spatial scaler from its texture slot's rectangle to its image slot's (see above): 1 when drawn, 0 to run the shader +#define FG_PART 1 +#include "nvn_framegen.inc" +#undef FG_PART static int up_mfx_encode(NvShader *cs) { int ti = -1, oi = -1; for (int i = 0; i < cs->rf.nb; i++) { int k = cs->rf.b[i].kind; if (k == NVN_RB_TEX && ti < 0) ti = i; else if (k == NVN_RB_IMG && oi < 0) oi = i; else if (k == NVN_RB_TEX || k == NVN_RB_IMG) return 0; } @@ -15753,6 +15761,7 @@ static int up_mfx_encode(NvShader *cs) { args_dirty_all(); g_up_mfx_kept++; return 0; // only the scratch image was scaled; the game's shader can still produce the output } args_dirty_all(); + if (!direct) fg_frame(sc, color, out, &orr, cw, ch); // scratch (DM_FRAMEGEN) if (!g_up_mfx_n++) MLOG("upscaler: first dispatch of %s drawn by MetalFX, %dx%d at %d,%d of %lux%lu (format %lu) → %dx%d at %d,%d of %lux%lu (format %lu)%s", cs->key, ir.width, ir.height, ir.xoffset, ir.yoffset, (unsigned long)in.width, (unsigned long)in.height, (unsigned long)in.pixelFormat, orr.width, orr.height, orr.xoffset, orr.yoffset, (unsigned long)out.width, (unsigned long)out.height, (unsigned long)out.pixelFormat, direct ? ", straight" : srgb ? ", through RGBA16F, decoded from sRGB" : ", through RGBA16F"); @@ -15943,10 +15952,12 @@ static int present_now(const tex_t *t, double period, uint64_t frame, uint64_t t #if TARGET_OS_SIMULATOR // The iOS simulator SDK has no presentation-time callback and no minimum display duration (used only for picture acceptance, display-link ticks are not counted) (void)period; [dr present]; #else + double fg_adj = g_fg_pres_adj; // scratch (DM_FRAMEGEN): a real frame held behind its generated frame reads as shown when it would have been without (after framegen-stage1-v4's pres_adj) uint64_t pn = ticket; // Record the present time by the game's present sequence number (ticket number): hle's present statistics are taken by "the game's nth present"; in the display link path the actual present count != the game's present count (old frames get merged and dropped) [dr addPresentedHandler:^(id d) { // Actual presentation time (the time this frame is really put on screen, CACurrentMediaTime base) CFTimeInterval tt = d.presentedTime; - g_pres[pn % PRING] = (pres_ent){pn, tt > 0 ? tt : -1}; + g_pres[pn % PRING] = (pres_ent){pn, tt > 0 ? tt + fg_adj : -1}; + fg_shown(0, tt, g_fg_tin[pn % 64]); // scratch (DM_FRAMEGEN) }]; // Each frame is displayed for at least one present period: when the GPU completion time jitters around a 120 Hz refresh boundary, the paired "25 ms + 8.33 ms" stutter no longer appears if (target_ts > 0) [dr present]; @@ -15955,6 +15966,9 @@ static int present_now(const tex_t *t, double period, uint64_t frame, uint64_t t (void)frame; return 1; } +#define FG_PART 2 +#include "nvn_framegen.inc" +#undef FG_PART // ---------- Present thread ---------- // NVN semantics: QueuePresentTexture only queues the present and returns immediately (the game often calls it while holding a lock; iOS real device: synchronously waiting for nextDrawable inside it // takes about 30 ms, the lock is held about 90% of the time, and the loading thread starves). Taking the drawable, blit and present are all done on this thread in enqueue order. @@ -15962,7 +15976,7 @@ static int present_now(const tex_t *t, double period, uint64_t frame, uint64_t t // Swapchain texture return: before AcquireTexture gets a texture it waits for the ticket of that texture's previous present to be processed (mtl_present_wait), after which the commands the game draws into it // are necessarily ordered after that blit on the GPU queue. #define PQ 8 -typedef struct { tex_t t; double period; uint64_t frame; int crop[4]; double t_in; uint64_t ev_val, snap_v; } pq_item; // snap_v: the snapshot copy this present shows (snap_commit_wait) // t_in: the time the game handed this frame into the present queue (CACurrentMediaTime); ev_val: see g_ext_ev +typedef struct { tex_t t; double period; uint64_t frame; int crop[4]; double t_in; uint64_t ev_val, snap_v, gen; } pq_item; // gen: the generated frame shown before this one (fg_compose), 0 = none // snap_v: the snapshot copy this present shows (snap_commit_wait) // t_in: the time the game handed this frame into the present queue (CACurrentMediaTime); ev_val: see g_ext_ev // External producer (the Vulkan layer, dm/gfx/vk) renders on its own queue and signals this event when a frame's rendering is done: the present of that frame waits for the value on g_q4 static id g_ext_ev; static void ext_wait(uint64_t v) { if (v && g_ext_ev) [g_q4 waitForEvent:g_ext_ev value:v]; } @@ -16171,11 +16185,20 @@ static void present_drop_for_suspend(void) { pthread_mutex_lock(&g_pq_mu); while (g_pq_read == g_pq_in || g_link_owns_layer) pthread_cond_wait(&g_pq_cv, &g_pq_mu); // In display-link mode the display-link callback takes them; an empty queue is judged by "taken"; "done" has to wait for the feedback of the present submit and lags behind taken pq_item it = g_pq[g_pq_read % PQ]; + int fg_waiting = (int)(g_pq_in - g_pq_read); uint64_t ticket = ++g_pq_read; g_classic_busy = 1; pthread_mutex_unlock(&g_pq_mu); int submitted; ext_wait(it.ev_val); snap_commit_wait(it.snap_v); - @autoreleasepool { submitted = present_now(&it.t, it.period, it.frame, ticket, it.crop, nil, 0); mtl_tex_unpin(it.t.serial); } + // scratch (DM_FRAMEGEN): the generated frame first, then the real one, each for half the period + int fg_show = 0; tex_t fg_t; memset(&fg_t, 0, sizeof fg_t); fg_t.magic = MAGIC('T', 'E', 'X', 0); fg_t.serial = it.gen; + fg_on(); g_fg_tin[ticket % 64] = it.t_in; + if (it.gen) fg_shots(it.frame, &fg_t, &it.t, it.crop); + if (it.gen && fg_pace(it.t_in, it.period, fg_waiting)) { @autoreleasepool { fg_show = present_gen(&fg_t, it.period * 0.5, it.crop); } } + else if (!it.gen) fg_pace(it.t_in, it.period, fg_waiting); + g_fg_pres_adj = fg_show ? -(it.period > 0 ? it.period * 0.5 : 1.0 / 60) : 0; + @autoreleasepool { submitted = present_now(&it.t, fg_show ? it.period * 0.5 : it.period, it.frame, ticket, it.crop, nil, 0); mtl_tex_unpin(it.t.serial); } + g_fg_pres_adj = 0; pthread_mutex_lock(&g_pq_mu); g_classic_busy = 0; pthread_cond_broadcast(&g_pq_cv); pthread_mutex_unlock(&g_pq_mu); if (!submitted) { pthread_mutex_lock(&g_pq_mu); @@ -16337,6 +16360,8 @@ static uint64_t present_enqueue(const tex_t *t, double period, const int crop[4] else if (st == 2 && now - t0 >= at + dur) { st = 3; mtl_link_pause(0); MLOG("DM_TEST_LINK_STALL: resuming the display link (frame %llu)", (unsigned long long)atomic_load(&g_cur_frame)); } } if (ws_on()) { static uint64_t f0; uint64_t f = atomic_load(&g_cur_frame); if (!f0) f0 = f; else if (f >= f0 + 300) { ws_report(f, f - f0); f0 = f; } } + uint64_t fg_gen = 0; int fg_slot = 0; + if (!ev_val) fg_gen = fg_compose(t, crop, &fg_slot); // scratch (DM_FRAMEGEN): from the game's own texture, before a snapshot replaces it tex_t snap; uint64_t ss = 0; int slot = 0; if (!ev_val && mtl_snap_on()) { slot = (int)(atomic_load(&g_snap_next) % (uint64_t)g_snap_n); @@ -16348,6 +16373,7 @@ static uint64_t present_enqueue(const tex_t *t, double period, const int crop[4] pq_item *it = &g_pq[g_pq_in % PQ]; *it = (pq_item){*t, period, atomic_load(&g_cur_frame), {0}, CACurrentMediaTime(), ev_val, ss ? g_snap[slot].ev : 0}; if (crop) memcpy(it->crop, crop, sizeof it->crop); // The current frame's state is enqueued with the ticket number, rather than reading the mutable window late on the present thread uint64_t tk = ++g_pq_in; + if (fg_gen) { it->gen = fg_gen; FG.win_ticket[fg_slot] = tk; FG.win_next++; } if (ss) { g_snap[slot].ticket = tk; atomic_fetch_add(&g_snap_next, 1); if (copy) *copy = g_snap[slot].ev; } pthread_cond_broadcast(&g_pq_cv); pthread_mutex_unlock(&g_pq_mu); From 3604e200a08e657e74af002864d9fe80648603f1 Mon Sep 17 00:00:00 2001 From: seedds Date: Wed, 11 Nov 2026 20:17:23 +0100 Subject: [PATCH 2/9] DM_UPSCALER=metalfx takes an upscaler whose colour is a square root: the profile key upscaler_sqrt names it, its input is squared into MetalFX's (0400486024DEC000: 1.12 ms of GPU a frame to about 0.1) The upscale step of agl_ndsr_vr_light.pck samples colour the dispatch before it stored as square roots and writes v * |v|, so its code has no sRGB curve for the classifier and the option replaced nothing. DM_NVN_SKIP_CS (diagnostic) drops named compute dispatches, to measure one inside an encoder. --- dm/games/0400486024DEC000/README.md | 2 ++ dm/games/0400486024DEC000/game.tsv | 4 ++++ dm/games/README.md | 1 + dm/games/game.py | 12 ++++++++-- dm/gfx/msl/copy.metal | 3 +++ dm/gfx/nvn/nvn_mtl.m | 37 ++++++++++++++++++++--------- dm/gfx/tests/t_upscaler_mfx.m | 8 +++++++ dm/tools/tests/test_game_profile.py | 11 +++++---- docs/agents/symptoms.md | 1 + docs/env_switches.tsv | 1 + docs/reference/rendering.md | 9 +++++++ 11 files changed, 72 insertions(+), 17 deletions(-) diff --git a/dm/games/0400486024DEC000/README.md b/dm/games/0400486024DEC000/README.md index 1b54d1a048..f2dbd6f5da 100644 --- a/dm/games/0400486024DEC000/README.md +++ b/dm/games/0400486024DEC000/README.md @@ -31,6 +31,7 @@ One [NUP container](../../../docs/development.md#the-packing-script-in-detail) o Measured on an M6 Mac with 16 GB, headless, on one walk with camera turns (`walk.sh` in the work folder). - Current on an M5 Pro MacBook Pro (16-inch, 20-core GPU, 24 GB, macOS 27.2, 2026-11-09), `kokiri-deck` from a deck autosave with the full library of chain `dedc9391c30b`, at full resolution (Dynamic resolution off), two runs a variant: in a window a locked 60 fps (59.8–60.0, frame p99 16.8–19.4 ms, GPU p50 10.2 ms). In fullscreen on its ProMotion screen it held 40 fps (39.6–40.1) at the same GPU time, with every present a 120 Hz refresh late and every other link tick skipped: [The 60 Hz link at 40 fps in fullscreen on a ProMotion screen](../../../docs/reference/frame-rate.md#the-60-hz-link-at-40-fps-in-fullscreen-on-a-promotion-screen). With the 120 Hz link there, 58.8–59.6 fps, frame p99 25.3 ms; with the screen held at its fixed 60 Hz mode, 60.0 fps, frame p99 16.9 ms, which fullscreen now switches to. MetalFX present upscaling to the 3456-wide fullscreen cost no measurable GPU time (59.4–59.7 fps). Resident about 10 GB, GPU allocated 11.2–11.4 GB; the prepare step took 7.3 s on a first start. +- Current: `DM_UPSCALER=metalfx` draws the upscale step with MetalFX (M4 Pro, 20-core GPU, 2026-11-11, Kakariko Village from a save, chain `b8856126d238`, `DM_FPS_HOLD=off`). In play the game anti-aliases and upscales with four dispatches of `agl_ndsr_vr.pck` and `agl_ndsr_vr_light.pck` (`f8965bffbdf51d78`, `d17eefaff4f0bc23`, `a335c1db6270c355`, `fa3033d05c0a41d6`), one encoder of 1.25 ms; only the last is the upscale (texture slot 0 at 1536x864 to image slot 1 at 2048x1152). With its dispatches dropped (`DM_NVN_SKIP_CS`, the picture stale) the frame's compute fell 3.91-3.93 → 2.78-2.80 ms: 1.12 ms. Its colour, from its translated code: the dispatch before it stores each channel's square root, packed as RGB9E5 bits in a single-channel image; the step samples that, adds its network's residual and writes `v * |v|`, linear, with no sRGB curve, so the host's classifier kept the shader. The profile names it ([`upscaler_sqrt`](game.tsv)): [An upscaler with square-root colour](../../../docs/reference/rendering.md#an-upscaler-with-square-root-colour). Two runs a side in turn: headless, compute 3.91 and 3.95 → 2.80 and 2.77 ms, the frame's GPU work 14.84 and 15.61 → 13.64 and 13.48 ms (median sum of encoder times over 37-38 profiled frames); in a window, compute 3.90 and 3.89 → 2.76 and 2.79 ms, summed GPU p50 15.59 and 15.13 → 14.63 and 14.46 ms, 60.0 fps on both. The end shots' mean colours agree (15.9 / 25.3 / 29.4 against 15.8 / 24.8 / 28.4 of 255; two shots of the game's own differ as much, by the moving grass); MetalFX's picture is sharper, the network's softer, which the player preferred. The three anti-aliasing dispatches still run. Open: not judged in motion, nor with a render scale, the dynamic resolution or a cutscene. - Current: the title and menu, a movie behind them, are a locked 60 fps (p50 16.67 ms, GPU span 15.6 ms). - Game speed: the game steps its time by a frame ratio, its base rate (60, or 30) / its int frame rate, whatever the frame took (`main+0x6e6cc4` on v131072), so frames longer than a vsync played in slow motion (seen by the user on a slower Mac; [issue 1375](https://github.com/seedds/nxnative/issues/1375)'s M1 Max ran the title at about 28 fps). With `fps_target 60` the ratio is the host's frame step × base / 60 ([A fixed frame step plays in slow motion on a slow Mac](../../../docs/reference/frame-rate.md#a-fixed-frame-step-plays-in-slow-motion-on-a-slow-mac)), a stub placed by its function's fingerprint ([`patches.py`](patches.py)). Mac runs on the M6 (2026-11-08, a test host on the libraries of chain `3ee493b2c3a4`): the user's play sessions held 60 fps with 2- and 3-vsync holds while areas loaded, game time 1.000 of real time by the host's count. That count is the stub's own; no run on a Mac slow enough to show the slow motion before and after. - Sizes since the host answers graphics scale tier 0 docked ([The graphics scale does not follow the operation mode](../../../docs/reference/services.md#the-graphics-scale-does-not-follow-the-operation-mode)): the game's output is 2560x1440 and its top internal resolution 1920x1080. Every size and time in this note was taken with tier 1, at 0.8 of that (2048x1152 and 1536x864, and the dynamic resolution's levels with them), and has not been taken again. With tier 0 on `main` (two contributors' runs, only that answer changed): M4 Pro (20-core GPU), docked, fullscreen, GPU p50 15.0 ms against 12.9-14.0 ms, 59.0 fps against 55.8-59.8; M5 Pro (20-core GPU, 48 GB), Dynamic resolution off, 24 minutes of play, GPU p50 12.7 ms, 58.4 fps. Open: a slower Mac, and the routes of [`bench.tsv`](bench.tsv), with tier 0. @@ -126,6 +127,7 @@ Its 37 kernels that correct samples, NDSR's network among them, gave about 0.25 ## Found on this game +- [An upscaler with square-root colour](../../../docs/reference/rendering.md#an-upscaler-with-square-root-colour): `DM_UPSCALER=metalfx` replaced no dispatch; with the profile's `upscaler_sqrt` the upscale step's 1.12 ms of GPU goes to MetalFX (Kakariko Village, M4 Pro). - [A shader build's Python finishes one shader at a time](../../../docs/reference/shaders.md#a-shader-builds-python-finishes-one-shader-at-a-time): resolved, the whole Mac map's cold build took 58 minutes on an M3 Max with 4 to 5 of its 16 cores busy; 30 minutes with worker processes. - [Reads past a uniform binding's size](../../../docs/reference/shaders.md#reads-past-a-uniform-bindings-size): open ([GitHub issue #2218](https://github.com/seedds/nxnative/issues/2218)), long white streaks from the horses at Lon Lon Ranch and from Link's slingshot, and flickering parts of the slingshot and of the ReDeads' faces, reported with the uniform read bound on. The bound now takes the size the driver gives the GPU, rounded up to 256 bytes; the game binds sizes that are not multiples of 256. Not reproduced here before or after (the slingshot at Link's house, the ReDeads of adult Castle Town), so the reporters' retest decides. - [The blend helper's output reference on iOS](../../../docs/reference/shaders.md#the-blend-helpers-output-reference-on-ios): resolved, 22,296 of 42,848 Mac shaders failed to compile on macOS 26 ([GitHub issue #2272](https://github.com/seedds/nxnative/issues/2272)); the Mac compile now names the host's minimum macOS. diff --git a/dm/games/0400486024DEC000/game.tsv b/dm/games/0400486024DEC000/game.tsv index 9c4b09b052..3db6a57a63 100644 --- a/dm/games/0400486024DEC000/game.tsv +++ b/dm/games/0400486024DEC000/game.tsv @@ -48,3 +48,7 @@ draw_barriers game # As Tears of the Kingdom: the terrain's height, blend and hole maps are written by the CPU into textures after they are created. Without these keys they stay zero: holes in the ground and the wrong materials tex_flush_upload on tex_dirty_upload on +# The upscale step of agl_ndsr_vr_light.pck, the last of the four dispatches the game anti-aliases and upscales with in play: it samples colour the dispatch before it stored as square +# roots (RGB9E5) and writes v * |v|, linear, so its code has none of the sRGB constants the host reads a colour shape from. Named here, DM_UPSCALER=metalfx draws it with MetalFX +# (its share of the frame in Kakariko Village: 1.12 ms of GPU on an Apple M4 Pro, 2026-11-11) +upscaler_sqrt fa3033d05c0a41d6 diff --git a/dm/games/README.md b/dm/games/README.md index faad835f38..b4865ce16d 100644 --- a/dm/games/README.md +++ b/dm/games/README.md @@ -252,6 +252,7 @@ One row per profile; the notes are each profile's `README.md`. | `tex_dirty_upload` | `on` / `off` (the default), needs `tex_flush_upload`: a registered texture whose pages the CPU writes after `nvnTextureInitialize` without any NVN flush is uploaded as well (its pages are write-protected at registration, checked on first use; `nvn_ft_written` in `hle_nvn.c`). Tears of the Kingdom fills its ASTC terrain material arrays this way; without it the cave is magenta. FINAL FANTASY TACTICS - The Ivalice Chronicles loads its UI textures straight into CPU-uncached pools, which need no flush, after creating them; without both keys its screen stays black. The Great Ace Attorney Chronicles copies every texture into a CPU-cached pool after creating it and never flushes (its `main` has no cache maintenance at all): black too. The Legend of Zelda: Ocarina of Time writes its terrain tiles' height, blend and hole maps this way, and without the keys has holes in the ground | | `tex_dyn_slots` | `on` / `off` (the default): a shader whose buffer-texture handle the translator could not trace to one slot gets Ryujinx's fallback arrays (`__cb0_40000000`: 32 buffer textures and samplers indexed by the handle's texture / sampler ID; any other sampled texture indexes the whole pool without a key, [Texture handles from game memory index the pools](../../docs/reference/shaders.md#texture-handles-from-game-memory-index-the-pools)). With the key the replay puts slot indices in that shader's c2 texture and sampler tables (texture slot r → r, sampler slot r → r << 20) and binds slot k at element k, so the shader's own arithmetic picks the binding (`NVN_RB_TEXD` / `NVN_RB_SAMPD` in `nvn_mtl.m`). Right only where the handle comes from c2's tables: Octopath Traveler 0's 61 such vertex shaders read separate texture slots 0-10 and samplers 0-4 there and nothing else (FINAL FANTASY RESONANCE's 2,077 the texture table words 0x168-0x1c0 and sampler table words 0x568-0x590). Off, such shaders do not load. Not part of the shader chain version | | `draw_barriers` | `full` (the default) / `game`: how draw passes are ordered. `full`: each waits for all earlier GPU work. `game`: by the game's own barriers (NVN asks for one before results of rendering are read), so a pass's vertex work overlaps the previous pass's fragments. Ocarina of Time's GPU time 25 → 18 ms, about 180 full barriers a frame halved. Only vertex-stage reads of the previous pass's output without a barrier change order. `DM_NVN_BARRIER` overrides it | +| `upscaler_sqrt` | comma-separated 16-digit keys of upscaler compute shaders, outside `agl_ndsr.pck`, that sample square-root-encoded colour and write it squared (linear): `DM_UPSCALER=metalfx` squares their input into MetalFX's and runs it in HDR mode ([An upscaler with square-root colour](../../docs/reference/rendering.md#an-upscaler-with-square-root-colour)). Read the shape from the shader's translated code before naming one | | `vconv_range` | `on` / `off` (the default): vertex attributes the replay converts on the CPU (integer shader inputs fed float-type formats and the like, `vconv_bind` in `nvn_mtl.m`) are converted only over the vertices each draw reads (first / count, or the smallest and largest index of a direct indexed draw, kept while the index pages stay unwritten) instead of the whole bound range. Age of Calamity binds whole mesh pools: in a battle that was millions of vertices a frame. Indirect draws, per-instance streams and indices with a GPU write queued on them keep the whole range. `DM_NVN_VCONV_CHECK=1` scans the indices again at every reuse and reports differences; `DM_NVN_VCONV_RANGE=0` / `1` overrides it | | `shader_key_skip` | `+` (none by default): control-section bytes every shader key hashes as zeros, in `shader_split.py` (`key_ctl`) and the host (`hle_nvn.c` `keys_hash`), for an engine that shares one control section among shaders that differ only there. Batman: Arkham Knight: `0x770+0x78`, GLSLC's compile information (+0x778..+0x791 and +0x7d0..+0x7e7); without it the game asks for shaders the map does not have, each the control section of one record with the code area of another. It must stay clear of `+0x700..+0x75f`, which the tools read; it is a shader chain input, as every `shader_*` key | | `shader_groups`, `shader_note` | `shader_split.py` source groups and the map header note (Engage: `engine` for `data.unity3d` and `Resources/`, `assets` for the Addressables bundles). Both are shader inputs, the note too: changing either one, even by a word, builds every shader of the game again. Keep them as they are when a provisional profile moves into `dm/games` | diff --git a/dm/games/game.py b/dm/games/game.py index 9426822407..10517c7d5a 100644 --- a/dm/games/game.py +++ b/dm/games/game.py @@ -108,11 +108,19 @@ def by_build(main_id): # the profile whose build rows register this main build def upscaler_keys(g): """the game's own upscaler shaders, from its shader map (shader_split.py's source column): sorted 16-digit keys, none without a map""" p = g.table('shader_map.tsv') - if not p.exists(): return [] + if not p.exists(): return upscaler_sqrt_keys(g) keys = set() for line in p.read_text(encoding='utf-8').splitlines(): c = line.split('\t') if len(c) >= 5 and c[2] == 'compute' and UPSCALER_SOURCE.search(c[4]) and re.fullmatch(r'[0-9a-f]{16}', c[3]): keys.add(c[3]) + return sorted(keys | set(upscaler_sqrt_keys(g))) + + +def upscaler_sqrt_keys(g): + """upscaler shaders outside agl_ndsr.pck whose colour is no sRGB curve but a square root: the profile's upscaler_sqrt (comma-separated 16-digit keys), read from each shader's translated code""" + keys = [k for k in g.get('upscaler_sqrt', '').split(',') if k] + for k in keys: + if not re.fullmatch(r'[0-9a-f]{16}', k): sys.exit(f'{g.id}: upscaler_sqrt must be comma-separated 16-digit shader keys: {k}') return sorted(keys) @@ -260,7 +268,7 @@ def header_one(g): f'#define DM_GAME_TEX_DIRTY_UPLOAD {int(tdu == "on")}', f'#define DM_GAME_TEX_DYN_SLOTS {int(tds == "on")}', f'#define DM_GAME_DRAW_BARRIERS_GAME {int(dbr == "game")}', - f'#define DM_GAME_UPSCALER_KEYS {q(",".join(ups))}', f'#define DM_GAME_UPSCALER_N {len(ups)}', + f'#define DM_GAME_UPSCALER_KEYS {q(",".join(ups))}', f'#define DM_GAME_UPSCALER_N {len(ups)}', f'#define DM_GAME_UPSCALER_SQRT_KEYS {q(",".join(upscaler_sqrt_keys(g)))}', f'#define DM_GAME_VCONV_RANGE {int(vcr == "on")}', *([f'#define DM_GAME_NVN_API_MAJOR {api.split(".")[0]}', f'#define DM_GAME_NVN_API_MINOR {api.split(".")[1]}'] if g.get('device_api', '') else []), # without the key: by the SDK version (hle_nvn.c nvn_api) ''] diff --git a/dm/gfx/msl/copy.metal b/dm/gfx/msl/copy.metal index 466939c8e9..5177d37261 100644 --- a/dm/gfx/msl/copy.metal +++ b/dm/gfx/msl/copy.metal @@ -6,6 +6,9 @@ vertex V vs(uint id [[vertex_id]]) { float2 uv = float2((id << 1) & 2, id & 2); // p[0] = source origin (texels) + source texels per destination texel, p[1] = destination origin + source level size fragment float4 fs(V in [[stage_in]], texture2d t [[texture(0)]], sampler s [[sampler(0)]], constant float4 *p [[buffer(0)]]) { float2 st = p[0].xy + (in.p.xy - p[1].xy) * p[0].zw; return t.sample(s, st / p[1].zw, level(0)); } +// The same with the colour squared, sign kept: a square-root-encoded source into a linear destination (up_mfx_encode's input of an upscaler that squares its own result) +fragment float4 fssq(V in [[stage_in]], texture2d t [[texture(0)]], sampler s [[sampler(0)]], constant float4 *p [[buffer(0)]]) { + float2 st = p[0].xy + (in.p.xy - p[1].xy) * p[0].zw; float4 c = t.sample(s, st / p[1].zw, level(0)); return float4(c.rgb * abs(c.rgb), c.a); } struct D { float d [[depth(any)]]; }; fragment D fsd(V in [[stage_in]], depth2d t [[texture(0)]], sampler s [[sampler(0)]], constant float4 *p [[buffer(0)]]) { float2 st = p[0].xy + (in.p.xy - p[1].xy) * p[0].zw; D o; o.d = t.read(uint2(st)); return o; } diff --git a/dm/gfx/nvn/nvn_mtl.m b/dm/gfx/nvn/nvn_mtl.m index 1db5ed4f05..dd1dfc236a 100644 --- a/dm/gfx/nvn/nvn_mtl.m +++ b/dm/gfx/nvn/nvn_mtl.m @@ -14530,6 +14530,12 @@ static void dispatch(const p_dispatch *dp) { rp_flush(); // A full clear with no draw after it waits for the next pass on its targets, but it comes before this dispatch, which may read or write them as images: land it first, as copy_t2t does image_prepare(prog_get(S.cprog), 0x20); if (ndve_engine_dispatch(cs, dp)) return; // nn::ndve's movie network as the host's kernels (DM_NDVE_ENGINE=1) + { // DM_NVN_SKIP_CS=[,…] (diagnostic, up to 16): drops the dispatches of those compute shaders, to measure what one costs inside an encoder that holds several; what it writes stays stale + static char sk[16][20]; static int nsk = -1; + if (nsk < 0) { nsk = 0; for (const char *q = getenv("DM_NVN_SKIP_CS"); q && *q && nsk < 16; q = strchr(q, ',') ? strchr(q, ',') + 1 : q + strlen(q)) { size_t n = strcspn(q, ","); if (n > 16) n = 16; memcpy(sk[nsk], q, n); sk[nsk++][n] = 0; } + if (nsk) MLOG("DM_NVN_SKIP_CS: dropping the dispatches of %d compute shaders (diagnostic; the picture is wrong)", nsk); } + for (int i = 0; i < nsk; i++) if (!strncmp(cs->key, sk[i], strlen(sk[i]))) return; + } if (up_mfx_wanted(cs) && !dp->ind && up_mfx_encode(cs)) return; // the game's upscaler, drawn by MetalFX instead (DM_UPSCALER=metalfx) // Threadgroup memory past Metal's 32 KB, in a device buffer (dmsh_rewrite.py tg_spill): a slice for every workgroup of the grid. An indirect dispatch's grid is in GPU memory, written earlier // in the frame: the replay waits for the GPU (as an indirect-count multi-draw does, mdi_wait) and reads it. Mario Kart World's spilled dispatches are all direct, of 1 to 3 workgroups @@ -14764,31 +14770,33 @@ static int fmt_int_or_ds(MTLPixelFormat f) { default: return 0; } } +enum { COPY_DRAW_SQUARE = 1 << 30 }; // in mirror, beside NVN's MIRROR bits: the colour squared on the way (copy.metal fssq) static int copy_t2t_draw(id s, id d, unsigned sl, unsigned ss, unsigned dl, unsigned ds, const NVNcopyRegion *sr, const NVNcopyRegion *dr, int linear, int mirror) { - static NSMutableDictionary> *pso; static id vf, ff, fd, fds; static id samp[2]; static id at; static id dss, dss_s; static int failed; + static NSMutableDictionary> *pso; static id vf, ff, fd, fds, fsq; static id samp[2]; static id at; static id dss, dss_s; static int failed; if (failed || sr->width <= 0 || sr->height <= 0 || dr->width <= 0 || dr->height <= 0) return 0; int depth = (s.pixelFormat == MTLPixelFormatDepth32Float || s.pixelFormat == MTLPixelFormatDepth32Float_Stencil8 || s.pixelFormat == MTLPixelFormatDepth16Unorm) && d.pixelFormat == s.pixelFormat, stencil = depth && d.pixelFormat == MTLPixelFormatDepth32Float_Stencil8; if (!depth && (fmt_int_or_ds(s.pixelFormat) || fmt_int_or_ds(d.pixelFormat))) return 0; if (mtl_fmt_block(d.pixelFormat) || !(d.usage & MTLTextureUsageRenderTarget) || s.sampleCount > 1 || d.sampleCount > 1) return 0; if (!pso) { NSError *e = nil; id lib = host_lib("copy", &e); - vf = [lib newFunctionWithName:@"vs"]; ff = [lib newFunctionWithName:@"fs"]; fd = [lib newFunctionWithName:@"fsd"]; fds = [lib newFunctionWithName:@"fsds"]; + vf = [lib newFunctionWithName:@"vs"]; ff = [lib newFunctionWithName:@"fs"]; fd = [lib newFunctionWithName:@"fsd"]; fds = [lib newFunctionWithName:@"fsds"]; fsq = [lib newFunctionWithName:@"fssq"]; MTLDepthStencilDescriptor *dd = [MTLDepthStencilDescriptor new]; dd.depthCompareFunction = MTLCompareFunctionAlways; dd.depthWriteEnabled = YES; dss = [g_dev newDepthStencilStateWithDescriptor:dd]; MTLStencilDescriptor *sd_ = [MTLStencilDescriptor new]; sd_.stencilCompareFunction = MTLCompareFunctionAlways; sd_.depthStencilPassOperation = MTLStencilOperationReplace; // Replace writes the [[stencil]] output dd.frontFaceStencil = dd.backFaceStencil = sd_; dss_s = [g_dev newDepthStencilStateWithDescriptor:dd]; MTL4ArgumentTableDescriptor *ad = [MTL4ArgumentTableDescriptor new]; ad.maxTextureBindCount = 2; ad.maxSamplerStateBindCount = 1; ad.maxBufferBindCount = 1; at = rec_table([g_dev newArgumentTableWithDescriptor:ad error:&e]); for (int i = 0; i < 2; i++) { MTLSamplerDescriptor *sd = [MTLSamplerDescriptor new]; sd.minFilter = sd.magFilter = i ? MTLSamplerMinMagFilterLinear : MTLSamplerMinMagFilterNearest; sd.sAddressMode = sd.tAddressMode = MTLSamplerAddressModeClampToEdge; sd.supportArgumentBuffers = YES; samp[i] = [g_dev newSamplerStateWithDescriptor:sd]; } - if (!vf || !ff || !fd || !fds || !dss || !dss_s || !at || !samp[0] || !samp[1]) { failed = 1; MLOG("[RED] cannot create the pipelines for format-converting / scaling texture copies: %s", e.localizedDescription.UTF8String); return 0; } + if (!vf || !ff || !fd || !fds || !fsq || !dss || !dss_s || !at || !samp[0] || !samp[1]) { failed = 1; MLOG("[RED] cannot create the pipelines for format-converting / scaling texture copies: %s", e.localizedDescription.UTF8String); return 0; } pso = [NSMutableDictionary dictionary]; } // Stencil: read through a stencil-only view of the source (one per layer below); without one it is cleared to 0 id stv0 = stencil && s.pixelFormat == MTLPixelFormatDepth32Float_Stencil8 ? [s newTextureViewWithPixelFormat:MTLPixelFormatX32_Stencil8 textureType:MTLTextureType2D levels:NSMakeRange(sl, 1) slices:NSMakeRange(ss + sr->zoffset, 1)] : nil; int scopy = stv0 != nil; - NSNumber *pk = @(d.pixelFormat * 2 + scopy); + int square = !depth && !!(mirror & COPY_DRAW_SQUARE); + NSNumber *pk = @(d.pixelFormat * 4 + scopy + 2 * square); id ps = pso[pk]; if (!ps) { - MTLRenderPipelineDescriptor *pd = [MTLRenderPipelineDescriptor new]; pd.vertexFunction = vf; pd.fragmentFunction = depth ? (scopy ? fds : fd) : ff; + MTLRenderPipelineDescriptor *pd = [MTLRenderPipelineDescriptor new]; pd.vertexFunction = vf; pd.fragmentFunction = depth ? (scopy ? fds : fd) : square ? fsq : ff; if (depth) { pd.depthAttachmentPixelFormat = d.pixelFormat; if (stencil) pd.stencilAttachmentPixelFormat = d.pixelFormat; } else pd.colorAttachments[0].pixelFormat = d.pixelFormat; NSError *e = nil; ps = [g_dev newRenderPipelineStateWithDescriptor:pd error:&e]; if (!ps) { RED_ONCE(83, "format-converting / scaling texture copy: cannot create the pipeline for destination format %lu: %s", (unsigned long)d.pixelFormat, e.localizedDescription.UTF8String); return 0; } @@ -16162,7 +16170,7 @@ static int up_mfx_on(void) { // Bananza's all encode linear input (0.0031308, 1 / 2.4 as floats) and come in threes: no decode at the end (the output stays encoded), a decode clamped to 0-1, a decode up to 128. Splatoon Raiders' all read colour that is encoded // already and come in twos: no decode, a decode clamped to 0-1 (as half floats: 0.04045, 2.4 and 1 / 12.92 rounded to 0.0404358, 2.3984375 and 0.0774536). Only the variants with linear output are drawn, in these two shapes; a variant // with encoded output, or any other shape (no pack has shown one), keeps the game's shader. A constant counts from 8 uses, far fewer than a curve per channel of every output tile: two bytes also match by chance -enum { UP_COLOR_UNREAD, UP_COLOR_LINEAR, UP_COLOR_SRGB, UP_COLOR_KEEP }; // linear in and out: HDR mode; sRGB in, linear out: perceptual mode, then decoded; UP_COLOR_KEEP, UP_COLOR_UNREAD: the game's shader +enum { UP_COLOR_UNREAD, UP_COLOR_LINEAR, UP_COLOR_SRGB, UP_COLOR_KEEP, UP_COLOR_SQRT }; // linear in and out: HDR mode; sRGB in, linear out: perceptual mode, then decoded; square-root input, linear out: squared into the scaler's input, HDR mode; UP_COLOR_KEEP, UP_COLOR_UNREAD: the game's shader static int up_mfx_uses(const void *code, size_t n, const void *v, size_t w) { const uint8_t *p = code, *end = p + n; int hits = 0; while (hits < 8 && (p = memmem(p, end - p, v, w))) { hits++; p += w; } @@ -16182,8 +16190,15 @@ static int up_mfx_named(NvShader *cs) { for (const char *k = DM_GAME_UPSCALER_KEYS; *k; k += k[16] == ',' ? 17 : 16) if (!strncmp(k, cs->key, 16)) return 1; return 0; } +// A third shape has no constant to find: agl_ndsr_vr_light.pck's upscale step (Ocarina of Time) samples colour the dispatch before it stored as square roots (HDR, packed as RGB9E5) and writes +// v * |v|, linear. The profile names such shaders (upscaler_sqrt, DM_GAME_UPSCALER_SQRT_KEYS), read from their translated code +static int up_mfx_sqrt_named(NvShader *cs) { + for (const char *k = DM_GAME_UPSCALER_SQRT_KEYS; *k; k += k[16] == ',' ? 17 : 16) if (!strncmp(k, cs->key, 16)) return 1; + return 0; +} static void up_mfx_read_color(NvShader *cs, const NVNshaderData *sd) { if (cs->up_color || !DM_GAME_UPSCALER_N || !up_mfx_on() || !up_mfx_named(cs)) return; + if (up_mfx_sqrt_named(cs)) { cs->up_color = UP_COLOR_SQRT; MLOG("upscaler %s: square-root input, linear output (the profile's upscaler_sqrt), MetalFX in HDR mode on the squared input", cs->key); return; } uint32_t nctl, ncode = 0; if (sd->control) nvn_ctl_lens(sd->control, &nctl, &ncode); cs->up_color = sd->data && ncode && ncode < (1u << 24) ? up_mfx_color((const void *)(uintptr_t)sd->data, ncode) : UP_COLOR_KEEP; MLOG("upscaler %s: %s", cs->key, cs->up_color == UP_COLOR_LINEAR ? "linear input and output, MetalFX in HDR mode" : cs->up_color == UP_COLOR_SRGB ? "sRGB-encoded input, linear output, MetalFX in perceptual mode" : "neither of the colour shapes MetalFX draws; the game's shader runs"); @@ -16193,7 +16208,7 @@ static int up_mfx_wanted(NvShader *cs) { if (cs->up_mfx) return cs->up_mfx == 1; cs->up_mfx = 2; if (!DM_GAME_UPSCALER_N || !up_mfx_on()) return 0; - if (up_mfx_named(cs) && (cs->up_color == UP_COLOR_LINEAR || cs->up_color == UP_COLOR_SRGB)) cs->up_mfx = 1; + if (up_mfx_named(cs) && (cs->up_color == UP_COLOR_LINEAR || cs->up_color == UP_COLOR_SRGB || cs->up_color == UP_COLOR_SQRT)) cs->up_mfx = 1; return cs->up_mfx == 1; } static up_mfx_scaler *up_mfx_scaler_for(NSUInteger iw, NSUInteger ih, NSUInteger ow, NSUInteger oh, int srgb) { @@ -16252,11 +16267,11 @@ static int up_mfx_encode(NvShader *cs) { if (!in || !out || in == out || in.textureType != MTLTextureType2D || out.textureType != MTLTextureType2D || in.sampleCount != 1 || out.sampleCount != 1 || fmt_int_or_ds(in.pixelFormat) || fmt_int_or_ds(out.pixelFormat) || mtl_fmt_block(out.pixelFormat) || !up_mfx_rect(v, in.width, in.height, &ir) || !up_mfx_rect(v + 4, out.width, out.height, &orr) || orr.width < ir.width || orr.height < ir.height || (orr.width == ir.width && orr.height == ir.height)) { g_up_mfx_kept++; return 0; } - int srgb = cs->up_color == UP_COLOR_SRGB; + int srgb = cs->up_color == UP_COLOR_SRGB, sq = cs->up_color == UP_COLOR_SQRT; up_mfx_scaler *e = up_mfx_scaler_for((NSUInteger)ir.width, (NSUInteger)ir.height, (NSUInteger)orr.width, (NSUInteger)orr.height, srgb); if (!e) { g_up_mfx_kept++; return 0; } id sc = e->sc; // Straight in and out when MetalFX can take the game's textures as they are; through the scaler's own textures otherwise - int direct = !srgb && (NSUInteger)ir.width == in.width && (NSUInteger)ir.height == in.height && out.storageMode == MTLStorageModePrivate && + int direct = !srgb && !sq && (NSUInteger)ir.width == in.width && (NSUInteger)ir.height == in.height && out.storageMode == MTLStorageModePrivate && in.pixelFormat == MTLPixelFormatRGBA16Float && out.pixelFormat == MTLPixelFormatRGBA16Float && !ir.xoffset && !ir.yoffset && !orr.xoffset && !orr.yoffset && (NSUInteger)orr.width == out.width && (NSUInteger)orr.height == out.height && (in.usage & sc.colorTextureUsage) == sc.colorTextureUsage && (out.usage & sc.outputTextureUsage) == sc.outputTextureUsage; if (!direct && !(out.usage & MTLTextureUsageRenderTarget)) { g_up_mfx_kept++; return 0; } @@ -16265,7 +16280,7 @@ static int up_mfx_encode(NvShader *cs) { ubd_land_compute(); NVNcopyRegion at0 = {0, 0, 0, ir.width, ir.height, 1}, at0o = {0, 0, 0, orr.width, orr.height, 1}; id color = direct ? in : e->in, result = direct ? out : e->out; - if (!direct && !copy_t2t_draw(in, color, 0, 0, 0, 0, &ir, &at0, 0, 0)) { g_up_mfx_kept++; return 0; } // into RGBA16F at the origin (a nearest draw: same size) + if (!direct && !copy_t2t_draw(in, color, 0, 0, 0, 0, &ir, &at0, 0, sq ? COPY_DRAW_SQUARE : 0)) { g_up_mfx_kept++; return 0; } // into RGBA16F at the origin (a nearest draw: same size) id ce = [g_cb computeCommandEncoder]; // as dlss_eval: MetalFX opens encoders of its own, which take no barrier [ce barrierAfterQueueStages:MTLStageAll beforeStages:MTLStageBlit visibilityOptions:resource_vis()]; [ce barrierAfterStages:MTLStageBlit beforeQueueStages:MTLStageAll visibilityOptions:resource_vis()]; @@ -16282,7 +16297,7 @@ static int up_mfx_encode(NvShader *cs) { args_dirty_all(); if (!g_up_mfx_n++) MLOG("upscaler: first dispatch of %s drawn by MetalFX, %dx%d at %d,%d of %lux%lu (format %lu) → %dx%d at %d,%d of %lux%lu (format %lu)%s", cs->key, ir.width, ir.height, ir.xoffset, ir.yoffset, (unsigned long)in.width, (unsigned long)in.height, (unsigned long)in.pixelFormat, orr.width, orr.height, orr.xoffset, orr.yoffset, (unsigned long)out.width, (unsigned long)out.height, - (unsigned long)out.pixelFormat, direct ? ", straight" : srgb ? ", through RGBA16F, decoded from sRGB" : ", through RGBA16F"); + (unsigned long)out.pixelFormat, direct ? ", straight" : srgb ? ", through RGBA16F, decoded from sRGB" : sq ? ", through RGBA16F, squared on the way in" : ", through RGBA16F"); return 1; } // The present pass: src's valid rectangle drawn into dst (the drawable, or a screenshot's texture) as present_encode draws it, through MetalFX first where fx_encode takes it diff --git a/dm/gfx/tests/t_upscaler_mfx.m b/dm/gfx/tests/t_upscaler_mfx.m index f7875d36a3..435b6204e3 100644 --- a/dm/gfx/tests/t_upscaler_mfx.m +++ b/dm/gfx/tests/t_upscaler_mfx.m @@ -129,6 +129,14 @@ int main(void) { @autoreleasepool { CHECK(up_mfx_encode(named), "linear input after sRGB at the same sizes"); CHECK(mtl_wait(mtl_sub_commit(), 5000000000ull) && !mtl_last_commit_error(), "the second linear submission completes"); CHECK(matches(written, colour, MTLRegionMake2D(0, 0, 1920, 1080), black), "the same sizes in linear colour are not decoded: each colour has its own scaler"); + // Square-root input and linear output (Ocarina of Time's agl_ndsr_vr_light step, named by its profile's upscaler_sqrt): the image holds the colour squared + named->up_color = UP_COLOR_SQRT; + const float squared[4] = {0.25f, 0.0625f, 0.5625f, 1.0f}; + fill(written, MTLClearColorMake(0, 0, 0, 1)); mtl_sub_begin(); + CHECK(up_mfx_encode(named), "MetalFX takes square-root-encoded input"); + CHECK(mtl_wait(mtl_sub_commit(), 5000000000ull) && !mtl_last_commit_error(), "the square-root submission completes"); + CHECK(matches(written, squared, MTLRegionMake2D(0, 0, 1920, 1080), black), "the square-root input comes out squared"); + named->up_color = UP_COLOR_LINEAR; NvShader *encoded_out = upscaler(key); encoded_out->up_color = UP_COLOR_KEEP; CHECK(!up_mfx_wanted(encoded_out), "a named shader whose output stays encoded is left to its shader"); S.tex[5][0] = 701; diff --git a/dm/tools/tests/test_game_profile.py b/dm/tools/tests/test_game_profile.py index ca9b229999..a94957df0c 100644 --- a/dm/tools/tests/test_game_profile.py +++ b/dm/tools/tests/test_game_profile.py @@ -49,11 +49,11 @@ def test_rejected_tokens(self): 'cs_other.bin\t5\tcompute\t0123456789abcdef\t/Lib/agl/agl_resource.Nin_Ounce_NVN2.release.sarc.zs/agl_common.sharcb#BAHS@0x40\n') -def macros(shader_map): +def macros(shader_map, extra=''): """The upscaler macros of a synthetic profile's header, with this shader map (None: none)""" with tempfile.TemporaryDirectory() as d: Path(d, ID).mkdir() - (Path(d, ID) / 'game.tsv').write_text(TSV) + (Path(d, ID) / 'game.tsv').write_text(TSV + extra) if shader_map is not None: (Path(d, ID) / 'shader_map.tsv').write_text(shader_map) subprocess.run([sys.executable, str(GAME), 'header', ID, str(Path(d, ID) / 'game.h')], env={**os.environ, 'DM_GAMES_DIR': d}, check=True, capture_output=True) return [l for l in (Path(d, ID) / 'game.h').read_text().splitlines() if 'UPSCALER' in l] @@ -61,10 +61,13 @@ def macros(shader_map): class UpscalerKeys(unittest.TestCase): def test_from_shader_map(self): - none = ['#define DM_GAME_UPSCALER_KEYS ""', '#define DM_GAME_UPSCALER_N 0'] # the launcher hides the setting + sq = '#define DM_GAME_UPSCALER_SQRT_KEYS ""' + none = ['#define DM_GAME_UPSCALER_KEYS ""', '#define DM_GAME_UPSCALER_N 0', sq] # the launcher hides the setting self.assertEqual(macros(None), none) self.assertEqual(macros(MAP.split('cs_other.bin')[0]), none) - self.assertEqual(macros(MAP), ['#define DM_GAME_UPSCALER_KEYS "7629d857a0a93833,c56db3d5a0f1b5af"', '#define DM_GAME_UPSCALER_N 2']) + self.assertEqual(macros(MAP), ['#define DM_GAME_UPSCALER_KEYS "7629d857a0a93833,c56db3d5a0f1b5af"', '#define DM_GAME_UPSCALER_N 2', sq]) + self.assertEqual(macros(None, 'upscaler_sqrt\tfa3033d05c0a41d6\n'), # a shader the profile names for its square-root colour is an upscaler too + ['#define DM_GAME_UPSCALER_KEYS "fa3033d05c0a41d6"', '#define DM_GAME_UPSCALER_N 1', '#define DM_GAME_UPSCALER_SQRT_KEYS "fa3033d05c0a41d6"']) if __name__ == '__main__': diff --git a/docs/agents/symptoms.md b/docs/agents/symptoms.md index 20128beeed..0afcd782f1 100644 --- a/docs/agents/symptoms.md +++ b/docs/agents/symptoms.md @@ -820,6 +820,7 @@ One line per symptom in the [subsystem references](../reference/): what you see - a Switch 2 game using DLSS crashes after minutes of play, at a moment it makes a DLSS network again or evaluates one at new sizes (`[nvn] DLSS network …` and a new `DLSS: MetalFX temporal scaler …` line, for example a 1920x972 output), with `*** Terminating app due to uncaught exception 'NSInvalidArgumentException', reason: '-[AGXG17XFamilyMLContext executable]: unrecognized selector sent to instance …'` in `-[_MTL4MachineLearningCommandEncoder encodeToCommandQueue:]` under `rec_commit` / `mtl_sub_commit` (the host exits with 134). The class named varies with what took the freed memory, and many runs pass. → [DLSS scalers released under their encoders](../reference/rendering.md#dlss-scalers-released-under-their-encoders) - a Switch 2 game's frame in its heavy scenes is dominated by one compute dispatch between the render-resolution passes and the output-resolution ones: Donkey Kong Bananza's upscaler takes 3.5-5 ms of a 24 ms GPU frame on an M1 in handheld mode (960x540 → 1920x1080, `DM_NVN_GPUPROF`), and the game cannot reach its 60 fps interval there. → [A game's own upscaler drawn by MetalFX](../reference/rendering.md#a-games-own-upscaler-drawn-by-metalfx) - with `DM_UPSCALER=metalfx`, a game's picture is washed out wherever it upscales: lifted shadows and low contrast, the scene only, with the UI drawn after it correct; with `DM_UPSCALER=game`, or at a size the game does not upscale, it is right. → [An upscaler's colour read from its code](../reference/rendering.md#an-upscalers-colour-read-from-its-code) +- `DM_UPSCALER=metalfx` changes nothing in a game that upscales with agl's network: the log names the upscale shader with `neither of the colour shapes MetalFX draws; the game's shader runs`, or names no shader at all because the game's upscaler is not in `agl_ndsr.pck`. → [An upscaler with square-root colour](../reference/rendering.md#an-upscaler-with-square-root-colour) - tall blue / textured panels that flickered frame to frame: the fountain's vertex shader pushed the water surface to the top of the screen. → [Metal 4 does not order operations inside one encoder](../reference/rendering.md#metal-4-does-not-order-operations-inside-one-encoder) - a buffer copy, inline update or fill completes, but a following copy through another Metal buffer over the same allocation reads stale bytes. → [Buffer copies through aliased Metal addresses](../reference/rendering.md#buffer-copies-through-aliased-metal-addresses) - many render passes with one draw each and high submit time: Metroid Dread's cutscene frames had about 180 passes with one draw each (about 5 ms of submit time per frame on the Mac). → [Inline uniform updates inside a render pass](../reference/rendering.md#inline-uniform-updates-inside-a-render-pass) diff --git a/docs/env_switches.tsv b/docs/env_switches.tsv index 435f71eebd..7e4d7d14f4 100644 --- a/docs/env_switches.tsv +++ b/docs/env_switches.tsv @@ -202,6 +202,7 @@ DM_NVN_SHDUMP diagnostic `=` saves programs missing from the shader map as DM_NVN_SHMAP path The shader map (`shader_map.tsv`); `c1_full.tsv` and `pso_manifest.tsv` are taken from the same folder. Default: the game profile's folder on the Mac; the iOS app sets it DM_NVN_SHOT diagnostic `=[,…]:` saves the swapchain texture as an image at those frames (`%d` in the path = frame number) DM_NVN_SHOT_OUT diagnostic `=x` makes `DM_NVN_SHOT` capture the picture as the present pass upscales it to that size, the crop included, instead of the raw swapchain texture +DM_NVN_SKIP_CS diagnostic `=[,…]` (up to 16) drops all dispatches of those compute shaders, to measure one dispatch's GPU cost inside an encoder that holds several; what they write stays stale, so the picture is wrong DM_NVN_SKIP_FS diagnostic `=[,…]` (up to 16) drops all draws of those fragment shaders, to measure their GPU cost or bisect a wrong draw; the picture is wrong DM_NVN_SKIP_MESH diagnostic `=1` drops every task and mesh draw, to measure what they cost in all; the picture is wrong DM_NVN_SKIP_SMALL diagnostic `=` drops all draws whose target width is at most that, to measure the cost of tiny render passes; the picture is wrong diff --git a/docs/reference/rendering.md b/docs/reference/rendering.md index 657d15be64..251c0d9dfb 100644 --- a/docs/reference/rendering.md +++ b/docs/reference/rendering.md @@ -1294,6 +1294,15 @@ The replay on Metal (`dm/gfx/nvn/nvn_mtl.m`) and the Vulkan and OpenGL layers. E - **Check:** `make -C dm/gfx/tests GAME=0400380020056000 check_t_upscaler_mfx`: the classifier on synthetic code in each shape, and an sRGB input, full and as a packed subrectangle, comes out decoded, with linear input at the same sizes not decoded. The classifier against the translated shaders of all 44 variants of the two packs: no mismatch (Bananza 16 linear, 8 kept; Splatoon Raiders 10 sRGB, 10 kept). **Open:** Mario Kart World's 30 variants have not been read; whatever shape they are, an unknown one keeps its shader. - **Found on:** [Splatoon Raiders](../../dm/games/0400380020056000/README.md) ([#1156](https://github.com/seedds/nxnative/issues/1156)), the first level with `c1b8edc09e3313b1` and `30d8ee33de9dee85`: the washed-out picture reproduced, and with the fix the sky and scene means match the game's upscaler within 0.6 of 255 per channel, against about 50 too high before. + +### An upscaler with square-root colour + +- **Symptom:** `DM_UPSCALER=metalfx` changes nothing in a game that upscales with agl's network: the log names the upscale shader with `neither of the colour shapes MetalFX draws; the game's shader runs`, or names no shader at all because the game's upscaler is not in `agl_ndsr.pck`. +- **Cause:** two things. The shader map's pattern takes only `agl_ndsr.pck`; Ocarina of Time upscales in play with four dispatches of `agl_ndsr_vr.pck` and `agl_ndsr_vr_light.pck`, of which the last alone is the upscale (one texture in at slot 0, one image out at slot 1, the rectangles in bank 24: the shape `up_mfx_encode` takes) and the three before it are anti-aliasing. And that step's colour has no sRGB curve for [the classifier](#an-upscalers-colour-read-from-its-code) to find: the dispatch before it stores each channel's square root, packed as RGB9E5, and the step writes `v * |v|` at its end, linear. +- **Fix:** the profile key `upscaler_sqrt` names such shaders (`dm/games/game.py`: they join `DM_GAME_UPSCALER_KEYS` and are listed in `DM_GAME_UPSCALER_SQRT_KEYS`), read from their translated code, since no constant marks the shape. `up_mfx_read_color` gives them `UP_COLOR_SQRT`: the draw of the input rectangle into the scaler's RGBA16F texture squares the colour (`copy.metal` `fssq`, `COPY_DRAW_SQUARE`), MetalFX runs in HDR mode, and its result is drawn into the game's image as it is; never straight, as the input must be converted. A per-game key because the shape was read by a person, not derived: a general reader of the translated code's last multiply would replace it. +- **Check:** `make -C dm/gfx/tests GAME=0400486024DEC000 check_t_upscaler_mfx` (a square-root input of 0.5, 0.25, 0.75 comes out 0.25, 0.0625, 0.5625); `test_game_profile.py` (the key's macros, with and without a shader map). +- **Found on:** [The Legend of Zelda: Ocarina of Time](../../dm/games/0400486024DEC000/README.md#frame-rate): `fa3033d05c0a41d6`, 1536x864 → 2048x1152, 1.12 ms of GPU a frame on an M4 Pro, about 0.1 ms through MetalFX; the end shots' mean colours agree with the game's upscaler within 1 of 255. **Open:** the picture in motion is not judged yet. + ### Metal 4 does not order operations inside one encoder - **Symptom:** tall blue / textured panels that flickered frame to frame: the fountain's vertex shader pushed the water surface to the top of the screen. From 6f9212ec72771e46f5526d44d915f894d2088dde Mon Sep 17 00:00:00 2001 From: seedds Date: Wed, 11 Nov 2026 21:11:28 +0100 Subject: [PATCH 3/9] Upscaler sharpness (DM_UPSCALER_SHARPNESS, the profile's upscaler_sharpness): MetalFX's picture blended with a bilinear one of the same input, 0.7 for 0400486024DEC000, whose eight square-root upscale variants are named MetalFX's spatial scaler has no strength setting and is crisper than agl's networks. The draw of its result into the game's image blends in the scaler's input, sampled bilinearly, by 1 - sharpness; 1.0 (every other game) is the picture as before. Front-ends: new: the settings list has DM_UPSCALER_SHARPNESS (range 0.0-1.0, shown while DM_UPSCALER is metalfx, default from the game profile) --- dm/games/0400486024DEC000/README.md | 3 ++- dm/games/0400486024DEC000/game.tsv | 7 +++-- dm/games/README.md | 1 + dm/games/game.py | 3 +++ dm/games/settings.tsv | 1 + dm/gfx/msl/copy.metal | 10 ++++--- dm/gfx/nvn/nvn_mtl.m | 42 ++++++++++++++++++++--------- dm/gfx/tests/t_upscaler_mfx.m | 5 ++++ dm/tools/tests/test_game_profile.py | 10 +++---- docs/agents/symptoms.md | 1 + docs/env_switches.tsv | 1 + docs/reference/rendering.md | 13 +++++++-- docs/settings.md | 10 ++++++- 13 files changed, 81 insertions(+), 26 deletions(-) diff --git a/dm/games/0400486024DEC000/README.md b/dm/games/0400486024DEC000/README.md index f2dbd6f5da..8fcd8766db 100644 --- a/dm/games/0400486024DEC000/README.md +++ b/dm/games/0400486024DEC000/README.md @@ -31,7 +31,7 @@ One [NUP container](../../../docs/development.md#the-packing-script-in-detail) o Measured on an M6 Mac with 16 GB, headless, on one walk with camera turns (`walk.sh` in the work folder). - Current on an M5 Pro MacBook Pro (16-inch, 20-core GPU, 24 GB, macOS 27.2, 2026-11-09), `kokiri-deck` from a deck autosave with the full library of chain `dedc9391c30b`, at full resolution (Dynamic resolution off), two runs a variant: in a window a locked 60 fps (59.8–60.0, frame p99 16.8–19.4 ms, GPU p50 10.2 ms). In fullscreen on its ProMotion screen it held 40 fps (39.6–40.1) at the same GPU time, with every present a 120 Hz refresh late and every other link tick skipped: [The 60 Hz link at 40 fps in fullscreen on a ProMotion screen](../../../docs/reference/frame-rate.md#the-60-hz-link-at-40-fps-in-fullscreen-on-a-promotion-screen). With the 120 Hz link there, 58.8–59.6 fps, frame p99 25.3 ms; with the screen held at its fixed 60 Hz mode, 60.0 fps, frame p99 16.9 ms, which fullscreen now switches to. MetalFX present upscaling to the 3456-wide fullscreen cost no measurable GPU time (59.4–59.7 fps). Resident about 10 GB, GPU allocated 11.2–11.4 GB; the prepare step took 7.3 s on a first start. -- Current: `DM_UPSCALER=metalfx` draws the upscale step with MetalFX (M4 Pro, 20-core GPU, 2026-11-11, Kakariko Village from a save, chain `b8856126d238`, `DM_FPS_HOLD=off`). In play the game anti-aliases and upscales with four dispatches of `agl_ndsr_vr.pck` and `agl_ndsr_vr_light.pck` (`f8965bffbdf51d78`, `d17eefaff4f0bc23`, `a335c1db6270c355`, `fa3033d05c0a41d6`), one encoder of 1.25 ms; only the last is the upscale (texture slot 0 at 1536x864 to image slot 1 at 2048x1152). With its dispatches dropped (`DM_NVN_SKIP_CS`, the picture stale) the frame's compute fell 3.91-3.93 → 2.78-2.80 ms: 1.12 ms. Its colour, from its translated code: the dispatch before it stores each channel's square root, packed as RGB9E5 bits in a single-channel image; the step samples that, adds its network's residual and writes `v * |v|`, linear, with no sRGB curve, so the host's classifier kept the shader. The profile names it ([`upscaler_sqrt`](game.tsv)): [An upscaler with square-root colour](../../../docs/reference/rendering.md#an-upscaler-with-square-root-colour). Two runs a side in turn: headless, compute 3.91 and 3.95 → 2.80 and 2.77 ms, the frame's GPU work 14.84 and 15.61 → 13.64 and 13.48 ms (median sum of encoder times over 37-38 profiled frames); in a window, compute 3.90 and 3.89 → 2.76 and 2.79 ms, summed GPU p50 15.59 and 15.13 → 14.63 and 14.46 ms, 60.0 fps on both. The end shots' mean colours agree (15.9 / 25.3 / 29.4 against 15.8 / 24.8 / 28.4 of 255; two shots of the game's own differ as much, by the moving grass); MetalFX's picture is sharper, the network's softer, which the player preferred. The three anti-aliasing dispatches still run. Open: not judged in motion, nor with a render scale, the dynamic resolution or a cutscene. +- Current: `DM_UPSCALER=metalfx` draws the upscale step with MetalFX (M4 Pro, 20-core GPU, 2026-11-11, Kakariko Village from a save, chain `b8856126d238`, `DM_FPS_HOLD=off`). In play the game anti-aliases and upscales with four dispatches of `agl_ndsr_vr.pck` and `agl_ndsr_vr_light.pck` (`f8965bffbdf51d78`, `d17eefaff4f0bc23`, `a335c1db6270c355`, `fa3033d05c0a41d6`), one encoder of 1.25 ms; only the last is the upscale (texture slot 0 at 1536x864 to image slot 1 at 2048x1152). With its dispatches dropped (`DM_NVN_SKIP_CS`, the picture stale) the frame's compute fell 3.91-3.93 → 2.78-2.80 ms: 1.12 ms. Its colour, from its translated code: the dispatch before it stores each channel's square root, packed as RGB9E5 bits in a single-channel image; the step samples that, adds its network's residual and writes `v * |v|`, linear, with no sRGB curve, so the host's classifier kept the shader. The profile names it ([`upscaler_sqrt`](game.tsv)): [An upscaler with square-root colour](../../../docs/reference/rendering.md#an-upscaler-with-square-root-colour). Two runs a side in turn: headless, compute 3.91 and 3.95 → 2.80 and 2.77 ms, the frame's GPU work 14.84 and 15.61 → 13.64 and 13.48 ms (median sum of encoder times over 37-38 profiled frames); in a window, compute 3.90 and 3.89 → 2.76 and 2.79 ms, summed GPU p50 15.59 and 15.13 → 14.63 and 14.46 ms, 60.0 fps on both. The end shots' mean colours agree (15.9 / 25.3 / 29.4 against 15.8 / 24.8 / 28.4 of 255; two shots of the game's own differ as much, by the moving grass); MetalFX's picture is crisper, the network's softer, which the player preferred: the profile's `upscaler_sharpness 0.7` blends 30 % of a bilinear picture in, the closest of seven variants in 2x crops ([MetalFX's picture is crisper than a game's own upscaler](../../../docs/reference/rendering.md#metalfxs-picture-is-crisper-than-a-games-own-upscaler)), at the same GPU time. The profile names the pack's eight variants that end as this one does; only this one was drawn and compared (`d99fbeeb53d38847`, the upscaler of an earlier Kokiri Forest run, is among the seven others). The three anti-aliasing dispatches still run. Open: not judged in motion, nor with a render scale, the dynamic resolution or a cutscene. - Current: the title and menu, a movie behind them, are a locked 60 fps (p50 16.67 ms, GPU span 15.6 ms). - Game speed: the game steps its time by a frame ratio, its base rate (60, or 30) / its int frame rate, whatever the frame took (`main+0x6e6cc4` on v131072), so frames longer than a vsync played in slow motion (seen by the user on a slower Mac; [issue 1375](https://github.com/seedds/nxnative/issues/1375)'s M1 Max ran the title at about 28 fps). With `fps_target 60` the ratio is the host's frame step × base / 60 ([A fixed frame step plays in slow motion on a slow Mac](../../../docs/reference/frame-rate.md#a-fixed-frame-step-plays-in-slow-motion-on-a-slow-mac)), a stub placed by its function's fingerprint ([`patches.py`](patches.py)). Mac runs on the M6 (2026-11-08, a test host on the libraries of chain `3ee493b2c3a4`): the user's play sessions held 60 fps with 2- and 3-vsync holds while areas loaded, game time 1.000 of real time by the host's count. That count is the stub's own; no run on a Mac slow enough to show the slow motion before and after. - Sizes since the host answers graphics scale tier 0 docked ([The graphics scale does not follow the operation mode](../../../docs/reference/services.md#the-graphics-scale-does-not-follow-the-operation-mode)): the game's output is 2560x1440 and its top internal resolution 1920x1080. Every size and time in this note was taken with tier 1, at 0.8 of that (2048x1152 and 1536x864, and the dynamic resolution's levels with them), and has not been taken again. With tier 0 on `main` (two contributors' runs, only that answer changed): M4 Pro (20-core GPU), docked, fullscreen, GPU p50 15.0 ms against 12.9-14.0 ms, 59.0 fps against 55.8-59.8; M5 Pro (20-core GPU, 48 GB), Dynamic resolution off, 24 minutes of play, GPU p50 12.7 ms, 58.4 fps. Open: a slower Mac, and the routes of [`bench.tsv`](bench.tsv), with tier 0. @@ -128,6 +128,7 @@ Its 37 kernels that correct samples, NDSR's network among them, gave about 0.25 ## Found on this game - [An upscaler with square-root colour](../../../docs/reference/rendering.md#an-upscaler-with-square-root-colour): `DM_UPSCALER=metalfx` replaced no dispatch; with the profile's `upscaler_sqrt` the upscale step's 1.12 ms of GPU goes to MetalFX (Kakariko Village, M4 Pro). +- [MetalFX's picture is crisper than a game's own upscaler](../../../docs/reference/rendering.md#metalfxs-picture-is-crisper-than-a-games-own-upscaler): the launcher's Upscaler sharpness, 70 % here by the player's eye. - [A shader build's Python finishes one shader at a time](../../../docs/reference/shaders.md#a-shader-builds-python-finishes-one-shader-at-a-time): resolved, the whole Mac map's cold build took 58 minutes on an M3 Max with 4 to 5 of its 16 cores busy; 30 minutes with worker processes. - [Reads past a uniform binding's size](../../../docs/reference/shaders.md#reads-past-a-uniform-bindings-size): open ([GitHub issue #2218](https://github.com/seedds/nxnative/issues/2218)), long white streaks from the horses at Lon Lon Ranch and from Link's slingshot, and flickering parts of the slingshot and of the ReDeads' faces, reported with the uniform read bound on. The bound now takes the size the driver gives the GPU, rounded up to 256 bytes; the game binds sizes that are not multiples of 256. Not reproduced here before or after (the slingshot at Link's house, the ReDeads of adult Castle Town), so the reporters' retest decides. - [The blend helper's output reference on iOS](../../../docs/reference/shaders.md#the-blend-helpers-output-reference-on-ios): resolved, 22,296 of 42,848 Mac shaders failed to compile on macOS 26 ([GitHub issue #2272](https://github.com/seedds/nxnative/issues/2272)); the Mac compile now names the host's minimum macOS. diff --git a/dm/games/0400486024DEC000/game.tsv b/dm/games/0400486024DEC000/game.tsv index 3db6a57a63..e4fe984a3e 100644 --- a/dm/games/0400486024DEC000/game.tsv +++ b/dm/games/0400486024DEC000/game.tsv @@ -50,5 +50,8 @@ tex_flush_upload on tex_dirty_upload on # The upscale step of agl_ndsr_vr_light.pck, the last of the four dispatches the game anti-aliases and upscales with in play: it samples colour the dispatch before it stored as square # roots (RGB9E5) and writes v * |v|, linear, so its code has none of the sRGB constants the host reads a colour shape from. Named here, DM_UPSCALER=metalfx draws it with MetalFX -# (its share of the frame in Kakariko Village: 1.12 ms of GPU on an Apple M4 Pro, 2026-11-11) -upscaler_sqrt fa3033d05c0a41d6 +# (fa3033d05c0a41d6 in Kakariko Village: 1.12 ms of GPU a frame on an Apple M4 Pro, 2026-11-11). The eight variants of the pack whose translated code ends that way; its seven others +# encode their result (x 12.92 after the square) and keep their shader, as do agl_ndsr_vr.pck's fifteen, which take linear colour and have not been seen in play +upscaler_sqrt 04135bb812a0d66e,7c511cc23ff85d14,8df148d58b7cbe8e,aad7ea7e5c02f41d,b106afba1a18faa9,c8b20adc9d4978e0,d99fbeeb53d38847,fa3033d05c0a41d6 +# The player compared MetalFX's picture with the network's in 2x crops and found it too crisp; with 30 % of a bilinear picture blended in it came closest (2026-11-11) +upscaler_sharpness 0.7 diff --git a/dm/games/README.md b/dm/games/README.md index b4865ce16d..dee456d914 100644 --- a/dm/games/README.md +++ b/dm/games/README.md @@ -253,6 +253,7 @@ One row per profile; the notes are each profile's `README.md`. | `tex_dyn_slots` | `on` / `off` (the default): a shader whose buffer-texture handle the translator could not trace to one slot gets Ryujinx's fallback arrays (`__cb0_40000000`: 32 buffer textures and samplers indexed by the handle's texture / sampler ID; any other sampled texture indexes the whole pool without a key, [Texture handles from game memory index the pools](../../docs/reference/shaders.md#texture-handles-from-game-memory-index-the-pools)). With the key the replay puts slot indices in that shader's c2 texture and sampler tables (texture slot r → r, sampler slot r → r << 20) and binds slot k at element k, so the shader's own arithmetic picks the binding (`NVN_RB_TEXD` / `NVN_RB_SAMPD` in `nvn_mtl.m`). Right only where the handle comes from c2's tables: Octopath Traveler 0's 61 such vertex shaders read separate texture slots 0-10 and samplers 0-4 there and nothing else (FINAL FANTASY RESONANCE's 2,077 the texture table words 0x168-0x1c0 and sampler table words 0x568-0x590). Off, such shaders do not load. Not part of the shader chain version | | `draw_barriers` | `full` (the default) / `game`: how draw passes are ordered. `full`: each waits for all earlier GPU work. `game`: by the game's own barriers (NVN asks for one before results of rendering are read), so a pass's vertex work overlaps the previous pass's fragments. Ocarina of Time's GPU time 25 → 18 ms, about 180 full barriers a frame halved. Only vertex-stage reads of the previous pass's output without a barrier change order. `DM_NVN_BARRIER` overrides it | | `upscaler_sqrt` | comma-separated 16-digit keys of upscaler compute shaders, outside `agl_ndsr.pck`, that sample square-root-encoded colour and write it squared (linear): `DM_UPSCALER=metalfx` squares their input into MetalFX's and runs it in HDR mode ([An upscaler with square-root colour](../../docs/reference/rendering.md#an-upscaler-with-square-root-colour)). Read the shape from the shader's translated code before naming one | +| `upscaler_sharpness` | `0.0` to `1.0` (the default): with `DM_UPSCALER=metalfx`, the share of MetalFX's own picture in the game's image, the rest the scaler's input scaled bilinearly; the launcher's **Upscaler sharpness** starts from it (`DM_UPSCALER_SHARPNESS`). Lower it for a game whose own network is softer than MetalFX, by eye against the game's picture | | `vconv_range` | `on` / `off` (the default): vertex attributes the replay converts on the CPU (integer shader inputs fed float-type formats and the like, `vconv_bind` in `nvn_mtl.m`) are converted only over the vertices each draw reads (first / count, or the smallest and largest index of a direct indexed draw, kept while the index pages stay unwritten) instead of the whole bound range. Age of Calamity binds whole mesh pools: in a battle that was millions of vertices a frame. Indirect draws, per-instance streams and indices with a GPU write queued on them keep the whole range. `DM_NVN_VCONV_CHECK=1` scans the indices again at every reuse and reports differences; `DM_NVN_VCONV_RANGE=0` / `1` overrides it | | `shader_key_skip` | `+` (none by default): control-section bytes every shader key hashes as zeros, in `shader_split.py` (`key_ctl`) and the host (`hle_nvn.c` `keys_hash`), for an engine that shares one control section among shaders that differ only there. Batman: Arkham Knight: `0x770+0x78`, GLSLC's compile information (+0x778..+0x791 and +0x7d0..+0x7e7); without it the game asks for shaders the map does not have, each the control section of one record with the code area of another. It must stay clear of `+0x700..+0x75f`, which the tools read; it is a shader chain input, as every `shader_*` key | | `shader_groups`, `shader_note` | `shader_split.py` source groups and the map header note (Engage: `engine` for `data.unity3d` and `Resources/`, `assets` for the Addressables bundles). Both are shader inputs, the note too: changing either one, even by a word, builds every shader of the game again. Keep them as they are when a provisional profile moves into `dm/games` | diff --git a/dm/games/game.py b/dm/games/game.py index 10517c7d5a..bdc4818d87 100644 --- a/dm/games/game.py +++ b/dm/games/game.py @@ -211,6 +211,8 @@ def header_one(g): if vcr not in ('on', 'off'): sys.exit(f'{g.id}: vconv_range must be on or off: {vcr}') mml = g.get('mem_map_limit', 'off') # MapPhysicalMemory refused beyond the application pool, as on Horizon (hle_svc.c): on / off if mml not in ('on', 'off'): sys.exit(f'{g.id}: mem_map_limit must be on or off: {mml}') + ush = g.get('upscaler_sharpness', '1.0') # DM_UPSCALER=metalfx: the share of MetalFX's own picture, the rest a bilinear one (nvn_mtl.m up_mfx_sharpness; DM_UPSCALER_SHARPNESS overrides): 0.0-1.0 + if not re.fullmatch(r'[01]\.\d{1,2}', ush) or float(ush) > 1: sys.exit(f'{g.id}: upscaler_sharpness must be 0.0 to 1.0: {ush}') ups = upscaler_keys(g) # the game's own upscaler shaders, which DM_UPSCALER=metalfx (the launcher's Game's upscaler) replaces with MetalFX's spatial scaler (nvn_mtl.m up_mfx_encode) dbr = g.get('draw_barriers', 'full') # how draw passes are ordered (nvn_mtl.m barrier_game): full = each waits for all earlier GPU work, game = by the game's own barriers, so vertex work overlaps the previous pass's fragments if dbr not in ('full', 'game'): sys.exit(f'{g.id}: draw_barriers must be full or game: {dbr}') @@ -269,6 +271,7 @@ def header_one(g): f'#define DM_GAME_TEX_DYN_SLOTS {int(tds == "on")}', f'#define DM_GAME_DRAW_BARRIERS_GAME {int(dbr == "game")}', f'#define DM_GAME_UPSCALER_KEYS {q(",".join(ups))}', f'#define DM_GAME_UPSCALER_N {len(ups)}', f'#define DM_GAME_UPSCALER_SQRT_KEYS {q(",".join(upscaler_sqrt_keys(g)))}', + f'#define DM_GAME_UPSCALER_SHARPNESS {ush}', f'#define DM_GAME_VCONV_RANGE {int(vcr == "on")}', *([f'#define DM_GAME_NVN_API_MAJOR {api.split(".")[0]}', f'#define DM_GAME_NVN_API_MINOR {api.split(".")[1]}'] if g.get('device_api', '') else []), # without the key: by the SDK version (hle_nvn.c nvn_api) ''] diff --git a/dm/games/settings.tsv b/dm/games/settings.tsv index 15f4824107..ab5760525e 100644 --- a/dm/games/settings.tsv +++ b/dm/games/settings.tsv @@ -20,6 +20,7 @@ DM_RENDER_SCALE range 0.5:1.0:0.05 1.0 fixups:RENDER_SCALE Game Render scale le DM_OPMODE choice 1=Docked,0=Handheld DM_GAME_OPMODE always Game Console mode level=basic DM_PRESENT_UPSCALE choice off=Standard,metalfx=MetalFX off always Game Upscaling Uses more GPU time and can increase heat. DM_UPSCALER choice game=Game's own,metalfx=MetalFX game DM_GAME_UPSCALER_N!=0 Game Game's upscaler +DM_UPSCALER_SHARPNESS range 0.0:1.0:0.05 DM_GAME_UPSCALER_SHARPNESS DM_GAME_UPSCALER_N!=0 Game Upscaler sharpness needs=DM_UPSCALER=metalfx DM_LANG choice languages build:DM_LANG languages>1 Game Game language level=basic DM_FULLSCREEN choice 0=Window,1=Fullscreen 0 platform:mac Game Display mode level=basic DM_PICTURE_FIT choice fit=Fit,fill=Fill fit platform:ios Game Picture Fill crops the game's 16:9 picture to cover the screen. Health bars, subtitles or maps near the edges can be cut off. A full-screen picture without cropping needs a patch for each game. diff --git a/dm/gfx/msl/copy.metal b/dm/gfx/msl/copy.metal index 5177d37261..b12bf63533 100644 --- a/dm/gfx/msl/copy.metal +++ b/dm/gfx/msl/copy.metal @@ -6,9 +6,13 @@ vertex V vs(uint id [[vertex_id]]) { float2 uv = float2((id << 1) & 2, id & 2); // p[0] = source origin (texels) + source texels per destination texel, p[1] = destination origin + source level size fragment float4 fs(V in [[stage_in]], texture2d t [[texture(0)]], sampler s [[sampler(0)]], constant float4 *p [[buffer(0)]]) { float2 st = p[0].xy + (in.p.xy - p[1].xy) * p[0].zw; return t.sample(s, st / p[1].zw, level(0)); } -// The same with the colour squared, sign kept: a square-root-encoded source into a linear destination (up_mfx_encode's input of an upscaler that squares its own result) -fragment float4 fssq(V in [[stage_in]], texture2d t [[texture(0)]], sampler s [[sampler(0)]], constant float4 *p [[buffer(0)]]) { - float2 st = p[0].xy + (in.p.xy - p[1].xy) * p[0].zw; float4 c = t.sample(s, st / p[1].zw, level(0)); return float4(c.rgb * abs(c.rgb), c.a); } +// The copies around MetalFX's spatial scaler (up_mfx_encode). p[2].x = blend with t1, a smooth picture of the same rectangle (0 = none); p[2].y = 1: the colour squared, sign kept +// (square-root-encoded colour into linear); p[2].zw = t1's coordinate per source texel of the rectangle +fragment float4 fsup(V in [[stage_in]], texture2d t [[texture(0)]], texture2d t1 [[texture(1)]], sampler s [[sampler(0)]], constant float4 *p [[buffer(0)]]) { + float2 st = p[0].xy + (in.p.xy - p[1].xy) * p[0].zw; float4 c = t.sample(s, st / p[1].zw, level(0)); + if (p[2].x > 0) c = mix(c, t1.sample(s, (st - p[0].xy) * p[2].zw, level(0)), p[2].x); + if (p[2].y > 0) c.rgb = c.rgb * abs(c.rgb); + return c; } struct D { float d [[depth(any)]]; }; fragment D fsd(V in [[stage_in]], depth2d t [[texture(0)]], sampler s [[sampler(0)]], constant float4 *p [[buffer(0)]]) { float2 st = p[0].xy + (in.p.xy - p[1].xy) * p[0].zw; D o; o.d = t.read(uint2(st)); return o; } diff --git a/dm/gfx/nvn/nvn_mtl.m b/dm/gfx/nvn/nvn_mtl.m index dd1dfc236a..3d0890e0ba 100644 --- a/dm/gfx/nvn/nvn_mtl.m +++ b/dm/gfx/nvn/nvn_mtl.m @@ -14770,33 +14770,34 @@ static int fmt_int_or_ds(MTLPixelFormat f) { default: return 0; } } -enum { COPY_DRAW_SQUARE = 1 << 30 }; // in mirror, beside NVN's MIRROR bits: the colour squared on the way (copy.metal fssq) +enum { COPY_DRAW_UP = 1 << 30 }; // in mirror, beside NVN's MIRROR bits: drawn by copy.metal fsup with g_copy_up's parameters, bilinear +static struct { float mix, square; __unsafe_unretained id t1; } g_copy_up; // set by up_mfx_encode around each such copy static int copy_t2t_draw(id s, id d, unsigned sl, unsigned ss, unsigned dl, unsigned ds, const NVNcopyRegion *sr, const NVNcopyRegion *dr, int linear, int mirror) { - static NSMutableDictionary> *pso; static id vf, ff, fd, fds, fsq; static id samp[2]; static id at; static id dss, dss_s; static int failed; + static NSMutableDictionary> *pso; static id vf, ff, fd, fds, fup; static id samp[2]; static id at; static id dss, dss_s; static int failed; if (failed || sr->width <= 0 || sr->height <= 0 || dr->width <= 0 || dr->height <= 0) return 0; int depth = (s.pixelFormat == MTLPixelFormatDepth32Float || s.pixelFormat == MTLPixelFormatDepth32Float_Stencil8 || s.pixelFormat == MTLPixelFormatDepth16Unorm) && d.pixelFormat == s.pixelFormat, stencil = depth && d.pixelFormat == MTLPixelFormatDepth32Float_Stencil8; if (!depth && (fmt_int_or_ds(s.pixelFormat) || fmt_int_or_ds(d.pixelFormat))) return 0; if (mtl_fmt_block(d.pixelFormat) || !(d.usage & MTLTextureUsageRenderTarget) || s.sampleCount > 1 || d.sampleCount > 1) return 0; if (!pso) { NSError *e = nil; id lib = host_lib("copy", &e); - vf = [lib newFunctionWithName:@"vs"]; ff = [lib newFunctionWithName:@"fs"]; fd = [lib newFunctionWithName:@"fsd"]; fds = [lib newFunctionWithName:@"fsds"]; fsq = [lib newFunctionWithName:@"fssq"]; + vf = [lib newFunctionWithName:@"vs"]; ff = [lib newFunctionWithName:@"fs"]; fd = [lib newFunctionWithName:@"fsd"]; fds = [lib newFunctionWithName:@"fsds"]; fup = [lib newFunctionWithName:@"fsup"]; MTLDepthStencilDescriptor *dd = [MTLDepthStencilDescriptor new]; dd.depthCompareFunction = MTLCompareFunctionAlways; dd.depthWriteEnabled = YES; dss = [g_dev newDepthStencilStateWithDescriptor:dd]; MTLStencilDescriptor *sd_ = [MTLStencilDescriptor new]; sd_.stencilCompareFunction = MTLCompareFunctionAlways; sd_.depthStencilPassOperation = MTLStencilOperationReplace; // Replace writes the [[stencil]] output dd.frontFaceStencil = dd.backFaceStencil = sd_; dss_s = [g_dev newDepthStencilStateWithDescriptor:dd]; MTL4ArgumentTableDescriptor *ad = [MTL4ArgumentTableDescriptor new]; ad.maxTextureBindCount = 2; ad.maxSamplerStateBindCount = 1; ad.maxBufferBindCount = 1; at = rec_table([g_dev newArgumentTableWithDescriptor:ad error:&e]); for (int i = 0; i < 2; i++) { MTLSamplerDescriptor *sd = [MTLSamplerDescriptor new]; sd.minFilter = sd.magFilter = i ? MTLSamplerMinMagFilterLinear : MTLSamplerMinMagFilterNearest; sd.sAddressMode = sd.tAddressMode = MTLSamplerAddressModeClampToEdge; sd.supportArgumentBuffers = YES; samp[i] = [g_dev newSamplerStateWithDescriptor:sd]; } - if (!vf || !ff || !fd || !fds || !fsq || !dss || !dss_s || !at || !samp[0] || !samp[1]) { failed = 1; MLOG("[RED] cannot create the pipelines for format-converting / scaling texture copies: %s", e.localizedDescription.UTF8String); return 0; } + if (!vf || !ff || !fd || !fds || !fup || !dss || !dss_s || !at || !samp[0] || !samp[1]) { failed = 1; MLOG("[RED] cannot create the pipelines for format-converting / scaling texture copies: %s", e.localizedDescription.UTF8String); return 0; } pso = [NSMutableDictionary dictionary]; } // Stencil: read through a stencil-only view of the source (one per layer below); without one it is cleared to 0 id stv0 = stencil && s.pixelFormat == MTLPixelFormatDepth32Float_Stencil8 ? [s newTextureViewWithPixelFormat:MTLPixelFormatX32_Stencil8 textureType:MTLTextureType2D levels:NSMakeRange(sl, 1) slices:NSMakeRange(ss + sr->zoffset, 1)] : nil; int scopy = stv0 != nil; - int square = !depth && !!(mirror & COPY_DRAW_SQUARE); - NSNumber *pk = @(d.pixelFormat * 4 + scopy + 2 * square); + int up = !depth && !!(mirror & COPY_DRAW_UP); + NSNumber *pk = @(d.pixelFormat * 4 + scopy + 2 * up); id ps = pso[pk]; if (!ps) { - MTLRenderPipelineDescriptor *pd = [MTLRenderPipelineDescriptor new]; pd.vertexFunction = vf; pd.fragmentFunction = depth ? (scopy ? fds : fd) : square ? fsq : ff; + MTLRenderPipelineDescriptor *pd = [MTLRenderPipelineDescriptor new]; pd.vertexFunction = vf; pd.fragmentFunction = depth ? (scopy ? fds : fd) : up ? fup : ff; if (depth) { pd.depthAttachmentPixelFormat = d.pixelFormat; if (stencil) pd.stencilAttachmentPixelFormat = d.pixelFormat; } else pd.colorAttachments[0].pixelFormat = d.pixelFormat; NSError *e = nil; ps = [g_dev newRenderPipelineStateWithDescriptor:pd error:&e]; if (!ps) { RED_ONCE(83, "format-converting / scaling texture copy: cannot create the pipeline for destination format %lu: %s", (unsigned long)d.pixelFormat, e.localizedDescription.UTF8String); return 0; } @@ -14808,12 +14809,14 @@ static int copy_t2t_draw(id s, id d, unsigned sl, unsign rp_end(); if (g_blit) { enc_b_end(g_blit); g_blit = nil; } for (int z = 0; z < nz; z++) { id sv = [s newTextureViewWithPixelFormat:s.pixelFormat textureType:MTLTextureType2D levels:NSMakeRange(sl, 1) slices:NSMakeRange(ss + sr->zoffset + z, 1)]; - id pb; size_t po = arena_alloc(32, &pb); // parameters in the frame arena (resident already): a fresh 32-byte buffer per copy was a residency add, a commit and a release (Link's Awakening, about 11 a frame) + id pb; size_t po = arena_alloc(48, &pb); // parameters in the frame arena (resident already): a fresh 32-byte buffer per copy was a residency add, a commit and a release (Link's Awakening, about 11 a frame) if (!sv || !pb) { RED_ONCE(84, "format-converting / scaling texture copy: cannot create the source view / argument block"); return z > 0; } float *p = (float *)((uint8_t *)pb.contents + po); p[0] = sr->xoffset; p[1] = sr->yoffset; p[2] = (float)sr->width / dr->width; p[3] = (float)sr->height / dr->height; if (mirror & NVN_COPY_FLAGS_MIRROR_X_BIT) { p[0] += sr->width; p[2] = -p[2]; } // mirrored: walk the source region from its far edge if (mirror & NVN_COPY_FLAGS_MIRROR_Y_BIT) { p[1] += sr->height; p[3] = -p[3]; } p[4] = dr->xoffset; p[5] = dr->yoffset; p[6] = sw; p[7] = sh; + id t1 = up ? g_copy_up.t1 : nil; + p[8] = t1 ? g_copy_up.mix : 0; p[9] = up ? g_copy_up.square : 0; p[10] = 1.0f / sr->width; p[11] = 1.0f / sr->height; MTL4RenderPassDescriptor *rp = [MTL4RenderPassDescriptor new]; MTLRenderPassAttachmentDescriptor *a = depth ? (MTLRenderPassAttachmentDescriptor *)rp.depthAttachment : (MTLRenderPassAttachmentDescriptor *)rp.colorAttachments[0]; int whole = !dr->xoffset && !dr->yoffset && (unsigned)dr->width == dw && (unsigned)dr->height == dh; // the copy draws every pixel of the level (no blending, no discard): its old content is not loaded (Link's Awakening copies its scene this way every frame) @@ -14823,7 +14826,7 @@ static int copy_t2t_draw(id s, id d, unsigned sl, unsign ser_gate(); id en = enc_r_begin(rp); if (depth) [en setDepthStencilState:scopy ? dss_s : dss]; [en setArgumentTable:at atStages:MTLRenderStageFragment]; - [at setTexture:sv.gpuResourceID atIndex:0]; if (stv) [at setTexture:stv.gpuResourceID atIndex:1]; [at setSamplerState:samp[!!linear && !depth].gpuResourceID atIndex:0]; [at setAddress:pb.gpuAddress + po atIndex:0]; + [at setTexture:sv.gpuResourceID atIndex:0]; if (stv) [at setTexture:stv.gpuResourceID atIndex:1]; if (up) [at setTexture:(t1 ?: sv).gpuResourceID atIndex:1]; [at setSamplerState:samp[(!!linear || up) && !depth].gpuResourceID atIndex:0]; [at setAddress:pb.gpuAddress + po atIndex:0]; [en setViewport:(MTLViewport){dr->xoffset, dr->yoffset, dr->width, dr->height, 0, 1}]; [en setScissorRect:(MTLScissorRect){(NSUInteger)dr->xoffset, (NSUInteger)dr->yoffset, (NSUInteger)dr->width, (NSUInteger)dr->height}]; [en setRenderPipelineState:ps]; [en drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:3]; enc_r_end(en); @@ -16196,6 +16199,17 @@ static int up_mfx_sqrt_named(NvShader *cs) { for (const char *k = DM_GAME_UPSCALER_SQRT_KEYS; *k; k += k[16] == ',' ? 17 : 16) if (!strncmp(k, cs->key, 16)) return 1; return 0; } +// How much of MetalFX's own picture the game's image gets (DM_UPSCALER_SHARPNESS, the launcher's Upscaler sharpness; the profile's upscaler_sharpness when unset; 1 = all of it): the rest +// is the scaler's input scaled bilinearly, blended in by the draw into the game's image. MetalFX's spatial scaler has no strength of its own (its one setting is the colour mode), and +// agl's networks give a softer picture than it does. Not for sRGB-encoded input, whose result is decoded only by that draw +static float g_up_sharp = -1; +static float up_mfx_sharpness(void) { + if (g_up_sharp >= 0) return g_up_sharp; + const char *e = getenv("DM_UPSCALER_SHARPNESS"); float v = e && *e ? strtof(e, NULL) : DM_GAME_UPSCALER_SHARPNESS; + g_up_sharp = v >= 0 && v <= 1 ? v : 1; + if (g_up_sharp < 1) MLOG("upscaler sharpness %.2f: MetalFX's picture blended with %.0f %% of a bilinear one", g_up_sharp, 100 * (1 - g_up_sharp)); + return g_up_sharp; +} static void up_mfx_read_color(NvShader *cs, const NVNshaderData *sd) { if (cs->up_color || !DM_GAME_UPSCALER_N || !up_mfx_on() || !up_mfx_named(cs)) return; if (up_mfx_sqrt_named(cs)) { cs->up_color = UP_COLOR_SQRT; MLOG("upscaler %s: square-root input, linear output (the profile's upscaler_sqrt), MetalFX in HDR mode on the squared input", cs->key); return; } @@ -16268,10 +16282,11 @@ static int up_mfx_encode(NvShader *cs) { fmt_int_or_ds(in.pixelFormat) || fmt_int_or_ds(out.pixelFormat) || mtl_fmt_block(out.pixelFormat) || !up_mfx_rect(v, in.width, in.height, &ir) || !up_mfx_rect(v + 4, out.width, out.height, &orr) || orr.width < ir.width || orr.height < ir.height || (orr.width == ir.width && orr.height == ir.height)) { g_up_mfx_kept++; return 0; } int srgb = cs->up_color == UP_COLOR_SRGB, sq = cs->up_color == UP_COLOR_SQRT; + float soft = srgb ? 0 : 1 - up_mfx_sharpness(); up_mfx_scaler *e = up_mfx_scaler_for((NSUInteger)ir.width, (NSUInteger)ir.height, (NSUInteger)orr.width, (NSUInteger)orr.height, srgb); if (!e) { g_up_mfx_kept++; return 0; } id sc = e->sc; // Straight in and out when MetalFX can take the game's textures as they are; through the scaler's own textures otherwise - int direct = !srgb && !sq && (NSUInteger)ir.width == in.width && (NSUInteger)ir.height == in.height && out.storageMode == MTLStorageModePrivate && + int direct = !srgb && !sq && soft <= 0 && (NSUInteger)ir.width == in.width && (NSUInteger)ir.height == in.height && out.storageMode == MTLStorageModePrivate && in.pixelFormat == MTLPixelFormatRGBA16Float && out.pixelFormat == MTLPixelFormatRGBA16Float && !ir.xoffset && !ir.yoffset && !orr.xoffset && !orr.yoffset && (NSUInteger)orr.width == out.width && (NSUInteger)orr.height == out.height && (in.usage & sc.colorTextureUsage) == sc.colorTextureUsage && (out.usage & sc.outputTextureUsage) == sc.outputTextureUsage; if (!direct && !(out.usage & MTLTextureUsageRenderTarget)) { g_up_mfx_kept++; return 0; } @@ -16280,7 +16295,8 @@ static int up_mfx_encode(NvShader *cs) { ubd_land_compute(); NVNcopyRegion at0 = {0, 0, 0, ir.width, ir.height, 1}, at0o = {0, 0, 0, orr.width, orr.height, 1}; id color = direct ? in : e->in, result = direct ? out : e->out; - if (!direct && !copy_t2t_draw(in, color, 0, 0, 0, 0, &ir, &at0, 0, sq ? COPY_DRAW_SQUARE : 0)) { g_up_mfx_kept++; return 0; } // into RGBA16F at the origin (a nearest draw: same size) + g_copy_up.mix = 0; g_copy_up.square = 1; g_copy_up.t1 = nil; + if (!direct && !copy_t2t_draw(in, color, 0, 0, 0, 0, &ir, &at0, 0, sq ? COPY_DRAW_UP : 0)) { g_up_mfx_kept++; return 0; } // into RGBA16F at the origin (a nearest draw: same size) id ce = [g_cb computeCommandEncoder]; // as dlss_eval: MetalFX opens encoders of its own, which take no barrier [ce barrierAfterQueueStages:MTLStageAll beforeStages:MTLStageBlit visibilityOptions:resource_vis()]; [ce barrierAfterStages:MTLStageBlit beforeQueueStages:MTLStageAll visibilityOptions:resource_vis()]; @@ -16291,13 +16307,15 @@ static int up_mfx_encode(NvShader *cs) { [sc encodeToCommandBuffer:cb]; }); g_last_end_relaxed = 0; // the next pass waits for the scaler whole (enc_r_begin_pass) - if (!direct && !copy_t2t_draw(e->read, out, 0, 0, 0, 0, &at0o, &orr, 0, 0)) { + g_copy_up.mix = soft; g_copy_up.square = 0; g_copy_up.t1 = color; // the softer picture: the scaler's own input, linear as its result is + if (!direct && !copy_t2t_draw(e->read, out, 0, 0, 0, 0, &at0o, &orr, 0, soft > 0 ? COPY_DRAW_UP : 0)) { args_dirty_all(); g_up_mfx_kept++; return 0; // only the scratch image was scaled; the game's shader can still produce the output } args_dirty_all(); if (!g_up_mfx_n++) MLOG("upscaler: first dispatch of %s drawn by MetalFX, %dx%d at %d,%d of %lux%lu (format %lu) → %dx%d at %d,%d of %lux%lu (format %lu)%s", cs->key, ir.width, ir.height, ir.xoffset, ir.yoffset, (unsigned long)in.width, (unsigned long)in.height, (unsigned long)in.pixelFormat, orr.width, orr.height, orr.xoffset, orr.yoffset, (unsigned long)out.width, (unsigned long)out.height, (unsigned long)out.pixelFormat, direct ? ", straight" : srgb ? ", through RGBA16F, decoded from sRGB" : sq ? ", through RGBA16F, squared on the way in" : ", through RGBA16F"); + g_copy_up.t1 = nil; return 1; } // The present pass: src's valid rectangle drawn into dst (the drawable, or a screenshot's texture) as present_encode draws it, through MetalFX first where fx_encode takes it diff --git a/dm/gfx/tests/t_upscaler_mfx.m b/dm/gfx/tests/t_upscaler_mfx.m index 435b6204e3..5b26b05128 100644 --- a/dm/gfx/tests/t_upscaler_mfx.m +++ b/dm/gfx/tests/t_upscaler_mfx.m @@ -136,6 +136,11 @@ int main(void) { @autoreleasepool { CHECK(up_mfx_encode(named), "MetalFX takes square-root-encoded input"); CHECK(mtl_wait(mtl_sub_commit(), 5000000000ull) && !mtl_last_commit_error(), "the square-root submission completes"); CHECK(matches(written, squared, MTLRegionMake2D(0, 0, 1920, 1080), black), "the square-root input comes out squared"); + g_up_sharp = 0.5f; fill(written, MTLClearColorMake(0, 0, 0, 1)); mtl_sub_begin(); // half of a bilinear picture of the same squared input blended in: a flat colour stays itself + CHECK(up_mfx_encode(named), "MetalFX's picture blended with a bilinear one (sharpness 0.5)"); + CHECK(mtl_wait(mtl_sub_commit(), 5000000000ull) && !mtl_last_commit_error(), "the blended submission completes"); + CHECK(matches(written, squared, MTLRegionMake2D(0, 0, 1920, 1080), black), "the blend of two pictures of one colour is that colour, squared once"); + g_up_sharp = 1; named->up_color = UP_COLOR_LINEAR; NvShader *encoded_out = upscaler(key); encoded_out->up_color = UP_COLOR_KEEP; CHECK(!up_mfx_wanted(encoded_out), "a named shader whose output stays encoded is left to its shader"); diff --git a/dm/tools/tests/test_game_profile.py b/dm/tools/tests/test_game_profile.py index a94957df0c..f19f777ae7 100644 --- a/dm/tools/tests/test_game_profile.py +++ b/dm/tools/tests/test_game_profile.py @@ -61,13 +61,13 @@ def macros(shader_map, extra=''): class UpscalerKeys(unittest.TestCase): def test_from_shader_map(self): - sq = '#define DM_GAME_UPSCALER_SQRT_KEYS ""' - none = ['#define DM_GAME_UPSCALER_KEYS ""', '#define DM_GAME_UPSCALER_N 0', sq] # the launcher hides the setting + sq, sh = '#define DM_GAME_UPSCALER_SQRT_KEYS ""', '#define DM_GAME_UPSCALER_SHARPNESS 1.0' + none = ['#define DM_GAME_UPSCALER_KEYS ""', '#define DM_GAME_UPSCALER_N 0', sq, sh] # the launcher hides the setting self.assertEqual(macros(None), none) self.assertEqual(macros(MAP.split('cs_other.bin')[0]), none) - self.assertEqual(macros(MAP), ['#define DM_GAME_UPSCALER_KEYS "7629d857a0a93833,c56db3d5a0f1b5af"', '#define DM_GAME_UPSCALER_N 2', sq]) - self.assertEqual(macros(None, 'upscaler_sqrt\tfa3033d05c0a41d6\n'), # a shader the profile names for its square-root colour is an upscaler too - ['#define DM_GAME_UPSCALER_KEYS "fa3033d05c0a41d6"', '#define DM_GAME_UPSCALER_N 1', '#define DM_GAME_UPSCALER_SQRT_KEYS "fa3033d05c0a41d6"']) + self.assertEqual(macros(MAP), ['#define DM_GAME_UPSCALER_KEYS "7629d857a0a93833,c56db3d5a0f1b5af"', '#define DM_GAME_UPSCALER_N 2', sq, sh]) + self.assertEqual(macros(None, 'upscaler_sqrt\tfa3033d05c0a41d6\nupscaler_sharpness\t0.7\n'), # a shader the profile names for its square-root colour is an upscaler too + ['#define DM_GAME_UPSCALER_KEYS "fa3033d05c0a41d6"', '#define DM_GAME_UPSCALER_N 1', '#define DM_GAME_UPSCALER_SQRT_KEYS "fa3033d05c0a41d6"', '#define DM_GAME_UPSCALER_SHARPNESS 0.7']) if __name__ == '__main__': diff --git a/docs/agents/symptoms.md b/docs/agents/symptoms.md index 0afcd782f1..0c9fae112a 100644 --- a/docs/agents/symptoms.md +++ b/docs/agents/symptoms.md @@ -821,6 +821,7 @@ One line per symptom in the [subsystem references](../reference/): what you see - a Switch 2 game's frame in its heavy scenes is dominated by one compute dispatch between the render-resolution passes and the output-resolution ones: Donkey Kong Bananza's upscaler takes 3.5-5 ms of a 24 ms GPU frame on an M1 in handheld mode (960x540 → 1920x1080, `DM_NVN_GPUPROF`), and the game cannot reach its 60 fps interval there. → [A game's own upscaler drawn by MetalFX](../reference/rendering.md#a-games-own-upscaler-drawn-by-metalfx) - with `DM_UPSCALER=metalfx`, a game's picture is washed out wherever it upscales: lifted shadows and low contrast, the scene only, with the UI drawn after it correct; with `DM_UPSCALER=game`, or at a size the game does not upscale, it is right. → [An upscaler's colour read from its code](../reference/rendering.md#an-upscalers-colour-read-from-its-code) - `DM_UPSCALER=metalfx` changes nothing in a game that upscales with agl's network: the log names the upscale shader with `neither of the colour shapes MetalFX draws; the game's shader runs`, or names no shader at all because the game's upscaler is not in `agl_ndsr.pck`. → [An upscaler with square-root colour](../reference/rendering.md#an-upscaler-with-square-root-colour) +- with `DM_UPSCALER=metalfx` the colours are right but fine texture and edges look harsher than with the game's network: a woven cap's threads, grass blades, the rims of stones. → [MetalFX's picture is crisper than a game's own upscaler](../reference/rendering.md#metalfxs-picture-is-crisper-than-a-games-own-upscaler) - tall blue / textured panels that flickered frame to frame: the fountain's vertex shader pushed the water surface to the top of the screen. → [Metal 4 does not order operations inside one encoder](../reference/rendering.md#metal-4-does-not-order-operations-inside-one-encoder) - a buffer copy, inline update or fill completes, but a following copy through another Metal buffer over the same allocation reads stale bytes. → [Buffer copies through aliased Metal addresses](../reference/rendering.md#buffer-copies-through-aliased-metal-addresses) - many render passes with one draw each and high submit time: Metroid Dread's cutscene frames had about 180 passes with one draw each (about 5 ms of submit time per frame on the Mac). → [Inline uniform updates inside a render pass](../reference/rendering.md#inline-uniform-updates-inside-a-render-pass) diff --git a/docs/env_switches.tsv b/docs/env_switches.tsv index 7e4d7d14f4..0f260fa152 100644 --- a/docs/env_switches.tsv +++ b/docs/env_switches.tsv @@ -393,6 +393,7 @@ DM_UDC_SELFTEST diagnostic `=1` runs a start-up check of DontCare load and store DM_UE_SHADERS path Folder of the Unreal shader code archives decoded by `uepak.py`, for `shader_split.py` (default `ue_shaders` beside the RomFS); needed when the RomFS has Unreal paks DM_UE_TEXTURE_POOL switch Megabytes Unreal's texture streaming pool (`r.Streaming.PoolSize`) is raised to, overriding the profile's `ue_texture_pool`; `0` keeps the game's own pool. On iOS only this switch turns it on (`Documents/env.txt`) DM_UPSCALER player `metalfx` draws the game's own upscaler shaders (agl's NDSR network, found by the shader map) with MetalFX's spatial scaler; `game` (the default) runs them +DM_UPSCALER_SHARPNESS player `=<0.0-1.0>`: with `DM_UPSCALER=metalfx`, the share of MetalFX's own picture in the game's image, the rest a bilinear picture of the same input (softer); the profile's `upscaler_sharpness` when unset, 1.0 without one DM_VK_CAPTURE path `=` (Mac) writes every SPIR-V module the game creates to `/.spv`; the NSP script's Vulkan shader capture stage sets it DM_VK_LOG diagnostic `=1` logs Vulkan object creation in the Vulkan-on-Metal layer DM_VSYNC_HZ switch `=`: rate at which a host thread fires the vi vsync event when the display link does not drive it (default 60) diff --git a/docs/reference/rendering.md b/docs/reference/rendering.md index 251c0d9dfb..1c4a81af34 100644 --- a/docs/reference/rendering.md +++ b/docs/reference/rendering.md @@ -1299,10 +1299,19 @@ The replay on Metal (`dm/gfx/nvn/nvn_mtl.m`) and the Vulkan and OpenGL layers. E - **Symptom:** `DM_UPSCALER=metalfx` changes nothing in a game that upscales with agl's network: the log names the upscale shader with `neither of the colour shapes MetalFX draws; the game's shader runs`, or names no shader at all because the game's upscaler is not in `agl_ndsr.pck`. - **Cause:** two things. The shader map's pattern takes only `agl_ndsr.pck`; Ocarina of Time upscales in play with four dispatches of `agl_ndsr_vr.pck` and `agl_ndsr_vr_light.pck`, of which the last alone is the upscale (one texture in at slot 0, one image out at slot 1, the rectangles in bank 24: the shape `up_mfx_encode` takes) and the three before it are anti-aliasing. And that step's colour has no sRGB curve for [the classifier](#an-upscalers-colour-read-from-its-code) to find: the dispatch before it stores each channel's square root, packed as RGB9E5, and the step writes `v * |v|` at its end, linear. -- **Fix:** the profile key `upscaler_sqrt` names such shaders (`dm/games/game.py`: they join `DM_GAME_UPSCALER_KEYS` and are listed in `DM_GAME_UPSCALER_SQRT_KEYS`), read from their translated code, since no constant marks the shape. `up_mfx_read_color` gives them `UP_COLOR_SQRT`: the draw of the input rectangle into the scaler's RGBA16F texture squares the colour (`copy.metal` `fssq`, `COPY_DRAW_SQUARE`), MetalFX runs in HDR mode, and its result is drawn into the game's image as it is; never straight, as the input must be converted. A per-game key because the shape was read by a person, not derived: a general reader of the translated code's last multiply would replace it. -- **Check:** `make -C dm/gfx/tests GAME=0400486024DEC000 check_t_upscaler_mfx` (a square-root input of 0.5, 0.25, 0.75 comes out 0.25, 0.0625, 0.5625); `test_game_profile.py` (the key's macros, with and without a shader map). +- **Fix:** the profile key `upscaler_sqrt` names such shaders (`dm/games/game.py`: they join `DM_GAME_UPSCALER_KEYS` and are listed in `DM_GAME_UPSCALER_SQRT_KEYS`), read from their translated code, since no constant marks the shape. `up_mfx_read_color` gives them `UP_COLOR_SQRT`: the draw of the input rectangle into the scaler's RGBA16F texture squares the colour (`copy.metal` `fsup`, `COPY_DRAW_UP`), MetalFX runs in HDR mode, and its result is drawn into the game's image; never straight, as the input must be converted. A per-game key because the shape was read by a person, not derived: a general reader of the translated code's last multiply would replace it. +- **The two packs' variants** (Ocarina of Time v131072, the 30 compute shaders with the upscale shape, by the chain of their last image store): `agl_ndsr_vr_light.pck` has 15, of which 8 store `v * |v|` (named in the profile) and 7 multiply by 12.92 after it, an sRGB encode (their shader runs, as every variant with encoded output); `agl_ndsr_vr.pck` has 15 that take their own square root of linear input, 5 storing `v * |v|`, 5 that clamped and 5 encoded. Open: the linear ones of `agl_ndsr_vr.pck` are not named anywhere (the pattern would take them as linear in and out), since no run has shown one in use; only `fa3033d05c0a41d6` of the eight has been drawn and compared. +- **Check:** `make -C dm/gfx/tests GAME=0400486024DEC000 check_t_upscaler_mfx` (a square-root input of 0.5, 0.25, 0.75 comes out 0.25, 0.0625, 0.5625, also under a sharpness of 0.5); `test_game_profile.py` (the key's macros, with and without a shader map). - **Found on:** [The Legend of Zelda: Ocarina of Time](../../dm/games/0400486024DEC000/README.md#frame-rate): `fa3033d05c0a41d6`, 1536x864 → 2048x1152, 1.12 ms of GPU a frame on an M4 Pro, about 0.1 ms through MetalFX; the end shots' mean colours agree with the game's upscaler within 1 of 255. **Open:** the picture in motion is not judged yet. +### MetalFX's picture is crisper than a game's own upscaler + +- **Symptom:** with `DM_UPSCALER=metalfx` the colours are right but fine texture and edges look harsher than with the game's network: a woven cap's threads, grass blades, the rims of stones. +- **Cause:** MetalFX's spatial scaler sharpens as it scales and has no strength setting: its descriptor takes sizes, formats and a colour mode (perceptual, linear, HDR) and nothing else (the macOS 27 SDK's `MTLFXSpatialScaler.h`). agl's networks give a softer picture. +- **Fix:** `up_mfx_sharpness` (`DM_UPSCALER_SHARPNESS`, the launcher's **Upscaler sharpness** under Game's upscaler; the profile's `upscaler_sharpness` when unset, 1.0 without one): the draw of MetalFX's result into the game's image blends it with the scaler's own input sampled bilinearly (`copy.metal` `fsup`), by 1 - sharpness. Not for sRGB-encoded input, whose result only that draw decodes. Tried on Ocarina of Time in 2x crops of the same spot against the game's network, one run each: a blend of 30 % closest (the player's pick, the profile's 0.7), 60 % softer than the network; the input blurred before the scaler instead (each texel mixed half and half with its four neighbours) close too, fully blurred too soft; perceptual mode on the square roots, squared after the scaler, hardly different from HDR mode on the squared input, with or without the blend. None changed the GPU time (compute 2.77-2.81 ms a frame in all seven). +- **Check:** `check_t_upscaler_mfx` as above; `test_settings.py` (the row is offered with the game's upscaler, starting from the profile's value). +- **Found on:** [The Legend of Zelda: Ocarina of Time](../../dm/games/0400486024DEC000/README.md#frame-rate). **Open:** stills of one night scene; not judged in motion, and no other game's value was tried (theirs stay 1.0). + ### Metal 4 does not order operations inside one encoder - **Symptom:** tall blue / textured panels that flickered frame to frame: the fountain's vertex shader pushed the water surface to the top of the screen. diff --git a/docs/settings.md b/docs/settings.md index 95ef082f48..947bb4523d 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -3,7 +3,7 @@ The screen before the game starts shows the settings the game supports. A setting that only matters while another is on appears under it and disappears when that one is turned off. Tap **Play** to start; the app remembers your choices for the next start. All settings are on one scrollable screen, under **Game**, **Controls** and **Performance**. Tap a menu or move a slider there; no settings page needs opening. **Upscaling** is in **Game**, beside **Console mode**. A Mac app shows the same settings in a [window of its own](#the-mac-settings-window). - [Tested defaults](#tested-defaults) -- [Game](#game): [Frame rate](#frame-rate), [Render scale](#render-scale), [Console mode](#console-mode), [Upscaling](#upscaling), [Game's upscaler](#games-upscaler), [Game language](#game-language), [Picture](#picture), [Frame-rate counter](#frame-rate-counter) +- [Game](#game): [Frame rate](#frame-rate), [Render scale](#render-scale), [Console mode](#console-mode), [Upscaling](#upscaling), [Game's upscaler](#games-upscaler), [Upscaler sharpness](#upscaler-sharpness), [Game language](#game-language), [Picture](#picture), [Frame-rate counter](#frame-rate-counter) - [Controls](#controls): [Touch feedback](#touch-feedback), [Game rumble](#game-rumble), [Rumble strength](#rumble-strength), [Rumble cutoff](#rumble-cutoff), [Press a button to join](#press-a-button-to-join) - [Performance](#performance): [Frame buffering](#frame-buffering), [Metal 4 pipelines](#metal-4-pipelines), [Pipeline prebuild](#pipeline-prebuild), [Encode thread](#encode-thread), [Hold frames when late](#hold-frames-when-late), [Performance HUD](#performance-hud), [Reset to tested defaults](#reset-to-tested-defaults) - [The Mac settings window](#the-mac-settings-window): [Display mode](#display-mode) @@ -77,6 +77,13 @@ flowchart LR - **Cost:** none in GPU time (it saves 2.5-4 ms a frame in that case); the picture is Apple's scaler's rather than the game's. Frames where the game renders at its output size are not upscaled and keep the game's shader either way. Devices without MetalFX keep the game's upscaler. - **Not the same as [Upscaling](#upscaling):** that one scales the finished picture, HUD included, to the screen; this one replaces the game's own step inside the frame. Both can be on. +### Upscaler sharpness + +Shown under [Game's upscaler](#games-upscaler) while it is on **MetalFX**. MetalFX's spatial scaler gives a crisper picture than the networks games upscale with, and has no strength setting of its own, so the app blends its picture with a plain smooth one of the same input: 100 % is MetalFX's picture alone, lower values are softer. The starting value is the one tested against that game's own upscaler (70 % for The Legend of Zelda: Ocarina of Time; 100 % where none was tested). + +- **When to change it:** by taste. Lower it when edges and fine textures look harsher than with the game's own upscaler, raise it when the picture looks soft. +- **Cost:** none measurable; it changes one draw the swap already makes. + ### Game language The game's own languages. The app was packed with one of them; pick another here without repacking. A game packed from a dump or a NUP may come without its list: every language is offered then, with a note, and a game that lacks the one you pick may show another. @@ -281,6 +288,7 @@ A line like `DM_PRESENT_SNAP=0` in `Documents/env.txt` (in the Files app, under | Press a button to join | `DM_PAD_PRESS_TO_JOIN` | `1`, `0` | | Upscaling | `DM_PRESENT_UPSCALE` | `off`, `metalfx` | | Game's upscaler | `DM_UPSCALER` | `game`, `metalfx` | +| Upscaler sharpness | `DM_UPSCALER_SHARPNESS` | `0.0` to `1.0` | | Frame buffering | `DM_PRESENT_SNAP` | `1`, `0` | | Metal 4 pipelines | `DM_PSO_MTL4` | `1`, `0` | | Pipeline prebuild | `DM_PSO_PREBUILD` | `1`, `0` | From 3e0bc1cc0741f7db51e22980952649bdaa8cfbd3 Mon Sep 17 00:00:00 2001 From: seedds Date: Wed, 11 Nov 2026 21:34:10 +0100 Subject: [PATCH 4/9] 0400486024DEC000's notes: six of the eight named upscale variants seen in play, by input size, and drawn by MetalFX; no other variant dispatched in eight situations --- dm/games/0400486024DEC000/README.md | 2 +- docs/reference/rendering.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dm/games/0400486024DEC000/README.md b/dm/games/0400486024DEC000/README.md index 8fcd8766db..4be9609105 100644 --- a/dm/games/0400486024DEC000/README.md +++ b/dm/games/0400486024DEC000/README.md @@ -31,7 +31,7 @@ One [NUP container](../../../docs/development.md#the-packing-script-in-detail) o Measured on an M6 Mac with 16 GB, headless, on one walk with camera turns (`walk.sh` in the work folder). - Current on an M5 Pro MacBook Pro (16-inch, 20-core GPU, 24 GB, macOS 27.2, 2026-11-09), `kokiri-deck` from a deck autosave with the full library of chain `dedc9391c30b`, at full resolution (Dynamic resolution off), two runs a variant: in a window a locked 60 fps (59.8–60.0, frame p99 16.8–19.4 ms, GPU p50 10.2 ms). In fullscreen on its ProMotion screen it held 40 fps (39.6–40.1) at the same GPU time, with every present a 120 Hz refresh late and every other link tick skipped: [The 60 Hz link at 40 fps in fullscreen on a ProMotion screen](../../../docs/reference/frame-rate.md#the-60-hz-link-at-40-fps-in-fullscreen-on-a-promotion-screen). With the 120 Hz link there, 58.8–59.6 fps, frame p99 25.3 ms; with the screen held at its fixed 60 Hz mode, 60.0 fps, frame p99 16.9 ms, which fullscreen now switches to. MetalFX present upscaling to the 3456-wide fullscreen cost no measurable GPU time (59.4–59.7 fps). Resident about 10 GB, GPU allocated 11.2–11.4 GB; the prepare step took 7.3 s on a first start. -- Current: `DM_UPSCALER=metalfx` draws the upscale step with MetalFX (M4 Pro, 20-core GPU, 2026-11-11, Kakariko Village from a save, chain `b8856126d238`, `DM_FPS_HOLD=off`). In play the game anti-aliases and upscales with four dispatches of `agl_ndsr_vr.pck` and `agl_ndsr_vr_light.pck` (`f8965bffbdf51d78`, `d17eefaff4f0bc23`, `a335c1db6270c355`, `fa3033d05c0a41d6`), one encoder of 1.25 ms; only the last is the upscale (texture slot 0 at 1536x864 to image slot 1 at 2048x1152). With its dispatches dropped (`DM_NVN_SKIP_CS`, the picture stale) the frame's compute fell 3.91-3.93 → 2.78-2.80 ms: 1.12 ms. Its colour, from its translated code: the dispatch before it stores each channel's square root, packed as RGB9E5 bits in a single-channel image; the step samples that, adds its network's residual and writes `v * |v|`, linear, with no sRGB curve, so the host's classifier kept the shader. The profile names it ([`upscaler_sqrt`](game.tsv)): [An upscaler with square-root colour](../../../docs/reference/rendering.md#an-upscaler-with-square-root-colour). Two runs a side in turn: headless, compute 3.91 and 3.95 → 2.80 and 2.77 ms, the frame's GPU work 14.84 and 15.61 → 13.64 and 13.48 ms (median sum of encoder times over 37-38 profiled frames); in a window, compute 3.90 and 3.89 → 2.76 and 2.79 ms, summed GPU p50 15.59 and 15.13 → 14.63 and 14.46 ms, 60.0 fps on both. The end shots' mean colours agree (15.9 / 25.3 / 29.4 against 15.8 / 24.8 / 28.4 of 255; two shots of the game's own differ as much, by the moving grass); MetalFX's picture is crisper, the network's softer, which the player preferred: the profile's `upscaler_sharpness 0.7` blends 30 % of a bilinear picture in, the closest of seven variants in 2x crops ([MetalFX's picture is crisper than a game's own upscaler](../../../docs/reference/rendering.md#metalfxs-picture-is-crisper-than-a-games-own-upscaler)), at the same GPU time. The profile names the pack's eight variants that end as this one does; only this one was drawn and compared (`d99fbeeb53d38847`, the upscaler of an earlier Kokiri Forest run, is among the seven others). The three anti-aliasing dispatches still run. Open: not judged in motion, nor with a render scale, the dynamic resolution or a cutscene. +- Current: `DM_UPSCALER=metalfx` draws the upscale step with MetalFX (M4 Pro, 20-core GPU, 2026-11-11, Kakariko Village from a save, chain `b8856126d238`, `DM_FPS_HOLD=off`). In play the game anti-aliases and upscales with four dispatches of `agl_ndsr_vr.pck` and `agl_ndsr_vr_light.pck` (`f8965bffbdf51d78`, `d17eefaff4f0bc23`, `a335c1db6270c355`, `fa3033d05c0a41d6`), one encoder of 1.25 ms; only the last is the upscale (texture slot 0 at 1536x864 to image slot 1 at 2048x1152). With its dispatches dropped (`DM_NVN_SKIP_CS`, the picture stale) the frame's compute fell 3.91-3.93 → 2.78-2.80 ms: 1.12 ms. Its colour, from its translated code: the dispatch before it stores each channel's square root, packed as RGB9E5 bits in a single-channel image; the step samples that, adds its network's residual and writes `v * |v|`, linear, with no sRGB curve, so the host's classifier kept the shader. The profile names it ([`upscaler_sqrt`](game.tsv)): [An upscaler with square-root colour](../../../docs/reference/rendering.md#an-upscaler-with-square-root-colour). Two runs a side in turn: headless, compute 3.91 and 3.95 → 2.80 and 2.77 ms, the frame's GPU work 14.84 and 15.61 → 13.64 and 13.48 ms (median sum of encoder times over 37-38 profiled frames); in a window, compute 3.90 and 3.89 → 2.76 and 2.79 ms, summed GPU p50 15.59 and 15.13 → 14.63 and 14.46 ms, 60.0 fps on both. The end shots' mean colours agree (15.9 / 25.3 / 29.4 against 15.8 / 24.8 / 28.4 of 255; two shots of the game's own differ as much, by the moving grass); MetalFX's picture is crisper, the network's softer, which the player preferred: the profile's `upscaler_sharpness 0.7` blends 30 % of a bilinear picture in, the closest of seven variants in 2x crops ([MetalFX's picture is crisper than a game's own upscaler](../../../docs/reference/rendering.md#metalfxs-picture-is-crisper-than-a-games-own-upscaler)), at the same GPU time. The profile names the pack's eight variants that end as this one does; which one runs goes by the input size: `fa3033d05c0a41d6` at full size docked (also in the castle town, a dungeon and Kokiri Forest) and handheld (1440x810 → 1920x1080), `d99fbeeb53d38847` at render scale 0.5 (781x439), and with the dynamic resolution on `aad7ea7e5c02f41d`, `b106afba1a18faa9`, `04135bb812a0d66e` and `7c511cc23ff85d14` on the way down to its floor (nine sizes from 1536x864 to 763x429 in one run). All six were drawn by MetalFX with no red line, the end shots' mean colours within 1 of 255 of the game's upscaler in the same mode; `8df148d58b7cbe8e` and `c8b20adc9d4978e0` were not seen. None of the pack's seven encoding variants, of `agl_ndsr_vr.pck`'s fifteen or of `agl_ndsr.pck`'s was dispatched in those eight situations (pass dumps; the title uses none). What MetalFX saves grows with the upscale: compute 4.01 → 2.58 ms at render scale 0.5 and 3.25 → 2.27 ms handheld (one run a side). The three anti-aliasing dispatches still run. Open: not judged in motion, nor with a render scale, the dynamic resolution or a cutscene. - Current: the title and menu, a movie behind them, are a locked 60 fps (p50 16.67 ms, GPU span 15.6 ms). - Game speed: the game steps its time by a frame ratio, its base rate (60, or 30) / its int frame rate, whatever the frame took (`main+0x6e6cc4` on v131072), so frames longer than a vsync played in slow motion (seen by the user on a slower Mac; [issue 1375](https://github.com/seedds/nxnative/issues/1375)'s M1 Max ran the title at about 28 fps). With `fps_target 60` the ratio is the host's frame step × base / 60 ([A fixed frame step plays in slow motion on a slow Mac](../../../docs/reference/frame-rate.md#a-fixed-frame-step-plays-in-slow-motion-on-a-slow-mac)), a stub placed by its function's fingerprint ([`patches.py`](patches.py)). Mac runs on the M6 (2026-11-08, a test host on the libraries of chain `3ee493b2c3a4`): the user's play sessions held 60 fps with 2- and 3-vsync holds while areas loaded, game time 1.000 of real time by the host's count. That count is the stub's own; no run on a Mac slow enough to show the slow motion before and after. - Sizes since the host answers graphics scale tier 0 docked ([The graphics scale does not follow the operation mode](../../../docs/reference/services.md#the-graphics-scale-does-not-follow-the-operation-mode)): the game's output is 2560x1440 and its top internal resolution 1920x1080. Every size and time in this note was taken with tier 1, at 0.8 of that (2048x1152 and 1536x864, and the dynamic resolution's levels with them), and has not been taken again. With tier 0 on `main` (two contributors' runs, only that answer changed): M4 Pro (20-core GPU), docked, fullscreen, GPU p50 15.0 ms against 12.9-14.0 ms, 59.0 fps against 55.8-59.8; M5 Pro (20-core GPU, 48 GB), Dynamic resolution off, 24 minutes of play, GPU p50 12.7 ms, 58.4 fps. Open: a slower Mac, and the routes of [`bench.tsv`](bench.tsv), with tier 0. diff --git a/docs/reference/rendering.md b/docs/reference/rendering.md index 1c4a81af34..835213250d 100644 --- a/docs/reference/rendering.md +++ b/docs/reference/rendering.md @@ -1300,7 +1300,7 @@ The replay on Metal (`dm/gfx/nvn/nvn_mtl.m`) and the Vulkan and OpenGL layers. E - **Symptom:** `DM_UPSCALER=metalfx` changes nothing in a game that upscales with agl's network: the log names the upscale shader with `neither of the colour shapes MetalFX draws; the game's shader runs`, or names no shader at all because the game's upscaler is not in `agl_ndsr.pck`. - **Cause:** two things. The shader map's pattern takes only `agl_ndsr.pck`; Ocarina of Time upscales in play with four dispatches of `agl_ndsr_vr.pck` and `agl_ndsr_vr_light.pck`, of which the last alone is the upscale (one texture in at slot 0, one image out at slot 1, the rectangles in bank 24: the shape `up_mfx_encode` takes) and the three before it are anti-aliasing. And that step's colour has no sRGB curve for [the classifier](#an-upscalers-colour-read-from-its-code) to find: the dispatch before it stores each channel's square root, packed as RGB9E5, and the step writes `v * |v|` at its end, linear. - **Fix:** the profile key `upscaler_sqrt` names such shaders (`dm/games/game.py`: they join `DM_GAME_UPSCALER_KEYS` and are listed in `DM_GAME_UPSCALER_SQRT_KEYS`), read from their translated code, since no constant marks the shape. `up_mfx_read_color` gives them `UP_COLOR_SQRT`: the draw of the input rectangle into the scaler's RGBA16F texture squares the colour (`copy.metal` `fsup`, `COPY_DRAW_UP`), MetalFX runs in HDR mode, and its result is drawn into the game's image; never straight, as the input must be converted. A per-game key because the shape was read by a person, not derived: a general reader of the translated code's last multiply would replace it. -- **The two packs' variants** (Ocarina of Time v131072, the 30 compute shaders with the upscale shape, by the chain of their last image store): `agl_ndsr_vr_light.pck` has 15, of which 8 store `v * |v|` (named in the profile) and 7 multiply by 12.92 after it, an sRGB encode (their shader runs, as every variant with encoded output); `agl_ndsr_vr.pck` has 15 that take their own square root of linear input, 5 storing `v * |v|`, 5 that clamped and 5 encoded. Open: the linear ones of `agl_ndsr_vr.pck` are not named anywhere (the pattern would take them as linear in and out), since no run has shown one in use; only `fa3033d05c0a41d6` of the eight has been drawn and compared. +- **The two packs' variants** (Ocarina of Time v131072, the 30 compute shaders with the upscale shape, by the chain of their last image store): `agl_ndsr_vr_light.pck` has 15, of which 8 store `v * |v|` (named in the profile) and 7 multiply by 12.92 after it, an sRGB encode (their shader runs, as every variant with encoded output); `agl_ndsr_vr.pck` has 15 that take their own square root of linear input, 5 storing `v * |v|`, 5 that clamped and 5 encoded. Open: the linear ones of `agl_ndsr_vr.pck` are not named anywhere (the pattern would take them as linear in and out), since no run has shown one in use: pass dumps of eight situations (four areas docked, handheld, render scale 0.5, the dynamic resolution, the title) had only variants of the eight, six of them by input size, each then drawn by MetalFX (down to 763x429 → 2048x1152) with the picture's mean colour within 1 of 255 of the game's. - **Check:** `make -C dm/gfx/tests GAME=0400486024DEC000 check_t_upscaler_mfx` (a square-root input of 0.5, 0.25, 0.75 comes out 0.25, 0.0625, 0.5625, also under a sharpness of 0.5); `test_game_profile.py` (the key's macros, with and without a shader map). - **Found on:** [The Legend of Zelda: Ocarina of Time](../../dm/games/0400486024DEC000/README.md#frame-rate): `fa3033d05c0a41d6`, 1536x864 → 2048x1152, 1.12 ms of GPU a frame on an M4 Pro, about 0.1 ms through MetalFX; the end shots' mean colours agree with the game's upscaler within 1 of 255. **Open:** the picture in motion is not judged yet. From 2df12bd1758029dcb618657ca7d0427153d63e35 Mon Sep 17 00:00:00 2001 From: seedds Date: Wed, 11 Nov 2026 21:43:17 +0100 Subject: [PATCH 5/9] The sharpness setting is named after its parent in the launcher: Game's upscaler sharpness --- dm/games/0400486024DEC000/README.md | 2 +- dm/games/README.md | 2 +- dm/games/settings.tsv | 2 +- dm/gfx/nvn/nvn_mtl.m | 2 +- docs/reference/rendering.md | 2 +- docs/settings.md | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dm/games/0400486024DEC000/README.md b/dm/games/0400486024DEC000/README.md index 4be9609105..c50a4b732e 100644 --- a/dm/games/0400486024DEC000/README.md +++ b/dm/games/0400486024DEC000/README.md @@ -128,7 +128,7 @@ Its 37 kernels that correct samples, NDSR's network among them, gave about 0.25 ## Found on this game - [An upscaler with square-root colour](../../../docs/reference/rendering.md#an-upscaler-with-square-root-colour): `DM_UPSCALER=metalfx` replaced no dispatch; with the profile's `upscaler_sqrt` the upscale step's 1.12 ms of GPU goes to MetalFX (Kakariko Village, M4 Pro). -- [MetalFX's picture is crisper than a game's own upscaler](../../../docs/reference/rendering.md#metalfxs-picture-is-crisper-than-a-games-own-upscaler): the launcher's Upscaler sharpness, 70 % here by the player's eye. +- [MetalFX's picture is crisper than a game's own upscaler](../../../docs/reference/rendering.md#metalfxs-picture-is-crisper-than-a-games-own-upscaler): the launcher's Game's upscaler sharpness, 70 % here by the player's eye. - [A shader build's Python finishes one shader at a time](../../../docs/reference/shaders.md#a-shader-builds-python-finishes-one-shader-at-a-time): resolved, the whole Mac map's cold build took 58 minutes on an M3 Max with 4 to 5 of its 16 cores busy; 30 minutes with worker processes. - [Reads past a uniform binding's size](../../../docs/reference/shaders.md#reads-past-a-uniform-bindings-size): open ([GitHub issue #2218](https://github.com/seedds/nxnative/issues/2218)), long white streaks from the horses at Lon Lon Ranch and from Link's slingshot, and flickering parts of the slingshot and of the ReDeads' faces, reported with the uniform read bound on. The bound now takes the size the driver gives the GPU, rounded up to 256 bytes; the game binds sizes that are not multiples of 256. Not reproduced here before or after (the slingshot at Link's house, the ReDeads of adult Castle Town), so the reporters' retest decides. - [The blend helper's output reference on iOS](../../../docs/reference/shaders.md#the-blend-helpers-output-reference-on-ios): resolved, 22,296 of 42,848 Mac shaders failed to compile on macOS 26 ([GitHub issue #2272](https://github.com/seedds/nxnative/issues/2272)); the Mac compile now names the host's minimum macOS. diff --git a/dm/games/README.md b/dm/games/README.md index dee456d914..dff166e69d 100644 --- a/dm/games/README.md +++ b/dm/games/README.md @@ -253,7 +253,7 @@ One row per profile; the notes are each profile's `README.md`. | `tex_dyn_slots` | `on` / `off` (the default): a shader whose buffer-texture handle the translator could not trace to one slot gets Ryujinx's fallback arrays (`__cb0_40000000`: 32 buffer textures and samplers indexed by the handle's texture / sampler ID; any other sampled texture indexes the whole pool without a key, [Texture handles from game memory index the pools](../../docs/reference/shaders.md#texture-handles-from-game-memory-index-the-pools)). With the key the replay puts slot indices in that shader's c2 texture and sampler tables (texture slot r → r, sampler slot r → r << 20) and binds slot k at element k, so the shader's own arithmetic picks the binding (`NVN_RB_TEXD` / `NVN_RB_SAMPD` in `nvn_mtl.m`). Right only where the handle comes from c2's tables: Octopath Traveler 0's 61 such vertex shaders read separate texture slots 0-10 and samplers 0-4 there and nothing else (FINAL FANTASY RESONANCE's 2,077 the texture table words 0x168-0x1c0 and sampler table words 0x568-0x590). Off, such shaders do not load. Not part of the shader chain version | | `draw_barriers` | `full` (the default) / `game`: how draw passes are ordered. `full`: each waits for all earlier GPU work. `game`: by the game's own barriers (NVN asks for one before results of rendering are read), so a pass's vertex work overlaps the previous pass's fragments. Ocarina of Time's GPU time 25 → 18 ms, about 180 full barriers a frame halved. Only vertex-stage reads of the previous pass's output without a barrier change order. `DM_NVN_BARRIER` overrides it | | `upscaler_sqrt` | comma-separated 16-digit keys of upscaler compute shaders, outside `agl_ndsr.pck`, that sample square-root-encoded colour and write it squared (linear): `DM_UPSCALER=metalfx` squares their input into MetalFX's and runs it in HDR mode ([An upscaler with square-root colour](../../docs/reference/rendering.md#an-upscaler-with-square-root-colour)). Read the shape from the shader's translated code before naming one | -| `upscaler_sharpness` | `0.0` to `1.0` (the default): with `DM_UPSCALER=metalfx`, the share of MetalFX's own picture in the game's image, the rest the scaler's input scaled bilinearly; the launcher's **Upscaler sharpness** starts from it (`DM_UPSCALER_SHARPNESS`). Lower it for a game whose own network is softer than MetalFX, by eye against the game's picture | +| `upscaler_sharpness` | `0.0` to `1.0` (the default): with `DM_UPSCALER=metalfx`, the share of MetalFX's own picture in the game's image, the rest the scaler's input scaled bilinearly; the launcher's **Game's upscaler sharpness** starts from it (`DM_UPSCALER_SHARPNESS`). Lower it for a game whose own network is softer than MetalFX, by eye against the game's picture | | `vconv_range` | `on` / `off` (the default): vertex attributes the replay converts on the CPU (integer shader inputs fed float-type formats and the like, `vconv_bind` in `nvn_mtl.m`) are converted only over the vertices each draw reads (first / count, or the smallest and largest index of a direct indexed draw, kept while the index pages stay unwritten) instead of the whole bound range. Age of Calamity binds whole mesh pools: in a battle that was millions of vertices a frame. Indirect draws, per-instance streams and indices with a GPU write queued on them keep the whole range. `DM_NVN_VCONV_CHECK=1` scans the indices again at every reuse and reports differences; `DM_NVN_VCONV_RANGE=0` / `1` overrides it | | `shader_key_skip` | `+` (none by default): control-section bytes every shader key hashes as zeros, in `shader_split.py` (`key_ctl`) and the host (`hle_nvn.c` `keys_hash`), for an engine that shares one control section among shaders that differ only there. Batman: Arkham Knight: `0x770+0x78`, GLSLC's compile information (+0x778..+0x791 and +0x7d0..+0x7e7); without it the game asks for shaders the map does not have, each the control section of one record with the code area of another. It must stay clear of `+0x700..+0x75f`, which the tools read; it is a shader chain input, as every `shader_*` key | | `shader_groups`, `shader_note` | `shader_split.py` source groups and the map header note (Engage: `engine` for `data.unity3d` and `Resources/`, `assets` for the Addressables bundles). Both are shader inputs, the note too: changing either one, even by a word, builds every shader of the game again. Keep them as they are when a provisional profile moves into `dm/games` | diff --git a/dm/games/settings.tsv b/dm/games/settings.tsv index ab5760525e..08accd8d39 100644 --- a/dm/games/settings.tsv +++ b/dm/games/settings.tsv @@ -20,7 +20,7 @@ DM_RENDER_SCALE range 0.5:1.0:0.05 1.0 fixups:RENDER_SCALE Game Render scale le DM_OPMODE choice 1=Docked,0=Handheld DM_GAME_OPMODE always Game Console mode level=basic DM_PRESENT_UPSCALE choice off=Standard,metalfx=MetalFX off always Game Upscaling Uses more GPU time and can increase heat. DM_UPSCALER choice game=Game's own,metalfx=MetalFX game DM_GAME_UPSCALER_N!=0 Game Game's upscaler -DM_UPSCALER_SHARPNESS range 0.0:1.0:0.05 DM_GAME_UPSCALER_SHARPNESS DM_GAME_UPSCALER_N!=0 Game Upscaler sharpness needs=DM_UPSCALER=metalfx +DM_UPSCALER_SHARPNESS range 0.0:1.0:0.05 DM_GAME_UPSCALER_SHARPNESS DM_GAME_UPSCALER_N!=0 Game Game's upscaler sharpness needs=DM_UPSCALER=metalfx DM_LANG choice languages build:DM_LANG languages>1 Game Game language level=basic DM_FULLSCREEN choice 0=Window,1=Fullscreen 0 platform:mac Game Display mode level=basic DM_PICTURE_FIT choice fit=Fit,fill=Fill fit platform:ios Game Picture Fill crops the game's 16:9 picture to cover the screen. Health bars, subtitles or maps near the edges can be cut off. A full-screen picture without cropping needs a patch for each game. diff --git a/dm/gfx/nvn/nvn_mtl.m b/dm/gfx/nvn/nvn_mtl.m index 3d0890e0ba..0e3834c2c7 100644 --- a/dm/gfx/nvn/nvn_mtl.m +++ b/dm/gfx/nvn/nvn_mtl.m @@ -16199,7 +16199,7 @@ static int up_mfx_sqrt_named(NvShader *cs) { for (const char *k = DM_GAME_UPSCALER_SQRT_KEYS; *k; k += k[16] == ',' ? 17 : 16) if (!strncmp(k, cs->key, 16)) return 1; return 0; } -// How much of MetalFX's own picture the game's image gets (DM_UPSCALER_SHARPNESS, the launcher's Upscaler sharpness; the profile's upscaler_sharpness when unset; 1 = all of it): the rest +// How much of MetalFX's own picture the game's image gets (DM_UPSCALER_SHARPNESS, the launcher's Game's upscaler sharpness; the profile's upscaler_sharpness when unset; 1 = all of it): the rest // is the scaler's input scaled bilinearly, blended in by the draw into the game's image. MetalFX's spatial scaler has no strength of its own (its one setting is the colour mode), and // agl's networks give a softer picture than it does. Not for sRGB-encoded input, whose result is decoded only by that draw static float g_up_sharp = -1; diff --git a/docs/reference/rendering.md b/docs/reference/rendering.md index 835213250d..c97dcf8a0d 100644 --- a/docs/reference/rendering.md +++ b/docs/reference/rendering.md @@ -1308,7 +1308,7 @@ The replay on Metal (`dm/gfx/nvn/nvn_mtl.m`) and the Vulkan and OpenGL layers. E - **Symptom:** with `DM_UPSCALER=metalfx` the colours are right but fine texture and edges look harsher than with the game's network: a woven cap's threads, grass blades, the rims of stones. - **Cause:** MetalFX's spatial scaler sharpens as it scales and has no strength setting: its descriptor takes sizes, formats and a colour mode (perceptual, linear, HDR) and nothing else (the macOS 27 SDK's `MTLFXSpatialScaler.h`). agl's networks give a softer picture. -- **Fix:** `up_mfx_sharpness` (`DM_UPSCALER_SHARPNESS`, the launcher's **Upscaler sharpness** under Game's upscaler; the profile's `upscaler_sharpness` when unset, 1.0 without one): the draw of MetalFX's result into the game's image blends it with the scaler's own input sampled bilinearly (`copy.metal` `fsup`), by 1 - sharpness. Not for sRGB-encoded input, whose result only that draw decodes. Tried on Ocarina of Time in 2x crops of the same spot against the game's network, one run each: a blend of 30 % closest (the player's pick, the profile's 0.7), 60 % softer than the network; the input blurred before the scaler instead (each texel mixed half and half with its four neighbours) close too, fully blurred too soft; perceptual mode on the square roots, squared after the scaler, hardly different from HDR mode on the squared input, with or without the blend. None changed the GPU time (compute 2.77-2.81 ms a frame in all seven). +- **Fix:** `up_mfx_sharpness` (`DM_UPSCALER_SHARPNESS`, the launcher's **Game's upscaler sharpness** under Game's upscaler; the profile's `upscaler_sharpness` when unset, 1.0 without one): the draw of MetalFX's result into the game's image blends it with the scaler's own input sampled bilinearly (`copy.metal` `fsup`), by 1 - sharpness. Not for sRGB-encoded input, whose result only that draw decodes. Tried on Ocarina of Time in 2x crops of the same spot against the game's network, one run each: a blend of 30 % closest (the player's pick, the profile's 0.7), 60 % softer than the network; the input blurred before the scaler instead (each texel mixed half and half with its four neighbours) close too, fully blurred too soft; perceptual mode on the square roots, squared after the scaler, hardly different from HDR mode on the squared input, with or without the blend. None changed the GPU time (compute 2.77-2.81 ms a frame in all seven). - **Check:** `check_t_upscaler_mfx` as above; `test_settings.py` (the row is offered with the game's upscaler, starting from the profile's value). - **Found on:** [The Legend of Zelda: Ocarina of Time](../../dm/games/0400486024DEC000/README.md#frame-rate). **Open:** stills of one night scene; not judged in motion, and no other game's value was tried (theirs stay 1.0). diff --git a/docs/settings.md b/docs/settings.md index 947bb4523d..dd841d3fee 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -3,7 +3,7 @@ The screen before the game starts shows the settings the game supports. A setting that only matters while another is on appears under it and disappears when that one is turned off. Tap **Play** to start; the app remembers your choices for the next start. All settings are on one scrollable screen, under **Game**, **Controls** and **Performance**. Tap a menu or move a slider there; no settings page needs opening. **Upscaling** is in **Game**, beside **Console mode**. A Mac app shows the same settings in a [window of its own](#the-mac-settings-window). - [Tested defaults](#tested-defaults) -- [Game](#game): [Frame rate](#frame-rate), [Render scale](#render-scale), [Console mode](#console-mode), [Upscaling](#upscaling), [Game's upscaler](#games-upscaler), [Upscaler sharpness](#upscaler-sharpness), [Game language](#game-language), [Picture](#picture), [Frame-rate counter](#frame-rate-counter) +- [Game](#game): [Frame rate](#frame-rate), [Render scale](#render-scale), [Console mode](#console-mode), [Upscaling](#upscaling), [Game's upscaler](#games-upscaler), [Game's upscaler sharpness](#games-upscaler-sharpness), [Game language](#game-language), [Picture](#picture), [Frame-rate counter](#frame-rate-counter) - [Controls](#controls): [Touch feedback](#touch-feedback), [Game rumble](#game-rumble), [Rumble strength](#rumble-strength), [Rumble cutoff](#rumble-cutoff), [Press a button to join](#press-a-button-to-join) - [Performance](#performance): [Frame buffering](#frame-buffering), [Metal 4 pipelines](#metal-4-pipelines), [Pipeline prebuild](#pipeline-prebuild), [Encode thread](#encode-thread), [Hold frames when late](#hold-frames-when-late), [Performance HUD](#performance-hud), [Reset to tested defaults](#reset-to-tested-defaults) - [The Mac settings window](#the-mac-settings-window): [Display mode](#display-mode) @@ -77,7 +77,7 @@ flowchart LR - **Cost:** none in GPU time (it saves 2.5-4 ms a frame in that case); the picture is Apple's scaler's rather than the game's. Frames where the game renders at its output size are not upscaled and keep the game's shader either way. Devices without MetalFX keep the game's upscaler. - **Not the same as [Upscaling](#upscaling):** that one scales the finished picture, HUD included, to the screen; this one replaces the game's own step inside the frame. Both can be on. -### Upscaler sharpness +### Game's upscaler sharpness Shown under [Game's upscaler](#games-upscaler) while it is on **MetalFX**. MetalFX's spatial scaler gives a crisper picture than the networks games upscale with, and has no strength setting of its own, so the app blends its picture with a plain smooth one of the same input: 100 % is MetalFX's picture alone, lower values are softer. The starting value is the one tested against that game's own upscaler (70 % for The Legend of Zelda: Ocarina of Time; 100 % where none was tested). @@ -288,7 +288,7 @@ A line like `DM_PRESENT_SNAP=0` in `Documents/env.txt` (in the Files app, under | Press a button to join | `DM_PAD_PRESS_TO_JOIN` | `1`, `0` | | Upscaling | `DM_PRESENT_UPSCALE` | `off`, `metalfx` | | Game's upscaler | `DM_UPSCALER` | `game`, `metalfx` | -| Upscaler sharpness | `DM_UPSCALER_SHARPNESS` | `0.0` to `1.0` | +| Game's upscaler sharpness | `DM_UPSCALER_SHARPNESS` | `0.0` to `1.0` | | Frame buffering | `DM_PRESENT_SNAP` | `1`, `0` | | Metal 4 pipelines | `DM_PSO_MTL4` | `1`, `0` | | Pipeline prebuild | `DM_PSO_PREBUILD` | `1`, `0` | From 06505909a77c107787ef5762e8069339b36119fa Mon Sep 17 00:00:00 2001 From: seedds Date: Tue, 10 Nov 2026 20:06:18 -0800 Subject: [PATCH 6/9] Scratch: frame generation for Ocarina of Time (0400486024DEC000) from the scene before the HUD, its depth and the game's camera (DM_FRAMEGEN, off by default) A prototype for evaluation, not for merging as it is. Between two frames the game draws, the host shows one made by MetalFX's frame interpolator (Metal 4) from the 1440x810 scene before the HUD, the scene's depth and motion vectors reprojected from the game's camera block; the generated scene goes through the game's spatial upscale a second time and is composed under the real frame's HUD, which is never interpolated. The generated frame is presented ahead of its real frame on the present thread (DM_PRESENT_LINK=0), each for half the period; the pacing follows #2241, the presented-time shift and the delayed first engagement an outside patch (framegen-stage1-v4) read as reference. Specific to Ocarina of Time: the camera block's layout and the far-plane test that picks the scene camera, read from one build and not keyed by build ID yet. Played on the Ocarina performance line (cut6) on an M2 MacBook Air: 45-53 shown frames a second from a 30 fps cap, the game's frame rising from about 30 ms to about 37 ms. On main: the host compiles for 0400486024DEC000; no game run and no regress run. With DM_FRAMEGEN unset the hooks return at once. Co-Authored-By: Claude --- dm/gfx/nvn/nvn_framegen.inc | 345 ++++++++++++++++++++++++++++++++++++ dm/gfx/nvn/nvn_mtl.m | 32 +++- 2 files changed, 374 insertions(+), 3 deletions(-) create mode 100644 dm/gfx/nvn/nvn_framegen.inc diff --git a/dm/gfx/nvn/nvn_framegen.inc b/dm/gfx/nvn/nvn_framegen.inc new file mode 100644 index 0000000000..98ad0af71a --- /dev/null +++ b/dm/gfx/nvn/nvn_framegen.inc @@ -0,0 +1,345 @@ +// ---------- scratch (2026-11-10): frame generation for Ocarina of Time, DM_FRAMEGEN=1. An evaluation build, not for main as it is. ---------- +// A game held at 30 fps shown at 60: MetalFX's frame interpolator makes the frame between the previous scene and this one, from the scene before the HUD (the upscaler's +// input, 1440x810 linear), the scene depth and motion vectors reprojected from the game's camera block (zero where the depth says a pixel is not still scenery). +// The generated scene is upscaled by the same MetalFX spatial scaler (a second run), and the HUD is not interpolated: at the present, a pixel of the canvas that changed +// after the upscale (the HUD passes) keeps the real frame's picture, every other pixel takes the generated scene. The present thread (DM_PRESENT_LINK=0) shows the +// generated frame, then the real one, each for half the present's period; generated frames pause while frames come late (as seedds/nxnative#2241's FG_STEADY). +// Scratch limits: OoT only (the camera block is vertex uniform 0 of 13,568 bytes, the scene camera the binding with far plane 25000; the canvas is presented 1:1 inside the +// window texture's crop); DM_UPSCALER=metalfx, DM_NVN_ENC_THREAD=0 and DM_PRESENT_LINK=0 only; the constants belong in a profile if this goes further. +// Pacing and accounting borrow from seedds/nxnative#2241 (pause while late, own present allocators) and from framegen-stage1-v4.patch (the held frame's presented time shifted +// back by its hold; the first engagement delayed past boot). +// Switches: DM_FRAMEGEN=0|1 (0 still logs the shown-frame statistics, for comparison); DM_FRAMEGEN_PAUSE=0 shows every generated frame, late or not; +// DM_FRAMEGEN_VIEW=1 the generated scene without the HUD split, 2 = HUD pixels in magenta; DM_FRAMEGEN_FLIP=1 flips the canvas rows against the window's; +// DM_FRAMEGEN_SHOT=:- saves the generated and the real picture of frames a..b (slow: it waits for the GPU). +#if FG_PART == 0 +static int g_fg_mode = -1; // -1 unread, 0 off, 1 on +static int fg_on(void); static void fg_cam_note(const float *c); static void fg_depth_note(MTL4RenderPassDescriptor *pd); +static void fg_shown(int gen, double tt, double tin); +static __thread double g_fg_pres_adj; // present thread: seconds added to the presented time recorded for the real frame being presented (negative: it was held behind its generated frame) +static double g_fg_tin[64]; // by ticket: when the game queued the present (present_main), for the enqueue-to-screen time of real frames +#elif FG_PART == 1 +#define FG_WIN 3 +static int g_fg_seen, g_fg_view, g_fg_flip, g_fg_pause = 1; +static int g_fg_upto = 9; // DM_FRAMEGEN_UPTO=, to split the cost in play: 0 nothing drawn, 1 the three copies, 2 + motion pass, 3 + interpolation, 4 + second upscale, 5 + window picture (not shown), 9 all +static struct { + id fi; NSUInteger iw, ih, ow, oh; MTLPixelFormat cvf; int bad; + id c[2], d[2], mv, gen, gen_up, cv0, cv, depth_last, win[FG_WIN]; + uint64_t win_serial[FG_WIN], win_ticket[FG_WIN], win_next; + id walloc[FG_WIN]; id wcb[FG_WIN]; id wpar[FG_WIN]; + id margs, wargs; id mv_pso, comp_pso; MTLPixelFormat comp_fmt; id vf, f_mv, f_comp; + int cur, warm, prev_ok, gen_ready; long last_frame, cam_frame; float cam[96]; + uint64_t made, resets, nocam, nodepth, busy, late, behind, settling; +} FG; +static const char *g_fg_shot_dir; static long g_fg_shot_a = -1, g_fg_shot_b = -1; +static int fg_on(void) { + if (g_fg_mode >= 0) return g_fg_mode; + const char *e = getenv("DM_FRAMEGEN"), *x; int v = e && atoi(e) > 0; g_fg_seen = e != NULL; + if ((x = getenv("DM_FRAMEGEN_VIEW"))) g_fg_view = atoi(x); + if ((x = getenv("DM_FRAMEGEN_FLIP"))) g_fg_flip = atoi(x); + if ((x = getenv("DM_FRAMEGEN_PAUSE"))) g_fg_pause = atoi(x); + if ((x = getenv("DM_FRAMEGEN_UPTO"))) g_fg_upto = atoi(x); + if ((x = getenv("DM_FRAMEGEN_SHOT"))) { const char *c = strrchr(x, ':'); if (c && sscanf(c + 1, "%ld-%ld", &g_fg_shot_a, &g_fg_shot_b) == 2) g_fg_shot_dir = strndup(x, (size_t)(c - x)); } + FG.last_frame = -2; FG.cam_frame = -1; + if (v && !(g_q4 && [MTLFXFrameInterpolatorDescriptor supportsMetal4FX:g_dev])) { MLOG("[RED] frame generation (DM_FRAMEGEN): this GPU has no Metal 4 MetalFX frame interpolator; frames are shown as the game draws them"); v = 0; } + if (v && rec_on()) { MLOG("[RED] frame generation (scratch build): not with DM_NVN_ENC_THREAD=1; frames are shown as the game draws them"); v = 0; } + if (v) MLOG("frame generation (DM_FRAMEGEN=1, scratch build): the scene before the HUD is interpolated from depth and the camera, upscaled a second time, and the HUD is kept from the real frame; " + "pause while late %d, view %d, flip %d, stages up to %d", g_fg_pause, g_fg_view, g_fg_flip, g_fg_upto); + else if (g_fg_seen) MLOG("frame generation off (DM_FRAMEGEN=0): the shown-frame statistics are still logged, for comparison"); + return g_fg_mode = v; +} +// The camera block as the game binds it (encode_args_): the scene camera's binding has far plane 25000; the environment probe's (far 10000, one cube face a frame) is not taken +static void fg_cam_note(const float *c) { if (!fg_on() || !(c[57] > 20000.0f)) return; memcpy(FG.cam, c, sizeof FG.cam); FG.cam_frame = (long)atomic_load(&g_cur_frame); } +// Each render pass (td_note): the last large D32F depth attachment before the upscaler is the scene depth +static void fg_depth_note(MTL4RenderPassDescriptor *pd) { + if (!g_fg_mode || !pd || !fg_on()) return; + id d = pd.depthAttachment.texture; + if (d && d.pixelFormat == MTLPixelFormatDepth32Float && d.textureType == MTLTextureType2D && d.width >= 1024) FG.depth_last = d; +} +static int fg_inv4(const double a_[16], double r[16]) { + double a[16]; memcpy(a, a_, sizeof a); for (int i = 0; i < 16; i++) r[i] = i % 5 == 0; + for (int c = 0; c < 4; c++) { int p = c; for (int k = c + 1; k < 4; k++) if (fabs(a[k * 4 + c]) > fabs(a[p * 4 + c])) p = k; + if (fabs(a[p * 4 + c]) < 1e-12) return 0; + if (p != c) for (int k = 0; k < 4; k++) { double t = a[c * 4 + k]; a[c * 4 + k] = a[p * 4 + k]; a[p * 4 + k] = t; t = r[c * 4 + k]; r[c * 4 + k] = r[p * 4 + k]; r[p * 4 + k] = t; } + double s = 1.0 / a[c * 4 + c]; for (int k = 0; k < 4; k++) { a[c * 4 + k] *= s; r[c * 4 + k] *= s; } + for (int q = 0; q < 4; q++) if (q != c) { double f = a[q * 4 + c]; if (f != 0) for (int k = 0; k < 4; k++) { a[q * 4 + k] -= f * a[c * 4 + k]; r[q * 4 + k] -= f * r[c * 4 + k]; } } } + return 1; +} +static void fg_mul4(const double a[16], const double b[16], double r[16]) { for (int i = 0; i < 4; i++) for (int j = 0; j < 4; j++) { double s = 0; for (int k = 0; k < 4; k++) s += a[i * 4 + k] * b[k * 4 + j]; r[i * 4 + j] = s; } } +static NSString *const FG_MSL = @"#include \nusing namespace metal;\n" + "struct V { float4 p [[position]]; };\n" + "vertex V fg_vs(uint id [[vertex_id]]) { float2 uv = float2((id << 1) & 2, id & 2); V o; o.p = float4(uv * float2(2, -2) + float2(-1, 1), 0, 1); return o; }\n" + "struct MP { float4x4 reproj; float2 size; float2 inv_size; };\n" + "static float vz(float d) { return 1.0 / max(1.0 - d, 1e-7); }\n" + // Motion in pixels toward the previous position: this pixel's depth reprojected through the previous view-projection; zero where the previous depth there disagrees (a moving + // object, or a disocclusion): the test of MacNativeGames' mfx_mv (MIT), as the offline test used it + "fragment float4 fg_mv(V in [[stage_in]], depth2d d [[texture(0)]], depth2d pd [[texture(1)]], constant MP *p [[buffer(0)]]) {\n" + " uint2 g = uint2(in.p.xy); float2 uv = (float2(g) + 0.5) * p->inv_size; float d0 = d.read(g); float2 m = 0.0; int still = -1;\n" + " float4 c = p->reproj * float4(uv.x * 2.0 - 1.0, 1.0 - uv.y * 2.0, d0 * 2.0 - 1.0, 1.0);\n" + " if (c.w > 1e-7) { float2 n = c.xy / c.w; float2 prev = float2(n.x * 0.5 + 0.5, 0.5 - 0.5 * n.y); m = (prev - uv) * p->size; int2 qq = int2(floor(prev * p->size));\n" + " if (d0 < 1.0 && all(qq >= int2(1)) && all(qq < int2(p->size) - int2(1))) { float ze = vz(c.z / c.w * 0.5 + 0.5), zmin = 3.4e38, zmax = 0.0; bool nr = false;\n" + " for (int y = -1; y <= 1; y++) for (int x = -1; x <= 1; x++) { float z = vz(pd.read(uint2(qq + int2(x, y)))); zmin = min(zmin, z); zmax = max(zmax, z); nr = nr || fabs(z - ze) <= ze * 0.01; }\n" + " still = (zmax <= zmin * 1.05 ? (ze >= zmin * 0.99 && ze <= zmax * 1.01) : nr) ? 1 : 0; } }\n" + " if (still == 0) m = 0.0; if (any(isnan(m)) || any(isinf(m))) m = 0.0; return float4(m, 0.0, 0.0); }\n" + "struct CP { int2 org; int2 csize; int flip; int view; int encode; int pad; };\n" + "static float3 oetf(float3 c) { c = saturate(c); return mix(12.92 * c, 1.055 * pow(c, float3(1.0 / 2.4)) - 0.055, step(float3(0.0031308), c)); }\n" + // The generated frame's window picture: win = the real frame's window texture (encoded bytes), gen = the generated scene upscaled (linear), cv = the canvas now (with the HUD), + // cv0 = the canvas right after the real frame's upscale. A canvas pixel that changed is HUD: it keeps the real frame's picture. Any other takes the real picture plus the + // difference between the generated and the real scene, both encoded, so a still scene comes out exactly as the real frame whatever the game's last pass does to the colour + "fragment float4 fg_comp(V in [[stage_in]], texture2d win [[texture(0)]], texture2d gen [[texture(1)]], texture2d cv [[texture(2)]], texture2d cv0 [[texture(3)]], constant CP *p [[buffer(0)]]) {\n" + " uint2 g = uint2(in.p.xy); float4 w = win.read(g); int2 q = int2(g) - p->org; if (p->flip != 0) q.y = p->csize.y - 1 - q.y;\n" + " if (any(q < int2(0)) || any(q >= p->csize)) return w;\n" + " float3 a = cv.read(uint2(q)).rgb; float3 b = cv0.read(uint2(q)).rgb; bool hud = any(a != b);\n" + " if (p->view == 2 && hud) return float4(1.0, 0.0, 1.0, 1.0);\n" + " if (hud && p->view != 1) return w;\n" + " float3 c = gen.read(uint2(q)).rgb;\n" + " if (p->encode == 0) return float4(c, w.a);\n" + " if (p->view == 1) return float4(oetf(c), w.a);\n" + " return float4(saturate(w.rgb + oetf(c) - oetf(b)), w.a); }\n"; +static int fg_pipes(void) { + static int st; if (st) return st > 0; + st = -1; NSError *e = nil; uint64_t t0 = now_ns(); + id lib = [g_dev newLibraryWithSource:FG_MSL options:nil error:&e]; + if (!lib) { MLOG("[RED] frame generation: its shaders do not compile: %s", e.localizedDescription.UTF8String); return 0; } + FG.vf = [lib newFunctionWithName:@"fg_vs"]; FG.f_mv = [lib newFunctionWithName:@"fg_mv"]; FG.f_comp = [lib newFunctionWithName:@"fg_comp"]; + MTLRenderPipelineDescriptor *pd = [MTLRenderPipelineDescriptor new]; pd.vertexFunction = FG.vf; pd.fragmentFunction = FG.f_mv; pd.colorAttachments[0].pixelFormat = MTLPixelFormatRG16Float; + FG.mv_pso = FG.vf && FG.f_mv ? [g_dev newRenderPipelineStateWithDescriptor:pd error:&e] : nil; + MTL4ArgumentTableDescriptor *ad = [MTL4ArgumentTableDescriptor new]; ad.maxTextureBindCount = 4; ad.maxBufferBindCount = 1; + FG.margs = rec_table([g_dev newArgumentTableWithDescriptor:ad error:&e]); FG.wargs = [g_dev newArgumentTableWithDescriptor:ad error:&e]; + if (!FG.mv_pso || !FG.f_comp || !FG.margs || !FG.wargs) { MLOG("[RED] frame generation: no pipeline or argument table: %s", e.localizedDescription.UTF8String); return 0; } + MLOG("frame generation: shaders compiled in %.1f ms", (now_ns() - t0) / 1e6); + st = 1; return 1; +} +static id fg_tex(MTLPixelFormat f, NSUInteger w, NSUInteger h, MTLTextureUsage u, NSString *label) { + MTLTextureDescriptor *td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:f width:w height:h mipmapped:NO]; td.storageMode = MTLStorageModePrivate; td.usage = u; + id t = [g_dev newTextureWithDescriptor:td]; if (t) { t.label = label; res_add(t); } return t; +} +// The interpolator and its textures for a scene of iw x ih upscaled to ow x oh (made again when a size changes: dynamic resolution) +static int fg_setup(NSUInteger iw, NSUInteger ih, NSUInteger ow, NSUInteger oh, MTLPixelFormat cvf, id sc) { + if (FG.iw == iw && FG.ih == ih && FG.ow == ow && FG.oh == oh && FG.cvf == cvf) return !FG.bad; + id old[] = {FG.c[0], FG.c[1], FG.d[0], FG.d[1], FG.mv, FG.gen, FG.gen_up, FG.cv0}; + for (unsigned i = 0; i < sizeof old / sizeof *old; i++) if (old[i]) { res_remove(old[i]); defer_obj_release(old[i]); } + FG.c[0] = FG.c[1] = FG.d[0] = FG.d[1] = FG.mv = FG.gen = FG.gen_up = FG.cv0 = nil; FG.fi = nil; + FG.iw = iw; FG.ih = ih; FG.ow = ow; FG.oh = oh; FG.cvf = cvf; FG.bad = 1; FG.warm = 0; FG.prev_ok = 0; FG.gen_ready = 0; FG.last_frame = -2; + NSError *err = nil; uint64_t t0 = now_ns(); + if (!g_fx_comp) g_fx_comp = [g_dev newCompilerWithDescriptor:[MTL4CompilerDescriptor new] error:&err]; + MTLFXFrameInterpolatorDescriptor *d = [MTLFXFrameInterpolatorDescriptor new]; + d.colorTextureFormat = MTLPixelFormatRGBA16Float; d.outputTextureFormat = MTLPixelFormatRGBA16Float; d.depthTextureFormat = MTLPixelFormatDepth32Float; d.motionTextureFormat = MTLPixelFormatRG16Float; + d.inputWidth = iw; d.inputHeight = ih; d.outputWidth = iw; d.outputHeight = ih; + id fi = g_fx_comp ? [d newFrameInterpolatorWithDevice:g_dev compiler:g_fx_comp] : nil; + if (!fi) { MLOG("[RED] frame generation: MetalFX makes no frame interpolator for %lux%lu RGBA16Float; frames are shown as the game draws them", (unsigned long)iw, (unsigned long)ih); return 0; } + MTLTextureUsage rd = MTLTextureUsageShaderRead; + for (int i = 0; i < 2; i++) { FG.c[i] = fg_tex(MTLPixelFormatRGBA16Float, iw, ih, fi.colorTextureUsage | rd, @"framegen colour"); FG.d[i] = fg_tex(MTLPixelFormatDepth32Float, iw, ih, fi.depthTextureUsage | rd, @"framegen depth"); } + FG.mv = fg_tex(MTLPixelFormatRG16Float, iw, ih, fi.motionTextureUsage | MTLTextureUsageRenderTarget | rd, @"framegen motion"); + FG.gen = fg_tex(MTLPixelFormatRGBA16Float, iw, ih, fi.outputTextureUsage | sc.colorTextureUsage | rd, @"framegen generated scene"); + FG.gen_up = fg_tex(MTLPixelFormatRGBA16Float, ow, oh, sc.outputTextureUsage | rd, @"framegen generated scene, upscaled"); + FG.cv0 = fg_tex(cvf, ow, oh, rd, @"framegen canvas before the HUD"); + res_commit(); + if (!FG.c[0] || !FG.c[1] || !FG.d[0] || !FG.d[1] || !FG.mv || !FG.gen || !FG.gen_up || !FG.cv0) { MLOG("[RED] frame generation: a texture could not be made; frames are shown as the game draws them"); return 0; } + FG.fi = fi; FG.bad = 0; + MLOG("frame generation: MetalFX frame interpolator %lux%lu RGBA16Float, generated scene upscaled to %lux%lu (canvas format %lu), made in %.1f ms; about %.0f MB of textures", + (unsigned long)iw, (unsigned long)ih, (unsigned long)ow, (unsigned long)oh, (unsigned long)cvf, (now_ns() - t0) / 1e6, (iw * ih * (8 * 3 + 4 * 3) + ow * oh * (8 + 4)) / 1048576.0); + return 1; +} +static void fg_barrier(void) { // as up_mfx_encode: MetalFX opens encoders of its own, which take no barrier + id ce = [g_cb computeCommandEncoder]; + [ce barrierAfterQueueStages:MTLStageAll beforeStages:MTLStageBlit visibilityOptions:resource_vis()]; + [ce barrierAfterStages:MTLStageBlit beforeQueueStages:MTLStageAll visibilityOptions:resource_vis()]; + [ce endEncoding]; +} +// After the game's upscale of this frame (up_mfx_encode): color = the scene, linear RGBA16F (the scaler's input), out = the canvas the upscaled scene was just drawn into +static void fg_frame(id sc, id color, id out, const NVNcopyRegion *orr, NSUInteger cw, NSUInteger ch) { + if (!fg_on()) return; + long f = (long)atomic_load(&g_cur_frame); NSUInteger ow = (NSUInteger)orr->width, oh = (NSUInteger)orr->height; + if (orr->xoffset || orr->yoffset || ow != out.width || oh != out.height || cw != color.width || ch != color.height) { + static int once; if (!once++) MLOG("[RED] frame generation: the upscale is not whole textures (%lux%lu of %lux%lu into %lux%lu of %lux%lu); frames are shown as the game draws them", + (unsigned long)cw, (unsigned long)ch, (unsigned long)color.width, (unsigned long)color.height, (unsigned long)ow, (unsigned long)oh, (unsigned long)out.width, (unsigned long)out.height); + return; } + // Not in the first 300 upscaled frames: its shaders and the interpolator are made at first use, and making them under the boot's compile load is what framegen-stage1-v4 + // blames a crash in the shader compiler's cache on (a base M5). A stopgap as that patch's: the pipelines belong in the prebuilt set if this goes further + { static int seen; if (seen < 300) { seen++; return; } } + if (!fg_pipes() || !fg_setup(cw, ch, ow, oh, out.pixelFormat, sc)) return; + if (g_fg_upto < 1) return; + id dep = FG.depth_last; + int have_d = dep && dep.width == cw && dep.height == ch, have_cam = FG.cam_frame == f, cur = FG.cur, prev = cur ^ 1; + if (!have_d) FG.nodepth++; if (!have_cam) FG.nocam++; + MTLOrigin o0 = MTLOriginMake(0, 0, 0); + id b = blit(); + enc_b_copy_texture(b, color, 0, 0, o0, MTLSizeMake(cw, ch, 1), FG.c[cur], 0, 0, o0); + if (have_d) enc_b_copy_texture(b, dep, 0, 0, o0, MTLSizeMake(cw, ch, 1), FG.d[cur], 0, 0, o0); + enc_b_copy_texture(b, out, 0, 0, o0, MTLSizeMake(ow, oh, 1), FG.cv0, 0, 0, o0); + FG.cv = out; FG.gen_ready = 0; + int ok = have_d && have_cam && FG.prev_ok && FG.last_frame == f - 1; double m[16]; + if (ok) { // reprojection = projection x previous view (floats 84-95 of this frame's block) x inverse(view-projection) + double cam[96], ic[16], pvp[16]; for (int i = 0; i < 96; i++) cam[i] = FG.cam[i]; + double pv[16] = {cam[84], cam[85], cam[86], cam[87], cam[88], cam[89], cam[90], cam[91], cam[92], cam[93], cam[94], cam[95], 0, 0, 0, 1}; + if (fg_inv4(&cam[12], ic)) { fg_mul4(&cam[28], pv, pvp); fg_mul4(pvp, ic, m); } else ok = 0; + } + if (ok && g_fg_upto >= 2) { + rp_end(); if (g_blit) { enc_b_end(g_blit); g_blit = nil; } + id pb; size_t po = arena_alloc(96, &pb); float *p = (float *)((uint8_t *)pb.contents + po); + for (int c = 0; c < 4; c++) for (int r = 0; r < 4; r++) p[c * 4 + r] = (float)m[r * 4 + c]; + p[16] = cw; p[17] = ch; p[18] = 1.0f / cw; p[19] = 1.0f / ch; + MTL4RenderPassDescriptor *rp = [MTL4RenderPassDescriptor new]; + rp.colorAttachments[0].texture = FG.mv; rp.colorAttachments[0].loadAction = MTLLoadActionDontCare; rp.colorAttachments[0].storeAction = MTLStoreActionStore; + ser_gate(); id en = enc_r_begin(rp); + [en setArgumentTable:FG.margs atStages:MTLRenderStageFragment]; + [FG.margs setTexture:FG.d[cur].gpuResourceID atIndex:0]; [FG.margs setTexture:FG.d[prev].gpuResourceID atIndex:1]; [FG.margs setAddress:pb.gpuAddress + po atIndex:0]; + [en setViewport:(MTLViewport){0, 0, cw, ch, 0, 1}]; [en setScissorRect:(MTLScissorRect){0, 0, cw, ch}]; + [en setRenderPipelineState:FG.mv_pso]; [en drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:3]; enc_r_end(en); + id fi = FG.fi; id c1 = FG.c[cur], c0 = FG.c[prev], d1 = FG.d[cur], mv = FG.mv, gen = FG.gen, gup = FG.gen_up; + int reset = FG.warm == 0; float nearp = FG.cam[56], farp = FG.cam[57], fov = FG.cam[70] * 57.29578f, aspect = FG.cam[62]; + if (g_fg_upto >= 3) { + fg_barrier(); + rec_cb_block(g_cb, ^(id cb) { + fi.colorTexture = c1; fi.prevColorTexture = c0; fi.depthTexture = d1; fi.motionTexture = mv; fi.outputTexture = gen; + fi.motionVectorScaleX = 1; fi.motionVectorScaleY = 1; fi.jitterOffsetX = 0; fi.jitterOffsetY = 0; fi.deltaTime = 1.0f / 30; + fi.nearPlane = nearp; fi.farPlane = farp; fi.fieldOfView = fov; fi.aspectRatio = aspect; fi.depthReversed = NO; fi.shouldResetHistory = reset; + [fi encodeToCommandBuffer:cb]; + }); + } + if (g_fg_upto >= 4) { + fg_barrier(); + rec_cb_block(g_cb, ^(id cb) { // the generated scene through the game's own upscale, a second time + sc.colorTexture = gen; sc.outputTexture = gup; sc.inputContentWidth = cw; sc.inputContentHeight = ch; + [sc encodeToCommandBuffer:cb]; + }); + } + g_last_end_relaxed = 0; args_dirty_all(); g_pso_dirty = g_dss_dirty = 1; + FG.made++; + if (++FG.warm >= 3 && g_fg_upto >= 5) FG.gen_ready = 1; // the first two outputs after a new history are the new frame unchanged + } else if (!ok) { if (FG.warm) FG.resets++; FG.warm = 0; } + FG.prev_ok = have_d; FG.last_frame = f; FG.cur = prev; +} +// At the present (present_enqueue, the game's thread): the generated frame's window picture into a ring texture of the host's, in a command buffer of its own committed +// behind the frame's work. Returns the texture's serial (0 = no generated frame for this present) and its ring slot +static uint64_t fg_compose(const tex_t *t, const int crop[4], int *slot) { + if (g_fg_mode <= 0 || !FG.gen_ready) return 0; + FG.gen_ready = 0; + if (g_dm_present_link) { static int once; if (!once++) MLOG("[RED] frame generation (scratch build): DM_PRESENT_LINK is not 0, generated frames are not shown (it paces on the present thread only)"); return 0; } + id src = tex_mtl(t); if (!src) return 0; + id raw = present_raw_src(src); NSUInteger r[4]; + if (!present_crop_rect(src, crop, r)) return 0; + if (r[2] != FG.ow || r[3] != FG.oh) { static int once; if (!once++) MLOG("[RED] frame generation: the present shows %lux%lu of the window texture, not the %lux%lu canvas 1:1; frames are shown as the game draws them", + (unsigned long)r[2], (unsigned long)r[3], (unsigned long)FG.ow, (unsigned long)FG.oh); return 0; } + int k = (int)(FG.win_next % FG_WIN); + pthread_mutex_lock(&g_pq_mu); int busy = FG.win_ticket[k] > g_pq_done; pthread_mutex_unlock(&g_pq_mu); + if (busy) { FG.busy++; return 0; } + NSError *e = nil; + if (!FG.comp_pso || FG.comp_fmt != raw.pixelFormat) { + MTLRenderPipelineDescriptor *pd = [MTLRenderPipelineDescriptor new]; pd.vertexFunction = FG.vf; pd.fragmentFunction = FG.f_comp; pd.colorAttachments[0].pixelFormat = raw.pixelFormat; + FG.comp_pso = [g_dev newRenderPipelineStateWithDescriptor:pd error:&e]; FG.comp_fmt = raw.pixelFormat; + if (!FG.comp_pso) { MLOG("[RED] frame generation: no pipeline for the window format %lu: %s", (unsigned long)raw.pixelFormat, e.localizedDescription.UTF8String); return 0; } + } + if (!FG.comp_pso) return 0; + id dst = FG.win[k]; + if (!dst || dst.width != src.width || dst.height != src.height || dst.pixelFormat != raw.pixelFormat) { + dst = fg_tex(raw.pixelFormat, src.width, src.height, MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead, @"framegen window picture"); + if (!dst) { MLOG("[RED] frame generation: no texture for the generated frame's window picture"); return 0; } + if (!FG.win_serial[k]) { static uint64_t n; FG.win_serial[k] = (1ull << 61) + (1ull << 40) + ++n; } + serial_put(FG.win_serial[k], dst); FG.win[k] = dst; + MLOG("frame generation: window picture ring slot %d %lux%lu format %lu", k, (unsigned long)dst.width, (unsigned long)dst.height, (unsigned long)dst.pixelFormat); + } + if (!FG.walloc[k]) { FG.walloc[k] = [g_dev newCommandAllocator]; FG.wcb[k] = [g_dev newCommandBuffer]; FG.wpar[k] = [g_dev newBufferWithLength:64 options:MTLResourceStorageModeShared]; + if (!FG.walloc[k] || !FG.wcb[k] || !FG.wpar[k]) { MLOG("[RED] frame generation: no command allocator or buffer for the window picture"); FG.walloc[k] = nil; return 0; } + res_add(FG.wpar[k]); } + res_commit(); + int32_t *p = (int32_t *)FG.wpar[k].contents; p[0] = (int32_t)r[0]; p[1] = (int32_t)r[1]; p[2] = (int32_t)FG.ow; p[3] = (int32_t)FG.oh; p[4] = g_fg_flip; p[5] = g_fg_view; + p[6] = raw.pixelFormat == MTLPixelFormatRGBA8Unorm || raw.pixelFormat == MTLPixelFormatBGRA8Unorm; p[7] = 0; + [FG.walloc[k] reset]; + id cb = FG.wcb[k]; + [cb beginCommandBufferWithAllocator:FG.walloc[k]]; + MTL4RenderPassDescriptor *rp = [MTL4RenderPassDescriptor new]; + rp.colorAttachments[0].texture = dst; rp.colorAttachments[0].loadAction = MTLLoadActionDontCare; rp.colorAttachments[0].storeAction = MTLStoreActionStore; + id en = [cb renderCommandEncoderWithDescriptor:rp]; + if (!en) { [cb endCommandBuffer]; MLOG("[RED] frame generation: no render encoder for the window picture"); return 0; } + [en barrierAfterQueueStages:MTLStageAll beforeStages:MTLStageAll visibilityOptions:resource_vis()]; + [en setArgumentTable:FG.wargs atStages:MTLRenderStageFragment]; + [FG.wargs setTexture:raw.gpuResourceID atIndex:0]; [FG.wargs setTexture:FG.gen_up.gpuResourceID atIndex:1]; [FG.wargs setTexture:FG.cv.gpuResourceID atIndex:2]; [FG.wargs setTexture:FG.cv0.gpuResourceID atIndex:3]; + [FG.wargs setAddress:FG.wpar[k].gpuAddress atIndex:0]; + [en setViewport:(MTLViewport){0, 0, dst.width, dst.height, 0, 1}]; [en setScissorRect:(MTLScissorRect){0, 0, dst.width, dst.height}]; + [en setRenderPipelineState:FG.comp_pso]; [en drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:3]; + [en endEncoding]; + [cb endCommandBuffer]; + pthread_mutex_lock(&g_res_mu); [g_q4 commit:&cb count:1]; pthread_mutex_unlock(&g_res_mu); + *slot = k; + return FG.win_serial[k]; +} +// The present thread: whether this present's generated frame is shown. Not while frames come late or the queue is behind, until three presents in a row came in time +static int fg_pace(double t_in, double period, int waiting) { + static double prev; static int steady; + double per = period > 0 ? period : 1.0 / 30; int late = prev > 0 && t_in - prev > per * 1.25; prev = t_in; + if (g_fg_upto < 9) return 0; + if (!g_fg_pause) return 1; + if (late || waiting > 2) { steady = 0; if (late) FG.late++; else FG.behind++; return 0; } + if (steady < 3) { steady++; FG.settling++; return 0; } + return 1; +} +// Each frame put on screen (the drawable's presented handler): the intervals between shown frames and, for real frames, the time from the game's enqueue to the screen +static void fg_shown(int gen, double tt, double tin) { + static pthread_mutex_t mu = PTHREAD_MUTEX_INITIALIZER; static double last, first, iv[600], lat[600]; static int n, nl, ngen; + if (!g_fg_seen || !(tt > 0)) return; + pthread_mutex_lock(&mu); + if (last > 0 && tt > last) { if (!n) first = last; iv[n++] = (tt - last) * 1e3; ngen += gen; } + if (tt > last) last = tt; + if (!gen && tin > 0 && nl < 600) lat[nl++] = (tt - tin) * 1e3; + if (n == 600) { + for (int i = 1; i < n; i++) { double x = iv[i]; int j = i - 1; while (j >= 0 && iv[j] > x) { iv[j + 1] = iv[j]; j--; } iv[j + 1] = x; } + for (int i = 1; i < nl; i++) { double x = lat[i]; int j = i - 1; while (j >= 0 && lat[j] > x) { lat[j + 1] = lat[j]; j--; } lat[j + 1] = x; } + int over = 0; for (int i = 0; i < n; i++) over += iv[i] > 25.0; + MLOG("frame generation %s: 600 shown frames at %.1f a second, %d generated; interval between shown frames p50 %.2f ms, p95 %.2f, max %.2f, over 25 ms %d; game enqueue to screen (real frames) p50 %.1f ms, p95 %.1f; " + "interpolations %llu, history restarts %llu, no camera %llu, no depth %llu, ring slot busy %llu; generated frames not shown: late %llu, queue behind %llu, settling %llu", + g_fg_mode > 0 ? "on" : "off", 600.0 / (last - first), ngen, iv[299], iv[569], iv[599], over, nl ? lat[nl / 2] : 0.0, nl ? lat[nl * 95 / 100] : 0.0, + (unsigned long long)FG.made, (unsigned long long)FG.resets, (unsigned long long)FG.nocam, (unsigned long long)FG.nodepth, (unsigned long long)FG.busy, + (unsigned long long)FG.late, (unsigned long long)FG.behind, (unsigned long long)FG.settling); + n = nl = ngen = 0; + } + pthread_mutex_unlock(&mu); +} +#elif FG_PART == 2 +// The generated frame onto the screen, ahead of its real frame (present_main; the present thread takes the drawable itself: DM_PRESENT_LINK=0). Command allocators of its +// own (two, in turn), as #2241 gives the picture after a generated frame one, so neither present waits for the other's GPU work +static id g_fg_palloc[2]; static id g_fg_pcb[2]; static id g_fg_puv[2]; static uint64_t g_fg_pfence[2], g_fg_pdone[2], g_fg_pseq; +static int present_gen(const tex_t *t, double hold, const int crop[4]) { + id src = tex_mtl(t); + if (!src || atomic_load(&g_gpu_lost)) return 0; + CAMetalLayer *layer = (__bridge CAMetalLayer *)front_metal_layer(); + if (!layer) return 0; + if (g_present_layer_resident != layer) { [g_q4 addResidencySet:layer.residencySet]; g_present_layer_resident = layer; } + present_pipeline(layer.pixelFormat); + uint64_t seq = ++g_fg_pseq; int i = (int)(seq & 1); + if (!g_fg_palloc[i]) { g_fg_palloc[i] = [g_dev newCommandAllocator]; g_fg_pcb[i] = [g_dev newCommandBuffer]; g_fg_puv[i] = [g_dev newBufferWithLength:sizeof(float) * 8 options:MTLResourceStorageModeShared]; + if (!g_fg_palloc[i] || !g_fg_pcb[i] || !g_fg_puv[i]) { g_fg_palloc[i] = nil; return 0; } + res_add(g_fg_puv[i]); } + res_commit(); + id dr = [layer nextDrawable]; + if (!dr) return 0; + if (g_fg_pfence[i]) { pthread_mutex_lock(&g_pq_mu); while (g_fg_pdone[i] < g_fg_pfence[i]) pthread_cond_wait(&g_pq_cv, &g_pq_mu); pthread_mutex_unlock(&g_pq_mu); } + [g_fg_palloc[i] reset]; + [g_q4 waitForDrawable:dr]; + id cb = g_fg_pcb[i]; + [cb beginCommandBufferWithAllocator:g_fg_palloc[i]]; + int kind = present_encode_m4(cb, src, dr.texture, crop, g_fg_puv[i], &g_fx_present); + [cb endCommandBuffer]; + if (!kind) { [g_q4 signalDrawable:dr]; return 0; } + MTL4CommitOptions *o = [MTL4CommitOptions new]; + [o addFeedbackHandler:^(id fb) { + if (fb.error) { MLOG("[RED] frame generation: display command buffer error: %s", fb.error.description.UTF8String); gpu_lost_check(fb.error, "display (generated frame)"); } + pthread_mutex_lock(&g_pq_mu); if (seq > g_fg_pdone[i]) g_fg_pdone[i] = seq; g_pq_inflight--; pthread_cond_broadcast(&g_pq_cv); pthread_mutex_unlock(&g_pq_mu); + }]; + pthread_mutex_lock(&g_pq_mu); g_pq_inflight++; pthread_mutex_unlock(&g_pq_mu); + [g_q4 commit:&cb count:1 options:o]; + [g_q4 signalDrawable:dr]; + g_fg_pfence[i] = seq; + [dr addPresentedHandler:^(id d) { fg_shown(1, d.presentedTime, 0); }]; + if (hold > 0) [dr presentAfterMinimumDuration:hold]; else [dr present]; + return 1; +} +// DM_FRAMEGEN_SHOT: the generated and the real picture of a frame as PNGs (waits for the GPU: a diagnostic) +static void fg_shots(uint64_t frame, const tex_t *gen, const tex_t *real, const int crop[4]) { + if (!g_fg_shot_dir || (long)frame < g_fg_shot_a || (long)frame > g_fg_shot_b) return; + char path[1024]; mtl_finish_gpu(); + id g = gen ? tex_mtl(gen) : nil, r = tex_mtl(real); + if (g) { snprintf(path, sizeof path, "%s/F%llu_generated.png", g_fg_shot_dir, (unsigned long long)frame); shot(g, path, 1, crop); } + if (r) { snprintf(path, sizeof path, "%s/F%llu_real.png", g_fg_shot_dir, (unsigned long long)frame); shot(r, path, 1, crop); } +} +#endif diff --git a/dm/gfx/nvn/nvn_mtl.m b/dm/gfx/nvn/nvn_mtl.m index 0e3834c2c7..e0acb22798 100644 --- a/dm/gfx/nvn/nvn_mtl.m +++ b/dm/gfx/nvn/nvn_mtl.m @@ -272,6 +272,9 @@ static void ws_report(uint64_t frame, uint64_t frames) { MLOG("[waits] up to frame %llu, per frame:%s", (unsigned long long)frame, o ? line : " none"); } #include "nvn_rec.h" // deferred encoding (DM_NVN_ENC_THREAD) +#define FG_PART 0 +#include "nvn_framegen.inc" // scratch: frame generation (DM_FRAMEGEN) +#undef FG_PART static NSHashTable *g_res_objects; static _Atomic unsigned g_image_heap_n, g_big_n; // g_big_n: large pools in chunks, whose buffers overlap (BIG_MIN) static inline MTL4VisibilityOptions resource_vis(void) { return MTL4VisibilityOptionDevice | (g_image_heap_n || g_big_n ? MTL4VisibilityOptionResourceAlias : 0); } @@ -11088,6 +11091,7 @@ static uint64_t cbuf_slot(NvShader *sh, int nvn_stage, const nvn_rbind *b, __str uint64_t g; const uint8_t *c; if (ubd_covers(u->addr, u->addr + ul)) gpu = ubd_shadow(u->addr, u->addr + ul); else if (ubp_get(u->addr, u->addr + ul, &g, &c)) gpu = g; else RED_ONCE(92, "a uniform update deferred inside the pass covers only part of the binding (stage %d cb%d): the content before the update is read", nvn_stage, reg); } + if (g_fg_mode && nvn_stage == 0 && reg == 0 && u->size == 13568) fg_cam_note((const float *)u->addr); // scratch (DM_FRAMEGEN): Ocarina of Time's camera block return gpu; } // The uniform slots' bound sizes (DMSH_CBSIZE_BANK), which the translator's reads at computed offsets compare with: one past its slot's size, rounded up to 256 bytes (ub_bound), gives 0, as on Maxwell. 0 for an unbound slot, @@ -15765,6 +15769,7 @@ static void shot(id src, const char *path, int screen, const int cro // DM_NVN_TEXDUMP=size=WxH:path instead picks the frame by content: the first frame that opens a render pass whose first color attachment level is WxH (for one-off work whose frame differs between runs) static unsigned g_td_w, g_td_h; void td_note(MTL4RenderPassDescriptor *pd, NSUInteger nc) { + fg_depth_note(pd); // scratch (DM_FRAMEGEN) if (g_td_frame == -2) { const char *e = getenv("DM_NVN_TEXDUMP"), *c = e ? strchr(e, ':') : NULL; g_td_frame = c ? atol(e) : -1; g_td_path = c ? c + 1 : NULL; g_td_list = [NSMutableArray array]; if (c && !strncmp(e, "size=", 5) && sscanf(e + 5, "%ux%u", &g_td_w, &g_td_h) == 2) g_td_frame = LONG_MAX; if (c && !strncmp(e, "layers=", 7)) { g_td_layers = atoi(e + 7); g_td_frame = (long)atomic_load(&g_cur_frame); } } @@ -16259,6 +16264,9 @@ static int up_mfx_rect(const uint32_t v[4], NSUInteger w, NSUInteger h, NVNcopyR if (v[0] > INT_MAX || v[1] > INT_MAX || rw > INT_MAX || rh > INT_MAX || (uint64_t)v[0] + rw > w || (uint64_t)v[1] + rh > h) return 0; *r = (NVNcopyRegion){(int)v[0], (int)v[1], 0, (int)rw, (int)rh, 1}; return 1; } +#define FG_PART 1 +#include "nvn_framegen.inc" +#undef FG_PART // The dispatch of upscaler cs as MetalFX's spatial scaler from its texture slot's rectangle to its image slot's (see above): 1 when drawn, 0 to run the shader static int up_mfx_encode(NvShader *cs) { int ti = -1, oi = -1; @@ -16312,6 +16320,7 @@ static int up_mfx_encode(NvShader *cs) { args_dirty_all(); g_up_mfx_kept++; return 0; // only the scratch image was scaled; the game's shader can still produce the output } args_dirty_all(); + if (!direct) fg_frame(sc, color, out, &orr, cw, ch); // scratch (DM_FRAMEGEN) if (!g_up_mfx_n++) MLOG("upscaler: first dispatch of %s drawn by MetalFX, %dx%d at %d,%d of %lux%lu (format %lu) → %dx%d at %d,%d of %lux%lu (format %lu)%s", cs->key, ir.width, ir.height, ir.xoffset, ir.yoffset, (unsigned long)in.width, (unsigned long)in.height, (unsigned long)in.pixelFormat, orr.width, orr.height, orr.xoffset, orr.yoffset, (unsigned long)out.width, (unsigned long)out.height, (unsigned long)out.pixelFormat, direct ? ", straight" : srgb ? ", through RGBA16F, decoded from sRGB" : sq ? ", through RGBA16F, squared on the way in" : ", through RGBA16F"); @@ -16503,10 +16512,12 @@ static int present_now(const tex_t *t, double period, uint64_t frame, uint64_t t #if TARGET_OS_SIMULATOR // The iOS simulator SDK has no presentation-time callback and no minimum display duration (used only for picture acceptance, display-link ticks are not counted) (void)period; [dr present]; #else + double fg_adj = g_fg_pres_adj; // scratch (DM_FRAMEGEN): a real frame held behind its generated frame reads as shown when it would have been without (after framegen-stage1-v4's pres_adj) uint64_t pn = ticket; // Record the present time by the game's present sequence number (ticket number): hle's present statistics are taken by "the game's nth present"; in the display link path the actual present count != the game's present count (old frames get merged and dropped) [dr addPresentedHandler:^(id d) { // Actual presentation time (the time this frame is really put on screen, CACurrentMediaTime base) CFTimeInterval tt = d.presentedTime; - g_pres[pn % PRING] = (pres_ent){pn, tt > 0 ? tt : -1}; + g_pres[pn % PRING] = (pres_ent){pn, tt > 0 ? tt + fg_adj : -1}; + fg_shown(0, tt, g_fg_tin[pn % 64]); // scratch (DM_FRAMEGEN) }]; // Each frame is displayed for at least one present period: when the GPU completion time jitters around a 120 Hz refresh boundary, the paired "25 ms + 8.33 ms" stutter no longer appears if (target_ts > 0) [dr present]; @@ -16515,6 +16526,9 @@ static int present_now(const tex_t *t, double period, uint64_t frame, uint64_t t (void)frame; return 1; } +#define FG_PART 2 +#include "nvn_framegen.inc" +#undef FG_PART // ---------- Present thread ---------- // NVN semantics: QueuePresentTexture only queues the present and returns immediately (the game often calls it while holding a lock; iOS real device: synchronously waiting for nextDrawable inside it // takes about 30 ms, the lock is held about 90% of the time, and the loading thread starves). Taking the drawable, blit and present are all done on this thread in enqueue order. @@ -16522,7 +16536,7 @@ static int present_now(const tex_t *t, double period, uint64_t frame, uint64_t t // Swapchain texture return: before AcquireTexture gets a texture it waits for the ticket of that texture's previous present to be processed (mtl_present_wait), after which the commands the game draws into it // are necessarily ordered after that blit on the GPU queue. #define PQ 8 -typedef struct { tex_t t; double period; uint64_t frame; int crop[4]; double t_in; uint64_t ev_val, snap_v; } pq_item; // snap_v: the snapshot copy this present shows (snap_commit_wait) // t_in: the time the game handed this frame into the present queue (CACurrentMediaTime); ev_val: see g_ext_ev +typedef struct { tex_t t; double period; uint64_t frame; int crop[4]; double t_in; uint64_t ev_val, snap_v, gen; } pq_item; // gen: the generated frame shown before this one (fg_compose), 0 = none // snap_v: the snapshot copy this present shows (snap_commit_wait) // t_in: the time the game handed this frame into the present queue (CACurrentMediaTime); ev_val: see g_ext_ev // External producer (the Vulkan layer, dm/gfx/vk) renders on its own queue and signals this event when a frame's rendering is done: the present of that frame waits for the value on g_q4 static id g_ext_ev; static void ext_wait(uint64_t v) { if (v && g_ext_ev) [g_q4 waitForEvent:g_ext_ev value:v]; } @@ -16731,11 +16745,20 @@ static void present_drop_for_suspend(void) { pthread_mutex_lock(&g_pq_mu); while (g_pq_read == g_pq_in || g_link_owns_layer) pthread_cond_wait(&g_pq_cv, &g_pq_mu); // In display-link mode the display-link callback takes them; an empty queue is judged by "taken"; "done" has to wait for the feedback of the present submit and lags behind taken pq_item it = g_pq[g_pq_read % PQ]; + int fg_waiting = (int)(g_pq_in - g_pq_read); uint64_t ticket = ++g_pq_read; g_classic_busy = 1; pthread_mutex_unlock(&g_pq_mu); int submitted; ext_wait(it.ev_val); snap_commit_wait(it.snap_v); - @autoreleasepool { submitted = present_now(&it.t, it.period, it.frame, ticket, it.crop, nil, 0); mtl_tex_unpin(it.t.serial); } + // scratch (DM_FRAMEGEN): the generated frame first, then the real one, each for half the period + int fg_show = 0; tex_t fg_t; memset(&fg_t, 0, sizeof fg_t); fg_t.magic = MAGIC('T', 'E', 'X', 0); fg_t.serial = it.gen; + fg_on(); g_fg_tin[ticket % 64] = it.t_in; + if (it.gen) fg_shots(it.frame, &fg_t, &it.t, it.crop); + if (it.gen && fg_pace(it.t_in, it.period, fg_waiting)) { @autoreleasepool { fg_show = present_gen(&fg_t, it.period * 0.5, it.crop); } } + else if (!it.gen) fg_pace(it.t_in, it.period, fg_waiting); + g_fg_pres_adj = fg_show ? -(it.period > 0 ? it.period * 0.5 : 1.0 / 60) : 0; + @autoreleasepool { submitted = present_now(&it.t, fg_show ? it.period * 0.5 : it.period, it.frame, ticket, it.crop, nil, 0); mtl_tex_unpin(it.t.serial); } + g_fg_pres_adj = 0; pthread_mutex_lock(&g_pq_mu); g_classic_busy = 0; pthread_cond_broadcast(&g_pq_cv); pthread_mutex_unlock(&g_pq_mu); if (!submitted) { pthread_mutex_lock(&g_pq_mu); @@ -16897,6 +16920,8 @@ static uint64_t present_enqueue(const tex_t *t, double period, const int crop[4] else if (st == 2 && now - t0 >= at + dur) { st = 3; mtl_link_pause(0); MLOG("DM_TEST_LINK_STALL: resuming the display link (frame %llu)", (unsigned long long)atomic_load(&g_cur_frame)); } } if (ws_on()) { static uint64_t f0; uint64_t f = atomic_load(&g_cur_frame); if (!f0) f0 = f; else if (f >= f0 + 300) { ws_report(f, f - f0); f0 = f; } } + uint64_t fg_gen = 0; int fg_slot = 0; + if (!ev_val) fg_gen = fg_compose(t, crop, &fg_slot); // scratch (DM_FRAMEGEN): from the game's own texture, before a snapshot replaces it tex_t snap; uint64_t ss = 0; int slot = 0; if (!ev_val && mtl_snap_on()) { slot = (int)(atomic_load(&g_snap_next) % (uint64_t)g_snap_n); @@ -16908,6 +16933,7 @@ static uint64_t present_enqueue(const tex_t *t, double period, const int crop[4] pq_item *it = &g_pq[g_pq_in % PQ]; *it = (pq_item){*t, period, atomic_load(&g_cur_frame), {0}, CACurrentMediaTime(), ev_val, ss ? g_snap[slot].ev : 0}; if (crop) memcpy(it->crop, crop, sizeof it->crop); // The current frame's state is enqueued with the ticket number, rather than reading the mutable window late on the present thread uint64_t tk = ++g_pq_in; + if (fg_gen) { it->gen = fg_gen; FG.win_ticket[fg_slot] = tk; FG.win_next++; } if (ss) { g_snap[slot].ticket = tk; atomic_fetch_add(&g_snap_next, 1); if (copy) *copy = g_snap[slot].ev; } pthread_cond_broadcast(&g_pq_cv); pthread_mutex_unlock(&g_pq_mu); From da4e6598ebb240d4281322adbcb6eee9da0b623e Mon Sep 17 00:00:00 2001 From: seedds Date: Wed, 11 Nov 2026 15:02:19 -0800 Subject: [PATCH 7/9] Frame generation scratch: a [RED] line when DM_FRAMEGEN=1 and no upscale of the game's is drawn by MetalFX, or when MetalFX scales straight between the game's textures Without the upscale the hook (up_mfx_encode_from, seedds/nxnative#1951's chained-upscaler path included) is never reached and nothing was generated, silently: devMlGUE's report on seedds/nxnative#2279. Compile-checked on main ccd834e0 + #1951; not run. Co-Authored-By: Claude --- dm/gfx/nvn/nvn_framegen.inc | 8 ++++++++ dm/gfx/nvn/nvn_mtl.m | 1 + 2 files changed, 9 insertions(+) diff --git a/dm/gfx/nvn/nvn_framegen.inc b/dm/gfx/nvn/nvn_framegen.inc index 98ad0af71a..b6bfafdeca 100644 --- a/dm/gfx/nvn/nvn_framegen.inc +++ b/dm/gfx/nvn/nvn_framegen.inc @@ -21,6 +21,7 @@ static double g_fg_tin[64]; // by ticket: when the game queued the present (pr #define FG_WIN 3 static int g_fg_seen, g_fg_view, g_fg_flip, g_fg_pause = 1; static int g_fg_upto = 9; // DM_FRAMEGEN_UPTO=, to split the cost in play: 0 nothing drawn, 1 the three copies, 2 + motion pass, 3 + interpolation, 4 + second upscale, 5 + window picture (not shown), 9 all +static _Atomic int g_fg_hooked; // the hook was reached: an upscale of the game's was drawn by MetalFX (up_mfx_encode_from -> fg_frame) static struct { id fi; NSUInteger iw, ih, ow, oh; MTLPixelFormat cvf; int bad; id c[2], d[2], mv, gen, gen_up, cv0, cv, depth_last, win[FG_WIN]; @@ -149,6 +150,7 @@ static void fg_barrier(void) { // as up_mfx_encode: MetalFX opens encoders of // After the game's upscale of this frame (up_mfx_encode): color = the scene, linear RGBA16F (the scaler's input), out = the canvas the upscaled scene was just drawn into static void fg_frame(id sc, id color, id out, const NVNcopyRegion *orr, NSUInteger cw, NSUInteger ch) { if (!fg_on()) return; + atomic_store(&g_fg_hooked, 1); long f = (long)atomic_load(&g_cur_frame); NSUInteger ow = (NSUInteger)orr->width, oh = (NSUInteger)orr->height; if (orr->xoffset || orr->yoffset || ow != out.width || oh != out.height || cw != color.width || ch != color.height) { static int once; if (!once++) MLOG("[RED] frame generation: the upscale is not whole textures (%lux%lu of %lux%lu into %lux%lu of %lux%lu); frames are shown as the game draws them", @@ -268,6 +270,12 @@ static uint64_t fg_compose(const tex_t *t, const int crop[4], int *slot) { static int fg_pace(double t_in, double period, int waiting) { static double prev; static int steady; double per = period > 0 ? period : 1.0 / 30; int late = prev > 0 && t_in - prev > per * 1.25; prev = t_in; + // The hook is the game's upscale drawn by MetalFX. Without one nothing is generated, and that was silent (seedds/nxnative#2279, devMlGUE: 0 generated on main, + // where Ocarina of Time's agl_ndsr_vr upscaler was not recognised): say so after 600 presents, and again every 3,600 while it lasts + { static uint64_t n; if (g_fg_mode > 0 && !atomic_load(&g_fg_hooked) && (++n == 600 || !(n % 3600))) + MLOG("[RED] frame generation (DM_FRAMEGEN=1): %llu presents and no upscale of the game's was drawn by MetalFX, so no frame is generated. It needs DM_UPSCALER=metalfx and " + "the game's upscaler among the profile's upscaler keys (the log then has \"upscaler: first dispatch of ... drawn by MetalFX\"); %llu upscaler dispatches were left to the game's shader", + (unsigned long long)n, (unsigned long long)g_up_mfx_kept); } if (g_fg_upto < 9) return 0; if (!g_fg_pause) return 1; if (late || waiting > 2) { steady = 0; if (late) FG.late++; else FG.behind++; return 0; } diff --git a/dm/gfx/nvn/nvn_mtl.m b/dm/gfx/nvn/nvn_mtl.m index e0acb22798..13c7495fac 100644 --- a/dm/gfx/nvn/nvn_mtl.m +++ b/dm/gfx/nvn/nvn_mtl.m @@ -16321,6 +16321,7 @@ static int up_mfx_encode(NvShader *cs) { } args_dirty_all(); if (!direct) fg_frame(sc, color, out, &orr, cw, ch); // scratch (DM_FRAMEGEN) + else if (fg_on()) { static int once; if (!once++) MLOG("[RED] frame generation: MetalFX scales straight between the game's own textures here (%s), which the scratch build does not hook; frames are shown as the game draws them", cs->key); } if (!g_up_mfx_n++) MLOG("upscaler: first dispatch of %s drawn by MetalFX, %dx%d at %d,%d of %lux%lu (format %lu) → %dx%d at %d,%d of %lux%lu (format %lu)%s", cs->key, ir.width, ir.height, ir.xoffset, ir.yoffset, (unsigned long)in.width, (unsigned long)in.height, (unsigned long)in.pixelFormat, orr.width, orr.height, orr.xoffset, orr.yoffset, (unsigned long)out.width, (unsigned long)out.height, (unsigned long)out.pixelFormat, direct ? ", straight" : srgb ? ", through RGBA16F, decoded from sRGB" : sq ? ", through RGBA16F, squared on the way in" : ", through RGBA16F"); From b5a508080b9eb521fadef4e546956fa937e9f063 Mon Sep 17 00:00:00 2001 From: seedds Date: Wed, 11 Nov 2026 15:36:41 -0800 Subject: [PATCH 8/9] Frame generation scratch: the generated frame gets the real frame's upscaler sharpness blend (seedds/nxnative#2366's upscaler_sharpness) The real frame's upscale blends a bilinear picture of the scaler's input into MetalFX's result; the generated scene was upscaled bare, so with a sharpness under 1 (Ocarina of Time: 0.7) generated frames would be crisper than real ones. fg_comp blends the generated scene before its upscale in by the same share. Compile-checked; not run. Co-Authored-By: Claude --- dm/gfx/nvn/nvn_framegen.inc | 14 +++++++++----- dm/gfx/nvn/nvn_mtl.m | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/dm/gfx/nvn/nvn_framegen.inc b/dm/gfx/nvn/nvn_framegen.inc index b6bfafdeca..bae56a47f2 100644 --- a/dm/gfx/nvn/nvn_framegen.inc +++ b/dm/gfx/nvn/nvn_framegen.inc @@ -21,6 +21,7 @@ static double g_fg_tin[64]; // by ticket: when the game queued the present (pr #define FG_WIN 3 static int g_fg_seen, g_fg_view, g_fg_flip, g_fg_pause = 1; static int g_fg_upto = 9; // DM_FRAMEGEN_UPTO=, to split the cost in play: 0 nothing drawn, 1 the three copies, 2 + motion pass, 3 + interpolation, 4 + second upscale, 5 + window picture (not shown), 9 all +static float g_fg_soft; // the real frame's upscale blends this share of a bilinear picture in (up_mfx_encode's soft); the generated frame's does the same (fg_comp) static _Atomic int g_fg_hooked; // the hook was reached: an upscale of the game's was drawn by MetalFX (up_mfx_encode_from -> fg_frame) static struct { id fi; NSUInteger iw, ih, ow, oh; MTLPixelFormat cvf; int bad; @@ -81,18 +82,21 @@ static NSString *const FG_MSL = @"#include \nusing namespace metal " for (int y = -1; y <= 1; y++) for (int x = -1; x <= 1; x++) { float z = vz(pd.read(uint2(qq + int2(x, y)))); zmin = min(zmin, z); zmax = max(zmax, z); nr = nr || fabs(z - ze) <= ze * 0.01; }\n" " still = (zmax <= zmin * 1.05 ? (ze >= zmin * 0.99 && ze <= zmax * 1.01) : nr) ? 1 : 0; } }\n" " if (still == 0) m = 0.0; if (any(isnan(m)) || any(isinf(m))) m = 0.0; return float4(m, 0.0, 0.0); }\n" - "struct CP { int2 org; int2 csize; int flip; int view; int encode; int pad; };\n" + "struct CP { int2 org; int2 csize; int flip; int view; int encode; float soft; };\n" "static float3 oetf(float3 c) { c = saturate(c); return mix(12.92 * c, 1.055 * pow(c, float3(1.0 / 2.4)) - 0.055, step(float3(0.0031308), c)); }\n" // The generated frame's window picture: win = the real frame's window texture (encoded bytes), gen = the generated scene upscaled (linear), cv = the canvas now (with the HUD), // cv0 = the canvas right after the real frame's upscale. A canvas pixel that changed is HUD: it keeps the real frame's picture. Any other takes the real picture plus the // difference between the generated and the real scene, both encoded, so a still scene comes out exactly as the real frame whatever the game's last pass does to the colour - "fragment float4 fg_comp(V in [[stage_in]], texture2d win [[texture(0)]], texture2d gen [[texture(1)]], texture2d cv [[texture(2)]], texture2d cv0 [[texture(3)]], constant CP *p [[buffer(0)]]) {\n" + "fragment float4 fg_comp(V in [[stage_in]], texture2d win [[texture(0)]], texture2d gen [[texture(1)]], texture2d cv [[texture(2)]], texture2d cv0 [[texture(3)]], texture2d gs [[texture(4)]], constant CP *p [[buffer(0)]]) {\n" " uint2 g = uint2(in.p.xy); float4 w = win.read(g); int2 q = int2(g) - p->org; if (p->flip != 0) q.y = p->csize.y - 1 - q.y;\n" " if (any(q < int2(0)) || any(q >= p->csize)) return w;\n" " float3 a = cv.read(uint2(q)).rgb; float3 b = cv0.read(uint2(q)).rgb; bool hud = any(a != b);\n" " if (p->view == 2 && hud) return float4(1.0, 0.0, 1.0, 1.0);\n" " if (hud && p->view != 1) return w;\n" + // soft: the share of a bilinear picture of the scaler's own input in the real frame's upscale (the profile's upscaler_sharpness, seedds/nxnative#2366): the generated + // scene (gs, before its upscale) is blended in the same way, or generated frames would be crisper than real ones " float3 c = gen.read(uint2(q)).rgb;\n" + " if (p->soft > 0.0) { constexpr sampler ls(coord::normalized, address::clamp_to_edge, filter::linear); c = mix(c, gs.sample(ls, (float2(q) + 0.5) / float2(p->csize)).rgb, p->soft); }\n" " if (p->encode == 0) return float4(c, w.a);\n" " if (p->view == 1) return float4(oetf(c), w.a);\n" " return float4(saturate(w.rgb + oetf(c) - oetf(b)), w.a); }\n"; @@ -104,7 +108,7 @@ static int fg_pipes(void) { FG.vf = [lib newFunctionWithName:@"fg_vs"]; FG.f_mv = [lib newFunctionWithName:@"fg_mv"]; FG.f_comp = [lib newFunctionWithName:@"fg_comp"]; MTLRenderPipelineDescriptor *pd = [MTLRenderPipelineDescriptor new]; pd.vertexFunction = FG.vf; pd.fragmentFunction = FG.f_mv; pd.colorAttachments[0].pixelFormat = MTLPixelFormatRG16Float; FG.mv_pso = FG.vf && FG.f_mv ? [g_dev newRenderPipelineStateWithDescriptor:pd error:&e] : nil; - MTL4ArgumentTableDescriptor *ad = [MTL4ArgumentTableDescriptor new]; ad.maxTextureBindCount = 4; ad.maxBufferBindCount = 1; + MTL4ArgumentTableDescriptor *ad = [MTL4ArgumentTableDescriptor new]; ad.maxTextureBindCount = 5; ad.maxBufferBindCount = 1; FG.margs = rec_table([g_dev newArgumentTableWithDescriptor:ad error:&e]); FG.wargs = [g_dev newArgumentTableWithDescriptor:ad error:&e]; if (!FG.mv_pso || !FG.f_comp || !FG.margs || !FG.wargs) { MLOG("[RED] frame generation: no pipeline or argument table: %s", e.localizedDescription.UTF8String); return 0; } MLOG("frame generation: shaders compiled in %.1f ms", (now_ns() - t0) / 1e6); @@ -246,7 +250,7 @@ static uint64_t fg_compose(const tex_t *t, const int crop[4], int *slot) { res_add(FG.wpar[k]); } res_commit(); int32_t *p = (int32_t *)FG.wpar[k].contents; p[0] = (int32_t)r[0]; p[1] = (int32_t)r[1]; p[2] = (int32_t)FG.ow; p[3] = (int32_t)FG.oh; p[4] = g_fg_flip; p[5] = g_fg_view; - p[6] = raw.pixelFormat == MTLPixelFormatRGBA8Unorm || raw.pixelFormat == MTLPixelFormatBGRA8Unorm; p[7] = 0; + p[6] = raw.pixelFormat == MTLPixelFormatRGBA8Unorm || raw.pixelFormat == MTLPixelFormatBGRA8Unorm; ((float *)p)[7] = g_fg_soft; [FG.walloc[k] reset]; id cb = FG.wcb[k]; [cb beginCommandBufferWithAllocator:FG.walloc[k]]; @@ -256,7 +260,7 @@ static uint64_t fg_compose(const tex_t *t, const int crop[4], int *slot) { if (!en) { [cb endCommandBuffer]; MLOG("[RED] frame generation: no render encoder for the window picture"); return 0; } [en barrierAfterQueueStages:MTLStageAll beforeStages:MTLStageAll visibilityOptions:resource_vis()]; [en setArgumentTable:FG.wargs atStages:MTLRenderStageFragment]; - [FG.wargs setTexture:raw.gpuResourceID atIndex:0]; [FG.wargs setTexture:FG.gen_up.gpuResourceID atIndex:1]; [FG.wargs setTexture:FG.cv.gpuResourceID atIndex:2]; [FG.wargs setTexture:FG.cv0.gpuResourceID atIndex:3]; + [FG.wargs setTexture:raw.gpuResourceID atIndex:0]; [FG.wargs setTexture:FG.gen_up.gpuResourceID atIndex:1]; [FG.wargs setTexture:FG.cv.gpuResourceID atIndex:2]; [FG.wargs setTexture:FG.cv0.gpuResourceID atIndex:3]; [FG.wargs setTexture:FG.gen.gpuResourceID atIndex:4]; [FG.wargs setAddress:FG.wpar[k].gpuAddress atIndex:0]; [en setViewport:(MTLViewport){0, 0, dst.width, dst.height, 0, 1}]; [en setScissorRect:(MTLScissorRect){0, 0, dst.width, dst.height}]; [en setRenderPipelineState:FG.comp_pso]; [en drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:3]; diff --git a/dm/gfx/nvn/nvn_mtl.m b/dm/gfx/nvn/nvn_mtl.m index 13c7495fac..0d26722cc6 100644 --- a/dm/gfx/nvn/nvn_mtl.m +++ b/dm/gfx/nvn/nvn_mtl.m @@ -16320,7 +16320,7 @@ static int up_mfx_encode(NvShader *cs) { args_dirty_all(); g_up_mfx_kept++; return 0; // only the scratch image was scaled; the game's shader can still produce the output } args_dirty_all(); - if (!direct) fg_frame(sc, color, out, &orr, cw, ch); // scratch (DM_FRAMEGEN) + if (!direct) { g_fg_soft = soft; fg_frame(sc, color, out, &orr, cw, ch); } // scratch (DM_FRAMEGEN) else if (fg_on()) { static int once; if (!once++) MLOG("[RED] frame generation: MetalFX scales straight between the game's own textures here (%s), which the scratch build does not hook; frames are shown as the game draws them", cs->key); } if (!g_up_mfx_n++) MLOG("upscaler: first dispatch of %s drawn by MetalFX, %dx%d at %d,%d of %lux%lu (format %lu) → %dx%d at %d,%d of %lux%lu (format %lu)%s", cs->key, ir.width, ir.height, ir.xoffset, ir.yoffset, (unsigned long)in.width, (unsigned long)in.height, (unsigned long)in.pixelFormat, orr.width, orr.height, orr.xoffset, orr.yoffset, (unsigned long)out.width, (unsigned long)out.height, From 76e6843e4ff8f661abf27c707a39ea58ddb6f195 Mon Sep 17 00:00:00 2001 From: seedds Date: Thu, 12 Nov 2026 23:47:20 -0800 Subject: [PATCH 9/9] Frame generation scratch: the scene depth is the newest recent depth attachment of exactly the scene's size, so it generates with dynamic resolution on The depth was the last D32F attachment at least 1024 wide. With dynamic resolution the scene drops below that (953x536, 854x480 on an M1 Pro), the last large one is then a shadow map, and every frame counted as no depth: 64 generated frames in a session of several minutes. Found, fixed and tested by devMlGUE (seedds/nxnative#2279): the last four depth attachments are kept and the newest of the scene's size is used; a log line names the choice when it changes. Their figures with the change, dynamic resolution on, M1 Pro: 157-219 generated per 600 shown, no depth 0, 37-44 shown frames a second. Their second rule (a larger depth texture of the scene's shape, with an offset) is left out: nothing exercised it. Checked here: nvn_mtl.m passes a syntax-only compile. Not run here: the host build, tests and a game run. Co-Authored-By: Claude Fable 5.2 --- dm/gfx/nvn/nvn_framegen.inc | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/dm/gfx/nvn/nvn_framegen.inc b/dm/gfx/nvn/nvn_framegen.inc index bae56a47f2..73c1bac083 100644 --- a/dm/gfx/nvn/nvn_framegen.inc +++ b/dm/gfx/nvn/nvn_framegen.inc @@ -25,7 +25,7 @@ static float g_fg_soft; // the real frame's upscale blends this share of a bil static _Atomic int g_fg_hooked; // the hook was reached: an upscale of the game's was drawn by MetalFX (up_mfx_encode_from -> fg_frame) static struct { id fi; NSUInteger iw, ih, ow, oh; MTLPixelFormat cvf; int bad; - id c[2], d[2], mv, gen, gen_up, cv0, cv, depth_last, win[FG_WIN]; + id c[2], d[2], mv, gen, gen_up, cv0, cv, depth_seen[4], win[FG_WIN]; uint64_t win_serial[FG_WIN], win_ticket[FG_WIN], win_next; id walloc[FG_WIN]; id wcb[FG_WIN]; id wpar[FG_WIN]; id margs, wargs; id mv_pso, comp_pso; MTLPixelFormat comp_fmt; id vf, f_mv, f_comp; @@ -51,11 +51,15 @@ static int fg_on(void) { } // The camera block as the game binds it (encode_args_): the scene camera's binding has far plane 25000; the environment probe's (far 10000, one cube face a frame) is not taken static void fg_cam_note(const float *c) { if (!fg_on() || !(c[57] > 20000.0f)) return; memcpy(FG.cam, c, sizeof FG.cam); FG.cam_frame = (long)atomic_load(&g_cur_frame); } -// Each render pass (td_note): the last large D32F depth attachment before the upscaler is the scene depth +// Each render pass (td_note): the last few D32F depth attachments, newest first. The scene depth is the newest of them that holds the scene being upscaled +// (fg_frame); with dynamic resolution the scene is narrower than a shadow map, so "the last large one" is not it static void fg_depth_note(MTL4RenderPassDescriptor *pd) { if (!g_fg_mode || !pd || !fg_on()) return; id d = pd.depthAttachment.texture; - if (d && d.pixelFormat == MTLPixelFormatDepth32Float && d.textureType == MTLTextureType2D && d.width >= 1024) FG.depth_last = d; + if (!d || d.pixelFormat != MTLPixelFormatDepth32Float || d.textureType != MTLTextureType2D || d == FG.depth_seen[0]) return; + int n = 3; for (int i = 1; i < 4; i++) if (FG.depth_seen[i] == d) { n = i; break; } + for (int i = n; i > 0; i--) FG.depth_seen[i] = FG.depth_seen[i - 1]; + FG.depth_seen[0] = d; } static int fg_inv4(const double a_[16], double r[16]) { double a[16]; memcpy(a, a_, sizeof a); for (int i = 0; i < 16; i++) r[i] = i % 5 == 0; @@ -165,9 +169,14 @@ static void fg_frame(id sc, id color, id dep = FG.depth_last; - int have_d = dep && dep.width == cw && dep.height == ch, have_cam = FG.cam_frame == f, cur = FG.cur, prev = cur ^ 1; + id dep = nil; // the newest recent depth attachment of exactly the scene's size (devMlGUE, seedds/nxnative#2279: with dynamic resolution the scene is 953x536 or 854x480 and the last large one is a shadow map) + for (int i = 0; i < 4 && !dep; i++) { id d = FG.depth_seen[i]; if (d && d.width == cw && d.height == ch) dep = d; } + int have_d = dep != nil, have_cam = FG.cam_frame == f, cur = FG.cur, prev = cur ^ 1; if (!have_d) FG.nodepth++; if (!have_cam) FG.nocam++; + { static NSUInteger lw, lh, ldw; static int n; NSUInteger dw = dep ? dep.width : 0; + if ((lw != cw || lh != ch || ldw != dw) && n < 40) { n++; lw = cw; lh = ch; ldw = dw; char b[96] = ""; int k = 0; + for (int i = 0; i < 4; i++) if (FG.depth_seen[i]) k += snprintf(b + k, sizeof b - k, " %lux%lu", (unsigned long)FG.depth_seen[i].width, (unsigned long)FG.depth_seen[i].height); + MLOG("frame generation: scene %lux%lu, depth used %lux%lu, recent depth attachments:%s", (unsigned long)cw, (unsigned long)ch, (unsigned long)dw, (unsigned long)(dep ? dep.height : 0), b); } } MTLOrigin o0 = MTLOriginMake(0, 0, 0); id b = blit(); enc_b_copy_texture(b, color, 0, 0, o0, MTLSizeMake(cw, ch, 1), FG.c[cur], 0, 0, o0);