diff --git a/dm/games/0400C3F00006E000/README.md b/dm/games/0400C3F00006E000/README.md index 55f0337a45..9830a0f917 100644 --- a/dm/games/0400C3F00006E000/README.md +++ b/dm/games/0400C3F00006E000/README.md @@ -94,6 +94,7 @@ Update 1.8.0 (v917504, SDK 21.4.0) is not on this Mac. Pull request #1116 ported ## Found on this game +- [Guest event queue ordering](../../../docs/reference/nvn-objects.md#guest-event-queue-ordering) - [A NUP with complete standalone updates stops at "no base game"](../../../docs/reference/packing.md#a-nup-with-complete-standalone-updates-stops-at-no-base-game) (packing reproduced with the base dump; 1.8.0 gameplay remains untested) - [Large sparse textures as Metal placement sparse textures](../../../docs/reference/nvn-objects.md#large-sparse-textures-as-metal-placement-sparse-textures) - [Physical pages mapped at several places of a placement sparse texture](../../../docs/reference/nvn-objects.md#physical-pages-mapped-at-several-places-of-a-placement-sparse-texture) diff --git a/dm/gfx/nvn/hle_nvn.c b/dm/gfx/nvn/hle_nvn.c index b1bceba609..fe486ad72a 100644 --- a/dm/gfx/nvn/hle_nvn.c +++ b/dm/gfx/nvn/hle_nvn.c @@ -17,6 +17,8 @@ #include "nvn_pace.h" #include "nvn_fstep.h" #include "nvn_timestamp.h" +#include "nvn_queue_schedule.h" +#include "nvn_obj.h" #include "../../host/front.h" #include "nvn_dirty.h" #include @@ -85,6 +87,14 @@ typedef struct { const char *name; void *fn; } nvn_impl; static int g_log; static pthread_mutex_t g_sub_mu = PTHREAD_MUTEX_INITIALIZER; +static _Atomic int g_event_waits; +static void qe_queue_fence(const queue_t *, sync_t *); +static int qe_wait(nvn_qs_ticket, uint64_t, int); +static int qe_wait_recorded(const sync_t *, uint64_t); +static void qe_flush_queue(void); +static void qe_finish(const queue_t *); +static void qe_cpu_signal(const ev_t *, int, uint32_t); +static void qe_sync_wait(const queue_t *, const sync_t *); static _Atomic uint64_t g_serial; uint64_t nvn_new_serial(void) { return ++g_serial; } void nvn_serial_floor(uint64_t s) { uint64_t c = atomic_load(&g_serial); while (c < s && !atomic_compare_exchange_weak(&g_serial, &c, s)) {} } @@ -736,10 +746,19 @@ NVN(nvnSyncInitialize, NVNboolean, NVNsync *s, NVNdevice *d) { (void)d; *(sync_t NVN(nvnSyncFinalize, void, NVNsync *s) { ((sync_t *)s)->magic = 0; } NVN(nvnSyncSetDebugLabel, void, NVNsync *s, const char *l) { (void)s; (void)l; } // Fence = "all commands previously submitted to the queue have completed": record the fence value of the most recent submit -NVN(nvnQueueFenceSync, void, NVNqueue *q, NVNsync *s, NVNsyncCondition c, int f) { (void)q; (void)c; (void)f; ((sync_t *)s)->fenced = 1; ((sync_t *)s)->fence = mtl_last_fence(); } +NVN(nvnQueueFenceSync, void, NVNqueue *q, NVNsync *s, NVNsyncCondition c, int f) { + (void)c; + (void)f; + qe_queue_fence((const queue_t *)q, (sync_t *)s); +} + // A sync object that was never fenced waits until timeout, as on a real device; a fenced one is released only after the GPU has actually completed up to that fence value -// A GPU-side wait on a sync: the host has a single Metal queue that executes submissions in order, so commands before the fence are always done first and nothing needs waiting. A sync not fenced yet (fenced later by another thread) is not waited for here; logged once -NVN(nvnQueueWaitSync, NVNboolean, NVNqueue *q, const NVNsync *s) { (void)q; static int once; if (!((const sync_t *)s)->fenced && !once++) LOG("[nvn] QueueWaitSync: the sync object has no fence yet, the host does not wait"); return 1; } +// A GPU-side wait yields its guest queue until the fenced prefix has been ordered on the shared Metal queue. +NVN(nvnQueueWaitSync, NVNboolean, NVNqueue *q, const NVNsync *s) { + qe_sync_wait((const queue_t *)q, (const sync_t *)s); + return 1; +} + // ZCull data: the host does no ZCull, so there is nothing to save / restore NVN(nvnCommandBufferRestoreZCullData, void, NVNcommandBuffer *c, NVNbufferAddress a, size_t n) { (void)c; (void)a; (void)n; } NVN(nvnCommandBufferSaveZCullData, void, NVNcommandBuffer *c, NVNbufferAddress a, size_t n) { (void)c; (void)a; (void)n; } @@ -748,12 +767,37 @@ NVN(nvnCommandBufferSaveZCullData, void, NVNcommandBuffer *c, NVNbufferAddress a NVN_NOPROTO(nvnCommandBufferSetStencilCullCriteria, void, NVNcommandBuffer *c, int func, int ref) { (void)c; (void)func; (void)ref; } static _Atomic uint64_t g_sync_ns; NVN(nvnSyncWait, NVNsyncWaitResult, const NVNsync *s, uint64_t timeout) { - const sync_t *o = (const sync_t *)s; - if (!o->fenced) return NVN_SYNC_WAIT_RESULT_TIMEOUT_EXPIRED; - int (*w)(uint64_t, uint64_t) = o->fenced == 2 ? mtl_wait_acq : mtl_wait; - if (w(o->fence, 0)) return NVN_SYNC_WAIT_RESULT_ALREADY_SIGNALED; - uint64_t t0 = now_ns(); int ok = w(o->fence, timeout); atomic_fetch_add(&g_sync_ns, now_ns() - t0); // the frame's blocked SyncWait time (@sync_ms) - return ok ? NVN_SYNC_WAIT_RESULT_CONDITION_SATISFIED : NVN_SYNC_WAIT_RESULT_TIMEOUT_EXPIRED; + const sync_t *sync = (const sync_t *)s; + if (!sync->fenced) { + return NVN_SYNC_WAIT_RESULT_TIMEOUT_EXPIRED; + } + if (sync->fenced == NVN_SYNC_FENCE_RECORDED_PENDING) { + if (qe_wait_recorded(sync, 0)) { + return NVN_SYNC_WAIT_RESULT_ALREADY_SIGNALED; + } + uint64_t started = now_ns(); + int completed = qe_wait_recorded(sync, timeout); + atomic_fetch_add(&g_sync_ns, now_ns() - started); + return completed ? NVN_SYNC_WAIT_RESULT_CONDITION_SATISFIED : NVN_SYNC_WAIT_RESULT_TIMEOUT_EXPIRED; + } + if (sync->fenced == NVN_SYNC_FENCE_QUEUE_PREFIX) { + nvn_qs_ticket ticket = {sync->queue, sync->fence}; + if (qe_wait(ticket, 0, 1)) { + return NVN_SYNC_WAIT_RESULT_ALREADY_SIGNALED; + } + uint64_t started = now_ns(); + int completed = qe_wait(ticket, timeout, 1); + atomic_fetch_add(&g_sync_ns, now_ns() - started); + return completed ? NVN_SYNC_WAIT_RESULT_CONDITION_SATISFIED : NVN_SYNC_WAIT_RESULT_TIMEOUT_EXPIRED; + } + int (*wait_for_fence)(uint64_t, uint64_t) = sync->fenced == NVN_SYNC_FENCE_ACQUIRE ? mtl_wait_acq : mtl_wait; + if (wait_for_fence(sync->fence, 0)) { + return NVN_SYNC_WAIT_RESULT_ALREADY_SIGNALED; + } + uint64_t started = now_ns(); + int completed = wait_for_fence(sync->fence, timeout); + atomic_fetch_add(&g_sync_ns, now_ns() - started); // the frame's blocked SyncWait time (@sync_ms) + return completed ? NVN_SYNC_WAIT_RESULT_CONDITION_SATISFIED : NVN_SYNC_WAIT_RESULT_TIMEOUT_EXPIRED; } NVN(nvnEventBuilderSetDefaults, void, NVNeventBuilder *b) { memset(b, 0, sizeof(ev_builder_t)); } NVN(nvnEventBuilderSetStorage, void, NVNeventBuilder *b, const NVNmemoryPool *p, int64_t off) { *(ev_builder_t *)b = (ev_builder_t){(const pool_t *)p, off}; } @@ -765,7 +809,9 @@ NVN(nvnEventInitialize, NVNboolean, NVNevent *e, const NVNeventBuilder *b) { NVN(nvnEventFinalize, void, NVNevent *e) { ((ev_t *)e)->magic = 0; } NVN(nvnEventGetValue, uint32_t, const NVNevent *e) { return *((const ev_t *)e)->val; } static void ev_signal(const ev_t *e, int mode, uint32_t v) { if (mode == NVN_EVENT_SIGNAL_MODE_ADD) *e->val += v; else *e->val = v; } -NVN(nvnEventSignal, void, NVNevent *e, NVNeventSignalMode m, uint32_t v) { ev_signal((const ev_t *)e, m, v); } +NVN(nvnEventSignal, void, NVNevent *event, NVNeventSignalMode mode, uint32_t value) { + qe_cpu_signal((const ev_t *)event, mode, value); +} // ---------- Queue ---------- #define QB(b) ((queue_builder_t *)(b)) @@ -795,13 +841,23 @@ NVN(nvnQueueInitialize, NVNboolean, NVNqueue *q, const NVNqueueBuilder *b) { OLOG("QueueInitialize %p", (void *)q); return 1; } NVN(nvnQueueFinalize, void, NVNqueue *q) { - pthread_mutex_lock(&g_sub_mu); mtl_queue_forget(((queue_t *)q)->serial); ((queue_t *)q)->magic = 0; pthread_mutex_unlock(&g_sub_mu); + qe_finish((const queue_t *)q); + pthread_mutex_lock(&g_sub_mu); + mtl_queue_forget(((queue_t *)q)->serial); + ((queue_t *)q)->magic = 0; + pthread_mutex_unlock(&g_sub_mu); } + NVN(nvnQueueGetError, NVNqueueGetErrorResult, NVNqueue *q, NVNqueueErrorInfo *info) { (void)q; if (info) memset(info, 0, sizeof *info); return NVN_QUEUE_GET_ERROR_RESULT_GPU_NO_ERROR; } -NVN(nvnQueueFlush, void, NVNqueue *q) { (void)q; } -NVN(nvnQueueFinish, void, NVNqueue *q) { (void)q; mtl_finish(); } +NVN(nvnQueueFlush, void, NVNqueue *q) { + (void)q; + qe_flush_queue(); +} +NVN(nvnQueueFinish, void, NVNqueue *q) { + qe_finish((const queue_t *)q); +} // ---------- Command buffer: record into the game's command memory, perform CPU-visible side effects at submit ---------- #define CB(c) ((cmd_t *)(c)) @@ -950,12 +1006,37 @@ NVN(nvnCommandBufferIsRecording, NVNboolean, const NVNcommandBuffer *c) { return NVN(nvnCommandBufferCopyCommands, void, NVNcommandBuffer *c, int n, const NVNcommandHandle *h) { for (int i = 0; i < n; i++) { uint8_t *p = cmd_alloc(CB(c), 16); put_hdr(p, OP_CALL, 16); memcpy(p + 8, &h[i], 8); } } -NVN(nvnCommandBufferSignalEvent, void, NVNcommandBuffer *c, const NVNevent *e, NVNeventSignalMode m, NVNeventSignalLocation loc, int flags, uint32_t v) { - (void)loc; (void)flags; - uint8_t *p = cmd_alloc(CB(c), 24); put_hdr(p, OP_SIGNAL_EVENT, 24); - const ev_t *ev = (const ev_t *)e; volatile uint32_t *val = ev->val; uint32_t mm = m; - memcpy(p + 8, &val, 8); memcpy(p + 16, &mm, 4); memcpy(p + 20, &v, 4); +NVN(nvnCommandBufferSignalEvent, void, NVNcommandBuffer *command_buffer, const NVNevent *event, NVNeventSignalMode mode, + NVNeventSignalLocation loc, int flags, uint32_t value) { + uint8_t *record = cmd_alloc(CB(command_buffer), 32); + put_hdr(record, OP_SIGNAL_EVENT, 32); + const ev_t *event_state = (const ev_t *)event; + volatile uint32_t *event_value = event_state->val; + uint32_t signal_mode = mode; + memcpy(record + 8, &event_value, 8); + memcpy(record + 16, &signal_mode, 4); + memcpy(record + 20, &value, 4); + uint32_t location = loc; + memcpy(record + 24, &location, 4); + memcpy(record + 28, &flags, 4); +} +NVN(nvnCommandBufferWaitEvent, void, NVNcommandBuffer *command_buffer, const NVNevent *event, NVNeventWaitMode mode, uint32_t value) { + uint8_t *record = cmd_alloc(CB(command_buffer), 24); + put_hdr(record, OP_WAIT_EVENT, 24); + volatile uint32_t *event_value = ((const ev_t *)event)->val; + uint32_t wait_mode = mode; + memcpy(record + 8, &event_value, 8); + memcpy(record + 16, &wait_mode, 4); + memcpy(record + 20, &value, 4); + atomic_store(&g_event_waits, 1); +} +NVN(nvnCommandBufferWaitSync, void, NVNcommandBuffer *command_buffer, const NVNsync *sync) { + uint8_t *record = cmd_alloc(CB(command_buffer), 16); + put_hdr(record, OP_WAIT_SYNC, 16); + memcpy(record + 8, &sync, 8); + atomic_store(&g_event_waits, 1); } + // A fence recorded in a command buffer: the replay fences the sync with the submission that reaches this point (OP_FENCE_SYNC). Unreal Engine's NVN RHI fences its command buffers this way (Octopath Traveler 0); Pokémon Legends: Z-A fences once a frame this way and waits on it NVN(nvnCommandBufferFenceSync, void, NVNcommandBuffer *c, NVNsync *s, NVNsyncCondition cond, int flags) { (void)cond; (void)flags; @@ -1297,34 +1378,53 @@ void dm_gfx_suspend(int on) { pthread_mutex_unlock(&g_susp_mu); if (on) { dm_vk_drain(); dm_gl_drain(); mtl_drain(); LOG("[nvn] graphics suspended: in-flight GPU work drained"); } else LOG("[nvn] graphics resumed"); // the Vulkan and GL queues first: presents on the NVN queue wait for their frames } +#include "nvn_queue_dispatch.inc" + // One submission at a time: the Metal replay keeps one command buffer and one upload batch for the submission in progress (nvn_mtl.m g_cb). Fire Emblem: Three Houses initializes two queues and -// submits on both from two threads; the second submission arrived while the first was replaying and the host stopped with "submit re-entered" at the ninth frame. NVN orders work on different queues only -// through syncs, so replaying the two queues' submissions in lock order is one valid order. +// submits on both from two threads; the second submission arrived while the first was replaying and the host stopped with "submit re-entered" at the ninth frame. Cross-queue events and syncs +// additionally constrain replay order; the scheduler suspends blocked prefixes while other guest queues supply their producers. // The first submission's prepare step (mtl_prepare: the prewarm and the pipeline prebuild) loads shaders and builds pipelines and submits no GPU work, so it runs under the lock but outside the // graphics gate: inside it, going to the background meanwhile (dm_gfx_suspend, on the main thread) waited for it, minutes on a first start. It stays host work for the guest scheduler. The lock // comes before the gate: a second submission waiting for it must not count as in flight, or the suspend waits for it while the first waits to enter the closed gate holding the lock NVN(nvnQueueSubmitCommands, void, NVNqueue *q, int n, const NVNcommandHandle *h) { dm_host_mutex_lock(&g_sub_mu); - dm_host_work++; mtl_prepare(); dm_host_work--; + dm_host_work++; + mtl_prepare(); + dm_host_work--; gfx_enter(); - uint64_t t0 = now_ns(), c0 = clock_gettime_nsec_np(CLOCK_THREAD_CPUTIME_ID), i0, y0, s0 = thr_sys_ns(); thr_counts(&i0, &y0); + uint64_t t0 = now_ns(), c0 = clock_gettime_nsec_np(CLOCK_THREAD_CPUTIME_ID), i0, y0, s0 = thr_sys_ns(); + thr_counts(&i0, &y0); ((queue_t *)q)->submits++; - mtl_queue_activate(((const queue_t *)q)->serial); - mtl_set_queue_origin_lower_left(((const queue_t *)q)->origin_ll); - nvn_cap_submit(n, (const uint64_t *)h); // DM_NVN_CAPTURE (nvn_capture.inc) - mtl_sub_begin(); + nvn_cap_submit(n, (const uint64_t *)h); // DM_NVN_CAPTURE (nvn_capture.inc) nvn_ft_poll(); - mtl_sub_index((const uint64_t *)h, n); - for (int i = 0; i < n; i++) { mtl_exec_rest((const uint64_t *)h + i + 1, n - i - 1); mtl_exec((const uint8_t *)h[i]); } - mtl_exec_rest(NULL, 0); - mtl_sub_commit(); + if (g_qs || atomic_load(&g_event_waits)) { + qe_submit((const queue_t *)q, n, (const uint64_t *)h); + } else { + mtl_queue_activate(((const queue_t *)q)->serial); + mtl_set_queue_origin_lower_left(((const queue_t *)q)->origin_ll); + mtl_sub_begin(); + mtl_sub_index((const uint64_t *)h, n); + for (int i = 0; i < n; i++) { + mtl_exec_rest((const uint64_t *)h + i + 1, n - i - 1); + mtl_exec((const uint8_t *)h[i]); + } + mtl_exec_rest(NULL, 0); + mtl_sub_commit(); + } pthread_mutex_unlock(&g_sub_mu); uint64_t dw = now_ns() - t0; - atomic_fetch_add(&g_nvn_ns, dw); atomic_fetch_add(&g_sub_wall_ns, dw); - uint64_t i1, y1; thr_counts(&i1, &y1); uint64_t dc = clock_gettime_nsec_np(CLOCK_THREAD_CPUTIME_ID) - c0, di = i1 - i0; - atomic_fetch_add(&g_cum_sub_cyc, y1 - y0); atomic_fetch_add(&g_cum_sub_sys_ns, thr_sys_ns() - s0); atomic_fetch_add(&g_cum_sub_wall_ns, dw); - atomic_fetch_add(&g_sub_cpu_ns, dc); atomic_fetch_add(&g_cum_sub_cpu_ns, dc); - atomic_fetch_add(&g_sub_instr, di); atomic_fetch_add(&g_cum_sub_instr, di); + atomic_fetch_add(&g_nvn_ns, dw); + atomic_fetch_add(&g_sub_wall_ns, dw); + uint64_t i1, y1; + thr_counts(&i1, &y1); + uint64_t dc = clock_gettime_nsec_np(CLOCK_THREAD_CPUTIME_ID) - c0, di = i1 - i0; + atomic_fetch_add(&g_cum_sub_cyc, y1 - y0); + atomic_fetch_add(&g_cum_sub_sys_ns, thr_sys_ns() - s0); + atomic_fetch_add(&g_cum_sub_wall_ns, dw); + atomic_fetch_add(&g_sub_cpu_ns, dc); + atomic_fetch_add(&g_cum_sub_cpu_ns, dc); + atomic_fetch_add(&g_sub_instr, di); + atomic_fetch_add(&g_cum_sub_instr, di); gfx_leave(); } @@ -1333,7 +1433,6 @@ static void nvn_nop(void) {} static const char *const g_nops[] = { "nvnCommandBufferSetPatchSize", "nvnCommandBufferSetDepthBounds", - "nvnCommandBufferWaitSync", "nvnCommandBufferSetShaderScratchMemory", "nvnCommandBufferPushDebugGroupDynamic", "nvnCommandBufferPopDebugGroup", @@ -1343,7 +1442,6 @@ static const char *const g_nops[] = { "nvnCommandBufferPushDebugGroupStatic", "nvnCommandBufferPopDebugGroupId", "nvnCommandBufferInsertDebugMarkerStatic", - "nvnCommandBufferWaitEvent", "nvnWindowFinalize", }; __attribute__((constructor)) static void nvn_nops_register(void) { @@ -1455,9 +1553,12 @@ NVN(nvnWindowAcquireTexture, NVNwindowAcquireTextureResult, NVNwindow *w, NVNsyn pace_ns += mtl_snap_acquire_wait(); // ring capacity is waited for at acquire, where the game does not hold its present lock g_acq_wait_ns += wait; trow(g_frame, "@late_ms", wait ? (now_ns() - at) / 1e6 : 0); // The previous present of this texture has not yet been processed by the present thread (display cannot keep up, a whole swapchain round behind): wait for it to be returned, then hand it to the game to draw + if (*idx < 4 && o->submit_queue[*idx]) { + qe_wait((nvn_qs_ticket){o->submit_queue[*idx], o->submit_seq[*idx]}, UINT64_MAX, 0); + } uint64_t tk = *idx < 4 ? o->ticket[*idx] : 0; if (tk) { uint64_t w0 = now_ns(); mtl_present_wait(tk); g_pq_wait_ns += now_ns() - w0; } - if (s) { ((sync_t *)s)->fenced = 2; ((sync_t *)s)->fence = mtl_last_fence(); } // The swapchain texture becomes available once the commits before this point have completed (fenced 2: an acquire sync, for the wait ledger) + if (s) { ((sync_t *)s)->fenced = NVN_SYNC_FENCE_ACQUIRE; ((sync_t *)s)->fence = mtl_last_fence(); } // The swapchain texture becomes available once the commits before this point have completed (fenced 2: an acquire sync, for the wait ledger) if (!g_fs_start) { g_fs_start = now_ns(); g_fs_wait = pace_ns; } // the frame starts when the first acquire after a present returns: on a display-link tick when on time return NVN_WINDOW_ACQUIRE_TEXTURE_RESULT_SUCCESS; } @@ -1598,6 +1699,9 @@ static void tsummary(uint64_t frame) { double v[TWIN]; memcpy(v, g_tw[k], n * sizeof *v); qsort(v, n, sizeof *v, dcmp); o += snprintf(line + o, sizeof line - o, " %s p50 %.2f / p95 %.2f / max %.2f ms (%u frames);", g_tname[k], v[n / 2], v[(n * 95) / 100 < n ? (n * 95) / 100 : n - 1], v[n - 1], n); } + { static uint64_t pf, pchunk, pmerged; double nf = (double)(frame - pf); // the guest queue scheduler's chunks (nvn_queue_dispatch.inc): how many host submissions a frame they cost, and how many DM_NVN_CHUNK_MERGE spared + if (g_qe_chunks && pf && frame > pf) o += snprintf(line + o, sizeof line - o, " scheduler chunks per frame %.2f, %.2f of them merged into an open command buffer;", (double)(g_qe_chunks - pchunk) / nf, (double)(g_qe_merged - pmerged) / nf); + pf = frame; pchunk = g_qe_chunks; pmerged = g_qe_merged; } LOG("[nvn] timing up to frame %llu:%s", frame, line); { void mtl_pso_line(char *, size_t); char pl[1024]; mtl_pso_line(pl, sizeof pl); LOG("[nvn] pipelines up to frame %llu: %s", frame, pl); } double tgt = 1000.0 / mtl_fps_target(); // milliseconds one frame should take: 16.7 at 60 fps, 33.3 when the game itself asks for 30 (the over-time / dropped-frame counts below use it) @@ -1998,14 +2102,20 @@ NVN(nvnQueuePresentTexture, void, NVNqueue *q, NVNwindow *w, int idx) { int lag = 0; for (int i = 0; i < lag_k; i++) if (o->presents % (uint64_t)lag_n[i] < (uint64_t)lag_run[i]) lag += lag_ms[i]; if (lag) usleep((useconds_t)lag * 1000); } + dm_host_mutex_lock(&g_sub_mu); gfx_enter(); uint64_t t0 = now_ns(); if (idx >= 0 && idx < o->ntex && idx < 4) { // enqueue only, return immediately const tex_t *t = CTX(o->tex[idx]); int lc[4]; const int *crop = present_crop(o->crop, (int)t->d.width, (int)t->d.height, lc); nvn_cap_present(t, crop); // DM_NVN_CAPTURE (nvn_capture.inc) - o->ticket[idx] = mtl_present(t, present_period(o) / 1e9, crop); + if (g_qs) { + qe_present((const queue_t *)q, o, idx, t, present_period(o) / 1e9, crop); + } else { + o->ticket[idx] = mtl_present(t, present_period(o) / 1e9, crop); + } } gfx_leave(); + pthread_mutex_unlock(&g_sub_mu); atomic_fetch_add(&g_nvn_ns, now_ns() - t0); uint64_t f = g_frame, ps, dr; mtl_stats(&ps, &dr); if (nvn_real[0] || nvn_real[1]) { // DM_NVN_PROF: record phase = time of all implemented functions except submit; submit phase = QueueSubmitCommands @@ -2985,6 +3095,6 @@ int nvn_host_write_texture(const void *nvntex, const void *src, int w, int h, si int nvn_host_signal_sync(void *nvnsync) { sync_t *o = nvnsync; if (!o || o->magic != MAGIC('S', 'Y', 'N', 'C')) return -1; - o->fence = 0; o->fenced = 1; return 0; + o->fence = 0; o->fenced = NVN_SYNC_FENCE_GPU; return 0; } NVN(nvnTextureFlushTexels, void, const NVNtexture *t, const NVNtextureView *v, const NVNcopyRegion *r) { (void)t; (void)v; (void)r; } // unified memory + dirty page tracking: nothing to flush diff --git a/dm/gfx/nvn/nvn_capture.c b/dm/gfx/nvn/nvn_capture.c index c75f0b6473..b47ae34d45 100644 --- a/dm/gfx/nvn/nvn_capture.c +++ b/dm/gfx/nvn/nvn_capture.c @@ -10,7 +10,7 @@ __asm__(".linker_option \"-lcompression\""); // libcompression (LZFSE) links i #define CAP_SIZE(T, n) _Static_assert(sizeof(T) == (n), "nvn_capture.c: " #T " changed size: update cap_rec_fields and cap_layout") #define CAP_OFF(T, f, n) _Static_assert(offsetof(T, f) == (n), "nvn_capture.c: " #T "." #f " moved: update cap_rec_fields and cap_layout") CAP_SIZE(tex_t, 128); CAP_OFF(tex_t, mem, 88); CAP_OFF(tex_t, pool, 96); CAP_OFF(tex_t, serial, 120); -CAP_SIZE(pool_t, 40); CAP_SIZE(dpool_t, 24); CAP_SIZE(prog_t, 112); CAP_SIZE(sync_t, 16); CAP_SIZE(samp_par, 40); CAP_SIZE(view_t, 16); +CAP_SIZE(pool_t, 40); CAP_SIZE(dpool_t, 24); CAP_SIZE(prog_t, 112); CAP_SIZE(sync_t, 24); CAP_SIZE(samp_par, 40); CAP_SIZE(view_t, 16); CAP_SIZE(p_set_rt, 240); CAP_OFF(p_set_rt, n, 0); CAP_OFF(p_set_rt, color, 8); CAP_OFF(p_set_rt, depth, 208); CAP_SIZE(p_bind_program, 16); CAP_OFF(p_bind_program, prog, 0); CAP_SIZE(NVNbufferRange, 16); CAP_SIZE(p_bind_vbuf, 264); CAP_OFF(p_bind_vbuf, n, 4); CAP_OFF(p_bind_vbuf, r, 8); @@ -33,25 +33,24 @@ CAP_SIZE(p_render_cond, 16); CAP_OFF(p_render_cond, mode, 0); CAP_OFF(p_render_c CAP_SIZE(p_multi_draw, 48); CAP_OFF(p_multi_draw, ind, 0); CAP_OFF(p_multi_draw, cnt, 8); CAP_OFF(p_multi_draw, ibuf, 16); CAP_OFF(p_multi_draw, stride, 24); CAP_OFF(p_multi_draw, max, 32); CAP_OFF(p_multi_draw, prim, 36); CAP_OFF(p_multi_draw, itype, 40); _Static_assert(OP_MULTI_DRAW == 55, "keep existing capture op numbers stable"); -_Static_assert(OP_COUNT_ == 56, "nvn_capture.c: a record op was added or removed: give it a case in cap_rec_fields"); +_Static_assert(OP_COUNT_ == 58, "nvn_capture.c: a record op was added or removed: give it a case in cap_rec_fields"); #define MULTI_DRAW_LAYOUT "op 55 p_multi_draw 48 ind 0 cnt 8 ibuf 16 stride 24 max 32 prim 36 itype 40\n" const char *cap_layout(void) { // sizes and pointer offsets as the walk knows them (the asserts above hold them to nvn_obj.h); bump this text with any change to the walk - return "format 1\nops 54\n" - "tex_t 128 mem 88 pool 96 serial 120\npool_t 40\ndpool_t 24\nprog_t 112\nsync_t 16\nsamp_par 40\nview_t 16\n" + return "format 1\nops 58\n" + "tex_t 128 mem 88 pool 96 serial 120\npool_t 40\ndpool_t 24\nprog_t 112\nsync_t 24\nsamp_par 40\nview_t 16\n" "p_set_rt 240 color 8 depth 208\np_bind_program 16\np_bind_vbuf 264 r 8\np_bind_vbuf1 16 addr 8\np_bind_buf 24 addr 8\np_bind_ubo_small 16 addr 8\n" "p_bind_handles 280 src 16\np_bind_tex1 24 nobase 0x100 short 16\np_bind_state short\np_draw 48 ind 24 mdi_cnt 32\np_draw_idx 56 ibuf 24 ind 32 mdi_cnt 40\np_dispatch 32 ind 24\np_set_pool 8\n" "p_copy_b2t 64 src 0 tex 8\np_copy_t2t 104 src 0 dst 8\np_clear_tex 72 tex 0\np_clear_buf 24 addr 0\np_tf_bind 24 addr 8\np_tf_state 16 ctl 8\np_render_cond 16 addr 8\n" - "signal_event 16 val 0\nreport_counter 16 addr 8\ncopy_b2b 24 src 0 dst 8\nupdate_ub dst 0\nfence_sync 8 sync 0\njump 8\ncall 8\n" + "signal_event 24 val 0 location 16 flags 20\nwait_event 16 val 0\nwait_sync 8 sync 0\nreport_counter 16 addr 8\ncopy_b2b 24 src 0 dst 8\nupdate_ub dst 0\nfence_sync 8 sync 0\njump 8\ncall 8\n" MULTI_DRAW_LAYOUT; } int cap_layout_matches(const char *layout, size_t n) { - // The new opcode was appended. Captures with the complete previous layout still use the same op numbers and payloads. + // Event scheduling changes the sync-object size and signal payload. const char *current = cap_layout(); - size_t full = strlen(current), previous = full - (sizeof MULTI_DRAW_LAYOUT - 1); - return (n == full || n == previous) && !memcmp(layout, current, n); + return n == strlen(current) && !memcmp(layout, current, n); } // One 8-byte field at byte offset off of the payload, present only when the record is long enough to hold it @@ -75,10 +74,12 @@ int cap_rec_fields(uint32_t op, uint8_t *payload, uint32_t size, const cap_ops * case OP_DEPTH_CLAMP: case OP_RAST_DISCARD: case OP_VP_SWIZZLE: case OP_PRIM_RESTART: case OP_ALPHA_REF: case OP_PUSH_CONST: case OP_DRAW_TEX: // OP_DRAW_TEX: a texture handle, read through the pools at replay return 1; case OP_JUMP: case OP_CALL: return HAS(0) && ops->obj(ctx, CAP_K_SEG, F(0), payload, (uint32_t)plen); + case OP_WAIT_EVENT: case OP_SIGNAL_EVENT: return HAS(0) && ops->addr(ctx, F(0), 4, 1); // volatile uint32_t *val, mode, value case OP_REPORT_COUNTER: return HAS(8) && ops->addr(ctx, F(8), 16, 1); // type, address of the 16-byte report case OP_COPY_B2B: { if (!HAS(16)) return 0; uint64_t n = u64_at(payload + 16); return ops->addr(ctx, F(0), n, 0) & ops->addr(ctx, F(8), n, 1); } case OP_UPDATE_UB: { if (!HAS(8)) return 0; return ops->addr(ctx, F(0), u64_at(payload + 8), 1); } // dst, n, inline data + case OP_WAIT_SYNC: case OP_FENCE_SYNC: return HAS(0) && ops->obj(ctx, CAP_K_SYNC, F(0), payload, (uint32_t)plen); case OP_SET_RT: { int32_t n = HAS(0) ? i32_at(payload) : -1; if (n < 0 || n > 8 || !HAS(offsetof(p_set_rt, depth))) return 0; diff --git a/dm/gfx/nvn/nvn_mtl.h b/dm/gfx/nvn/nvn_mtl.h index 1c960a3253..bbde380718 100644 --- a/dm/gfx/nvn/nvn_mtl.h +++ b/dm/gfx/nvn/nvn_mtl.h @@ -35,7 +35,32 @@ void mtl_res_snapshot(mtl_res_count *out); // One submit: begin -> several operations -> commit (returns the fence value at which this submit completes) void mtl_prepare(void); // the first submission's prepare step (prewarm, pipeline prebuild): no GPU work, called outside the graphics gate void mtl_sub_begin(void); +void mtl_queue_seal(void); // own persistent target snapshots before record retirement +void mtl_sub_resume(void); // continuation after a guest event boundary +// Serialized by the submit lock. Event chunks append to one allocator lease; +// pressure may revoke its completed prefix so blocked submissions cannot pin +// the bounded allocator pool forever. Unencoded continuations reacquire it. +typedef struct mtl_submit_alloc mtl_submit_alloc; +mtl_submit_alloc *mtl_submit_alloc_new(void); +void mtl_submit_alloc_close(mtl_submit_alloc *alloc); // last chunk committed, or owner released +void mtl_submit_alloc_release(mtl_submit_alloc *alloc); +void mtl_sub_begin_alloc(mtl_submit_alloc *alloc, int resume); +// Submission-mode timestamps share one clock across host event chunks. The +// logical submission owns the token; set_clock retains it until completion. +// Segment-mode GPU timestamps and direct submits without a token are unchanged. +typedef struct mtl_submit_clock mtl_submit_clock; +mtl_submit_clock *mtl_submit_clock_new(void); +void mtl_submit_clock_release(mtl_submit_clock *clock); +void mtl_sub_set_clock(mtl_submit_clock *clock); // after begin/resume, before encoding reports +uint64_t mtl_completed_fence(void); // post-operations are visible through this fence void mtl_queue_activate(uint64_t serial); // submit lock held: restore this queue's persistent GPU state +// DM_NVN_CHUNK_MERGE (nvn_queue_dispatch.inc qe_pump): chunks the scheduler offers in a row share one command buffer. mtl_chunk_merge: may the next chunk, with this +// logical submission's clock, go into the open buffer (open: one is); mtl_chunk_end / mtl_chunk_begin: a chunk boundary inside the open buffer (what commit and begin do +// for the chunk's own state, without a commit or a new buffer); mtl_pending_fence: the fence the open buffer commits with +int mtl_chunk_merge(mtl_submit_clock *clock, int open); +void mtl_chunk_end(void); +void mtl_chunk_begin(int resume); +uint64_t mtl_pending_fence(void); void mtl_queue_forget(uint64_t serial); // finalize: discard its state, including guest object pointers void mtl_sub_copy(uint64_t src, uint64_t dst, uint64_t n); // Buffer copy in GPU ordering void mtl_sub_event(volatile uint32_t *val, uint32_t mode, uint32_t v); // Write the event value after completion (0 write / 1 add) diff --git a/dm/gfx/nvn/nvn_mtl.m b/dm/gfx/nvn/nvn_mtl.m index ac60cc6f71..2e901c673f 100644 --- a/dm/gfx/nvn/nvn_mtl.m +++ b/dm/gfx/nvn/nvn_mtl.m @@ -97,7 +97,7 @@ static inline uint64_t tex_slot_h(const uint64_t *tex, const uint64_t *img, unsi static dispatch_queue_t g_feedback_q; static _Atomic uint32_t g_present_qos; #define ALLOC_MAX 8 -static id g_alloc4[ALLOC_MAX]; static int g_alloc_n, g_alloc_cur; // Command allocator pool of the main submission: at start pick one the GPU has finished with, create a new one if there is none (wait only at the cap) +static id g_alloc4[ALLOC_MAX]; static int g_alloc_n, g_alloc_cur; // Bounded pool: logical event chunks append through a lease; reset only after every committed use completes static id g_up_alloc4[3]; static id g_present_alloc4[3]; static id g_present_cb[3]; @@ -174,9 +174,9 @@ static void cp_report(void) { static pthread_mutex_t g_res_mu = PTHREAD_MUTEX_INITIALIZER; // Dedicated lock for the residency set: res_add is called on paths that hold g_mu (arena_new), so it cannot share g_mu // Wait ledger (DM_NVN_WAITS=1): every place a thread of the host or the game waits on the CPU for the GPU or for the encode thread, counted per frame (calls, calls that blocked, // the time they blocked) and logged every 300 frames with the frame's GPU idle. The game's own SyncWait is split by how far its fence is behind the newest one (0: the last submission) -enum { WS_SYNC, WS_SYNC_NEWEST, WS_SYNC_POLL, WS_ACQ, WS_FINISH, WS_ALLOC, WS_VIS, WS_PG, WS_MDI, WS_READTEX, WS_T2T, WS_DIAG, WS_SNAP, WS_UPZERO, WS_UPALLOC, WS_DRAIN, WS_UBC, WS_OLDQ, WS_N }; +enum { WS_SYNC, WS_SYNC_NEWEST, WS_SYNC_POLL, WS_ACQ, WS_FINISH, WS_ALLOC, WS_VIS, WS_PG, WS_MDI, WS_READTEX, WS_T2T, WS_DIAG, WS_SNAP, WS_UPZERO, WS_UPALLOC, WS_DRAIN, WS_UBC, WS_OLDQ, WS_POST, WS_N }; static const char *const ws_name[WS_N] = {"SyncWait (game, older fence)", "SyncWait (game, newest fence)", "SyncWait polls", "SyncWait on an acquire sync", "QueueFinish", "allocator", "occlusion ring", "PASSGPU ring", - "indirect-count multi-draw", "texture readback", "texture copy t2t", "diagnostics", "snapshot ring", "upload batches", "upload slot", "encode thread drain", "uniform copy worker", "old-queue command buffer"}; + "indirect-count multi-draw", "texture readback", "texture copy t2t", "diagnostics", "snapshot ring", "upload batches", "upload slot", "encode thread drain", "uniform copy worker", "old-queue command buffer", "completion records"}; static _Atomic uint64_t g_ws_n[WS_N], g_ws_blk[WS_N], g_ws_ns[WS_N]; static int ws_on(void) { static int v = -1; if (v < 0) { const char *e = getenv("DM_NVN_WAITS"); v = e && atoi(e) > 0; } return v; } static inline uint64_t ws_t0(void) { return clock_gettime_nsec_np(CLOCK_UPTIME_RAW); } @@ -702,6 +702,7 @@ int mtl_gpu_addr(uint64_t addr, uint64_t len, void **buf, uint64_t *off, uint64_ static void mirror_rb_land(void *m); // kind 4: a GPU copy's bytes read back into the destination texture's storage (mirror_readback) static uint32_t g_seg; // Index of the segment being encoded in this submission (0 until the first sub_cut) typedef struct seg_clk seg_clk; static seg_clk *g_sclk; static void seg_clk_put(seg_clk *c); // Counter timestamps by segment (see gpu_ticks) +static mtl_submit_clock *g_submit_clock; // retained for this host chunk, transferred to its completion record // ---------- Conditional rendering (profile render_cond) ---------- // SetRenderEnableConditional(mode, rec) enables rendering if the 64-bit counter of the report at rec equals (RENDER_IF_EQUAL) or differs from (RENDER_IF_NOT_EQUAL) the one at // rec + 16, until the next SetRenderEnable(true). Tears of the Kingdom wraps nearly every ClearColor / ClearDepthStencil this way: rec is set to 1 from the CPU when the target @@ -750,16 +751,31 @@ static int rc_value(uint64_t a, uint64_t *v) { // 0 = unknown at encode time enum { VIS_RING = ALLOC_MAX }; static vis_buf g_vis_ring[VIS_RING]; static int g_vis_ring_n, g_vis_ring_i = -1; static id g_vis; static uint32_t g_vis_n; // This submission's visibility buffer and number of slots used +static int g_vis_ring_used; // the ring buffer was bound, even if overflow later replaced g_vis static int vis_in_ring(id b) { for (int i = 0; i < g_vis_ring_n; i++) if (g_vis_ring[i].buf == b) return 1; return 0; } +static int vis_buffer_pinned(id b); // active or saved queue's cumulative query still reads slots in it static int g_vis_on, g_vis_counting, g_vis_cur = -1; static uint64_t g_vis_cur_pass; static vis_slot *g_vs; static int g_nvs, g_vscap; // Slots since the last reset static uint64_t g_cnt_report[32], g_cnt_reset[32]; // Report / reset counts per counter type (diagnostics) static void vis_clear(void) { for (int i = 0; i < g_nvs; i++) CFRelease(g_vs[i].buf); g_nvs = 0; g_vis_cur = -1; } static vis_list *vis_snapshot(void) { - vis_list *l = malloc(sizeof *l + g_nvs * sizeof(vis_slot)); l->n = g_nvs; - for (int i = 0; i < g_nvs; i++) { l->s[i] = g_vs[i]; CFRetain(g_vs[i].buf); } + vis_list *l = malloc(sizeof *l + g_nvs * sizeof(vis_slot)); + l->n = g_nvs; + uint64_t fence = g_fence + 1; + for (int i = 0; i < g_nvs; i++) { + l->s[i] = g_vs[i]; + CFRetain(g_vs[i].buf); + // A reset can drop the cumulative list before this report completes. + // Protect its CPU callback's reads as well as the buffer's GPU writes. + for (int k = 0; k < g_vis_ring_n; k++) { + if ((__bridge void *)g_vis_ring[k].buf == g_vs[i].buf && g_vis_ring[k].fence < fence) { + g_vis_ring[k].fence = fence; + } + } + } return l; } + // Target ranges of buffer writes (copies / inline uniform updates) already encoded in this submission but not yet executed on the GPU: reading these addresses on the CPU side at encode time yields old values (uv_proof avoids them when reading uniforms) // Inline updates keep where their data sits in the update ring (g_pend_upd: index + 1 into g_upd), so a bindless handle they wrote can be read as the GPU will see it (pend_read). // Monster Hunter Rise queues more than 256 writes in a submission (the old limit, after which every address counted as written). The writes are indexed by 4 KB page (a chain per hash @@ -860,8 +876,86 @@ static int pend_read(uint64_t a, uint64_t n, void *out, int *kind) { static struct { uint64_t fence, lo, hi; } g_gsr[GSR_N]; static uint64_t g_gsr_n; static int g_gml_on; static pthread_mutex_t g_gsr_mu = PTHREAD_MUTEX_INITIALIZER; static uint64_t g_alloc_fence[ALLOC_MAX]; static uint64_t g_alloc_waits, g_alloc_grown; +struct mtl_submit_alloc { + int slot; +}; +static mtl_submit_alloc *g_alloc_owner[ALLOC_MAX]; +mtl_submit_alloc *mtl_submit_alloc_new(void) { + mtl_submit_alloc *lease = malloc(sizeof *lease); + if (!lease) { + MDIE("submission allocator lease allocation failed"); + } + lease->slot = -1; + return lease; +} +void mtl_submit_alloc_close(mtl_submit_alloc *lease) { + if (!lease || lease->slot < 0) { + return; + } + if (g_alloc_owner[lease->slot] != lease) { + MDIE("submission allocator lease lost its owner"); + } + g_alloc_owner[lease->slot] = NULL; + lease->slot = -1; +} +void mtl_submit_alloc_release(mtl_submit_alloc *lease) { + mtl_submit_alloc_close(lease); + free(lease); +} int mtl_wait(uint64_t fence, uint64_t timeout_ns); static int mtl_wait_at(uint64_t fence, uint64_t timeout_ns, int site); +static int alloc_acquire(mtl_submit_alloc *owner) { + if (owner && owner->slot >= 0) { + int i = owner->slot; + if (g_alloc_owner[i] != owner) { + MDIE("submission allocator lease lost its slot"); + } + // endCommandBuffer disassociates the previous buffer. Its encoded bytes + // remain valid while the next buffer appends, even before GPU completion. + return i; + } + int slot = -1; + uint64_t done = g_done; + for (int i = 0; i < g_alloc_n; i++) { + if (!g_alloc_owner[i] && g_alloc_fence[i] <= done) { + slot = i; + break; + } + } + if (slot < 0 && g_alloc_n < ALLOC_MAX) { + NSError *error = nil; + MTL4CommandAllocatorDescriptor *descriptor = [MTL4CommandAllocatorDescriptor new]; + g_alloc4[g_alloc_n] = [g_dev newCommandAllocatorWithDescriptor:descriptor error:&error]; + if (!g_alloc4[g_alloc_n]) { + MDIE("failed to create a new MTL4CommandAllocator: %s", error.localizedDescription.UTF8String); + } + slot = g_alloc_n++; + g_alloc_grown++; + } + if (slot < 0) { + // A logical submission can remain blocked forever. Revoke only its + // completed encoded prefix; future chunks have no allocator bytes yet. + // Waiting for logical END here would deadlock a ninth producer queue. + slot = 0; + for (int i = 1; i < g_alloc_n; i++) { + if (g_alloc_fence[i] < g_alloc_fence[slot]) { + slot = i; + } + } + if (g_alloc_fence[slot] > done) { + g_alloc_waits++; + mtl_wait_at(g_alloc_fence[slot], UINT64_MAX, WS_ALLOC); + } + mtl_submit_alloc_close(g_alloc_owner[slot]); + } + [g_alloc4[slot] reset]; + g_alloc_owner[slot] = owner; + if (owner) { + owner->slot = slot; + } + return slot; +} + static void up_flush(void); // The upload batch belongs only to the submit thread (the thread that called mtl_sub_begin): the batch's encoder cannot be used across threads. Every entry point asserts current thread == submit thread, // and on mismatch prints the stack and aborts on the spot - so that 'another thread touched the batch' is exposed deterministically on Mac too, not by timing (real iOS device stuck at the main menu: the present thread flushed the submit thread's batch via q_cb). @@ -894,43 +988,104 @@ void mtl_prepare(void) { if (!g_warm_started && prepare_on()) { mtl_init(); shmap_load(); } } static void sub_cut_begin(void); -void mtl_sub_begin(void) { - mtl_init(); g_sub_serial++; g_seg = 0; seg_clk_put(g_sclk); g_sclk = NULL; g_rc_npend = 0; g_rc_skip = 0; +static void sub_begin(int resume, mtl_submit_alloc *owner) { + mtl_init(); + g_sub_serial++; + g_seg = 0; + seg_clk_put(g_sclk); + g_sclk = NULL; + if (!resume) { + g_rc_npend = 0; + g_rc_skip = 0; + } sub_cut_begin(); defer_drain(); defer_reset(); - mtl_prepare(); // done by then in the submit entry; the replay tool (nvn_replay) reaches it here + mtl_prepare(); // done by then in the submit entry; the replay tool (nvn_replay) reaches it here warm_claim(); - if (g_cb) MDIE("submit re-entered"); - g_sub_thread = pthread_self(); atomic_store(&g_sub_thread_set, 1); + if (g_cb) { + MDIE("submit re-entered"); + } + mtl_submit_clock_release(g_submit_clock); + g_submit_clock = NULL; + g_sub_thread = pthread_self(); + atomic_store(&g_sub_thread_set, 1); atomic_store(&g_last_commit_error, 0); g_sub_pool = objc_autoreleasePoolPush(); g_cb = rec_cb_new(); // Do not wait for the GPU mid-frame (hot real device: with a fixed rotation of 3, when the GPU cannot keep up every frame blocks here, and the submit thread runs at low frequency after waking, instruction count −18% but CPU time +49%): // pick an allocator the GPU has finished with; if all are in use create a new one; only at the cap wait for the oldest one. How far ahead it runs is bounded by the wait where the game acquires the swapchain texture, same as the old interface. - int alloc_i = -1; uint64_t done = g_done; - for (int i = 0; i < g_alloc_n; i++) if (g_alloc_fence[i] <= done) { alloc_i = i; break; } - if (alloc_i < 0 && g_alloc_n < ALLOC_MAX) { NSError *ae = nil; MTL4CommandAllocatorDescriptor *ad = [MTL4CommandAllocatorDescriptor new]; - g_alloc4[g_alloc_n] = [g_dev newCommandAllocatorWithDescriptor:ad error:&ae]; if (!g_alloc4[g_alloc_n]) MDIE("failed to create a new MTL4CommandAllocator: %s", ae.localizedDescription.UTF8String); - alloc_i = g_alloc_n++; g_alloc_grown++; } - if (alloc_i < 0) { alloc_i = 0; for (int i = 1; i < g_alloc_n; i++) if (g_alloc_fence[i] < g_alloc_fence[alloc_i]) alloc_i = i; g_alloc_waits++; mtl_wait_at(g_alloc_fence[alloc_i], UINT64_MAX, WS_ALLOC); } + int alloc_i = alloc_acquire(owner); g_alloc_cur = alloc_i; - [g_alloc4[alloc_i] reset]; [g_cb beginCommandBufferWithAllocator:g_alloc4[alloc_i]]; g_commit_blocks = [NSMutableArray array]; // The overflow buffer created when the previous submission ran out of slots (see g_vis_n >= VIS_SLOTS): retire it. It used to be just cleared here and stayed in the residency set for good (from d2r-mobile 2eb575877) - if (g_vis && !vis_in_ring(g_vis)) defer_obj_release(g_vis); - g_npost = 0; g_vis = nil; g_vis_n = 0; g_vis_cur = -1; pend_reset(); - zskip_clear(); g_zskip_count = 0; + if (g_vis && !vis_in_ring(g_vis)) { + defer_obj_release(g_vis); + } + g_npost = 0; + g_vis = nil; + g_vis_n = 0; + g_vis_ring_used = 0; + g_vis_cur = -1; + pend_reset(); + zskip_clear(); + g_zskip_count = 0; int pick = -1; - for (int i = 0; i < g_vis_ring_n; i++) if (g_vis_ring[i].fence <= g_done) { pick = i; break; } - if (pick < 0 && g_vis_ring_n < VIS_RING) pick = g_vis_ring_n++; - if (pick < 0) { uint64_t oldest = UINT64_MAX; for (int i = 0; i < VIS_RING; i++) if (g_vis_ring[i].fence < oldest) { oldest = g_vis_ring[i].fence; pick = i; } if (oldest > g_done) mtl_wait_at(oldest, UINT64_MAX, WS_VIS); } - g_vis_ring_i = pick; g_vis_ring[pick].fence = UINT64_MAX; - if (!g_vis_ring[pick].buf) { g_vis_ring[pick].buf = [g_dev newBufferWithLength:VIS_SLOTS * 8 options:MTLResourceStorageModeShared]; res_add(g_vis_ring[pick].buf); } - g_vis = g_vis_ring[pick].buf; - void pg_sub_begin_ext(uint64_t fence); pg_sub_begin_ext(g_fence + 1); - void arena_begin(void); arena_begin(); + for (int i = 0; i < g_vis_ring_n; i++) { + if (g_vis_ring[i].fence <= g_done && !vis_buffer_pinned(g_vis_ring[i].buf)) { + pick = i; + break; + } + } + if (pick < 0 && g_vis_ring_n < VIS_RING) { + pick = g_vis_ring_n++; + } + if (pick < 0) { + uint64_t oldest = UINT64_MAX; + for (int i = 0; i < g_vis_ring_n; i++) { + if (!vis_buffer_pinned(g_vis_ring[i].buf) && g_vis_ring[i].fence < oldest) { + oldest = g_vis_ring[i].fence; + pick = i; + } + } + if (pick >= 0 && oldest > g_done) { + mtl_wait_at(oldest, UINT64_MAX, WS_VIS); + } + } + g_vis_ring_i = pick; + if (pick >= 0) { + g_vis_ring[pick].fence = UINT64_MAX; + if (!g_vis_ring[pick].buf) { + g_vis_ring[pick].buf = [g_dev newBufferWithLength:VIS_SLOTS * 8 options:MTLResourceStorageModeShared]; + res_add(g_vis_ring[pick].buf); + } + g_vis = g_vis_ring[pick].buf; + } else { + // Completion alone cannot recycle a cumulative query's slots: another + // queue or a resumed segment can report them after this GPU fence. If + // every ring buffer is pinned, use an overflow buffer until a reset or + // queue finalization releases slots; waiting cannot remove these pins. + g_vis = [g_dev newBufferWithLength:VIS_SLOTS * 8 options:MTLResourceStorageModeShared]; + res_add(g_vis); + } + if (!g_vis) { + MDIE("visibility buffer allocation failed"); + } + void pg_sub_begin_ext(uint64_t fence); + pg_sub_begin_ext(g_fence + 1); + void arena_begin(void); + arena_begin(); +} + +void mtl_sub_begin(void) { + sub_begin(0, NULL); +} +void mtl_sub_resume(void) { + sub_begin(1, NULL); +} +void mtl_sub_begin_alloc(mtl_submit_alloc *a, int resume) { + sub_begin(resume, a); } static void rp_end(void); @@ -1499,6 +1654,45 @@ static void post(post_op o) { // They used to come from CLOCK_MONOTONIC, which keeps running while the Mac sleeps while the guest clock (mach_absolute_time) does not, so the two differed by the sleep time; once timestamps were converted to ns, Mario halved its frame rate. In graphics unit tests guest_ns is a weak definition returning 0; fall back to the monotonic clock uint64_t guest_ns(void); static uint64_t gpu_ticks(void) { uint64_t g = guest_ns(); if (!g) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); g = (uint64_t)t.tv_sec * 1000000000ull + t.tv_nsec; } return nvn_timestamp_from_ns(g); } +// Event waits can split one guest submission into several host completions. +// Keep its configured submission-mode clock scoped to that guest submission, +// while publishing each report at its own completed point in the event stream. +struct mtl_submit_clock { + _Atomic unsigned refs; + _Atomic uint64_t ticks; +}; +mtl_submit_clock *mtl_submit_clock_new(void) { + mtl_submit_clock *clock = calloc(1, sizeof *clock); + if (!clock) { + MDIE("submission clock allocation failed"); + } + atomic_init(&clock->refs, 1); + atomic_init(&clock->ticks, UINT64_MAX); + return clock; +} +void mtl_submit_clock_release(mtl_submit_clock *clock) { + if (clock && atomic_fetch_sub(&clock->refs, 1) == 1) { + free(clock); + } +} +void mtl_sub_set_clock(mtl_submit_clock *clock) { + if (clock) { + atomic_fetch_add(&clock->refs, 1); + } + mtl_submit_clock_release(g_submit_clock); + g_submit_clock = clock; +} +static uint64_t submit_clock_ticks(mtl_submit_clock *clock) { + uint64_t ticks = atomic_load(&clock->ticks); + if (ticks == UINT64_MAX) { + uint64_t now = gpu_ticks(); + if (atomic_compare_exchange_strong(&clock->ticks, &ticks, now)) { + ticks = now; + } + } + return ticks; +} + // Counter timestamps by segment (profile gpu_timestamps segment; DM_NVN_GPU_TS=segment / submission overrides it). On hardware a timestamp report is written when the GPU reaches it, so two reports of one submission bracket the GPU work between them. // The host writes all reports of a submission in its completion handler; with one time for all of them a game that measures its GPU time that way reads 0 (Breath of the Wild: frame-interval controller and dynamic resolution). // In segment mode a report takes the GPU time of the segment it was encoded in (sub_cut): TIMESTAMP_TOP (0x10) that segment's GPU start, other types its GPU end. The writes stay in the last segment's handler, before the fence, @@ -1669,53 +1863,115 @@ int mtl_gpu_span(uint64_t f, uint64_t *t0, uint64_t *t1) { static void lc_land(void); // A submission's CPU-side writes at its end (events, samples-passed and counter reports): run once, by its feedback handler or, when a game's SyncWait wakes on the // GPU's own signal of the fence first, by that waiter (DM_NVN_FAST_WAKE, on by default): the feedback handler comes 0.26 ms after the GPU end on an M2 Max -typedef struct { _Atomic uint64_t fence; _Atomic int claimed; int n; post_op *ops; seg_clk *sc; uint32_t seg_last; uint64_t t_commit; } post_rec; // t_commit: guest ns when it was committed +typedef struct { + _Atomic uint64_t fence; + _Atomic int claimed; + int n; + post_op *ops; + seg_clk *segment_clock; + uint32_t seg_last; + uint64_t t_commit; + mtl_submit_clock *clock; +} post_rec; // t_commit: guest ns when it was committed #define POST_RING 128 static post_rec g_postr[POST_RING]; static pthread_mutex_t g_post_mu = PTHREAD_MUTEX_INITIALIZER; // A successful claim holds the lock until post_run finishes. Ownership alone // cannot publish completion: another waiter must see the actual writes first. -static int post_claim(post_rec *r, uint64_t fence) { +static int post_claim(post_rec *record, uint64_t fence) { pthread_mutex_lock(&g_post_mu); - if (r->fence != fence || r->claimed) { pthread_mutex_unlock(&g_post_mu); return 0; } + if (record->fence != fence || record->claimed) { + pthread_mutex_unlock(&g_post_mu); + return 0; + } return 1; } -static void post_run(post_rec *r, uint64_t c0, uint64_t c1) { +static void post_run(post_rec *record, uint64_t c0, uint64_t c1) { // A guest SyncWait may run completion on a thread whose outer pool never drains. // MTLBuffer.contents may autorelease its receiver; keep that lifetime inside the completed copy. @autoreleasepool { - int n = r->n; post_op *ops = r->ops; seg_clk *sc = r->sc; uint32_t seg_last = r->seg_last; - for (int i = 0; i < n; i++) { - if (ops[i].kind == 1) { if (ops[i].mode == 1) *ops[i].val += ops[i].v; else *ops[i].val = ops[i].v; } - else if (ops[i].kind == 3) { // Samples passed: the sum of all slots since the reset - vis_list *l = ops[i].vis; uint64_t sum = 0; - for (int k = 0; k < l->n; k++) { sum += ((const uint64_t *)((__bridge id)l->s[k].buf).contents)[l->s[k].slot]; CFRelease(l->s[k].buf); } - free(l); uint64_t rec[2] = {sum, gpu_ticks()}; memcpy((void *)ops[i].addr, rec, sizeof rec); + int n = record->n; + post_op *ops = record->ops; + seg_clk *segment_clock = record->segment_clock; + uint32_t seg_last = record->seg_last; + mtl_submit_clock *clock = record->clock; + int shared_time = clock && !seg_ts(); + for (int i = 0; i < n; i++) { + if (ops[i].kind == 1) { + if (ops[i].mode == 1) { + *ops[i].val += ops[i].v; + } else { + *ops[i].val = ops[i].v; + } + } else if (ops[i].kind == 3) { // Samples passed: the sum of all slots since the reset + vis_list *samples = ops[i].vis; + uint64_t sum = 0; + for (int k = 0; k < samples->n; k++) { + sum += ((const uint64_t *)((__bridge id)samples->s[k].buf).contents)[samples->s[k].slot]; + CFRelease(samples->s[k].buf); + } + free(samples); + uint64_t report[2] = {sum, shared_time ? submit_clock_ticks(clock) : gpu_ticks()}; + memcpy((void *)ops[i].addr, report, sizeof report); + } else if (ops[i].kind == 4) { + mirror_rb_land(ops[i].vis); + } else { + uint64_t ticks = shared_time ? submit_clock_ticks(clock) + : rep_ticks(&ops[i], segment_clock, seg_last, c0, c1); + // render_cond: timestamp reports have a zero counter field, + // as on the GPU (see rc_value). + int timestamp_counter = (ops[i].type == 0 || ops[i].type == 0x10) && !DM_GAME_RENDER_COND; + uint64_t report[2] = {timestamp_counter ? ticks : 0, ticks}; + memcpy((void *)ops[i].addr, report, sizeof report); + } } - else if (ops[i].kind == 4) mirror_rb_land(ops[i].vis); - else { uint64_t t = rep_ticks(&ops[i], sc, seg_last, c0, c1), rec[2] = {(ops[i].type == 0 || ops[i].type == 0x10) && !DM_GAME_RENDER_COND ? t : 0, t}; memcpy((void *)ops[i].addr, rec, sizeof rec); } // render_cond: a timestamp report's counter field is 0, as on the GPU (see rc_value) - } - free(ops); seg_clk_put(sc); r->ops = NULL; r->sc = NULL; - atomic_store(&r->claimed, 1); - pthread_mutex_unlock(&g_post_mu); + free(ops); + seg_clk_put(segment_clock); + mtl_submit_clock_release(clock); + record->ops = NULL; + record->segment_clock = NULL; + record->clock = NULL; + atomic_store(&record->claimed, 1); + pthread_mutex_unlock(&g_post_mu); } } // Both GPU feedback and an early waiter complete records in submission order. // Delayed callbacks carry their fence, never the identity of a recyclable slot. static void post_complete(uint64_t fence, uint64_t c0, uint64_t c1) { uint64_t now = guest_ns(); - for (uint64_t f = g_done + 1; f <= fence; f++) { - post_rec *r = &g_postr[f % POST_RING]; - if (!post_claim(r, f)) continue; - uint64_t start = r->t_commit; - if (r->sc && r->seg_last > 0 && r->seg_last <= SEG_CLK_N) { - uint64_t pe = atomic_load(&r->sc->t[r->seg_last - 1][1]); if (pe > start) start = pe; + for (uint64_t completed = g_done + 1; completed <= fence; completed++) { + post_rec *record = &g_postr[completed % POST_RING]; + if (!post_claim(record, completed)) { + continue; } - if (start > now) start = now; - post_run(r, f == fence && c1 ? c0 : seg_ts() ? start : 0, f == fence && c1 ? c1 : seg_ts() ? now : 0); + uint64_t start = record->t_commit; + if (record->segment_clock && record->seg_last > 0 && record->seg_last <= SEG_CLK_N) { + uint64_t previous_end = atomic_load(&record->segment_clock->t[record->seg_last - 1][1]); + if (previous_end > start) { + start = previous_end; + } + } + if (start > now) { + start = now; + } + uint64_t report_start = 0, report_end = 0; + if (completed == fence && c1) { + report_start = c0; + report_end = c1; + } else if (seg_ts()) { + report_start = start; + report_end = now; + } + post_run(record, report_start, report_end); + } + pthread_mutex_lock(&g_mu); + if (fence > g_done) { + g_done = fence; } - pthread_mutex_lock(&g_mu); if (fence > g_done) g_done = fence; pthread_cond_broadcast(&g_cv); pthread_mutex_unlock(&g_mu); + pthread_cond_broadcast(&g_cv); + pthread_mutex_unlock(&g_mu); } + uint64_t mtl_sub_commit(void) { lc_land(); // clears still waiting (the command list ends with them) defer_fallback(); // Deferred merged pass: at the end of the submission there are still P draws not replayed (the normal path never gets here) @@ -1726,9 +1982,19 @@ uint64_t mtl_sub_commit(void) { treg_finish_submission(); // Late descriptor views retire behind this submission's fence. uint64_t fence = ++g_fence; gw_commit(fence); - if (g_vis_ring_i >= 0) g_vis_ring[g_vis_ring_i].fence = fence; + // An unused query buffer contains no bytes referenced by this submission. + // Event-only chunks must not exhaust the visibility ring behind a busy GPU. + if (g_vis_ring_i >= 0) { + g_vis_ring[g_vis_ring_i].fence = g_vis_ring_used || g_vis_n ? fence : g_done; + } // CPU-visible effects after completion: copy the op table into the callback int n = g_npost; post_op *ops = n ? malloc(n * sizeof *ops) : NULL; post_rec *pr = &g_postr[fence % POST_RING]; + // A logical allocator can append more chunks than this bounded callback + // table holds. Finish only the already-committed record being reused; do + // not overwrite its event/report writes or wait for the logical submission. + if (pr->fence && !atomic_load(&pr->claimed)) { + mtl_wait_at(pr->fence, UINT64_MAX, WS_POST); + } if (ws_on()) for (int i = 0; i < n; i++) { int k = g_post[i].kind == 2 ? (g_post[i].type == 0 || g_post[i].type == 0x10 ? 3 : 4) : g_post[i].kind == 3 ? 2 : 1; atomic_fetch_add(&g_ws_post[k], 1); } // 1 event, 2 samples passed, 3 timestamp, 4 other counter if (n) memcpy(ops, g_post, n * sizeof *ops); NSArray *commit_blocks = [g_commit_blocks copy]; @@ -1738,8 +2004,18 @@ uint64_t mtl_sub_commit(void) { NSString *cctx = @(cctx_snap()); seg_clk *sc = g_sclk; uint32_t seg_last = g_seg; g_sclk = NULL; pthread_mutex_lock(&g_post_mu); - if (!atomic_load(&pr->claimed) && pr->fence) MDIE("post ring: the writes of fence %llu never ran", (unsigned long long)pr->fence); - pr->n = n; pr->ops = ops; pr->sc = sc; pr->seg_last = seg_last; pr->t_commit = guest_ns(); atomic_store(&pr->claimed, 0); atomic_store(&pr->fence, fence); + if (!atomic_load(&pr->claimed) && pr->fence) { + MDIE("post ring: the writes of fence %llu never ran", (unsigned long long)pr->fence); + } + pr->n = n; + pr->ops = ops; + pr->segment_clock = sc; + pr->seg_last = seg_last; + pr->t_commit = guest_ns(); + pr->clock = g_submit_clock; + g_submit_clock = NULL; + atomic_store(&pr->claimed, 0); + atomic_store(&pr->fence, fence); pthread_mutex_unlock(&g_post_mu); [o addFeedbackHandler:^(id cb) { gpu_tag_done(gpu_f, cb.GPUStartTime, cb.GPUEndTime); @@ -1765,6 +2041,9 @@ uint64_t mtl_sub_commit(void) { } uint64_t mtl_last_fence(void) { return g_fence; } +uint64_t mtl_completed_fence(void) { + return atomic_load(&g_done); +} int mtl_last_commit_error(void) { return atomic_load(&g_last_commit_error); } static int fast_wake(void) { static int v = -1; if (v < 0) { const char *e = getenv("DM_NVN_FAST_WAKE"); v = !(e && !strcmp(e, "0")); MLOG("game SyncWaits wake on the GPU's signal of the fence%s (DM_NVN_FAST_WAKE)", v ? "" : ": off, on the feedback handler"); } return v; } @@ -4269,6 +4548,9 @@ static void attach(MTLRenderPassAttachmentDescriptor *a, const tex_t *t, const v // Slots used up (one submission exceeds VIS_SLOTS counting passes): continue in a new overflow buffer. A ring buffer must not be retired (it is still used in rotation, and retiring removes it from the residency set); only a previous overflow buffer is if (!g_vis || g_vis_n >= VIS_SLOTS) { if (g_vis && !vis_in_ring(g_vis)) defer_obj_release(g_vis); g_vis = [g_dev newBufferWithLength:VIS_SLOTS * 8 options:MTLResourceStorageModeShared]; res_add(g_vis); g_vis_n = 0; } pd.visibilityResultBuffer = g_vis; + if (g_vis_ring_i >= 0 && g_vis == g_vis_ring[g_vis_ring_i].buf) { + g_vis_ring_used = 1; + } } g_enc_vis = 0; g_enc_has_vis = g_vis_on; // Layered rendering: a vertex shader that writes the layer selects the attachment layer per primitive only when the pass has renderTargetArrayLength; without it every layer went to the @@ -5761,29 +6043,139 @@ static void state_reset(void) { // GPU bindings belong to a queue, including across submissions. FFX/X-2 destroys its launcher's queue before creating a game's: a global S kept freed launcher programs in S.bst, // and X-2's first fragment-only bind dereferenced that stale vertex program in keys_hash. Separate queues also retain their own bindings when submissions alternate (Three Houses). -typedef struct queue_state { uint64_t serial; typeof(S) s; typeof(R) r; struct queue_state *next; } queue_state; +typedef struct queue_state { + uint64_t serial; + typeof(S) s; + typeof(R) r; + tex_t targets[9]; + int rc_skip, vis_on, vis_counting, nvs, vscap; + uint64_t *rc_pend; + uint32_t rc_npend, rc_pcap; + vis_slot *vs; + struct queue_state *next; +} queue_state; static queue_state *g_queue_states, *g_active_queue; -void mtl_queue_activate(uint64_t serial) { // caller holds the submission lock - if (g_active_queue && g_active_queue->serial == serial) return; - if (g_active_queue) { g_active_queue->s = S; g_active_queue->r = R; } +static int vis_buffer_pinned(id buffer) { + if (!buffer) { + return 0; + } + void *pointer = (__bridge void *)buffer; + for (int i = 0; i < g_nvs; i++) { + if (g_vs[i].buf == pointer) { + return 1; + } + } + for (queue_state *queue = g_queue_states; queue; queue = queue->next) { + if (queue != g_active_queue) { + for (int i = 0; i < queue->nvs; i++) { + if (queue->vs[i].buf == pointer) { + return 1; + } + } + } + } + return 0; +} + +void mtl_queue_activate(uint64_t serial) { // caller holds the submission lock + if (g_active_queue && g_active_queue->serial == serial) { + return; + } + if (g_active_queue) { + queue_state *old = g_active_queue; + old->s = S; + old->r = R; + old->rc_skip = g_rc_skip; + old->rc_pend = g_rc_pend; + old->rc_npend = g_rc_npend; + old->rc_pcap = g_rc_pcap; + old->vis_on = g_vis_on; + old->vis_counting = g_vis_counting; + old->vs = g_vs; + old->nvs = g_nvs; + old->vscap = g_vscap; + } queue_state *q = g_queue_states; - while (q && q->serial != serial) q = q->next; - if (q) { S = q->s; R = q->r; prog_resolve(); } - else { - q = calloc(1, sizeof *q); if (!q) MDIE("queue state allocation failed"); - q->serial = serial; q->next = g_queue_states; g_queue_states = q; - state_reset(); memset(&R, 0, sizeof R); + while (q && q->serial != serial) { + q = q->next; + } + if (q) { + S = q->s; + R = q->r; + prog_resolve(); + } else { + q = calloc(1, sizeof *q); + if (!q) { + MDIE("queue state allocation failed"); + } + q->serial = serial; + q->next = g_queue_states; + g_queue_states = q; + state_reset(); + memset(&R, 0, sizeof R); + } + g_rc_skip = q->rc_skip; + g_rc_pend = q->rc_pend; + g_rc_npend = q->rc_npend; + g_rc_pcap = q->rc_pcap; + g_vis_on = q->vis_on; + g_vis_counting = q->vis_counting; + g_vs = q->vs; + g_nvs = q->nvs; + g_vscap = q->vscap; + g_vis_cur = -1; + g_active_queue = q; + g_pso_dirty = g_dss_dirty = 1; + args_dirty_all(); +} +void mtl_queue_seal(void) { + if (!g_active_queue || !R.has_rt) { + return; + } + for (int i = 0; i < 8; i++) { + if (R.rt.color[i]) { + g_active_queue->targets[i] = *R.rt.color[i]; + R.rt.color[i] = &g_active_queue->targets[i]; + } + } + if (R.rt.depth) { + g_active_queue->targets[8] = *R.rt.depth; + R.rt.depth = &g_active_queue->targets[8]; } - g_active_queue = q; g_pso_dirty = g_dss_dirty = 1; args_dirty_all(); } -void mtl_queue_forget(uint64_t serial) { // caller holds the submission lock +void mtl_queue_forget(uint64_t serial) { // caller holds the submission lock queue_state **p = &g_queue_states; - while (*p && (*p)->serial != serial) p = &(*p)->next; - if (!*p) return; - queue_state *q = *p; *p = q->next; - if (g_active_queue == q) { g_active_queue = NULL; state_reset(); memset(&R, 0, sizeof R); args_dirty_all(); } + while (*p && (*p)->serial != serial) { + p = &(*p)->next; + } + if (!*p) { + return; + } + queue_state *q = *p; + *p = q->next; + if (g_active_queue == q) { + vis_clear(); + free(g_vs); + free(g_rc_pend); + g_vs = NULL; + g_nvs = g_vscap = g_vis_on = g_vis_counting = 0; + g_rc_pend = NULL; + g_rc_npend = g_rc_pcap = 0; + g_rc_skip = 0; + g_active_queue = NULL; + state_reset(); + memset(&R, 0, sizeof R); + args_dirty_all(); + } else { + for (int i = 0; i < q->nvs; i++) { + CFRelease(q->vs[i].buf); + } + free(q->vs); + free(q->rc_pend); + } free(q); } + // A finalized program leaves the bindings of every queue: NVN reads a program only when it is bound, so a game may finalize one that a queue still has bound and reuse its memory. // Fortune's Weave does this when its title screen hands over to the opening demo: the pointer stayed in the active queue's bindings, went into its saved state when another queue submitted, // and switching back to that queue combined its stages from the reused memory (keys_hash on float data) @@ -11942,6 +12334,38 @@ static void clear_tex(const p_clear_tex *c) { static uint64_t g_cut_draw0, g_cuts, g_copy_cuts; // The count starts with the submission (mtl_sub_begin). It used to start at the submission's first sub_cut, which only a render target change reaches (OP_SET_RT): the draws a // submission makes into the target it inherited, before its first target change, were never counted, and the first segment came that many draws late +// DM_NVN_CHUNK_MERGE=1 (0 / 1 / ab, alternated with the DM_AB phases; off by default): the queue scheduler (nvn_queue_dispatch.inc qe_pump) encodes the chunks it offers in a row into one +// command buffer instead of committing each as a submission of its own. Under #1245 a guest submission is cut into chunks at its event waits and signals and each chunk was a host +// submission: Ocarina of Time (Switch 2) went from 6 to 37 submissions a frame on an iPad Pro (M5), with the GPU draining between a chunk and the next while the host committed it (2.5 ms +// of idle inside a 22.9 ms frame), the 64-draw segment cuts reset at every chunk, and a drain of the encode thread per chunk. The order is the scheduler's either way (one host queue, the +// chunks in the order they were offered, a blocked queue's chunk still waits for its producer's offer), so a satisfied wait or a signal costs nothing: no commit, allocator, visibility +// buffer, completion record or drain. In submission timestamp mode (gpu_timestamps submission) one buffer holds one logical submission's clock (post_run), so a chunk of another logical +// submission opens a new buffer; in segment mode (Ocarina of Time) chunks of any submission share one. The open buffer's fence is g_fence + 1 (as OP_FENCE_SYNC and the PASSGPU ring take it); +// the scheduler records it for the chunk and the event projection, and the pump commits before it returns, so no thread waits on a fence not yet committed +static int chunk_merge_mode(void) { + static int m = -1; + if (m < 0) { m = sw_mode("DM_NVN_CHUNK_MERGE", 0); if (m) MLOG("DM_NVN_CHUNK_MERGE: the queue scheduler's chunks share one command buffer while they are offered in a row%s", m >= 2 ? " (alternated with the DM_AB phases)" : ""); } + return m; +} +int mtl_chunk_merge(mtl_submit_clock *clock, int open) { + if (!sw_on(chunk_merge_mode())) return 0; + if (open && !seg_ts() && g_submit_clock != clock) return 0; // submission-mode reports take the buffer's one clock (post_run) + return 1; +} +uint64_t mtl_pending_fence(void) { return g_fence + 1; } +void mtl_chunk_end(void) { // the chunk's work lands in the open buffer: what mtl_sub_commit does before it commits (waiting clears, the deferred pass, the open pass, uniform updates, the blit encoder) + lc_land(); + defer_fallback(); + rp_flush(); ubd_land(); + if (g_blit) { enc_b_end(g_blit); g_blit = nil; } + treg_finish_submission(); // this chunk's transient views (g_sub_serial advances per chunk) +} +void mtl_chunk_begin(int resume) { // the next chunk in the open buffer: sub_begin's per-chunk state, without a new buffer, allocator, visibility buffer, completion record or segment clock + g_sub_serial++; + if (!resume) { g_rc_npend = 0; g_rc_skip = 0; } + defer_reset(); + zskip_clear(); g_zskip_count = 0; +} static void sub_cut_begin(void) { g_cut_draw0 = g_stat_draw; } // DM_FSTEP_PROBE_RAMP=1 (default) / 0 / ab: a frame-step probe's grace is renewed while its frames keep getting cheaper (nvn_fstep.h FSTEP_RAMP), the time macOS's GPU clock takes to rise for the shorter deadline static int probe_ramp_mode(void) { static int m = -1; if (m < 0) m = sw_mode("DM_FSTEP_PROBE_RAMP", 1); return m; } @@ -12153,6 +12577,9 @@ void mtl_exec(const uint8_t *p) { case OP_CALL: { uint64_t t; memcpy(&t, a, 8); if (g_ret_n < 16) g_ret[g_ret_n] = p + h.size; g_ret_n++; // Return stack for lookahead: after a sub-list ends, continue looking at the caller mtl_exec((const uint8_t *)t); g_ret_n--; break; } + case OP_WAIT_EVENT: case OP_WAIT_SYNC: + MDIE("event/sync waits require the guest queue scheduler; raw capture replay cannot order them"); + break; case OP_SIGNAL_EVENT: { volatile uint32_t *val; uint32_t m, v; memcpy(&val, a, 8); memcpy(&m, (const uint8_t *)a + 8, 4); memcpy(&v, (const uint8_t *)a + 12, 4); PD("Y signal_event cmd=%p dst=%p mode=%u value=%u", p, val, m, v); mtl_sub_event(val, m, v); break; } case OP_REPORT_COUNTER: { uint64_t t, ad; memcpy(&t, a, 8); memcpy(&ad, (const uint8_t *)a + 8, 8); g_cnt_report[t & 31]++; @@ -12167,7 +12594,7 @@ void mtl_exec(const uint8_t *p) { g_rc_skip = known && (rc->mode ? x == y : x != y); PD("E %llu conditional render %s addr=%#llx values %llu / %llu%s%s", g_stat_pass, rc->mode ? "draw if not equal" : "draw if equal", (unsigned long long)rc->addr, (unsigned long long)x, (unsigned long long)y, known ? "" : " (unknown, drawn)", g_rc_skip ? " → not drawn" : ""); } break; } - case OP_FENCE_SYNC: { sync_t *s; memcpy(&s, a, 8); s->fence = g_fence + 1; s->fenced = 1; // the fence value this submission (segment) commits with + case OP_FENCE_SYNC: { sync_t *s; memcpy(&s, a, 8); s->fence = g_fence + 1; s->fenced = NVN_SYNC_FENCE_GPU; // the fence value this submission (segment) commits with PD("Y fence_sync cmd=%p sync=%p fence=%llu", p, (void *)s, (unsigned long long)s->fence); break; } case OP_RESET_COUNTER: { uint64_t t; memcpy(&t, a, 8); g_cnt_reset[t & 31]++; PD("Q reset type=%llu cmd=%p", (unsigned long long)t, p); if (t == NVN_COUNTER_TYPE_SAMPLES_PASSED) { diff --git a/dm/gfx/nvn/nvn_obj.h b/dm/gfx/nvn/nvn_obj.h index 672e06cb05..65fd0e6c32 100644 --- a/dm/gfx/nvn/nvn_obj.h +++ b/dm/gfx/nvn/nvn_obj.h @@ -56,14 +56,35 @@ typedef struct { uint32_t magic; samp_par p; } samp_t; // The per-stage shader keys are not kept in prog_t: the game's real NVNprogram may be smaller than the header's 0xC0 (growing prog_t corrupted the adjacent object in Mario); see prog_keys in hle_nvn.c typedef struct { uint32_t magic, nstage; NVNshaderData sd[6]; uint64_t serial; } prog_t; FITS(prog_t, NVNprogram); // serial: table key for the replay layer's resolved result -typedef struct { uint32_t magic, fenced; uint64_t fence; } sync_t; /* fenced 2: by WindowAcquireTexture */ FITS(sync_t, NVNsync); +enum { + NVN_SYNC_FENCE_NONE = 0, + NVN_SYNC_FENCE_GPU = 1, + NVN_SYNC_FENCE_ACQUIRE = 2, + NVN_SYNC_FENCE_QUEUE_PREFIX = 3, + NVN_SYNC_FENCE_RECORDED_PENDING = 4, +}; +typedef struct { + uint32_t magic, fenced; + uint64_t fence; // GPU fence, or logical sequence for NVN_SYNC_FENCE_QUEUE_PREFIX + uint64_t queue; // queue serial for a logical sequence +} sync_t; +FITS(sync_t, NVNsync); typedef struct { const pool_t *pool; int64_t off; } ev_builder_t; FITS(ev_builder_t, NVNeventBuilder); typedef struct { uint32_t magic; volatile uint32_t *val; } ev_t; FITS(ev_t, NVNevent); typedef struct { NVNdevice *dev; void *native; int32_t ntex, interval; NVNtexture *tex[4]; } win_builder_t; FITS(win_builder_t, NVNwindowBuilder); // avail[i]: time at which swapchain image i can be acquired again (ns, monotonic clock) = its display time + one present period; pace: target time of this window's present pacer -typedef struct { uint32_t magic; void *native; int32_t ntex, interval, next, crop[4]; NVNtexture *tex[4]; uint64_t presents, avail[4], pace, ticket[4]; } win_t; // ticket[i]: ticket of the most recent present of image i +typedef struct { + uint32_t magic; + void *native; + int32_t ntex, interval, next, crop[4]; + NVNtexture *tex[4]; + uint64_t presents, avail[4], pace; + uint64_t ticket[4]; // most recent present ticket for each image + uint64_t submit_queue[4]; // logical queue and sequence that schedule that present + uint64_t submit_seq[4]; +} win_t; FITS(win_t, NVNwindow); // Command buffer: commands are written into the command memory given by the game (AddCommandMemory), handle = segment start address; when it runs out, the game is called back for more memory @@ -107,7 +128,7 @@ enum { OP_BIND_PROGRAM, OP_BIND_STATE, OP_BIND_VATTR, OP_BIND_VSTREAM, OP_BIND_VBUF, OP_BIND_UBO, OP_BIND_SSBO, OP_BIND_TEX, OP_BIND_SAMP, OP_STENCIL, OP_POLY_OFFSET, OP_DRAW, OP_DRAW_IDX, OP_DISPATCH, OP_BARRIER, - OP_SET_TEX_POOL, OP_SET_SAMP_POOL, OP_COPY_B2T, OP_COPY_STRIDE, OP_BLEND_COLOR, OP_COPY_T2B, OP_BIND_IMG, OP_COPY_T2T, OP_RESET_COUNTER, OP_CLEAR_TEX, OP_CLEAR_BUF, OP_DEPTH_CLAMP, OP_RENDER_COND, OP_FENCE_SYNC, OP_TF_BIND, OP_TF_STATE, OP_RAST_DISCARD, OP_VP_SWIZZLE, OP_BIND_TEX1, OP_PRIM_RESTART, OP_BIND_UBO_SMALL, OP_BIND_VBUF1, OP_DRAW_TEX, OP_ALPHA_REF, OP_PUSH_CONST, OP_DLSS, OP_MULTI_DRAW, OP_COUNT_ + OP_SET_TEX_POOL, OP_SET_SAMP_POOL, OP_COPY_B2T, OP_COPY_STRIDE, OP_BLEND_COLOR, OP_COPY_T2B, OP_BIND_IMG, OP_COPY_T2T, OP_RESET_COUNTER, OP_CLEAR_TEX, OP_CLEAR_BUF, OP_DEPTH_CLAMP, OP_RENDER_COND, OP_FENCE_SYNC, OP_TF_BIND, OP_TF_STATE, OP_RAST_DISCARD, OP_VP_SWIZZLE, OP_BIND_TEX1, OP_PRIM_RESTART, OP_BIND_UBO_SMALL, OP_BIND_VBUF1, OP_DRAW_TEX, OP_ALPHA_REF, OP_PUSH_CONST, OP_DLSS, OP_MULTI_DRAW, OP_WAIT_EVENT, OP_WAIT_SYNC, OP_COUNT_ }; // Payload (immediately follows op_hdr) typedef struct { diff --git a/dm/gfx/nvn/nvn_queue_dispatch.inc b/dm/gfx/nvn/nvn_queue_dispatch.inc new file mode 100644 index 0000000000..0dadc7e28b --- /dev/null +++ b/dm/gfx/nvn/nvn_queue_dispatch.inc @@ -0,0 +1,554 @@ +// Guest queues share one Metal queue, but a forward event wait can suspend a +// guest submission until another guest queue supplies its producer. Never block +// the submitting CPU thread: it may itself submit that producer next. +#include "nvn_capture.h" + +typedef struct qe_value { + struct qe_value *next; + uint64_t address, fence; + uint32_t value; +} qe_value; +typedef struct qe_work { + struct qe_work *next; + uint8_t *bytes; + int first, origin; + uint64_t signal; + uint32_t mode, value; + mtl_submit_clock *clock; // borrowed from the logical submission + mtl_submit_alloc *alloc; // likewise; serialized by the submission lock + win_t *window; + int image, crop[4]; + tex_t texture; + double period; +} qe_work; +typedef struct { + qe_work *head, *tail; + mtl_submit_clock *clock; + mtl_submit_alloc *alloc; +} qe_submission; +static nvn_qs *g_qs; +static qe_value *g_qe_values; +static pthread_cond_t g_qe_cv = PTHREAD_COND_INITIALIZER; +static void *qe_worker(void *unused); +// DM_NVN_CHUNK_MERGE (nvn_mtl.m mtl_chunk_merge): the command buffer a chunk was encoded in stays open across the steps one pump offers, so the chunks after it (its own +// queue's, or another queue's whose wait the scheduler found satisfied by what is already in submission order) are encoded into the same buffer; it commits when the pump +// has no ready step left, before a present, or when the merge rule says no. Ordering is the one the scheduler chose either way (one host queue, chunks in offer order); +// what changes is that a satisfied wait or a signal no longer costs a commit, an allocator, a visibility buffer, a completion record and (encode thread) a drain, and the +// GPU is not left to drain between a chunk and the next while the host commits it. Ocarina of Time (Switch 2) cut a frame's 6 submissions into 37 chunks on an iPad Pro (M5), +// 2.5 ms of GPU idle inside a 22.9 ms frame. The fence of an open buffer is the one it will commit with (mtl_pending_fence); every CPU wait on a fence runs outside the pump +static int g_qe_open; // a chunk's command buffer is open (this pump's; committed by qe_open_commit before the pump returns) +static uint64_t g_qe_chunks, g_qe_merged; // chunks executed; chunks encoded into an already open buffer (the timing line) +static void qe_open_commit(void) { + if (!g_qe_open) { + return; + } + g_qe_open = 0; + mtl_sub_commit(); + mtl_queue_seal(); +} + +static qe_value *qe_value_find(uint64_t address, int add) { + qe_value *event = g_qe_values; + while (event && event->address != address) { + event = event->next; + } + if (!event && add) { + event = calloc(1, sizeof *event); + if (!event) { + die("NVN event state allocation failed"); + } + event->address = address; + event->next = g_qe_values; + g_qe_values = event; + } + return event; +} +static int qe_read(void *context, uint64_t address, uint32_t *value) { + (void)context; + if (address & 1) { // a sync wait: all work through its fence must be ordered + const sync_t *sync = (const sync_t *)(uintptr_t)(address & ~1ull); + *value = sync->fenced && sync->fenced != NVN_SYNC_FENCE_RECORDED_PENDING && + (sync->fenced != NVN_SYNC_FENCE_QUEUE_PREFIX || + nvn_qs_scheduled(g_qs, (nvn_qs_ticket){sync->queue, sync->fence})); + return 1; + } + qe_value *event = qe_value_find(address, 0); + // Once post-operations have landed, the mapped word is authoritative again; + // this includes CPU stores made directly through the memory pool. + if (event && event->fence > mtl_completed_fence()) { + *value = event->value; + } else { + *value = *(volatile uint32_t *)(uintptr_t)address; + } + return 1; +} +static void qe_release(void *context, nvn_qs_ticket ticket, void *payload) { + (void)context; + (void)ticket; + qe_submission *submission = payload; + if (!submission) { + return; + } + for (qe_work *work = submission->head, *next; work; work = next) { + next = work->next; + free(work->bytes); + free(work); + } + mtl_submit_clock_release(submission->clock); + mtl_submit_alloc_release(submission->alloc); + free(submission); +} +static void qe_init(void) { + if (!g_qs) { + g_qs = nvn_qs_create(qe_read, qe_release, NULL); + if (!g_qs) { + die("NVN queue scheduler allocation failed"); + } + pthread_t thread; + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, 4 * 1024 * 1024); + int err = pthread_create(&thread, &attr, qe_worker, NULL); + pthread_attr_destroy(&attr); + if (err) { + die("NVN queue worker creation failed: %d", err); + } + pthread_detach(thread); + LOG("[nvn] event queue ordering enabled: blocked guest queues yield to their producers"); + } +} +static void qe_pump(void) { // submission lock and graphics gate held + if (!g_qs) { + return; + } + nvn_qs_item item; + while (nvn_qs_peek(g_qs, &item)) { + uint64_t fence = 0; + if (item.step.kind == NVN_QS_WORK) { + qe_work *work = item.step.payload; + if (work->window) { + qe_open_commit(); // the present goes behind every chunk offered before it + work->window->ticket[work->image] = mtl_present(&work->texture, work->period, work->crop); + } else { + uint32_t signal_value = 0; + if (work->signal) { + uint32_t before; + qe_read(NULL, work->signal, &before); + if (!nvn_qs_signal_value(before, work->mode, work->value, &signal_value)) { + die("NVN invalid event signal mode"); + } + } + int merge = mtl_chunk_merge(work->clock, g_qe_open); + if (g_qe_open && !merge) { + qe_open_commit(); + } + if (g_qe_open) { + mtl_chunk_end(); // the open buffer's chunk lands before the queue state may change + mtl_queue_seal(); + } + mtl_queue_activate(item.ticket.queue); + mtl_set_queue_origin_lower_left(work->origin); + if (g_qe_open) { + mtl_chunk_begin(!work->first); + g_qe_merged++; + } else { + mtl_sub_begin_alloc(work->alloc, !work->first); + } + g_qe_chunks++; + mtl_sub_set_clock(work->clock); + uint64_t handle = (uint64_t)work->bytes; + mtl_sub_index(&handle, 1); + mtl_exec_rest(NULL, 0); + mtl_exec(work->bytes); + if (merge) { + g_qe_open = 1; + fence = mtl_pending_fence(); + } else { + mtl_sub_commit(); + mtl_queue_seal(); + fence = mtl_last_fence(); + } + if (!work->next) { + mtl_submit_alloc_close(work->alloc); + } + if (work->signal) { + qe_value *event = qe_value_find(work->signal, 1); + event->value = signal_value; + event->fence = fence; + } + } + } else if (item.step.kind == NVN_QS_WAIT) { + // The producer is ordered, but may still be running. A wait-only + // submission must retain that GPU completion dependency as well. + fence = g_qe_open ? mtl_pending_fence() : mtl_last_fence(); + } + // WAIT was satisfied at the ordered scheduling point. SIGNAL's WORK + // already committed its producer and published the projected value. + if (!nvn_qs_advance(g_qs, &item, fence)) { + die("NVN queue scheduler lost its offered step"); + } + } + qe_open_commit(); // nothing ready is left: the pending fence becomes the last one before any thread may wait on it + nvn_qs_complete(g_qs, mtl_completed_fence()); + pthread_cond_broadcast(&g_qe_cv); +} + +static void *qe_worker(void *unused) { + (void)unused; + pthread_setname_np("NVN event queues"); + pthread_mutex_lock(&g_sub_mu); + for (;;) { + if (nvn_qs_pending(g_qs)) { + // QueueSubmit returns even when every guest queue is blocked. Keep + // forward waits progressing when a CPU directly stores an event and + // then polls downstream work without making another NVN submission. + dm_host_work++; + mtl_prepare(); + dm_host_work--; + gfx_enter(); + qe_pump(); + gfx_leave(); + struct timespec delay = {0, 1000000}; + pthread_cond_timedwait_relative_np(&g_qe_cv, &g_sub_mu, &delay); + } else { + pthread_cond_wait(&g_qe_cv, &g_sub_mu); + } + } + return NULL; +} + +typedef struct { + size_t offset; + const uint8_t *source; + uint32_t size; +} qe_record; +typedef struct { + qe_submission *submission; + nvn_qs_step *steps; + size_t count, step_capacity; + uint8_t *bytes; + size_t size, capacity; + qe_record *records; + size_t record_count, record_capacity; + int first, origin; +} qe_builder; +static void qe_step(qe_builder *builder, nvn_qs_step step) { + if (builder->count == builder->step_capacity) { + builder->step_capacity = builder->step_capacity ? builder->step_capacity * 2 : 32; + nvn_qs_step *next = realloc(builder->steps, builder->step_capacity * sizeof *builder->steps); + if (!next) { + die("NVN queue step allocation failed"); + } + builder->steps = next; + } + builder->steps[builder->count++] = step; +} +static int qe_reloc_addr(void *context, uint64_t *field, uint64_t len, int writable) { + (void)context; + (void)field; + (void)len; + (void)writable; + return 1; +} +static int qe_reloc_obj(void *context, int kind, uint64_t *field, uint8_t *payload, uint32_t size) { + (void)kind; + (void)size; + const qe_record *record = context; + uint64_t address; + memcpy(&address, field, 8); + uintptr_t source = (uintptr_t)record->source; + if (address >= source && address - source < record->size) { + address = (uint64_t)(payload - 8) + (address - source); + memcpy(field, &address, 8); + } + return 1; +} +static void qe_flush(qe_builder *builder, uint64_t signal, uint32_t mode, uint32_t value) { + if (!builder->size) { + return; + } + qe_work *work = calloc(1, sizeof *work); + if (!work) { + die("NVN queue chunk allocation failed"); + } + work->bytes = malloc(builder->size + 8); + if (!work->bytes) { + die("NVN queue command allocation failed"); + } + memcpy(work->bytes, builder->bytes, builder->size); + put_hdr(work->bytes + builder->size, OP_END, 8); + const cap_ops reloc = {qe_reloc_addr, qe_reloc_obj}; + for (size_t i = 0; i < builder->record_count; i++) { + const qe_record *record = &builder->records[i]; + uint8_t *command = work->bytes + record->offset; + op_hdr header; + memcpy(&header, command, 8); + if (!cap_rec_fields(header.op, command + 8, header.size, &reloc, (void *)record)) { + die("NVN queue cannot relocate opcode %u", header.op); + } + } + work->first = builder->first; + builder->first = 0; + work->origin = builder->origin; + if (!builder->submission->clock) { + builder->submission->clock = mtl_submit_clock_new(); + } + work->clock = builder->submission->clock; + if (!builder->submission->alloc) { + builder->submission->alloc = mtl_submit_alloc_new(); + } + work->alloc = builder->submission->alloc; + work->signal = signal; + work->mode = mode; + work->value = value; + if (builder->submission->tail) { + builder->submission->tail->next = work; + } else { + builder->submission->head = work; + } + builder->submission->tail = work; + qe_step(builder, (nvn_qs_step){.kind = NVN_QS_WORK, .payload = work}); + builder->size = builder->record_count = 0; +} +static void qe_record_copy(qe_builder *builder, const uint8_t *command, uint32_t size) { + if (builder->size + size > builder->capacity) { + builder->capacity = (builder->size + size) * 2; + uint8_t *next = realloc(builder->bytes, builder->capacity); + if (!next) { + die("NVN queue recording allocation failed"); + } + builder->bytes = next; + } + if (builder->record_count == builder->record_capacity) { + builder->record_capacity = builder->record_capacity ? builder->record_capacity * 2 : 256; + qe_record *next = realloc(builder->records, builder->record_capacity * sizeof *builder->records); + if (!next) { + die("NVN queue relocation allocation failed"); + } + builder->records = next; + } + builder->records[builder->record_count++] = (qe_record){builder->size, command, size}; + memcpy(builder->bytes + builder->size, command, size); + builder->size += size; +} +static void qe_submit(const queue_t *queue, int handle_count, const uint64_t *handles) { + qe_init(); + qe_builder builder = {.submission = calloc(1, sizeof(qe_submission)), .first = 1, .origin = queue->origin_ll}; + if (!builder.submission) { + die("NVN queue submission allocation failed"); + } + for (int handle_index = 0; handle_index < handle_count; handle_index++) { + const uint8_t *command = (const uint8_t *)handles[handle_index], *returns[64]; + unsigned depth = 0; + size_t records = 0; + for (;;) { + op_hdr header; + memcpy(&header, command, 8); + if (header.size < 8 || (header.size & 7) || header.size > 64 * 1024 * 1024 || header.op >= OP_COUNT_ || + ++records > 10000000) { + die("NVN invalid scheduled command stream"); + } + if (header.op == OP_END) { + if (!depth) { + break; + } + command = returns[--depth]; + continue; + } + if (header.op == OP_CALL || header.op == OP_JUMP) { + if (header.size < 16) { + die("NVN short command link"); + } + if (header.op == OP_CALL) { + if (depth == 64) { + die("NVN command call nesting overflow"); + } + returns[depth++] = command + header.size; + } + memcpy(&command, command + 8, 8); + continue; + } + if (header.op == OP_WAIT_EVENT || header.op == OP_WAIT_SYNC) { + qe_flush(&builder, 0, 0, 0); + uint64_t address; + uint32_t mode = NVN_QS_EQUAL, value = 1; + memcpy(&address, command + 8, 8); + if (header.op == OP_WAIT_EVENT) { + memcpy(&mode, command + 16, 4); + memcpy(&value, command + 20, 4); + } else { + address |= 1; + } + qe_step(&builder, (nvn_qs_step){.kind = NVN_QS_WAIT, .event = address, .mode = mode, .value = value}); + } else { + qe_record_copy(&builder, command, header.size); + if (header.op == OP_SIGNAL_EVENT) { + uint64_t address; + uint32_t mode, value; + memcpy(&address, command + 8, 8); + memcpy(&mode, command + 16, 4); + memcpy(&value, command + 20, 4); + qe_flush(&builder, address, mode, value); + qe_step(&builder, + (nvn_qs_step){.kind = NVN_QS_SIGNAL, .event = address, .mode = mode, .value = value}); + } else if (header.op == OP_FENCE_SYNC) { + sync_t *sync; + memcpy(&sync, command + 8, 8); + sync->fence = 0; + sync->fenced = NVN_SYNC_FENCE_RECORDED_PENDING; + qe_flush(&builder, 0, 0, 0); + } + } + command += header.size; + } + } + qe_flush(&builder, 0, 0, 0); + nvn_qs_ticket ticket; + if (!nvn_qs_enqueue(g_qs, queue->serial, builder.steps, builder.count, builder.submission, &ticket)) { + die("NVN queue enqueue failed"); + } + free(builder.steps); + free(builder.bytes); + free(builder.records); + qe_pump(); +} +static void qe_queue_fence(const queue_t *queue, sync_t *sync) { + dm_host_mutex_lock(&g_sub_mu); + if (g_qs && nvn_qs_tail(g_qs, queue->serial).sequence) { + nvn_qs_ticket ticket = nvn_qs_tail(g_qs, queue->serial); + sync->fenced = NVN_SYNC_FENCE_QUEUE_PREFIX; + sync->fence = ticket.sequence; + sync->queue = ticket.queue; + } else { + sync->fenced = NVN_SYNC_FENCE_GPU; + sync->fence = mtl_last_fence(); + } + pthread_mutex_unlock(&g_sub_mu); +} +static int qe_wait(nvn_qs_ticket ticket, uint64_t timeout, int completed) { + uint64_t start = now_ns(), fence = 0; + dm_host_mutex_lock(&g_sub_mu); + for (;;) { + gfx_enter(); + qe_pump(); + gfx_leave(); + if (nvn_qs_fence(g_qs, ticket, &fence)) { + break; + } + uint64_t elapsed = now_ns() - start; + if (timeout != UINT64_MAX && elapsed >= timeout) { + pthread_mutex_unlock(&g_sub_mu); + return 0; + } + // New submissions and CPU signals wake this wait. The bounded fallback + // also observes mapped event stores made without an NVN API call. + uint64_t ns = timeout == UINT64_MAX ? 1000000 : timeout - elapsed; + if (ns > 1000000) { + ns = 1000000; + } + struct timespec delay = {0, (long)ns}; + pthread_cond_timedwait_relative_np(&g_qe_cv, &g_sub_mu, &delay); + } + pthread_mutex_unlock(&g_sub_mu); + if (!completed) { + return 1; + } + uint64_t elapsed = now_ns() - start; + return mtl_wait(fence, timeout == UINT64_MAX ? timeout : elapsed < timeout ? timeout - elapsed : 0); +} +static void qe_finish(const queue_t *queue) { + dm_host_mutex_lock(&g_sub_mu); + nvn_qs_ticket ticket = g_qs ? nvn_qs_tail(g_qs, queue->serial) : (nvn_qs_ticket){0}; + pthread_mutex_unlock(&g_sub_mu); + if (ticket.sequence) { + qe_wait(ticket, UINT64_MAX, 1); + } else { + mtl_finish(); + } +} +static void qe_cpu_signal(const ev_t *event, int mode, uint32_t value) { + dm_host_mutex_lock(&g_sub_mu); + qe_value *projection = qe_value_find((uint64_t)event->val, 0); + // Complete an older queued write before resetting the same CPU-visible word. + // The Metal post-operation mutex alone cannot establish that order. + if (projection && projection->fence > mtl_completed_fence()) { + mtl_wait(projection->fence, UINT64_MAX); + } + ev_signal(event, mode, value); + gfx_enter(); + qe_pump(); + gfx_leave(); + pthread_mutex_unlock(&g_sub_mu); +} +static void qe_sync_wait(const queue_t *queue, const sync_t *sync) { + dm_host_mutex_lock(&g_sub_mu); + qe_init(); + nvn_qs_step step = {.kind = NVN_QS_WAIT, .event = (uint64_t)sync | 1, .mode = NVN_QS_EQUAL, .value = 1}; + nvn_qs_ticket ticket; + if (!nvn_qs_enqueue(g_qs, queue->serial, &step, 1, NULL, &ticket)) { + die("NVN sync wait enqueue failed"); + } + gfx_enter(); + qe_pump(); + gfx_leave(); + pthread_mutex_unlock(&g_sub_mu); +} +static void qe_present(const queue_t *queue, win_t *window, int image, const tex_t *texture, double period, + const int crop[4]) { + qe_submission *submission = calloc(1, sizeof *submission); + qe_work *work = calloc(1, sizeof *work); + if (!submission || !work) { + die("NVN queued present allocation failed"); + } + submission->head = submission->tail = work; + work->window = window; + work->image = image; + work->texture = *texture; + work->period = period; + if (crop) { + memcpy(work->crop, crop, sizeof work->crop); + } + nvn_qs_step step = {.kind = NVN_QS_WORK, .payload = work}; + nvn_qs_ticket ticket; + if (!nvn_qs_enqueue(g_qs, queue->serial, &step, 1, submission, &ticket)) { + die("NVN present enqueue failed"); + } + window->submit_queue[image] = ticket.queue; + window->submit_seq[image] = ticket.sequence; + qe_pump(); +} + +static int qe_wait_recorded(const sync_t *sync, uint64_t timeout) { + uint64_t start = now_ns(), fence; + dm_host_mutex_lock(&g_sub_mu); + while (sync->fenced == NVN_SYNC_FENCE_RECORDED_PENDING) { + gfx_enter(); + qe_pump(); + gfx_leave(); + if (sync->fenced != NVN_SYNC_FENCE_RECORDED_PENDING) { + break; + } + uint64_t elapsed = now_ns() - start; + if (timeout != UINT64_MAX && elapsed >= timeout) { + pthread_mutex_unlock(&g_sub_mu); + return 0; + } + uint64_t ns = timeout == UINT64_MAX ? 1000000 : timeout - elapsed; + if (ns > 1000000) { + ns = 1000000; + } + struct timespec delay = {0, (long)ns}; + pthread_cond_timedwait_relative_np(&g_qe_cv, &g_sub_mu, &delay); + } + fence = sync->fence; + pthread_mutex_unlock(&g_sub_mu); + uint64_t elapsed = now_ns() - start; + return mtl_wait(fence, timeout == UINT64_MAX ? timeout : elapsed < timeout ? timeout - elapsed : 0); +} +static void qe_flush_queue(void) { + dm_host_mutex_lock(&g_sub_mu); + gfx_enter(); + qe_pump(); + gfx_leave(); + pthread_mutex_unlock(&g_sub_mu); +} diff --git a/dm/gfx/nvn/nvn_queue_schedule.c b/dm/gfx/nvn/nvn_queue_schedule.c new file mode 100644 index 0000000000..46f1b5977e --- /dev/null +++ b/dm/gfx/nvn/nvn_queue_schedule.c @@ -0,0 +1,291 @@ +#include "nvn_queue_schedule.h" +#include +#include + +typedef struct qs_job { + struct qs_job *next; + uint64_t sequence, order, fence; + size_t count, index; + int finished; + void *submission; + nvn_qs_step steps[]; +} qs_job; +typedef struct qs_queue { + struct qs_queue *next; + uint64_t serial, submitted, completed, scheduled, scheduled_fence, completed_fence; + qs_job *head, *tail, *cursor; +} qs_queue; +struct nvn_qs { + qs_queue *queues, *offered_queue; + qs_job *offered; + uint64_t order, dispatch, completed_fence; + size_t pending; + nvn_qs_read_event_fn read_event; + nvn_qs_release_fn release; + void *context; +}; + +int nvn_qs_wait_ready(uint32_t actual, uint32_t mode, uint32_t target) { + if (mode == NVN_QS_EQUAL) { + return actual == target; + } + if (mode == NVN_QS_GEQUAL_WRAP) { + return (uint32_t)(actual - target) < UINT32_C(0x80000000); + } + return 0; +} +int nvn_qs_signal_value(uint32_t before, uint32_t mode, uint32_t value, uint32_t *after) { + if (!after) { + return 0; + } + if (mode == NVN_QS_WRITE) { + *after = value; + } else if (mode == NVN_QS_ADD) { + *after = before + value; + } else { + return 0; + } + return 1; +} +static qs_queue *queue_find(const nvn_qs *scheduler, uint64_t serial) { + for (qs_queue *queue = scheduler->queues; queue; queue = queue->next) { + if (queue->serial == serial) { + return queue; + } + } + return NULL; +} +nvn_qs *nvn_qs_create(nvn_qs_read_event_fn read_event, nvn_qs_release_fn release, void *context) { + nvn_qs *scheduler = calloc(1, sizeof *scheduler); + if (scheduler) { + scheduler->read_event = read_event; + scheduler->release = release; + scheduler->context = context; + } + return scheduler; +} +int nvn_qs_destroy(nvn_qs *scheduler) { + if (!scheduler) { + return 1; + } + if (scheduler->pending) { + return 0; + } + while (scheduler->queues) { + qs_queue *queue = scheduler->queues; + scheduler->queues = queue->next; + free(queue); + } + free(scheduler); + return 1; +} +int nvn_qs_enqueue(nvn_qs *scheduler, uint64_t queue_serial, const nvn_qs_step *steps, size_t count, void *submission, + nvn_qs_ticket *ticket) { + if (ticket) { + *ticket = (nvn_qs_ticket){0}; + } + if (!scheduler || !queue_serial || !ticket || (count && !steps) || scheduler->order == UINT64_MAX || + scheduler->pending == SIZE_MAX || count > (SIZE_MAX - sizeof(qs_job)) / sizeof(nvn_qs_step)) { + return 0; + } + for (size_t i = 0; i < count; i++) { + const nvn_qs_step *step = &steps[i]; + if (step->kind > NVN_QS_SIGNAL || + ((step->kind == NVN_QS_WAIT || step->kind == NVN_QS_SIGNAL) && step->mode > 1)) { + return 0; + } + } + qs_queue *queue = queue_find(scheduler, queue_serial), *new_queue = NULL; + if (queue && queue->submitted == UINT64_MAX) { + return 0; + } + if (!queue) { + new_queue = calloc(1, sizeof *new_queue); + if (!new_queue) { + return 0; + } + queue = new_queue; + queue->serial = queue_serial; + } + qs_job *job = calloc(1, sizeof *job + count * sizeof *steps); + if (!job) { + free(new_queue); + return 0; + } + if (count) { + memcpy(job->steps, steps, count * sizeof *steps); + } + job->count = count; + job->submission = submission; + job->sequence = ++queue->submitted; + job->order = ++scheduler->order; + if (new_queue) { + queue->next = scheduler->queues; + scheduler->queues = queue; + } + if (queue->tail) { + queue->tail->next = job; + } else { + queue->head = job; + } + queue->tail = job; + if (!queue->cursor) { + queue->cursor = job; + } + scheduler->pending++; + *ticket = (nvn_qs_ticket){queue_serial, job->sequence}; + return 1; +} +static nvn_qs_step job_step(const qs_job *job) { + return job->index == job->count ? (nvn_qs_step){.kind = NVN_QS_END} : job->steps[job->index]; +} +static void offered_item(const nvn_qs *scheduler, nvn_qs_item *item) { + *item = (nvn_qs_item){.ticket = {scheduler->offered_queue->serial, scheduler->offered->sequence}, + .index = scheduler->offered->index, + .dispatch = scheduler->dispatch, + .step = job_step(scheduler->offered)}; +} +int nvn_qs_peek(nvn_qs *scheduler, nvn_qs_item *item) { + if (!item) { + return 0; + } + memset(item, 0, sizeof *item); + if (!scheduler) { + return 0; + } + if (scheduler->offered) { + offered_item(scheduler, item); + return 1; + } + qs_queue *best = NULL; + for (qs_queue *queue = scheduler->queues; queue; queue = queue->next) { + qs_job *job = queue->cursor; + if (!job || (best && best->cursor->order < job->order)) { + continue; + } + nvn_qs_step step = job_step(job); + if (step.kind == NVN_QS_WAIT) { + uint32_t value; + if (!scheduler->read_event || !scheduler->read_event(scheduler->context, step.event, &value) || + !nvn_qs_wait_ready(value, step.mode, step.value)) { + continue; + } + } + best = queue; + } + if (!best) { + return 0; + } + scheduler->offered_queue = best; + scheduler->offered = best->cursor; + if (!++scheduler->dispatch) { + ++scheduler->dispatch; // ticket + index also distinguish stale offers across wrap + } + offered_item(scheduler, item); + return 1; +} +static size_t retire(nvn_qs *scheduler) { + size_t retired = 0; + for (qs_queue *queue = scheduler->queues; queue; queue = queue->next) { + while (queue->head && queue->head->finished && queue->head->fence <= scheduler->completed_fence) { + qs_job *job = queue->head; + nvn_qs_ticket ticket = {queue->serial, job->sequence}; + void *payload = job->submission; + queue->head = job->next; + if (!queue->head) { + queue->tail = NULL; + } + queue->completed = job->sequence; + queue->completed_fence = job->fence; + scheduler->pending--; + retired++; + free(job); + if (scheduler->release) { + scheduler->release(scheduler->context, ticket, payload); + } + } + } + return retired; +} +int nvn_qs_advance(nvn_qs *scheduler, const nvn_qs_item *item, uint64_t gpu_fence) { + if (!scheduler || !item || !scheduler->offered || item->dispatch != scheduler->dispatch || + item->ticket.queue != scheduler->offered_queue->serial || + item->ticket.sequence != scheduler->offered->sequence || item->index != scheduler->offered->index || + (gpu_fence && gpu_fence < scheduler->offered->fence)) { + return 0; + } + qs_job *job = scheduler->offered; + if (gpu_fence) { + job->fence = gpu_fence; + } + if (job->index == job->count) { + qs_queue *queue = scheduler->offered_queue; + if (job->fence < queue->scheduled_fence) { + job->fence = queue->scheduled_fence; + } + queue->scheduled = job->sequence; + queue->scheduled_fence = job->fence; + job->finished = 1; + queue->cursor = job->next; + } else { + job->index++; + } + scheduler->offered = NULL; + scheduler->offered_queue = NULL; + retire(scheduler); + return 1; +} +size_t nvn_qs_complete(nvn_qs *scheduler, uint64_t gpu_fence) { + if (!scheduler) { + return 0; + } + if (gpu_fence > scheduler->completed_fence) { + scheduler->completed_fence = gpu_fence; + } + return retire(scheduler); +} +nvn_qs_ticket nvn_qs_tail(const nvn_qs *scheduler, uint64_t queue_serial) { + qs_queue *queue = scheduler ? queue_find(scheduler, queue_serial) : NULL; + return (nvn_qs_ticket){queue_serial, queue ? queue->submitted : 0}; +} +int nvn_qs_scheduled(const nvn_qs *scheduler, nvn_qs_ticket ticket) { + if (!ticket.sequence) { + return 1; + } + qs_queue *queue = scheduler ? queue_find(scheduler, ticket.queue) : NULL; + return queue && ticket.sequence <= queue->scheduled; +} +int nvn_qs_fence(const nvn_qs *scheduler, nvn_qs_ticket ticket, uint64_t *fence) { + if (!fence) { + return 0; + } + *fence = 0; + if (!ticket.sequence) { + return 1; + } + qs_queue *queue = scheduler ? queue_find(scheduler, ticket.queue) : NULL; + if (!queue || ticket.sequence > queue->scheduled) { + return 0; + } + if (ticket.sequence <= queue->completed) { + *fence = queue->completed_fence; + return 1; + } + for (qs_job *job = queue->head; job; job = job->next) { + if (job->sequence == ticket.sequence) { + *fence = job->fence; + return 1; + } + } + return 0; +} +int nvn_qs_done(const nvn_qs *scheduler, nvn_qs_ticket ticket) { + if (!ticket.sequence) { + return 1; + } + qs_queue *queue = scheduler ? queue_find(scheduler, ticket.queue) : NULL; + return queue && ticket.sequence <= queue->completed; +} +size_t nvn_qs_pending(const nvn_qs *scheduler) { + return scheduler ? scheduler->pending : 0; +} diff --git a/dm/gfx/nvn/nvn_queue_schedule.h b/dm/gfx/nvn/nvn_queue_schedule.h new file mode 100644 index 0000000000..63971405a6 --- /dev/null +++ b/dm/gfx/nvn/nvn_queue_schedule.h @@ -0,0 +1,78 @@ +// Guest queue ordering, independent of command decoding and Metal. +// The caller serializes all access, including the callbacks. WORK payloads and +// event keys are opaque; command streams and their resources remain caller-owned. +#pragma once +#include +#include + +typedef struct nvn_qs nvn_qs; +typedef struct { + uint64_t queue, sequence; +} nvn_qs_ticket; +enum { NVN_QS_WORK, NVN_QS_WAIT, NVN_QS_SIGNAL, NVN_QS_END }; +enum { NVN_QS_EQUAL, NVN_QS_GEQUAL_WRAP }; +enum { NVN_QS_WRITE, NVN_QS_ADD }; +typedef struct { + uint32_t kind, mode; + uint64_t event; + uint32_t value; + void *payload; +} nvn_qs_step; +typedef struct { + nvn_qs_ticket ticket; + size_t index; + uint64_t dispatch; // validates advance; copy the item unchanged + nvn_qs_step step; +} nvn_qs_item; + +// read_event returns 1 with the value at the integration's ordered scheduling +// point, or 0 when unknown. It must account for committed signals not yet CPU- +// visible; the core does not predict signals or mutate event storage itself. +// release runs exactly once after a submission's END and GPU fence complete. +// Neither callback may reenter or mutate this scheduler. +typedef int (*nvn_qs_read_event_fn)(void *ctx, uint64_t event, uint32_t *value); +typedef void (*nvn_qs_release_fn)(void *ctx, nvn_qs_ticket ticket, void *submission); +nvn_qs *nvn_qs_create(nvn_qs_read_event_fn read_event, nvn_qs_release_fn release, void *ctx); +// Refuses destruction while any submission remains, retaining its payloads. +int nvn_qs_destroy(nvn_qs *s); + +// Copies step descriptors, not payloads. queue must be a nonzero initialization +// serial. Appends an implicit END, including for an empty submission. On failure +// no ownership transfers and ticket is zeroed. Per-queue sequence never wraps. +int nvn_qs_enqueue(nvn_qs *s, uint64_t queue, const nvn_qs_step *steps, size_t count, void *submission, + nvn_qs_ticket *ticket); +// Offers the oldest ready submission head across queues; later submissions on +// a blocked queue cannot pass it. A successful offer latches WAIT satisfaction. +// Repeated peeks return that same item until advance, even if the event changes. +// Returns 0 when all queues are blocked or no unscheduled work remains. +int nvn_qs_peek(nvn_qs *s, nvn_qs_item *item); +// Consume the offered item only after its work/signal has been ordered on the +// host queue. SIGNAL's producer prefix must already be committed and reflected +// in read_event's state. END commits any remaining work before being consumed. +// gpu_fence is the latest single-host-queue fence protecting this submission's +// work, or 0 when this step adds none. Nonzero fences cannot decrease within a +// submission. Stale/mismatched items fail without changing the scheduler. +int nvn_qs_advance(nvn_qs *s, const nvn_qs_item *item, uint64_t gpu_fence); +// The host queue completed every fence <= gpu_fence. Retires finished submission +// prefixes per guest queue, invokes release, and returns the number retired. +// Older notifications are harmless. This is not an independent-queue fence API. +size_t nvn_qs_complete(nvn_qs *s, uint64_t gpu_fence); +nvn_qs_ticket nvn_qs_tail(const nvn_qs *s, uint64_t queue); +// True once this queue prefix has consumed END, whether or not the GPU finished. +int nvn_qs_scheduled(const nvn_qs *s, nvn_qs_ticket ticket); +// Returns a host fence protecting the scheduled queue prefix, including earlier +// submissions when this one has no GPU work. Fails and zeroes fence before END. +// After retirement the bound is the queue's latest completed fence: possibly +// newer than this ticket's own fence, but already complete. No per-ticket history +// is retained once its payload is released. A sequence-zero prefix has fence 0. +int nvn_qs_fence(const nvn_qs *s, nvn_qs_ticket ticket, uint64_t *fence); +// A sequence-zero ticket denotes an empty queue prefix and is already complete. +// Completed tickets remain queryable after their payload/metadata are released. +int nvn_qs_done(const nvn_qs *s, nvn_qs_ticket ticket); +size_t nvn_qs_pending(const nvn_qs *s); + +// Invalid modes fail closed. GEQUAL_WRAP compares the unsigned modular distance +// with half the 32-bit range, avoiding signed overflow or implementation-defined +// unsigned-to-signed conversion. ADD wraps modulo 2^32. +int nvn_qs_wait_ready(uint32_t actual, uint32_t mode, uint32_t target); +int nvn_qs_signal_value(uint32_t before, uint32_t mode, uint32_t value, uint32_t *after); diff --git a/dm/gfx/tests/Makefile b/dm/gfx/tests/Makefile index 02aa8ad4d4..e10c5799d4 100644 --- a/dm/gfx/tests/Makefile +++ b/dm/gfx/tests/Makefile @@ -51,6 +51,11 @@ t_gfx_suspend: t_gfx_suspend.c ../nvn/hle_nvn.c ../../host/crash.c ../../host/hl clang $(CFLAGS) -fobjc-arc -o $@ t_gfx_suspend.c ../../host/crash.c ../../host/hle_bigstack.c $(GFX) $(FW) t_nvn: t_nvn.c ../nvn/hle_nvn.c ../../host/crash.c ../../host/hle_bigstack.c $(GFX) $(GFX_H) clang $(CFLAGS) -fobjc-arc -o $@ t_nvn.c ../../host/crash.c ../../host/hle_bigstack.c $(GFX) $(FW) +# Its event tests go through the guest queue scheduler (qe_pump): once with a host submission per chunk, once with the chunks merged into one command buffer +.PHONY: check_t_nvn +check_t_nvn: t_nvn + ./t_nvn + DM_NVN_CHUNK_MERGE=1 ./t_nvn t_dirty_slots: t_dirty_slots.c ../nvn/nvn_dirty.c ../../host/crash.c ../nvn/nvn_dirty.h $(GAME_H) clang $(CFLAGS) -o $@ t_dirty_slots.c ../../host/crash.c ../nvn/nvn_dirty.c t_dirty_race: t_dirty_race.c ../nvn/nvn_dirty.c ../../host/crash.c ../nvn/nvn_dirty.h $(GAME_H) @@ -457,3 +462,12 @@ t_logic: t_logic.m $(MTL_T_DEP) clang $(CFLAGS) -fobjc-arc -o $@ t_logic.m $(MTL_T_SRC) $(FW) check_t_logic: t_logic @set -e; work=$$(mktemp -d); trap 'rm -rf "$$work"' EXIT; python3 test_dmsh_logic.py --fixture "$$work"; MTL_DEBUG_LAYER=1 ./t_logic "$$work" + +t_queue_vis: t_queue_vis.m $(MTL_T_DEP) + clang $(CFLAGS) -fobjc-arc -o $@ t_queue_vis.m $(MTL_T_SRC) $(FW) +check_t_queue_vis: RUN_ENV = MTL_DEBUG_LAYER=1 +t_queue_alloc: t_queue_alloc.m $(MTL_T_DEP) + clang $(CFLAGS) -fobjc-arc -o $@ t_queue_alloc.m $(MTL_T_SRC) $(FW) +check_t_queue_alloc: RUN_ENV = MTL_DEBUG_LAYER=1 +t_queue_schedule: t_queue_schedule.c ../nvn/nvn_queue_schedule.c ../nvn/nvn_queue_schedule.h + clang -std=c11 -O2 -g -Wall -Wextra -Werror -o $@ t_queue_schedule.c ../nvn/nvn_queue_schedule.c diff --git a/dm/gfx/tests/t_capture.c b/dm/gfx/tests/t_capture.c index 3ce23906db..aa1c2ddd34 100644 --- a/dm/gfx/tests/t_capture.c +++ b/dm/gfx/tests/t_capture.c @@ -119,7 +119,7 @@ int main(void) { cap_writer w; CHECK(cap_open(&w, path), "open for writing"); const char *lay = cap_layout(); const char *multi_layout = strstr(lay, "op 55 p_multi_draw"); - CHECK(multi_layout && cap_layout_matches(lay, (size_t)(multi_layout - lay)), "the complete legacy record layout remains readable"); + CHECK(multi_layout && !cap_layout_matches(lay, (size_t)(multi_layout - lay)), "the changed sync layout requires a new capture"); CHECK(cap_layout_matches(lay, strlen(lay)) && !cap_layout_matches(lay, strlen(lay) - 1), "accept the current layout, reject a truncated extension"); cap_put(&w, CAP_LAYT, 0, lay, strlen(lay), 0); for (int r = 0; r < nregs; r++) { uint8_t b[sizeof(cap_regn) + 65536]; cap_regn h = {(uint32_t)r, r == 2, regs[r].lo, regs[r].size}; diff --git a/dm/gfx/tests/t_nvn.c b/dm/gfx/tests/t_nvn.c index 25bf8927f8..0d2e95e83a 100644 --- a/dm/gfx/tests/t_nvn.c +++ b/dm/gfx/tests/t_nvn.c @@ -41,6 +41,87 @@ static NVNqueue *g_sq; static NVNcommandHandle g_sh; static NVNT_nvnQueueSubmitC static void *susp_thr(void *a) { (void)a; pthread_mutex_lock(&g_smu); atomic_store(&g_sstarted, 1); pthread_cond_broadcast(&g_scv); pthread_mutex_unlock(&g_smu); g_sub(g_sq, 1, &g_sh); atomic_store(&g_sdone, 1); return NULL; } static int wait_started(void) { struct timespec dl; clock_gettime(CLOCK_REALTIME, &dl); dl.tv_sec += 10; pthread_mutex_lock(&g_smu); while (!atomic_load(&g_sstarted) && pthread_cond_timedwait(&g_scv, &g_smu, &dl) == 0) {} int ok = atomic_load(&g_sstarted); pthread_mutex_unlock(&g_smu); return ok; } +static void *event_store_later(void *word) { + usleep(30000); + *(volatile uint32_t *)word = 1; + return NULL; +} + +// Poison a completed submission before the production release frees it. A +// later clear must use the queue's retained targets, not these retired records. +static int g_reloc_retired; +static void reloc_release(void *context, nvn_qs_ticket ticket, void *payload) { + qe_submission *submission = payload; + for (qe_work *work = submission->head; work; work = work->next) { + if (work->bytes) { + size_t size = 0; + op_hdr header; + do { + memcpy(&header, work->bytes + size, sizeof header); + size += header.size; + } while (header.op != OP_END); + volatile uint8_t *bytes = work->bytes; + for (size_t i = 0; i < size; i++) { + bytes[i] = 0xa5; // keep poisoning observable before free + } + } + } + g_reloc_retired++; + qe_release(context, ticket, payload); +} +static qe_builder reloc_clear(const NVNtexture *target, const float color[4]) { + qe_builder b = {.submission = calloc(1, sizeof(qe_submission)), .first = 1}; + _Alignas(8) uint8_t original[8 + SNAP_OFF(p_set_rt) + 2 * sizeof(tex_t)] = {0}; + uint8_t expected_snapshot[sizeof(tex_t)]; + if (target) { + put_hdr(original, OP_SET_RT, sizeof original); + p_set_rt *rt = (p_set_rt *)(original + 8); + rt->n = 1; + rt->color[0] = tex_snap(rt, SNAP_OFF(p_set_rt), 0, target, 0); + memcpy(expected_snapshot, original + 8 + SNAP_OFF(p_set_rt), sizeof expected_snapshot); + qe_record_copy(&b, original, sizeof original); + } + struct { + op_hdr h; + p_scissor s; + } scissor = {{OP_SCISSOR, sizeof scissor}, {0, 0, 1280, 720}}; + struct { + op_hdr h; + p_clear_color c; + } clear = {{OP_CLEAR_COLOR, sizeof clear}, {.index = 0, .mask = 15}}; + memcpy(clear.c.rgba, color, sizeof clear.c.rgba); + qe_record_copy(&b, (const uint8_t *)&scissor, sizeof scissor); + qe_record_copy(&b, (const uint8_t *)&clear, sizeof clear); + qe_flush(&b, 0, 0, 0); + if (target) { + const p_set_rt *owned = (const p_set_rt *)(b.submission->head->bytes + 8); + EXPECT("copied RT snapshot points inside owned record", + (uintptr_t)(b.submission->head->bytes + 8 + SNAP_OFF(p_set_rt)), (uintptr_t)owned->color[0]); + memset(original, 0, sizeof original); + const uint8_t *copied_snapshot = (const uint8_t *)owned->color[0]; + EXPECT("clearing original record preserves owned texture", 0, + memcmp(copied_snapshot, expected_snapshot, sizeof expected_snapshot)); + } + return b; +} +static void reloc_submit(qe_builder *b, const queue_t *q) { // submission lock and graphics gate held + nvn_qs_ticket ticket; + int enqueued = nvn_qs_enqueue(g_qs, q->serial, b->steps, b->count, b->submission, &ticket); + EXPECT("target snapshot work enqueues", 1, enqueued); + if (!enqueued) { + abort(); + } + free(b->steps); + free(b->bytes); + free(b->records); + qe_pump(); + uint64_t fence = 0; + EXPECT("target snapshot work schedules", 1, nvn_qs_fence(g_qs, ticket, &fence)); + EXPECT("target snapshot GPU work completes", 1, mtl_wait(fence, 5000000000ull)); + nvn_qs_complete(g_qs, mtl_completed_fence()); + EXPECT("target snapshot chunk retired before continuation", 1, nvn_qs_done(g_qs, ticket)); +} + static size_t tex_size3(int target, int fmt, int w, int h, int depth) { // a layered or 3D texture's storage size NVNtextureBuilder tb; P(nvnTextureBuilderSetDevice)(&tb, &dev); P(nvnTextureBuilderSetDefaults)(&tb); P(nvnTextureBuilderSetTarget)(&tb, target); @@ -620,7 +701,25 @@ int main(void) { EXPECT("wait after fence = already signaled (0)", 0, P(nvnSyncWait)(&s1, 0)); NVNtexture sc[2]; NVNtextureBuilder tb; P(nvnTextureBuilderSetDevice)(&tb, &dev); P(nvnTextureBuilderSetDefaults)(&tb); P(nvnTextureBuilderSetWidth)(&tb, 1280); P(nvnTextureBuilderSetHeight)(&tb, 720); P(nvnTextureBuilderSetFormat)(&tb, NVN_FORMAT_RGBA8); - P(nvnTextureBuilderSetStorage)(&tb, &pool, 0); P(nvnTextureInitialize)(&sc[0], &tb); P(nvnTextureInitialize)(&sc[1], &tb); + // Dedicated, correctly sized swapchain storage. The old 1 MB scratch pool was too small for a 1280x720 image and later tests wrote over it; + // with tex_dirty_upload that made these unused presentation fixtures look CPU-written (and tried to upload them on the present thread). + size_t swapchain_slot = (P(nvnTextureBuilderGetStorageSize)(&tb) + 65535u) & ~(size_t)65535u; + void *swapchain_memory = NULL; + if (posix_memalign(&swapchain_memory, (size_t)getpagesize(), swapchain_slot * 2)) { + abort(); + } + memset(swapchain_memory, 0, swapchain_slot * 2); + NVNmemoryPoolBuilder swapchain_pool_builder; + NVNmemoryPool swapchain_pool; + P(nvnMemoryPoolBuilderSetDevice)(&swapchain_pool_builder, &dev); + P(nvnMemoryPoolBuilderSetDefaults)(&swapchain_pool_builder); + P(nvnMemoryPoolBuilderSetStorage)(&swapchain_pool_builder, swapchain_memory, swapchain_slot * 2); + P(nvnMemoryPoolBuilderSetFlags)(&swapchain_pool_builder, 0x24); + P(nvnMemoryPoolInitialize)(&swapchain_pool, &swapchain_pool_builder); + P(nvnTextureBuilderSetStorage)(&tb, &swapchain_pool, 0); + P(nvnTextureInitialize)(&sc[0], &tb); + P(nvnTextureBuilderSetStorage)(&tb, &swapchain_pool, (ptrdiff_t)swapchain_slot); + P(nvnTextureInitialize)(&sc[1], &tb); EXPECT("texture width", 1280, P(nvnTextureGetWidth)(&sc[0])); NVNtexture *scp[2] = {&sc[0], &sc[1]}; NVNwindowBuilder wb; NVNwindow win; P(nvnWindowBuilderSetDevice)(&wb, &dev); P(nvnWindowBuilderSetDefaults)(&wb); @@ -946,6 +1045,242 @@ int main(void) { EXPECT("several rows whose alignment would break the wrapped address: refused", 0, nvn_tbuf_rows(20000, 4, 3, &w, &h, &row)); } + // Forward queue dependency: the consumer is submitted first on this same + // CPU thread. Its wait must yield so the producer can be submitted later. + { + NVNqueue producer; + EXPECT("event producer queue", 1, P(nvnQueueInitialize)(&producer, &qb)); + NVNevent a, b; + NVNeventBuilder builder; + P(nvnEventBuilderSetDefaults)(&builder); + P(nvnEventBuilderSetStorage)(&builder, &pool, 0x9000); + P(nvnEventInitialize)(&a, &builder); + P(nvnEventBuilderSetStorage)(&builder, &pool, 0x9040); + P(nvnEventInitialize)(&b, &builder); + P(nvnEventSignal)(&a, NVN_EVENT_SIGNAL_MODE_WRITE, 0); + P(nvnEventSignal)(&b, NVN_EVENT_SIGNAL_MODE_WRITE, 0); + uint32_t *source = (uint32_t *)(poolmem + 0xa000), *copy = (uint32_t *)(poolmem + 0xa100); + *source = *copy = 0; + P(nvnCommandBufferBeginRecording)(&cb); + P(nvnCommandBufferWaitEvent)(&cb, &a, NVN_EVENT_WAIT_MODE_GEQUAL_WRAP, 1); + P(nvnCommandBufferCopyBufferToBuffer)(&cb, (NVNbufferAddress)source, (NVNbufferAddress)copy, 4, 0); + P(nvnCommandBufferSignalEvent)(&cb, &b, NVN_EVENT_SIGNAL_MODE_WRITE, NVN_EVENT_SIGNAL_LOCATION_BOTTOM, 0, 1); + P(nvnCommandBufferWaitEvent)(&cb, &a, NVN_EVENT_WAIT_MODE_GEQUAL_WRAP, 2); + P(nvnCommandBufferCopyBufferToBuffer)(&cb, (NVNbufferAddress)source, (NVNbufferAddress)(copy + 1), 4, 0); + NVNcommandHandle consumer = P(nvnCommandBufferEndRecording)(&cb); + P(nvnQueueSubmitCommands)(&q, 1, &consumer); + NVNsync pending; + P(nvnSyncInitialize)(&pending, &dev); + P(nvnQueueFenceSync)(&q, &pending, NVN_SYNC_CONDITION_ALL_GPU_COMMANDS_COMPLETE, 0); + EXPECT("blocked queue fence does not complete early", NVN_SYNC_WAIT_RESULT_TIMEOUT_EXPIRED, + P(nvnSyncWait)(&pending, 0)); + EXPECT("forward waiter did not copy old input", 0, *copy); + P(nvnCommandBufferBeginRecording)(&cb); + uint32_t word = 0x1234abcd; + P(nvnCommandBufferUpdateUniformBuffer)(&cb, (NVNbufferAddress)source, 256, 0, 4, &word); + P(nvnCommandBufferSignalEvent)(&cb, &a, NVN_EVENT_SIGNAL_MODE_WRITE, NVN_EVENT_SIGNAL_LOCATION_BOTTOM, 0, 1); + P(nvnCommandBufferWaitEvent)(&cb, &b, NVN_EVENT_WAIT_MODE_EQUAL, 1); + word = 0x5678ef01; + P(nvnCommandBufferUpdateUniformBuffer)(&cb, (NVNbufferAddress)source, 256, 0, 4, &word); + P(nvnCommandBufferSignalEvent)(&cb, &a, NVN_EVENT_SIGNAL_MODE_ADD, NVN_EVENT_SIGNAL_LOCATION_BOTTOM, 0, 1); + NVNcommandHandle writer = P(nvnCommandBufferEndRecording)(&cb); + // Nested CALL plus existing 256-byte command segments exercise flattening. + P(nvnCommandBufferBeginRecording)(&cb); + P(nvnCommandBufferCopyCommands)(&cb, 1, &writer); + NVNcommandHandle outer = P(nvnCommandBufferEndRecording)(&cb); + P(nvnQueueSubmitCommands)(&producer, 1, &outer); + int result = P(nvnSyncWait)(&pending, 5000000000ull); + EXPECT("bidirectional event handshake fence", 1, result == 0 || result == 1); + EXPECT("first consumer reads first producer version", 0x1234abcd, copy[0]); + EXPECT("resumed consumer reads second producer version", 0x5678ef01, copy[1]); + EXPECT("GPU event ADD executes exactly once", 2, P(nvnEventGetValue)(&a)); + P(nvnQueueFinish)(&producer); + P(nvnQueueFinish)(&q); + // A queue containing only a wait still inherits the producer's GPU + // completion dependency, even though it emits no command buffer. + NVNqueue wait_only; + NVNsync producer_ready; + EXPECT("wait-only queue", 1, P(nvnQueueInitialize)(&wait_only, &qb)); + P(nvnSyncInitialize)(&producer_ready, &dev); + P(nvnQueueFenceSync)(&producer, &producer_ready, NVN_SYNC_CONDITION_ALL_GPU_COMMANDS_COMPLETE, 0); + uint64_t producer_fence = mtl_last_fence(); + EXPECT("wait-only fixture has a producer fence", 1, producer_fence != 0); + P(nvnQueueWaitSync)(&wait_only, &producer_ready); + pthread_mutex_lock(&g_sub_mu); + nvn_qs_ticket wait_ticket = nvn_qs_tail(g_qs, ((queue_t *)&wait_only)->serial); + uint64_t wait_fence = 0; + EXPECT("wait-only prefix is scheduled", 1, nvn_qs_fence(g_qs, wait_ticket, &wait_fence)); + EXPECT("wait-only prefix retains producer fence", producer_fence, wait_fence); + pthread_mutex_unlock(&g_sub_mu); + P(nvnQueueFinish)(&wait_only); + P(nvnQueueFinalize)(&wait_only); + P(nvnSyncFinalize)(&producer_ready); + P(nvnEventSignal)(&a, NVN_EVENT_SIGNAL_MODE_WRITE, 0); + EXPECT("CPU event reset supersedes completed projection", 0, P(nvnEventGetValue)(&a)); + NVNsync recorded; + P(nvnSyncInitialize)(&recorded, &dev); + P(nvnCommandBufferBeginRecording)(&cb); + P(nvnCommandBufferWaitEvent)(&cb, &a, NVN_EVENT_WAIT_MODE_EQUAL, 1); + P(nvnCommandBufferFenceSync)(&cb, &recorded, NVN_SYNC_CONDITION_ALL_GPU_COMMANDS_COMPLETE, 0); + P(nvnCommandBufferSignalEvent)(&cb, &b, NVN_EVENT_SIGNAL_MODE_WRITE, NVN_EVENT_SIGNAL_LOCATION_BOTTOM, 0, 2); + NVNcommandHandle deferred_fence = P(nvnCommandBufferEndRecording)(&cb); + P(nvnQueueSubmitCommands)(&q, 1, &deferred_fence); + EXPECT("command-buffer fence pending behind event", NVN_SYNC_WAIT_RESULT_TIMEOUT_EXPIRED, + P(nvnSyncWait)(&recorded, 0)); + pthread_t cpu_writer; + pthread_create(&cpu_writer, NULL, event_store_later, poolmem + 0x9000); + uint64_t started = now_ns(); + result = P(nvnSyncWait)(&recorded, 5000000000ull); + EXPECT("deferred recorded fence honors timeout and resumes", 1, result == 0 || result == 1); + EXPECT("recorded fence actually waited for producer", 1, now_ns() - started >= 15000000); + pthread_join(cpu_writer, NULL); + P(nvnQueueFinish)(&q); + P(nvnEventSignal)(&a, NVN_EVENT_SIGNAL_MODE_WRITE, 0); + P(nvnEventSignal)(&b, NVN_EVENT_SIGNAL_MODE_WRITE, 0); + P(nvnCommandBufferBeginRecording)(&cb); + P(nvnCommandBufferWaitEvent)(&cb, &a, NVN_EVENT_WAIT_MODE_EQUAL, 1); + P(nvnCommandBufferSignalEvent)(&cb, &b, NVN_EVENT_SIGNAL_MODE_WRITE, NVN_EVENT_SIGNAL_LOCATION_BOTTOM, 0, 3); + NVNcommandHandle autonomous = P(nvnCommandBufferEndRecording)(&cb); + P(nvnQueueSubmitCommands)(&q, 1, &autonomous); + *(volatile uint32_t *)(poolmem + 0x9000) = 1; + started = now_ns(); + while (*(volatile uint32_t *)(poolmem + 0x9040) != 3 && now_ns() - started < 5000000000ull) { + usleep(1000); + } + EXPECT("mapped event store progresses without another NVN call", 3, *(volatile uint32_t *)(poolmem + 0x9040)); + P(nvnQueueFinish)(&q); + + // More forward-waiting logical submissions than the allocator pool: + // completed prefixes may surrender their allocator lease, so this same + // CPU thread can still submit the late producer and resume every queue. + enum { BLOCKED_QUEUES = 12 }; + NVNqueue blocked[BLOCKED_QUEUES]; + P(nvnEventSignal)(&a, NVN_EVENT_SIGNAL_MODE_WRITE, 0); + *source = 0x31415926; + source[4] = 0; + memset(copy + 8, 0, BLOCKED_QUEUES * 2 * sizeof *copy); + for (int i = 0; i < BLOCKED_QUEUES; i++) { + EXPECT("blocked-prefix queue initializes", 1, P(nvnQueueInitialize)(&blocked[i], &qb)); + P(nvnCommandBufferBeginRecording)(&cb); + P(nvnCommandBufferCopyBufferToBuffer)(&cb, (NVNbufferAddress)source, (NVNbufferAddress)(copy + 8 + i), 4, 0); + P(nvnCommandBufferWaitEvent)(&cb, &a, NVN_EVENT_WAIT_MODE_EQUAL, 1); + P(nvnCommandBufferCopyBufferToBuffer)(&cb, (NVNbufferAddress)(source + 4), + (NVNbufferAddress)(copy + 8 + BLOCKED_QUEUES + i), 4, 0); + NVNcommandHandle prefix = P(nvnCommandBufferEndRecording)(&cb); + P(nvnQueueSubmitCommands)(&blocked[i], 1, &prefix); + } + for (int i = 0; i < BLOCKED_QUEUES; i++) { + EXPECT("blocked continuation has not executed", 0, copy[8 + BLOCKED_QUEUES + i]); + } + P(nvnCommandBufferBeginRecording)(&cb); + word = 0x27182818; + P(nvnCommandBufferUpdateUniformBuffer)(&cb, (NVNbufferAddress)source, 256, 16, 4, &word); + P(nvnCommandBufferSignalEvent)(&cb, &a, NVN_EVENT_SIGNAL_MODE_WRITE, NVN_EVENT_SIGNAL_LOCATION_BOTTOM, 0, 1); + NVNcommandHandle late = P(nvnCommandBufferEndRecording)(&cb); + P(nvnQueueSubmitCommands)(&producer, 1, &late); + for (int i = 0; i < BLOCKED_QUEUES; i++) { + P(nvnQueueFinish)(&blocked[i]); + EXPECT("reclaimed prefix retains GPU data", 0x31415926, copy[8 + i]); + EXPECT("resumed queue observes late producer", 0x27182818, copy[8 + BLOCKED_QUEUES + i]); + P(nvnQueueFinalize)(&blocked[i]); + } + P(nvnQueueFinish)(&producer); + + // Event splitting is an implementation boundary, not another guest + // submission. Its reports share one timestamp in submission mode even + // when an interleaved queue supplies a delayed dependency. + uint64_t *top = (uint64_t *)(poolmem + 0xa200), *bottom = top + 2, *interleaved = top + 4, *later = top + 6; + memset(top, 0, 8 * sizeof *top); + P(nvnEventSignal)(&a, NVN_EVENT_SIGNAL_MODE_WRITE, 0); + P(nvnEventSignal)(&b, NVN_EVENT_SIGNAL_MODE_WRITE, 0); + P(nvnCommandBufferBeginRecording)(&cb); + P(nvnCommandBufferReportCounter)(&cb, NVN_COUNTER_TYPE_TIMESTAMP_TOP, (NVNbufferAddress)top); + P(nvnCommandBufferSignalEvent)(&cb, &b, NVN_EVENT_SIGNAL_MODE_WRITE, NVN_EVENT_SIGNAL_LOCATION_BOTTOM, 0, 4); + P(nvnCommandBufferWaitEvent)(&cb, &a, NVN_EVENT_WAIT_MODE_EQUAL, 1); + P(nvnCommandBufferCopyBufferToBuffer)(&cb, (NVNbufferAddress)source, (NVNbufferAddress)(copy + 2), 4, 0); + P(nvnCommandBufferReportCounter)(&cb, NVN_COUNTER_TYPE_TIMESTAMP, (NVNbufferAddress)bottom); + NVNcommandHandle timed = P(nvnCommandBufferEndRecording)(&cb); + P(nvnQueueSubmitCommands)(&q, 1, &timed); + started = now_ns(); + while (*(volatile uint32_t *)(poolmem + 0x9040) != 4 && now_ns() - started < 5000000000ull) { + usleep(1000); + } + EXPECT("first timestamp publishes before blocked continuation", 1, top[1] != 0 && bottom[1] == 0); + usleep(30000); + P(nvnCommandBufferBeginRecording)(&cb); + word = 0x6688aacc; + P(nvnCommandBufferUpdateUniformBuffer)(&cb, (NVNbufferAddress)source, 256, 0, 4, &word); + P(nvnCommandBufferReportCounter)(&cb, NVN_COUNTER_TYPE_TIMESTAMP, (NVNbufferAddress)interleaved); + P(nvnCommandBufferSignalEvent)(&cb, &a, NVN_EVENT_SIGNAL_MODE_WRITE, NVN_EVENT_SIGNAL_LOCATION_BOTTOM, 0, 1); + NVNcommandHandle timed_producer = P(nvnCommandBufferEndRecording)(&cb); + P(nvnQueueSubmitCommands)(&producer, 1, &timed_producer); + P(nvnQueueFinish)(&producer); + P(nvnQueueFinish)(&q); + const char *ts_mode = getenv("DM_NVN_GPU_TS"); + int segment_times = ts_mode && *ts_mode ? !strcmp(ts_mode, "segment") : DM_GAME_GPU_TS_SEGMENT; + if (segment_times) { + EXPECT("segment timestamps preserve dependency interval", 1, bottom[1] > top[1]); + } else { + EXPECT("logical submission timestamps ignore host event splits", top[1], bottom[1]); + } + EXPECT("interleaved queue uses a separate timestamp", 1, interleaved[1] > top[1]); + EXPECT("timed continuation reads actual ordered producer data", 0x6688aacc, copy[2]); + P(nvnCommandBufferBeginRecording)(&cb); + P(nvnCommandBufferReportCounter)(&cb, NVN_COUNTER_TYPE_TIMESTAMP, (NVNbufferAddress)later); + NVNcommandHandle next_timed = P(nvnCommandBufferEndRecording)(&cb); + P(nvnQueueSubmitCommands)(&q, 1, &next_timed); + P(nvnQueueFinish)(&q); + EXPECT("later logical submission gets a fresh timestamp", 1, later[1] > top[1] && later[1] >= interleaved[1]); + P(nvnQueueFinalize)(&producer); + } + + // Copied inline RT snapshots must survive both original record reuse and + // retirement of the owned chunk that established the queue's bindings. + { + NVNqueue qa, qb_saved; + EXPECT("snapshot first queue", 1, P(nvnQueueInitialize)(&qa, &qb)); + EXPECT("snapshot second queue", 1, P(nvnQueueInitialize)(&qb_saved, &qb)); + const float red[4] = {1, 0, 0, 1}, blue[4] = {0, 0, 1, 1}, green[4] = {0, 1, 0, 1}; + qe_builder first = reloc_clear(&sc[0], red), other = reloc_clear(&sc[1], blue), + continuation = reloc_clear(NULL, green); + // The worker uses the same lock, so it cannot observe the temporary + // scheduler or race its destruction. Replay and retirement stay real. + pthread_mutex_lock(&g_sub_mu); + gfx_enter(); + qe_pump(); + nvn_qs *saved = g_qs; + g_qs = nvn_qs_create(qe_read, reloc_release, NULL); + if (!g_qs) { + abort(); + } + g_reloc_retired = 0; + reloc_submit(&first, (const queue_t *)&qa); + EXPECT("first target chunk poisoned and freed", 1, g_reloc_retired); + uint8_t pixel[4]; + NVNcopyRegion one = {0, 0, 0, 1, 1, 1}; + P(nvnTextureReadTexels)(&sc[0], NULL, &one, pixel); + EXPECT("owned copied target receives initial red clear", 0, memcmp(pixel, (uint8_t[]){255, 0, 0, 255}, 4)); + reloc_submit(&other, (const queue_t *)&qb_saved); + reloc_submit(&continuation, (const queue_t *)&qa); + EXPECT("all target chunks poisoned and freed", 3, g_reloc_retired); + EXPECT("snapshot scheduler destroys after retirement", 1, nvn_qs_destroy(g_qs)); + g_qs = saved; + gfx_leave(); + pthread_mutex_unlock(&g_sub_mu); + P(nvnTextureReadTexels)(&sc[0], NULL, &one, pixel); + EXPECT("resumed queue clears retained target without rebind", 0, memcmp(pixel, (uint8_t[]){0, 255, 0, 255}, 4)); + P(nvnTextureReadTexels)(&sc[1], NULL, &one, pixel); + EXPECT("other queue retains its independent blue target", 0, memcmp(pixel, (uint8_t[]){0, 0, 255, 255}, 4)); + P(nvnQueueFinalize)(&qa); + P(nvnQueueFinalize)(&qb_saved); + } + + P(nvnQueueFinish)(&q); + P(nvnTextureFinalize)(&sc[0]); + P(nvnTextureFinalize)(&sc[1]); + P(nvnMemoryPoolFinalize)(&swapchain_pool); + free(swapchain_memory); + printf("== %d checks, %d failed ==\n", g_n, g_fail); return g_fail != 0; } diff --git a/dm/gfx/tests/t_queue_alloc.m b/dm/gfx/tests/t_queue_alloc.m new file mode 100644 index 0000000000..f24d5e7328 --- /dev/null +++ b/dm/gfx/tests/t_queue_alloc.m @@ -0,0 +1,196 @@ +// Physical event chunks append into a logical submission's allocator. Real GPU +// copies exercise bytes retained before/after interleaving and lease pressure. +#include "../nvn/nvn_mtl.m" +void *front_metal_layer(void) { + return NULL; +} +module_t *module_of(const void *p) { + (void)p; + return NULL; +} +module_t *g_mod; +int g_nmod; +void svc_thread_dump(void) { +} +uint64_t now_ns(void) { + return bp_now(); +} +static int checks, failed, legacy; +#define CHECK(c, label) \ + do { \ + checks++; \ + if (!(c)) { \ + failed++; \ + fprintf(stderr, "FAIL: %s\n", label); \ + } \ + } while (0) + +static uint64_t copy_chunk(mtl_submit_alloc *a, uint64_t queue, id src, unsigned si, id dst, + unsigned di) { + mtl_queue_activate(queue); + // Fixture-only negative control: the previous scheduler called ordinary + // begin/resume for every chunk, acquiring/resetting a new allocator each time. + if (legacy) { + mtl_sub_resume(); + } else { + mtl_sub_begin_alloc(a, 1); + } + enc_b_copy_buffer(blit(), src, si * 4, dst, di * 4, 4); + uint64_t f = mtl_sub_commit(); + mtl_queue_seal(); + return f; +} +static void open_later(id gate) { + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, 500000000), dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0), + ^{ + gate.signaledValue = 1; + }); +} +static void wait_done(uint64_t f) { + CHECK(mtl_wait(f, UINT64_C(5000000000)), "GPU prefix completes"); +} + +int main(void) { + @autoreleasepool { + legacy = getenv("DM_TEST_ALLOC_LEGACY") != NULL; + setenv("DM_NVN_WAITS", "1", 1); + mtl_init(); + id src = [g_dev newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + id dst = [g_dev newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + res_add(src); + res_add(dst); + for (unsigned i = 0; i < 1024; i++) { + ((uint32_t *)src.contents)[i] = 0xa1300000u + i; + } + memset(dst.contents, 0, 4096); + // Warm the normal/direct path before a gate makes waits deterministic. + wait_done(copy_chunk(NULL, 940000, src, 0, dst, 0)); + + mtl_submit_alloc *a = mtl_submit_alloc_new(), *b = mtl_submit_alloc_new(); + id gate = [g_dev newSharedEvent]; + [g_q4 waitForEvent:gate value:1]; + uint64_t waits = g_alloc_waits, fence = 0, started = bp_now(); + int slot_a = -1, slot_b = -1; + for (unsigned i = 0; i < 32; i++) { + if (i == 8) { + open_later(gate); // old code blocks here until the gate opens + } + mtl_submit_alloc *owner = i & 1 ? b : a; + fence = copy_chunk(owner, 940001 + (i & 1), src, i, dst, i); + if (i == 0) { + slot_a = g_alloc_cur; + } + if (i == 1) { + slot_b = g_alloc_cur; + } + CHECK(g_alloc_cur == (i & 1 ? slot_b : slot_a), "interleaved logical submission keeps its allocator"); + } + CHECK(slot_a != slot_b, "independent logical submissions use separate leases"); + CHECK(g_alloc_waits == waits, "more than eight chunks do not introduce allocator waits"); + CHECK(gate.signaledValue == 0, "all split chunks encode while the GPU remains gated"); + CHECK(g_vis_ring_n == 1, "unused visibility buffers do not become an artificial chunk limit"); + printf("split chunks: 32, allocator waits: %llu, encode wall: %.3f ms, GPU gate: %s\n", + (unsigned long long)(g_alloc_waits - waits), (bp_now() - started) / 1e6, + gate.signaledValue ? "opened" : "closed"); + mtl_submit_alloc_close(a); + mtl_submit_alloc_close(b); + CHECK(a->slot == -1 && b->slot == -1, "logical END drops ownership before GPU completion"); + gate.signaledValue = 1; + wait_done(fence); + for (unsigned i = 0; i < 32; i++) { + CHECK(((uint32_t *)dst.contents)[i] == ((uint32_t *)src.contents)[i], + "all gated GPU copies survive later encoding"); + } + mtl_submit_alloc_release(a); + mtl_submit_alloc_release(b); + + // More blocked logical submissions than slots: none has reached logical + // END, and their first GPU prefixes cannot complete until the gate opens. + // The ninth must wait only for that GPU prefix, revoke its lease, and return + // so the same CPU thread can eventually submit the event producer. + enum { N = ALLOC_MAX + 4 }; + mtl_submit_alloc *blocked[N]; + gate = [g_dev newSharedEvent]; + [g_q4 waitForEvent:gate value:1]; + waits = g_alloc_waits; + for (unsigned i = 0; i < N; i++) { + blocked[i] = mtl_submit_alloc_new(); + if (i == ALLOC_MAX) { + open_later(gate); + } + fence = copy_chunk(blocked[i], 941000 + i, src, 64 + i, dst, 64 + i); + if (i + 1 == ALLOC_MAX) { + CHECK(g_alloc_waits == waits, "eight blocked prefixes fit without waiting"); + CHECK(gate.signaledValue == 0, "no live GPU prefix was reclaimed"); + } + } + CHECK(g_alloc_n == ALLOC_MAX, "allocator pool remains bounded with more logical submissions"); + CHECK(g_alloc_waits > waits, "capacity pressure waits for live GPU bytes"); + CHECK(blocked[0]->slot == -1, "completed suspended prefix lease is revoked under pressure"); + wait_done(fence); + for (unsigned i = 0; i < N; i++) { + CHECK(((uint32_t *)dst.contents)[64 + i] == ((uint32_t *)src.contents)[64 + i], + "reclaimed prefixes retain correct GPU output"); + } + mtl_submit_alloc *producer = mtl_submit_alloc_new(); + fence = copy_chunk(producer, 942000, src, 127, dst, 127); + mtl_submit_alloc_close(producer); + for (unsigned i = 0; i < N; i++) { + fence = copy_chunk(blocked[i], 941000 + i, dst, 127, dst, 96 + i); + mtl_submit_alloc_close(blocked[i]); + } + wait_done(fence); + for (unsigned i = 0; i < N; i++) { + CHECK(((uint32_t *)dst.contents)[96 + i] == ((uint32_t *)src.contents)[127], + "revoked continuation reads late producer data"); + mtl_submit_alloc_release(blocked[i]); + } + mtl_submit_alloc_release(producer); + for (int i = 0; i < g_alloc_n; i++) { + CHECK(!g_alloc_owner[i], "logical cleanup leaves no allocator pins"); + } + + // Reusing one allocator removes the old eight-chunk limit, but completion + // records remain bounded. A long submission must wait for an old physical + // fence before wrapping the table, preserving every event and report. + enum { LONG_CHUNKS = POST_RING + 8 }; + volatile uint32_t signals = 0; + uint64_t reports[LONG_CHUNKS][2] = {{0}}; + mtl_submit_alloc *long_submit = mtl_submit_alloc_new(); + gate = [g_dev newSharedEvent]; + [g_q4 waitForEvent:gate value:1]; + uint64_t post_waits = atomic_load(&g_ws_blk[WS_POST]); + waits = g_alloc_waits; + for (unsigned i = 0; i < LONG_CHUNKS; i++) { + if (i == POST_RING) { + open_later(gate); + } + mtl_sub_begin_alloc(long_submit, 1); + enc_b_copy_buffer(blit(), src, (256 + i) * 4, dst, (256 + i) * 4, 4); + mtl_sub_event(&signals, 1, 1); + mtl_sub_counter((uint64_t)(uintptr_t)reports[i], NVN_COUNTER_TYPE_TIMESTAMP_TOP); + fence = mtl_sub_commit(); + } + mtl_submit_alloc_close(long_submit); + wait_done(fence); + CHECK(atomic_load(&g_ws_blk[WS_POST]) > post_waits, "completion ring waits before overwriting live records"); + CHECK(g_alloc_waits == waits, "long logical submission does not wait on its own allocator"); + CHECK(signals == LONG_CHUNKS, "all wrapped completion events publish exactly once"); + for (unsigned i = 0; i < LONG_CHUNKS; i++) { + CHECK(reports[i][1] != 0, "every wrapped timestamp report publishes"); + CHECK(((uint32_t *)dst.contents)[256 + i] == ((uint32_t *)src.contents)[256 + i], + "long submission preserves GPU command bytes across callback wrap"); + } + mtl_submit_alloc_release(long_submit); + // Direct API still uses ordinary completed-fence-based allocator reuse. + wait_done(copy_chunk(NULL, 940000, src, 200, dst, 200)); + CHECK(((uint32_t *)dst.contents)[200] == ((uint32_t *)src.contents)[200], + "direct submission path retains its behavior"); + res_remove(src); + res_remove(dst); + mtl_drain(); + printf("queue allocators%s: %d checks, %d failed\n", legacy ? " (legacy negative control)" : "", checks, + failed); + return failed != 0; + } +} diff --git a/dm/gfx/tests/t_queue_schedule.c b/dm/gfx/tests/t_queue_schedule.c new file mode 100644 index 0000000000..e289c879b9 --- /dev/null +++ b/dm/gfx/tests/t_queue_schedule.c @@ -0,0 +1,385 @@ +// Pure-C queue dependency tests; no Metal, guest process, timing or sleeps. +// Build: cc -std=c11 -Wall -Wextra -Werror t_queue_schedule.c ../nvn/nvn_queue_schedule.c -o t_queue_schedule +#include "../nvn/nvn_queue_schedule.h" +#include +#include +#include + +static unsigned checks, failed; +#define CHECK(why, cond) \ + do { \ + checks++; \ + if (!(cond)) { \ + failed++; \ + fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, why); \ + } \ + } while (0) +enum { BASE = 11, COMPUTE = 12, AFTER = 13, LATER = 14 }; +typedef struct { + uint64_t id; + uint32_t projected, visible; +} event; +typedef struct { + uint64_t fence, event; + uint32_t value; +} publication; +typedef struct { + unsigned id; +} payload; +typedef struct { + event events[16]; + size_t nevents; + publication publications[128]; + size_t npublications, published; + unsigned work[128]; + size_t nwork; + unsigned released[32], nreleased; + uint64_t fence; +} host; +static event *event_find(host *h, uint64_t id) { + for (size_t i = 0; i < h->nevents; i++) { + if (h->events[i].id == id) { + return &h->events[i]; + } + } + return NULL; +} +static void event_add(host *h, uint64_t id, uint32_t value) { + CHECK("event table has space", h->nevents < 16); + if (h->nevents < 16) { + h->events[h->nevents++] = (event){id, value, value}; + } +} +static int event_read(void *ctx, uint64_t id, uint32_t *value) { + event *e = event_find(ctx, id); + if (!e) { + return 0; + } + *value = e->projected; + return 1; +} +static void release(void *ctx, nvn_qs_ticket ticket, void *data) { + host *h = ctx; + payload *p = data; + CHECK("release has a logical queue position", ticket.queue && ticket.sequence); + if (p) { + CHECK("release id has space", p->id < 32); + if (p->id < 32) { + CHECK("payload released once", ++h->released[p->id] == 1); + } + free(p); + } + h->nreleased++; +} +static nvn_qs_ticket enqueue(nvn_qs *s, uint64_t q, const nvn_qs_step *p, size_t n, unsigned id) { + payload *owned = malloc(sizeof *owned); + nvn_qs_ticket t = {0}; + CHECK("submission allocation", owned != NULL); + if (!owned) { + return t; + } + owned->id = id; + int ok = nvn_qs_enqueue(s, q, p, n, owned, &t); + CHECK("enqueue succeeds", ok); + if (!ok) { + free(owned); + } + return t; +} +static int run_one(nvn_qs *s, host *h) { + nvn_qs_item it; + if (!nvn_qs_peek(s, &it)) { + return 0; + } + if (it.step.kind == NVN_QS_WORK) { + CHECK("work log has space", h->nwork < 128); + if (h->nwork < 128) { + h->work[h->nwork++] = (unsigned)(uintptr_t)it.step.payload; + } + h->fence++; + } else if (it.step.kind == NVN_QS_SIGNAL) { + event *e = event_find(h, it.step.event); + CHECK("signal names known event", e != NULL); + if (!e) { + return 0; + } + CHECK("signal arithmetic valid", nvn_qs_signal_value(e->projected, it.step.mode, it.step.value, &e->projected)); + CHECK("publication log has space", h->npublications < 128); + if (h->npublications < 128) { + h->publications[h->npublications++] = (publication){++h->fence, e->id, e->projected}; + } + } else if (it.step.kind == NVN_QS_END) { + h->fence++; + } + CHECK("advance offered step", nvn_qs_advance(s, &it, h->fence)); + return 1; +} +static void drain(nvn_qs *s, host *h) { + unsigned steps = 0; + while (steps < 500 && run_one(s, h)) { + steps++; + } + CHECK("bounded scheduler progress", steps < 500); +} +static size_t complete(nvn_qs *s, host *h, uint64_t fence) { + while (h->published < h->npublications && h->publications[h->published].fence <= fence) { + const publication *p = &h->publications[h->published++]; + event_find(h, p->event)->visible = p->value; + } + return nvn_qs_complete(s, fence); +} +static nvn_qs_step work(unsigned id) { + return (nvn_qs_step){.kind = NVN_QS_WORK, .payload = (void *)(uintptr_t)id}; +} +static nvn_qs_step wait_for(uint64_t e, uint32_t mode, uint32_t v) { + return (nvn_qs_step){.kind = NVN_QS_WAIT, .mode = mode, .event = e, .value = v}; +} +static nvn_qs_step signal_step(uint64_t e, uint32_t mode, uint32_t v) { + return (nvn_qs_step){.kind = NVN_QS_SIGNAL, .mode = mode, .event = e, .value = v}; +} + +static void predicates(void) { + struct { + uint32_t actual, mode, target; + int ready; + } cases[] = {{0, NVN_QS_EQUAL, 0, 1}, + {2, NVN_QS_EQUAL, 1, 0}, + {0, NVN_QS_EQUAL, UINT32_MAX, 0}, + {1, NVN_QS_GEQUAL_WRAP, 1, 1}, + {2, NVN_QS_GEQUAL_WRAP, 1, 1}, + {0, NVN_QS_GEQUAL_WRAP, 1, 0}, + {0, NVN_QS_GEQUAL_WRAP, UINT32_MAX, 1}, + {UINT32_MAX, NVN_QS_GEQUAL_WRAP, 0, 0}, + {0x7fffffff, NVN_QS_GEQUAL_WRAP, 0, 1}, + {0x80000000, NVN_QS_GEQUAL_WRAP, 0, 0}, + {0, NVN_QS_GEQUAL_WRAP, 0x80000000, 0}, + {0, 99, 0, 0}}; + for (size_t i = 0; i < sizeof cases / sizeof *cases; i++) { + CHECK("wait predicate boundary", + nvn_qs_wait_ready(cases[i].actual, cases[i].mode, cases[i].target) == cases[i].ready); + } + uint32_t after = 77; + CHECK("WRITE replaces", nvn_qs_signal_value(17, NVN_QS_WRITE, 3, &after) && after == 3); + CHECK("ADD wraps at 32 bits", nvn_qs_signal_value(UINT32_MAX, NVN_QS_ADD, 2, &after) && after == 1); + CHECK("invalid signal mode preserves output", !nvn_qs_signal_value(0, 99, 0, &after) && after == 1); + CHECK("null signal output rejected", !nvn_qs_signal_value(0, NVN_QS_WRITE, 0, NULL)); +} +static void water_forward_wait(void) { + // Captured dependency: q6 waits for base event >=2 before compute; q1 is + // submitted later, rasterizes the base, then signals 2. q1 also waits on q6. + const uint64_t base = UINT64_C(0x7424cc7000), ack = UINT64_C(0x7424cc7040); + host h = {0}; + event_add(&h, base, 0); + event_add(&h, ack, 0); + nvn_qs *s = nvn_qs_create(event_read, release, &h); + CHECK("create", s != NULL); + nvn_qs_step a[] = {wait_for(base, NVN_QS_GEQUAL_WRAP, 1), signal_step(ack, NVN_QS_WRITE, 1), + wait_for(base, NVN_QS_GEQUAL_WRAP, 2), work(COMPUTE), signal_step(ack, NVN_QS_WRITE, 3)}; + nvn_qs_step later[] = {work(LATER)}; + nvn_qs_ticket ta = enqueue(s, 6, a, 5, 1), tl = enqueue(s, 6, later, 1, 2); + nvn_qs_item it; + CHECK("forward wait yields instead of blocking caller", !nvn_qs_peek(s, &it)); + CHECK("blocked head also holds next submit", nvn_qs_pending(s) == 2 && h.nwork == 0); + CHECK("tail includes blocked later submit", + nvn_qs_tail(s, 6).sequence == tl.sequence && tl.sequence == ta.sequence + 1); + uint64_t fence = 99; + CHECK("blocked ticket is not yet scheduled", !nvn_qs_scheduled(s, ta) && !nvn_qs_scheduled(s, tl)); + CHECK("blocked ticket has no GPU fence", !nvn_qs_fence(s, ta, &fence) && fence == 0); + CHECK("cannot destroy retained work", !nvn_qs_destroy(s)); + nvn_qs_step b[] = {signal_step(base, NVN_QS_WRITE, 1), work(BASE), signal_step(base, NVN_QS_WRITE, 2), + wait_for(ack, NVN_QS_GEQUAL_WRAP, 3), work(AFTER)}; + nvn_qs_ticket tb = enqueue(s, 1, b, 5, 3); + drain(s, &h); + CHECK("producer base runs before water compute", h.nwork == 4 && h.work[0] == BASE && h.work[1] == COMPUTE); + CHECK("consumer queue FIFO and reverse dependency", h.work[2] == LATER && h.work[3] == AFTER); + CHECK("scheduling signal does not publish CPU event", + event_find(&h, base)->visible == 0 && event_find(&h, ack)->visible == 0); + CHECK("dequeued submissions retain GPU lifetime", nvn_qs_pending(s) == 3 && h.nreleased == 0); + CHECK("logical fence waits for GPU completion", !nvn_qs_done(s, ta) && !nvn_qs_done(s, tb) && !nvn_qs_done(s, tl)); + CHECK("scheduled prefix exposes fence before completion", + nvn_qs_scheduled(s, ta) && nvn_qs_fence(s, ta, &fence) && fence > 0 && fence <= h.fence); + CHECK("all submissions retire", complete(s, &h, h.fence) == 3); + CHECK("CPU signals now visible", event_find(&h, base)->visible == 2 && event_find(&h, ack)->visible == 3); + CHECK("completed tickets survive payload release", nvn_qs_done(s, ta) && nvn_qs_done(s, tl) && nvn_qs_done(s, tb)); + CHECK("duplicate completion never repeats release", complete(s, &h, h.fence) == 0 && h.nreleased == 3); + CHECK("destroy drained scheduler", nvn_qs_destroy(s)); +} +static void handshake(void) { + host h = {0}; + event_add(&h, 10, 0); + event_add(&h, 20, 0); + nvn_qs *s = nvn_qs_create(event_read, release, &h); + nvn_qs_step a[10], b[10]; + for (uint32_t i = 0; i < 5; i++) { + a[2 * i] = wait_for(10, NVN_QS_GEQUAL_WRAP, i + 1); + a[2 * i + 1] = signal_step(20, NVN_QS_WRITE, i + 1); + b[2 * i] = signal_step(10, NVN_QS_WRITE, i + 1); + b[2 * i + 1] = wait_for(20, NVN_QS_GEQUAL_WRAP, i + 1); + } + nvn_qs_ticket ta = enqueue(s, 6, a, 10, 1), tb = enqueue(s, 1, b, 10, 2); + drain(s, &h); + CHECK("interleaved handshake reaches both tails", + h.npublications == 10 && event_find(&h, 10)->projected == 5 && event_find(&h, 20)->projected == 5); + CHECK("prefix signals progress before whole submit completes", h.nreleased == 0); + complete(s, &h, h.fence); + CHECK("both handshake fences complete", nvn_qs_done(s, ta) && nvn_qs_done(s, tb)); + CHECK("destroy handshake", nvn_qs_destroy(s)); +} +static void equality_and_unknown(void) { + host h = {0}; + event_add(&h, 10, 2); + nvn_qs *s = nvn_qs_create(event_read, release, &h); + nvn_qs_step a[] = {wait_for(10, NVN_QS_EQUAL, 1), work(AFTER)}; + enqueue(s, 1, a, 2, 1); + nvn_qs_item it, again; + CHECK("EQUAL overshoot does not satisfy wait", !nvn_qs_peek(s, &it)); + event_find(&h, 10)->projected = 1; + CHECK("CPU reset can make wait ready", nvn_qs_peek(s, &it) && it.step.kind == NVN_QS_WAIT); + event_find(&h, 10)->projected = 2; + CHECK("successful wait is latched", + nvn_qs_peek(s, &again) && again.dispatch == it.dispatch && again.index == it.index); + CHECK("latched wait advances after subsequent event change", nvn_qs_advance(s, &it, 0)); + CHECK("stale item rejected", !nvn_qs_advance(s, &it, 0)); + drain(s, &h); + complete(s, &h, h.fence); + nvn_qs_step unknown[] = {wait_for(99, NVN_QS_EQUAL, 0)}; + enqueue(s, 1, unknown, 1, 2); + CHECK("unknown event is not assumed zero", !nvn_qs_peek(s, &it)); + event_add(&h, 99, 0); + drain(s, &h); + complete(s, &h, h.fence); + CHECK("destroy equality scheduler", nvn_qs_destroy(s)); +} +static void cyclic_waits(void) { + host h = {0}; + event_add(&h, 10, 0); + event_add(&h, 20, 0); + nvn_qs *s = nvn_qs_create(event_read, release, &h); + nvn_qs_step a[] = {wait_for(20, NVN_QS_EQUAL, 1), signal_step(10, NVN_QS_WRITE, 1)}; + nvn_qs_step b[] = {wait_for(10, NVN_QS_EQUAL, 1), signal_step(20, NVN_QS_WRITE, 1)}; + enqueue(s, 1, a, 2, 1); + enqueue(s, 2, b, 2, 2); + nvn_qs_item it; + CHECK("genuine cycle remains blocked", !nvn_qs_peek(s, &it) && nvn_qs_pending(s) == 2); + CHECK("cycle does not discard work", !nvn_qs_destroy(s)); + event_find(&h, 20)->projected = 1; // an external CPU signal resolves it + drain(s, &h); + complete(s, &h, h.fence); + CHECK("external producer resolves cycle", nvn_qs_pending(s) == 0); + CHECK("destroy cycle scheduler", nvn_qs_destroy(s)); +} +static void lifetime_and_empty(void) { + host h = {0}; + nvn_qs *s = nvn_qs_create(NULL, release, &h); + nvn_qs_item it; + nvn_qs_ticket t = enqueue(s, 3, NULL, 0, 1); + CHECK("empty submission still has END", nvn_qs_peek(s, &it) && it.step.kind == NVN_QS_END); + CHECK("empty submit can fence earlier GPU work", nvn_qs_advance(s, &it, 7)); + CHECK("earlier GPU completion retains payload", + nvn_qs_complete(s, 6) == 0 && !nvn_qs_done(s, t) && h.nreleased == 0); + CHECK("matching GPU completion releases payload", nvn_qs_complete(s, 7) == 1 && nvn_qs_done(s, t)); + CHECK("late callback cannot move watermark backward", nvn_qs_complete(s, 2) == 0 && nvn_qs_done(s, t)); + nvn_qs_ticket t2 = enqueue(s, 3, NULL, 0, 2); + CHECK("queue sequence survives retirement", t2.sequence == t.sequence + 1); + CHECK("completed old fence excludes later pending submit", nvn_qs_done(s, t) && !nvn_qs_done(s, t2)); + CHECK("peek second empty END", nvn_qs_peek(s, &it)); + CHECK("no-GPU empty END retires immediately", nvn_qs_advance(s, &it, 0) && nvn_qs_done(s, t2)); + CHECK("unknown queue empty prefix is done", nvn_qs_done(s, nvn_qs_tail(s, 999))); + CHECK("unknown nonempty ticket is not done", !nvn_qs_done(s, (nvn_qs_ticket){999, 1})); + CHECK("future ticket is not done", !nvn_qs_done(s, (nvn_qs_ticket){3, t2.sequence + 1})); + CHECK("destroy lifetime scheduler", nvn_qs_destroy(s)); +} +static void descriptor_and_fence_validation(void) { + host h = {0}; + nvn_qs *s = nvn_qs_create(NULL, release, &h); + nvn_qs_step p = work(BASE); + nvn_qs_ticket t = enqueue(s, 9, &p, 1, 1); + p.payload = (void *)(uintptr_t)COMPUTE; + nvn_qs_item it, invalid; + CHECK("enqueue owns descriptor copy", nvn_qs_peek(s, &it) && (uintptr_t)it.step.payload == BASE); + invalid = it; + invalid.ticket.queue++; + CHECK("wrong queue item rejected", !nvn_qs_advance(s, &invalid, 5)); + invalid = it; + invalid.ticket.sequence++; + CHECK("wrong sequence item rejected", !nvn_qs_advance(s, &invalid, 5)); + invalid = it; + invalid.index++; + CHECK("wrong position rejected", !nvn_qs_advance(s, &invalid, 5)); + CHECK("valid WORK consumes", nvn_qs_advance(s, &it, 5)); + CHECK("END is offered", nvn_qs_peek(s, &it) && it.step.kind == NVN_QS_END); + CHECK("decreasing fence rejected without consuming END", !nvn_qs_advance(s, &it, 4)); + CHECK("zero fence preserves previous work's fence", nvn_qs_advance(s, &it, 0)); + CHECK("previous GPU work still protects payload", nvn_qs_complete(s, 4) == 0 && !nvn_qs_done(s, t)); + CHECK("previous GPU fence retires payload", nvn_qs_complete(s, 5) == 1 && nvn_qs_done(s, t)); + CHECK("destroy validation scheduler", nvn_qs_destroy(s)); +} +static void fence_queries(void) { + host h = {0}; + nvn_qs *s = nvn_qs_create(NULL, release, &h); + nvn_qs_item it; + uint64_t fence = 99; + nvn_qs_ticket a = enqueue(s, 4, NULL, 0, 1), b = enqueue(s, 4, NULL, 0, 2); + CHECK("first END offered", nvn_qs_peek(s, &it) && it.ticket.sequence == a.sequence); + CHECK("first prefix committed", nvn_qs_advance(s, &it, 10)); + CHECK("first prefix scheduled with exact fence", + nvn_qs_scheduled(s, a) && nvn_qs_fence(s, a, &fence) && fence == 10); + CHECK("later prefix not yet scheduled", !nvn_qs_scheduled(s, b) && !nvn_qs_fence(s, b, &fence) && fence == 0); + CHECK("second END offered", nvn_qs_peek(s, &it) && it.ticket.sequence == b.sequence); + CHECK("empty prefix scheduled without new GPU work", nvn_qs_advance(s, &it, 0)); + CHECK("empty prefix inherits earlier queue fence", nvn_qs_fence(s, b, &fence) && fence == 10); + CHECK("neither prefix completes early", nvn_qs_complete(s, 9) == 0 && !nvn_qs_done(s, b)); + CHECK("both prefixes retire together", nvn_qs_complete(s, 10) == 2 && nvn_qs_done(s, b)); + CHECK("retired prefix remains scheduled and queryable", + nvn_qs_scheduled(s, a) && nvn_qs_fence(s, a, &fence) && fence == 10); + nvn_qs_ticket c = enqueue(s, 4, NULL, 0, 3); + CHECK("third END offered", nvn_qs_peek(s, &it)); + CHECK("third prefix committed", nvn_qs_advance(s, &it, 17)); + CHECK("later scheduled work cannot raise old completed bound", nvn_qs_fence(s, a, &fence) && fence == 10); + CHECK("third prefix retires", nvn_qs_complete(s, 17) == 1 && nvn_qs_done(s, c)); + CHECK("old retired query returns safe completed watermark", nvn_qs_fence(s, a, &fence) && fence == 17); + CHECK("empty nonexistent prefix is scheduled", nvn_qs_scheduled(s, (nvn_qs_ticket){8, 0})); + CHECK("empty nonexistent prefix has zero fence", nvn_qs_fence(s, (nvn_qs_ticket){8, 0}, &fence) && fence == 0); + CHECK("unknown ticket is unscheduled", !nvn_qs_scheduled(s, (nvn_qs_ticket){8, 1})); + CHECK("unknown ticket query clears output", !nvn_qs_fence(s, (nvn_qs_ticket){8, 1}, &fence) && fence == 0); + CHECK("missing fence output rejected", !nvn_qs_fence(s, a, NULL)); + CHECK("destroy fence-query scheduler", nvn_qs_destroy(s)); +} +static void invalid_inputs(void) { + host h = {0}; + nvn_qs *s = nvn_qs_create(NULL, release, &h); + nvn_qs_ticket t = {7, 7}; + nvn_qs_step p = work(BASE); + nvn_qs_item it; + CHECK("null scheduler rejected", !nvn_qs_enqueue(NULL, 1, &p, 1, NULL, &t) && !t.queue && !t.sequence); + CHECK("zero queue serial rejected", !nvn_qs_enqueue(s, 0, &p, 1, NULL, &t)); + CHECK("missing descriptors rejected", !nvn_qs_enqueue(s, 1, NULL, 1, NULL, &t)); + CHECK("allocation size overflow rejected before reading", !nvn_qs_enqueue(s, 1, &p, SIZE_MAX, NULL, &t)); + CHECK("missing ticket rejected", !nvn_qs_enqueue(s, 1, &p, 1, NULL, NULL)); + p.kind = NVN_QS_END; + CHECK("explicit END rejected", !nvn_qs_enqueue(s, 1, &p, 1, NULL, &t)); + p = wait_for(0, 2, 0); + CHECK("unknown wait mode rejected", !nvn_qs_enqueue(s, 1, &p, 1, NULL, &t)); + p = signal_step(0, 2, 0); + CHECK("unknown signal mode rejected", !nvn_qs_enqueue(s, 1, &p, 1, NULL, &t)); + CHECK("failed enqueue transfers no ownership", + nvn_qs_pending(s) == 0 && h.nreleased == 0 && nvn_qs_tail(s, 1).sequence == 0); + CHECK("peek empty scheduler", !nvn_qs_peek(s, &it)); + CHECK("advance without offered item rejected", !nvn_qs_advance(s, &it, 0)); + CHECK("null output rejected", !nvn_qs_peek(s, NULL)); + CHECK("destroy empty scheduler", nvn_qs_destroy(s)); + CHECK("destroy null scheduler", nvn_qs_destroy(NULL)); +} +int main(void) { + predicates(); + water_forward_wait(); + handshake(); + equality_and_unknown(); + cyclic_waits(); + lifetime_and_empty(); + descriptor_and_fence_validation(); + fence_queries(); + invalid_inputs(); + printf("queue scheduling: %u checks, %u failed\n", checks, failed); + return failed != 0; +} diff --git a/dm/gfx/tests/t_queue_vis.m b/dm/gfx/tests/t_queue_vis.m new file mode 100644 index 0000000000..89c9c27e26 --- /dev/null +++ b/dm/gfx/tests/t_queue_vis.m @@ -0,0 +1,134 @@ +// Cumulative query slots outlive GPU submissions. Write real GPU buffer values +// in the same slots draw() retains, then report through the production callback. +// Alternating queues and more segments than the ring size must not overwrite +// counts that a later ReportCounter still needs. +#include "../nvn/nvn_mtl.m" +void *front_metal_layer(void) { + return NULL; +} +module_t *module_of(const void *p) { + (void)p; + return NULL; +} +module_t *g_mod; +int g_nmod; +void svc_thread_dump(void) { +} +uint64_t now_ns(void) { + return bp_now(); +} +static int checks, failed; +#define CHECK(c, label) \ + do { \ + checks++; \ + if (!(c)) { \ + failed++; \ + fprintf(stderr, "FAIL: %s\n", label); \ + } \ + } while (0) + +static id write_slot(uint64_t count, int *ring_index) { + mtl_sub_resume(); + id chosen = g_vis; + *ring_index = g_vis_ring_i; + id seed; + size_t off = arena_alloc(256, &seed); + memcpy((uint8_t *)seed.contents + off, &count, sizeof count); + enc_b_copy_buffer(blit(), seed, off, chosen, 0, sizeof count); + // These are the retained slot and counting state draw() records after it + // selects a new visibility slot for a pass. Values are produced on the GPU. + if (g_nvs == g_vscap) { + g_vscap = g_vscap ? g_vscap * 2 : 16; + vis_slot *next = realloc(g_vs, g_vscap * sizeof *g_vs); + if (!next) { + abort(); + } + g_vs = next; + } + g_vs[g_nvs++] = (vis_slot){(void *)CFBridgingRetain(chosen), 0}; + g_vis_n = 1; + g_vis_on = g_vis_counting = 1; + CHECK(mtl_wait(mtl_sub_commit(), UINT64_C(5000000000)), "slot write completes"); + return chosen; +} +static uint64_t report(void) { + uint64_t result[2] = {UINT64_MAX, 0}; + mtl_sub_resume(); + post((post_op){.kind = 3, .addr = (uint64_t)(uintptr_t)result, .vis = vis_snapshot()}); + g_vis_cur = -1; + CHECK(mtl_wait(mtl_sub_commit(), UINT64_C(5000000000)), "query report completes"); + return result[0]; +} +int main(void) { + @autoreleasepool { + mtl_init(); + const uint64_t qa = 930001, qb = 930002; + id slots[VIS_RING]; + uint64_t sum_a = 0, sum_b = 0; + mtl_queue_activate(qa); + for (int i = 0; i < VIS_RING; i++) { + if (i == VIS_RING / 2) { + mtl_queue_activate(qb); + } + int ring_index; + slots[i] = write_slot((uint64_t)i + 1, &ring_index); + CHECK(ring_index >= 0, "ring grows while unallocated slots remain"); + for (int j = 0; j < i; j++) { + CHECK(slots[i] != slots[j], "completed cumulative slot is not recycled"); + } + if (i < VIS_RING / 2) { + sum_a += i + 1; + } else { + sum_b += i + 1; + } + } + CHECK(g_vis_ring_n == VIS_RING, "both queues pin the whole ring"); + for (int i = 0; i < VIS_RING; i++) { + CHECK(vis_buffer_pinned(slots[i]), "active and saved queue slots both pin buffers"); + CHECK(((const uint64_t *)slots[i].contents)[0] == (uint64_t)i + 1, + "original GPU count survives later segments"); + } + mtl_queue_activate(qa); + int ring_index; + id overflow = write_slot(100, &ring_index); + sum_a += 100; + CHECK(ring_index == -1 && !vis_in_ring(overflow), + "all pinned uses overflow without waiting or indexing ring -1"); + CHECK(vis_buffer_pinned(overflow), "overflow remains retained by cumulative query"); + CHECK(report() == sum_a, "resumed first queue reports all original and overflow counts"); + CHECK(vis_buffer_pinned(overflow), "report does not reset cumulative query"); + + uint64_t reset_result[2] = {UINT64_MAX, 0}; + mtl_sub_resume(); + uint64_t report_fence = g_fence + 1; + post((post_op){.kind = 3, .addr = (uint64_t)(uintptr_t)reset_result, .vis = vis_snapshot()}); + vis_clear(); // ResetCounter may run before the report callback reads A's slots. + for (int i = 0; i < VIS_RING / 2; i++) { + CHECK(g_vis_ring[i].fence >= report_fence, + "snapshot protects older buffers through the report fence after reset"); + } + CHECK(mtl_wait(mtl_sub_commit(), UINT64_C(5000000000)), "report before reset completes"); + CHECK(reset_result[0] == sum_a, "reset does not change an earlier pending report"); + CHECK(!vis_buffer_pinned(overflow), "reset releases overflow pin"); + for (int i = 0; i < VIS_RING; i++) { + CHECK(vis_buffer_pinned(slots[i]) == (i >= VIS_RING / 2), "reset leaves other queue's slots pinned"); + } + id reused = write_slot(999, &ring_index); + CHECK(ring_index >= 0 && ring_index < VIS_RING / 2, "reset permits reuse of a completed ring buffer"); + CHECK(ring_index >= 0 && ring_index < VIS_RING && reused == slots[ring_index], + "reset reuses original allocation"); + CHECK(report() == 999, "post-reset report excludes earlier query counts"); + mtl_queue_activate(qb); + CHECK(report() == sum_b, "other queue's cumulative report survives first queue reset and reuse"); + mtl_queue_forget(qb); + for (int i = VIS_RING / 2; i < VIS_RING; i++) { + CHECK(!vis_buffer_pinned(slots[i]), "queue finalization releases saved counts"); + } + mtl_queue_activate(qa); + mtl_queue_forget(qa); + CHECK(!vis_buffer_pinned(reused), "finalizing remaining queue releases last pin"); + mtl_drain(); + printf("queue visibility: %d checks, %d failed\n", checks, failed); + return failed != 0; + } +} diff --git a/docs/agents/symptoms.md b/docs/agents/symptoms.md index e7939e06af..b582826bc0 100644 --- a/docs/agents/symptoms.md +++ b/docs/agents/symptoms.md @@ -515,6 +515,8 @@ One line per symptom in the [subsystem references](../reference/): what you see - red stub lines for NVN calls. → [NVN calls that were red stubs](../reference/nvn-objects.md#nvn-calls-that-were-red-stubs) - `nvnWindowGetPresentInterval`, `nvnCommandBufferSetCopyImageStride` missing. → [SDK 8 window and copy-stride entry points](../reference/nvn-objects.md#sdk-8-window-and-copy-stride-entry-points) - `[RED] texture bound to an image slot lacks the shader-write usage`, and compute image writes go to a dummy texture. → [Image bindings supply shader-write intent](../reference/nvn-objects.md#image-bindings-supply-shader-write-intent) +- a consumer submitted before another queue's producer uses stale input. In Mario Kart World, water jumps for one frame and the minimap retains fading wave artifacts. → [Guest event queue ordering](../reference/nvn-objects.md#guest-event-queue-ordering) +- with the guest event queue ordering (above), a game whose submissions carry many event signals and waits runs with the GPU idle between its chunks. Ocarina of Time (Switch 2) on an iPad Pro (M5), Docked 1536x864: 37 host submissions a frame where the game makes 6, 2.5 ms (p50) of GPU idle inside a 22.9 ms frame against none in a 20.4 ms frame with the waits ignored, at the same GPU time; the 64-draw segment commits fell from 6.4-6.9 to 1-3 a frame. → [Scheduler chunks committed one by one](../reference/nvn-objects.md#scheduler-chunks-committed-one-by-one) ## Rendering diff --git a/docs/env_switches.tsv b/docs/env_switches.tsv index f02bd21dfb..92767e6f48 100644 --- a/docs/env_switches.tsv +++ b/docs/env_switches.tsv @@ -114,6 +114,7 @@ DM_NVN_CAPTURE diagnostic `=:1:` writes that frame (from its first DM_NVN_CBDUMP diagnostic `=`: saves each draw's whole uniform blocks as `F_p_st_cb_
.bin`, and the source bytes of buffer-to-3D-texture copies, with `DM_NVN_PASSDUMP` DM_NVN_CBDUMP_KEY diagnostic `=`: limits `DM_NVN_CBDUMP` to draws using that shader; for a vertex shader key also saves the draw's vertex streams and index buffer DM_NVN_CBLOG diagnostic `=1` logs the command memory the game adds to command buffers (`AddCommandMemory`: the first 60, then every 500th) and the out-of-memory callbacks +DM_NVN_CHUNK_MERGE switch `1`: the chunks the guest-queue scheduler offers in a row (a submission cut at its event waits and signals, other queues' chunks whose waits it found satisfied) are encoded into one command buffer, committed when nothing ready is left, before a present, or at a logical submission change in submission timestamp mode; off by default (every chunk a host submission); `ab` alternates it with the `DM_AB` phases DM_NVN_CLEARLOG diagnostic `=1` logs the first 40 colour clears drawn as a triangle (inside a pass, with a partial channel mask or a partial scissor) DM_NVN_CMDSTAT diagnostic `=1` logs every 300 frames the command recording usage: bytes and records per frame, command memory segment sizes, out-of-memory callbacks, host segments, count and bytes per command type DM_NVN_COUNT diagnostic `1` counts every NVN call (the counting trampolines) and writes the per-frame call lines; off by default (on with a `DM_NVN_TSV` table that has call rows), the count is an atomic add all recording threads share diff --git a/docs/reference/nvn-objects.md b/docs/reference/nvn-objects.md index bb24a8c58f..ddcd03fff9 100644 --- a/docs/reference/nvn-objects.md +++ b/docs/reference/nvn-objects.md @@ -693,3 +693,31 @@ Games read this count as a position in their own command memory and size their h ### `nvnPrivate0`, an inlined command-buffer fast path flag - `nvnPrivate0`, a new SDK 20 import, needs nothing: it is a flag in `sdk`'s bss that enables an inlined command-buffer fast path when the NVN driver sets it (it stays 0). Found on [The Legend of Zelda: Breath of the Wild](../../dm/games/01007EF00011E000/README.md) (1.9.0, SDK 20.5.22). + +### Guest event queue ordering + +- **Symptom:** a consumer submitted before another queue's producer uses stale input. In Mario Kart World, water jumps for one frame and the minimap retains fading wave artifacts. + +- **Cause:** event and command-buffer sync waits were no-ops. One shared Metal queue orders already submitted work, but cannot satisfy a forward dependency by replaying guest submissions in CPU call order. Handshakes inside submissions require scheduling smaller prefixes. + +- **Fix:** record waits and schedule ready prefixes in FIFO order within each guest queue. A blocked queue yields, allowing the same CPU thread to submit its producer later. Committed producers supply projected event values to the scheduler; mapped event storage changes only after GPU completion. Completed projections yield to mapped storage, CPU API resets first complete older queued writes, and a worker observes direct CPU stores while work is pending. Logical tickets retain completion dependencies, including wait-only submissions. Presentation remains queued work, and acquire waits for its present marker to be scheduled. + +Copied records relocate embedded snapshots. Render targets, conditional rendering and cumulative visibility queries retain their queue state across event boundaries. Query buffers stay pinned until reset or finalization, and report callbacks retain their inputs through completion. + +Submission-mode timestamps share one reference-counted clock across the logical submission's chunks. This preserves the existing approximate clock mode without inserting host scheduling delays between reports. Segment-mode GPU timestamps retain their existing behavior. + +Chunks append to a leased command allocator without resetting GPU-owned bytes. The pool remains bounded at eight: under pressure, a completed prefix can release its lease so a later producer can run, and the suspended continuation reacquires an allocator. Reset requires completion of every committed use. The bounded completion-record table also completes its previous occupant before reuse. + +Signal location and flags are retained; CPU publication conservatively waits for the producer prefix to complete. Raw capture replay lacks guest queue identities and rejects waits. The updated sync and signal layouts require matching host and replay builds; older captures are rejected. + +- **Validation:** scheduler tests cover wrapping comparisons, forward and bidirectional dependencies, FIFO ordering, retirement and fence tickets. Metal fixtures cover CPU signals, wait-only fences, retained render targets, visibility queries, both timestamp modes, allocator pressure with twelve suspended submissions, and completion-record ring reuse. The wait-only fence regression test fails without propagating the producer's completion fence. + +- **Found on:** [Mario Kart World](../../dm/games/0400C3F00006E000/README.md). + +### Scheduler chunks committed one by one + +- **Symptom:** with the guest event queue ordering (above), a game whose submissions carry many event signals and waits runs with the GPU idle between its chunks. Ocarina of Time (Switch 2) on an iPad Pro (M5), Docked 1536x864: 37 host submissions a frame where the game makes 6, 2.5 ms (p50) of GPU idle inside a 22.9 ms frame against none in a 20.4 ms frame with the waits ignored, at the same GPU time; the 64-draw segment commits fell from 6.4-6.9 to 1-3 a frame. +- **Cause:** every chunk the scheduler offers (a submission cut at each event wait, signal and recorded fence, whether or not anything waits on the signal) was a host submission of its own: a command buffer, an allocator lease, a visibility buffer, a completion record, a commit and, with the encode thread, a drain. The GPU finishes a chunk and waits for the next one to be encoded and committed, and a 760-draw colour pass that used to be fed in 64-draw segments is committed whole (1.6 ms of idle before it in the GPU profile). +- **Fix:** `DM_NVN_CHUNK_MERGE=1` (`nvn_queue_dispatch.inc` `qe_pump`; `nvn_mtl.m` `mtl_chunk_end`, `mtl_chunk_begin`): the chunks one pump offers in a row are encoded into one command buffer, which commits when the pump has no ready step left, before a present, and in submission timestamp mode when the next chunk belongs to another logical submission (one clock per buffer). The open buffer's fence is `g_fence + 1` (`mtl_pending_fence`) for the chunk and for the signal's projection; every CPU wait on a fence runs its pump first, so no thread waits on a fence not yet committed. The order on the Metal queue is the scheduler's as before: a chunk whose wait is not satisfied still waits for its producer's offer. Off by default. The `[nvn] timing` line counts the scheduler's chunks per frame and how many were merged. +- **Found on:** Ocarina of Time (0400486024DEC000 v131072): the iPad numbers above, with the chunks committed one by one. The merged form has run the tests and the Mac host, not yet a device. +- **Check:** `check_t_nvn` runs the event tests with a submission per chunk and again with `DM_NVN_CHUNK_MERGE=1`; `t_queue_schedule`, `t_queue_vis`, `t_queue_alloc` and `t_capture` are unchanged.