From 645fd5e381c37582667dbd8ac44098d7dbb60436 Mon Sep 17 00:00:00 2001 From: Dylan Neve Date: Tue, 27 Oct 2026 14:49:46 +0000 Subject: [PATCH] SCGPU stream ring: the GL stream's writes no longer wait for the app Every write(2) of the GL stream on /dev/vgpu3d was a mailbox command (TRANSFER_TO_HOST_3D of the pipe) and a wait for its reply: a round trip per flush, and under video (YouTube in Safari) 4-10K commands a second with the app's device thread 60% busy. Now, when the app says it takes them (SCGPU_F_RING), the guest puts the bytes into a 4 MiB byte ring in the pool, as records of the context's pipe, and returns at once. It rings the app only if the app sleeps, and waits only for room. The app hands each record to its pipe (transfer_write_iov straight from the ring) and does so with everything in the ring before it runs any mailbox command, so a reply, an upload or a read still follows every byte written before it. bell_ring has a lock of its own now, since both the mailbox and the ring ring. Co-Authored-By: Claude Opus 5.5 --- xnu/files/bsd/dev/hakios.vgpu3d_xnu.c | 120 +++++++++++++++++++++++++- xnu/files/bsd/sys/hakios.scgpu.h | 23 ++++- 2 files changed, 138 insertions(+), 5 deletions(-) diff --git a/xnu/files/bsd/dev/hakios.vgpu3d_xnu.c b/xnu/files/bsd/dev/hakios.vgpu3d_xnu.c index 553a94c19..0027b2c1d 100644 --- a/xnu/files/bsd/dev/hakios.vgpu3d_xnu.c +++ b/xnu/files/bsd/dev/hakios.vgpu3d_xnu.c @@ -130,6 +130,7 @@ static struct bell_event *bell_events, *bell_status; static uint64_t bell_events_pa, bell_status_pa; static unsigned bell_status_next; static uint64_t bell_rung, bell_heard; +static LCK_MTX_DECLARE(bell_lock, &vgpu3d_lck_grp); /* the status queue: rung from the mailbox and the ring */ static int scgpu_reply_chan; /* (its address: what a waiting thread sleeps on) */ /* After every interrupt of the doorbell: the app rang (each event a ring); the waiter wakes. */ @@ -179,15 +180,17 @@ hakios_scgpu_bell_attach(vio_dev_t *v) return true; } -/* Ring the app (under scgpu_mailbox_lock): a status event, which it reads from its socket. */ +/* Ring the app: a status event, which it reads from its socket. */ static void bell_ring(void) { uint32_t len; + lck_mtx_lock(&bell_lock); while (vio_queue_get(bell_statusq, &len) != NULL) { } if (vio_queue_num_free(bell_statusq) == 0) { + lck_mtx_unlock(&bell_lock); return; /* rings not yet taken: it is awake anyway */ } unsigned i = bell_status_next++ % BELL_STATUS; @@ -198,6 +201,59 @@ bell_ring(void) vio_queue_add(bell_statusq, &sg, 1, (void *)(uintptr_t)(i + 1)); vio_queue_kick(bell_statusq); bell_rung++; + lck_mtx_unlock(&bell_lock); +} + +/* ---- the stream ring (sys/hakios.scgpu.h) ---- */ + +#define SCGPU_RING_SIZE (4u << 20) +static uint8_t *scgpu_ring; +static uint32_t scgpu_ring_size; +static LCK_MTX_DECLARE(scgpu_ring_lock, &vgpu3d_lck_grp); +static uint64_t ring_waits; + +/* Until the ring has room for `need` more bytes past `head`: spin, then asleep on the doorbell. */ +static int +ring_room(uint32_t head, uint32_t need) +{ + volatile uint32_t *tailp = &scgpu->ring_tail; + uint64_t now, spin_until, give_up; + + if (head - __atomic_load_n((uint32_t *)(uintptr_t)tailp, __ATOMIC_ACQUIRE) + need <= scgpu_ring_size) { + return 0; + } + ring_waits++; + now = mach_absolute_time(); + nanoseconds_to_absolutetime(SCGPU_BELL_SPIN_US * NSEC_PER_USEC, &spin_until); + nanoseconds_to_absolutetime(SCGPU_TIMEOUT_S * NSEC_PER_SEC, &give_up); + spin_until += now; + give_up += now; + while (head - __atomic_load_n((uint32_t *)(uintptr_t)tailp, __ATOMIC_ACQUIRE) + need > scgpu_ring_size) { + now = mach_absolute_time(); + if (now > give_up) { + printf("vgpu3d: SCGPU: the stream ring stayed full for %d s\n", SCGPU_TIMEOUT_S); + return ETIMEDOUT; + } + if (now <= spin_until) { + continue; + } + if (bell == NULL) { + delay_for_interval(SCGPU_NAP_US, NSEC_PER_USEC); + continue; + } + uint64_t deadline; + clock_interval_to_deadline(SCGPU_BELL_SAFETY_MS, kMillisecondScale, &deadline); + assert_wait_deadline((event_t)&scgpu_reply_chan, THREAD_UNINT, deadline); + __atomic_store_n((uint32_t *)(uintptr_t)&scgpu->guest_waiting, 1, __ATOMIC_RELAXED); + __atomic_thread_fence(__ATOMIC_SEQ_CST); + if (head - __atomic_load_n((uint32_t *)(uintptr_t)tailp, __ATOMIC_ACQUIRE) + need <= scgpu_ring_size) { + clear_wait(current_thread(), THREAD_AWAKENED); + } else { + thread_block(THREAD_CONTINUE_NULL); + } + __atomic_store_n((uint32_t *)(uintptr_t)&scgpu->guest_waiting, 0, __ATOMIC_RELAXED); + } + return 0; } /* One command through the mailbox, as hakios_vgpu_ctrl puts one on the control queue. */ @@ -343,8 +399,18 @@ scgpu_attach(void) if (scgpu_chunks > SCGPU_MAX_CHUNKS) { scgpu_chunks = SCGPU_MAX_CHUNKS; } - printf("vgpu3d: SCGPU: the app's GPU at %#llx (%llu MiB, a pool of %u KiB), %s\n", pa, size >> 20, - scgpu_chunks * (SCGPU_CHUNK >> 10), scgpu->flags & SCGPU_F_HOST_READY ? "ready" : "NOT READY"); + /* the stream ring, from the pool; the app finds it in the header (no ring: every write by mailbox) */ + uint64_t ring_pa; + scgpu_ring = scgpu->flags & SCGPU_F_RING ? scgpu_alloc(SCGPU_RING_SIZE, &ring_pa) : NULL; + if (scgpu_ring != NULL) { + scgpu->ring_off = (uint32_t)(ring_pa - scgpu_pa); + scgpu->ring_head = scgpu->ring_tail = 0; + scgpu_ring_size = SCGPU_RING_SIZE; + __atomic_store_n((uint32_t *)(uintptr_t)&scgpu->ring_size, SCGPU_RING_SIZE, __ATOMIC_RELEASE); + } + printf("vgpu3d: SCGPU: the app's GPU at %#llx (%llu MiB, a pool of %u KiB, a stream ring of %u KiB), %s\n", pa, + size >> 20, scgpu_chunks * (SCGPU_CHUNK >> 10), scgpu_ring_size >> 10, + scgpu->flags & SCGPU_F_HOST_READY ? "ready" : "NOT READY"); } /* ---- the ops ---------------------------------------------------------------------------- */ @@ -822,6 +888,49 @@ open_of(dev_t dev) return o && o->ready ? o : NULL; } +/* + * SCGPU's write: the bytes into the stream ring as records of the context's pipe, a pipe's worth at + * most each (gfxstream copies a record through the pipe's linear copy), and back at once; the app is + * rung if it sleeps. Only room is waited for. + */ +static int +scgpu_ring_write(struct vopen *o, struct uio *uio) +{ + uint32_t mask = scgpu_ring_size - 1; + int err = 0; + + lck_mtx_lock(&scgpu_ring_lock); + while (uio_resid(uio) > 0 && !err) { + uint32_t n = uio_resid(uio) > (user_ssize_t)o->g.pipe_size ? o->g.pipe_size : (uint32_t)uio_resid(uio); + uint32_t need = (uint32_t)sizeof(struct scgpu_record) + ((n + 15) & ~15u); + uint32_t head = scgpu->ring_head; /* (ours alone) */ + + if ((err = ring_room(head, need)) != 0) { + break; + } + struct scgpu_record rec = { o->g.pipe_res, o->g.ctx_id, n, 0 }; + uint32_t pos = head & mask; /* 16-aligned in a ring of a multiple of 16: the record never splits */ + memcpy(scgpu_ring + pos, &rec, sizeof(rec)); + uint32_t at = (pos + (uint32_t)sizeof(rec)) & mask; + uint32_t first = n < scgpu_ring_size - at ? n : scgpu_ring_size - at; + err = uiomove((caddr_t)(scgpu_ring + at), (int)first, uio); + if (!err && n > first) { + err = uiomove((caddr_t)scgpu_ring, (int)(n - first), uio); + } + if (err) { + break; + } + __atomic_store_n((uint32_t *)(uintptr_t)&scgpu->ring_head, head + need, __ATOMIC_RELEASE); + o->g.bytes_out += n; + __atomic_thread_fence(__ATOMIC_SEQ_CST); + if (bell != NULL && __atomic_load_n((uint32_t *)(uintptr_t)&scgpu->host_sleeping, __ATOMIC_RELAXED)) { + bell_ring(); + } + } + lck_mtx_unlock(&scgpu_ring_lock); + return err; +} + static int vgpu3d_dev_write(dev_t dev, struct uio *uio, __unused int flag) { @@ -832,6 +941,11 @@ vgpu3d_dev_write(dev_t dev, struct uio *uio, __unused int flag) return ENXIO; } lck_mtx_lock(&o->lock); + if (scgpu != NULL && scgpu_ring != NULL) { + err = scgpu_ring_write(o, uio); + lck_mtx_unlock(&o->lock); + return err; + } while (uio_resid(uio) > 0 && !err) { uint32_t n = uio_resid(uio) > (user_ssize_t)o->g.pipe_size ? o->g.pipe_size : (uint32_t)uio_resid(uio); err = uiomove((caddr_t)o->g.pipe_va, (int)n, uio); diff --git a/xnu/files/bsd/sys/hakios.scgpu.h b/xnu/files/bsd/sys/hakios.scgpu.h index ae6b38acb..846e9b435 100644 --- a/xnu/files/bsd/sys/hakios.scgpu.h +++ b/xnu/files/bsd/sys/hakios.scgpu.h @@ -31,6 +31,15 @@ * a full barrier on both sides, so one of the two always sees the other. Without the device (an older * app) both sides poll as before. * + * The stream ring: what user space writes to the pipe (the GL stream, write(2) on /dev/vgpu3d) does + * not go through the mailbox, one command per write and a wait for each, but into a byte ring in the + * pool (ring_off, ring_size, a power of two) as records -- struct scgpu_record, then the bytes, then + * padding to 16 -- and the guest goes on at once. ring_head (the guest's) and ring_tail (the app's) count + * bytes forever; their difference is what waits. The app hands each record to its resource (the + * context's pipe) and moves ring_tail on, and it does so with everything the ring holds before it runs + * any mailbox command: so a reply, an upload or a read still comes after every byte written before it. + * The guest waits only for room (and rings the app, or is rung, as for the mailbox). + * * Commands are virtio-gpu's (virtio 1.2, 5.7.6), with these meanings at the app: * GET_CAPSET_INFO, GET_CAPSET gfxstream's capsets (stream_renderer_get_cap_set/fill_caps) * CTX_CREATE, CTX_DESTROY stream_renderer_context_create/destroy @@ -60,6 +69,8 @@ /* the app's renderer is up and answers the mailbox */ #define SCGPU_F_HOST_READY (1u << 0) +/* ... and takes the stream ring's records (without it the guest makes no ring: every write by mailbox) */ +#define SCGPU_F_RING (1u << 1) /* * guest_flags: the guest has begun to show frames through the host (rcFBPost): the app can put the * renderer's window in front from now on (before, the shared framebuffer had the picture) @@ -89,8 +100,16 @@ struct scgpu_header { uint64_t commands, presents, host_busy_ns; uint32_t guest_flags; /* SCGPU_G_*, set by the guest */ uint32_t host_sleeping; /* the app waits for the doorbell, not polling: ring it */ - uint32_t guest_waiting; /* the guest waits for a reply asleep: ring it */ - uint32_t reserved[13]; + uint32_t guest_waiting; /* the guest waits for a reply (or for room in the ring) asleep: ring it */ + uint32_t ring_off, ring_size; /* the stream ring, in the region; size 0: no ring (every write by mailbox) */ + uint32_t ring_head, ring_tail; /* bytes put in (the guest's), bytes taken out (the app's), forever */ + uint32_t reserved[9]; +}; + +/* A record of the stream ring: the bytes follow, then padding to 16. */ +struct scgpu_record { + uint32_t resource_id, ctx_id; /* the pipe of a context */ + uint32_t len, reserved; }; /* struct virtio_gpu_transfer_host_3d's twin, with the staging buffer instead of the backing */