Skip to content

SCGPU stream ring: GL stream writes no longer wait for the app #31

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dylanneve1 merged 1 commit into from
Oct 27, 2026
Merged

SCGPU stream ring: GL stream writes no longer wait for the app #31

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 117 additions & 3 deletions xnu/files/bsd/dev/hakios.vgpu3d_xnu.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@ static struct bell_event *bell_events, *bell_status;
static uint64_t bell_events_pa, bell_status_pa;
static unsigned bell_status_next;
static uint64_t bell_rung, bell_heard;
static LCK_MTX_DECLARE(bell_lock, &vgpu3d_lck_grp); /* the status queue: rung from the mailbox and the ring */
static int scgpu_reply_chan; /* (its address: what a waiting thread sleeps on) */

/* After every interrupt of the doorbell: the app rang (each event a ring); the waiter wakes. */
Expand Down Expand Up @@ -179,15 +180,17 @@ hakios_scgpu_bell_attach(vio_dev_t *v)
return true;
}

/* Ring the app (under scgpu_mailbox_lock): a status event, which it reads from its socket. */
/* Ring the app: a status event, which it reads from its socket. */
static void
bell_ring(void)
{
uint32_t len;

lck_mtx_lock(&bell_lock);
while (vio_queue_get(bell_statusq, &len) != NULL) {
}
if (vio_queue_num_free(bell_statusq) == 0) {
lck_mtx_unlock(&bell_lock);
return; /* rings not yet taken: it is awake anyway */
}
unsigned i = bell_status_next++ % BELL_STATUS;
Expand All @@ -198,6 +201,59 @@ bell_ring(void)
vio_queue_add(bell_statusq, &sg, 1, (void *)(uintptr_t)(i + 1));
vio_queue_kick(bell_statusq);
bell_rung++;
lck_mtx_unlock(&bell_lock);
}

/* ---- the stream ring (sys/hakios.scgpu.h) ---- */

#define SCGPU_RING_SIZE (4u << 20)
static uint8_t *scgpu_ring;
static uint32_t scgpu_ring_size;
static LCK_MTX_DECLARE(scgpu_ring_lock, &vgpu3d_lck_grp);
static uint64_t ring_waits;

/* Until the ring has room for `need` more bytes past `head`: spin, then asleep on the doorbell. */
static int
ring_room(uint32_t head, uint32_t need)
{
volatile uint32_t *tailp = &scgpu->ring_tail;
uint64_t now, spin_until, give_up;

if (head - __atomic_load_n((uint32_t *)(uintptr_t)tailp, __ATOMIC_ACQUIRE) + need <= scgpu_ring_size) {
return 0;
}
ring_waits++;
now = mach_absolute_time();
nanoseconds_to_absolutetime(SCGPU_BELL_SPIN_US * NSEC_PER_USEC, &spin_until);
nanoseconds_to_absolutetime(SCGPU_TIMEOUT_S * NSEC_PER_SEC, &give_up);
spin_until += now;
give_up += now;
while (head - __atomic_load_n((uint32_t *)(uintptr_t)tailp, __ATOMIC_ACQUIRE) + need > scgpu_ring_size) {
now = mach_absolute_time();
if (now > give_up) {
printf("vgpu3d: SCGPU: the stream ring stayed full for %d s\n", SCGPU_TIMEOUT_S);
return ETIMEDOUT;
}
if (now <= spin_until) {
continue;
}
if (bell == NULL) {
delay_for_interval(SCGPU_NAP_US, NSEC_PER_USEC);
continue;
}
uint64_t deadline;
clock_interval_to_deadline(SCGPU_BELL_SAFETY_MS, kMillisecondScale, &deadline);
assert_wait_deadline((event_t)&scgpu_reply_chan, THREAD_UNINT, deadline);
__atomic_store_n((uint32_t *)(uintptr_t)&scgpu->guest_waiting, 1, __ATOMIC_RELAXED);
__atomic_thread_fence(__ATOMIC_SEQ_CST);
if (head - __atomic_load_n((uint32_t *)(uintptr_t)tailp, __ATOMIC_ACQUIRE) + need <= scgpu_ring_size) {
clear_wait(current_thread(), THREAD_AWAKENED);
} else {
thread_block(THREAD_CONTINUE_NULL);
}
__atomic_store_n((uint32_t *)(uintptr_t)&scgpu->guest_waiting, 0, __ATOMIC_RELAXED);
}
return 0;
}

/* One command through the mailbox, as hakios_vgpu_ctrl puts one on the control queue. */
Expand Down Expand Up @@ -343,8 +399,18 @@ scgpu_attach(void)
if (scgpu_chunks > SCGPU_MAX_CHUNKS) {
scgpu_chunks = SCGPU_MAX_CHUNKS;
}
printf("vgpu3d: SCGPU: the app's GPU at %#llx (%llu MiB, a pool of %u KiB), %s\n", pa, size >> 20,
scgpu_chunks * (SCGPU_CHUNK >> 10), scgpu->flags & SCGPU_F_HOST_READY ? "ready" : "NOT READY");
/* the stream ring, from the pool; the app finds it in the header (no ring: every write by mailbox) */
uint64_t ring_pa;
scgpu_ring = scgpu->flags & SCGPU_F_RING ? scgpu_alloc(SCGPU_RING_SIZE, &ring_pa) : NULL;
if (scgpu_ring != NULL) {
scgpu->ring_off = (uint32_t)(ring_pa - scgpu_pa);
scgpu->ring_head = scgpu->ring_tail = 0;
scgpu_ring_size = SCGPU_RING_SIZE;
__atomic_store_n((uint32_t *)(uintptr_t)&scgpu->ring_size, SCGPU_RING_SIZE, __ATOMIC_RELEASE);
}
printf("vgpu3d: SCGPU: the app's GPU at %#llx (%llu MiB, a pool of %u KiB, a stream ring of %u KiB), %s\n", pa,
size >> 20, scgpu_chunks * (SCGPU_CHUNK >> 10), scgpu_ring_size >> 10,
scgpu->flags & SCGPU_F_HOST_READY ? "ready" : "NOT READY");
}

/* ---- the ops ---------------------------------------------------------------------------- */
Expand Down Expand Up @@ -822,6 +888,49 @@ open_of(dev_t dev)
return o && o->ready ? o : NULL;
}

/*
* SCGPU's write: the bytes into the stream ring as records of the context's pipe, a pipe's worth at
* most each (gfxstream copies a record through the pipe's linear copy), and back at once; the app is
* rung if it sleeps. Only room is waited for.
*/
static int
scgpu_ring_write(struct vopen *o, struct uio *uio)
{
uint32_t mask = scgpu_ring_size - 1;
int err = 0;

lck_mtx_lock(&scgpu_ring_lock);
while (uio_resid(uio) > 0 && !err) {
uint32_t n = uio_resid(uio) > (user_ssize_t)o->g.pipe_size ? o->g.pipe_size : (uint32_t)uio_resid(uio);
uint32_t need = (uint32_t)sizeof(struct scgpu_record) + ((n + 15) & ~15u);
uint32_t head = scgpu->ring_head; /* (ours alone) */

if ((err = ring_room(head, need)) != 0) {
break;
}
struct scgpu_record rec = { o->g.pipe_res, o->g.ctx_id, n, 0 };
uint32_t pos = head & mask; /* 16-aligned in a ring of a multiple of 16: the record never splits */
memcpy(scgpu_ring + pos, &rec, sizeof(rec));
uint32_t at = (pos + (uint32_t)sizeof(rec)) & mask;
uint32_t first = n < scgpu_ring_size - at ? n : scgpu_ring_size - at;
err = uiomove((caddr_t)(scgpu_ring + at), (int)first, uio);
if (!err && n > first) {
err = uiomove((caddr_t)scgpu_ring, (int)(n - first), uio);
}
if (err) {
break;
}
__atomic_store_n((uint32_t *)(uintptr_t)&scgpu->ring_head, head + need, __ATOMIC_RELEASE);
o->g.bytes_out += n;
__atomic_thread_fence(__ATOMIC_SEQ_CST);
if (bell != NULL && __atomic_load_n((uint32_t *)(uintptr_t)&scgpu->host_sleeping, __ATOMIC_RELAXED)) {
bell_ring();
}
}
lck_mtx_unlock(&scgpu_ring_lock);
return err;
}

static int
vgpu3d_dev_write(dev_t dev, struct uio *uio, __unused int flag)
{
Expand All @@ -832,6 +941,11 @@ vgpu3d_dev_write(dev_t dev, struct uio *uio, __unused int flag)
return ENXIO;
}
lck_mtx_lock(&o->lock);
if (scgpu != NULL && scgpu_ring != NULL) {
err = scgpu_ring_write(o, uio);
lck_mtx_unlock(&o->lock);
return err;
}
while (uio_resid(uio) > 0 && !err) {
uint32_t n = uio_resid(uio) > (user_ssize_t)o->g.pipe_size ? o->g.pipe_size : (uint32_t)uio_resid(uio);
err = uiomove((caddr_t)o->g.pipe_va, (int)n, uio);
Expand Down
23 changes: 21 additions & 2 deletions xnu/files/bsd/sys/hakios.scgpu.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,15 @@
* a full barrier on both sides, so one of the two always sees the other. Without the device (an older
* app) both sides poll as before.
*
* The stream ring: what user space writes to the pipe (the GL stream, write(2) on /dev/vgpu3d) does
* not go through the mailbox, one command per write and a wait for each, but into a byte ring in the
* pool (ring_off, ring_size, a power of two) as records -- struct scgpu_record, then the bytes, then
* padding to 16 -- and the guest goes on at once. ring_head (the guest's) and ring_tail (the app's) count
* bytes forever; their difference is what waits. The app hands each record to its resource (the
* context's pipe) and moves ring_tail on, and it does so with everything the ring holds before it runs
* any mailbox command: so a reply, an upload or a read still comes after every byte written before it.
* The guest waits only for room (and rings the app, or is rung, as for the mailbox).
*
* Commands are virtio-gpu's (virtio 1.2, 5.7.6), with these meanings at the app:
* GET_CAPSET_INFO, GET_CAPSET gfxstream's capsets (stream_renderer_get_cap_set/fill_caps)
* CTX_CREATE, CTX_DESTROY stream_renderer_context_create/destroy
Expand Down Expand Up @@ -60,6 +69,8 @@

/* the app's renderer is up and answers the mailbox */
#define SCGPU_F_HOST_READY (1u << 0)
/* ... and takes the stream ring's records (without it the guest makes no ring: every write by mailbox) */
#define SCGPU_F_RING (1u << 1)
/*
* guest_flags: the guest has begun to show frames through the host (rcFBPost): the app can put the
* renderer's window in front from now on (before, the shared framebuffer had the picture)
Expand Down Expand Up @@ -89,8 +100,16 @@ struct scgpu_header {
uint64_t commands, presents, host_busy_ns;
uint32_t guest_flags; /* SCGPU_G_*, set by the guest */
uint32_t host_sleeping; /* the app waits for the doorbell, not polling: ring it */
uint32_t guest_waiting; /* the guest waits for a reply asleep: ring it */
uint32_t reserved[13];
uint32_t guest_waiting; /* the guest waits for a reply (or for room in the ring) asleep: ring it */
uint32_t ring_off, ring_size; /* the stream ring, in the region; size 0: no ring (every write by mailbox) */
uint32_t ring_head, ring_tail; /* bytes put in (the guest's), bytes taken out (the app's), forever */
uint32_t reserved[9];
};

/* A record of the stream ring: the bytes follow, then padding to 16. */
struct scgpu_record {
uint32_t resource_id, ctx_id; /* the pipe of a context */
uint32_t len, reserved;
};

/* struct virtio_gpu_transfer_host_3d's twin, with the staging buffer instead of the backing */
Expand Down
Loading