diff --git a/xnu/files/bsd/dev/hakios.vgpu3d_xnu.c b/xnu/files/bsd/dev/hakios.vgpu3d_xnu.c index 09a5ee44b..553a94c19 100644 --- a/xnu/files/bsd/dev/hakios.vgpu3d_xnu.c +++ b/xnu/files/bsd/dev/hakios.vgpu3d_xnu.c @@ -49,6 +49,7 @@ #include #include #include +#include #include #include @@ -103,6 +104,8 @@ static uint8_t *attach_scratch; /* ATTACH_BACKING commands (unde #define SCGPU_MAX_CHUNKS 8192u /* 512 MiB of pool at most */ #define SCGPU_SPIN_US 300 /* a reply is waited for by spinning this long, */ #define SCGPU_NAP_US 20 /* then in naps of this, */ +#define SCGPU_BELL_SPIN_US 60 /* ... or, with the doorbell, this long and then asleep, */ +#define SCGPU_BELL_SAFETY_MS 5 /* looking again at least this often (a lost ring) */ #define SCGPU_TIMEOUT_S 10 /* until this (a first shader compile can take a while) */ static volatile struct scgpu_header *scgpu; /* NULL: the virtio-gpu device, as before */ @@ -113,6 +116,90 @@ static uint8_t scgpu_used[SCGPU_MAX_CHUNKS / 8]; static LCK_MTX_DECLARE(scgpu_mailbox_lock, &vgpu3d_lck_grp); static LCK_MTX_DECLARE(scgpu_pool_lock, &vgpu3d_lck_grp); +/* + * The doorbell (sys/hakios.scgpu.h): the virtio-input device "scion-scgpu", handed over by the input + * driver (hakios_scgpu_bell_attach). The app rings by an input event, which comes as an interrupt; + * the guest rings by a status event, which crosvm writes into the app's socket. + */ +#define BELL_EVENTS 16 +#define BELL_STATUS 8 +struct bell_event { uint16_t type, code; uint32_t value; }; +static vio_dev_t *bell; +static vio_queue_t *bell_eventq, *bell_statusq; +static struct bell_event *bell_events, *bell_status; +static uint64_t bell_events_pa, bell_status_pa; +static unsigned bell_status_next; +static uint64_t bell_rung, bell_heard; +static int scgpu_reply_chan; /* (its address: what a waiting thread sleeps on) */ + +/* After every interrupt of the doorbell: the app rang (each event a ring); the waiter wakes. */ +static void +bell_service(__unused void *arg, __unused uint8_t isr) +{ + uint32_t len; + void *cookie; + bool any = false; + + while ((cookie = vio_queue_get(bell_eventq, &len)) != NULL) { + uintptr_t i = (uintptr_t)cookie - 1; + struct vio_sg sg = { bell_events_pa + i * sizeof(struct bell_event), sizeof(struct bell_event), true }; + + vio_queue_add(bell_eventq, &sg, 1, cookie); + any = true; + bell_heard++; + } + if (any) { + vio_queue_kick(bell_eventq); + thread_wakeup((event_t)&scgpu_reply_chan); + } +} + +/* From the virtio-input driver, with the device initialized: the doorbell is ours. */ +bool +hakios_scgpu_bell_attach(vio_dev_t *v) +{ + if (vio_queue_setup(v, 0, BELL_EVENTS, &bell_eventq) != 0 || vio_queue_setup(v, 1, BELL_STATUS, &bell_statusq) != 0 || + (bell_events = vio_dma_alloc(BELL_EVENTS * sizeof(struct bell_event), &bell_events_pa)) == NULL || + (bell_status = vio_dma_alloc(BELL_STATUS * sizeof(struct bell_event), &bell_status_pa)) == NULL) { + printf("vgpu3d: SCGPU: the doorbell could not be set up; polling\n"); + vio_fail(v); + return false; + } + for (uintptr_t i = 0; i < vio_queue_size(bell_eventq); i++) { + struct vio_sg sg = { bell_events_pa + i * sizeof(struct bell_event), sizeof(struct bell_event), true }; + + vio_queue_add(bell_eventq, &sg, 1, (void *)(i + 1)); + } + vio_intr_enable(v); + vio_driver_ok(v); + vio_queue_kick(bell_eventq); + vio_service_start(v, bell_service, NULL, 4 * NSEC_PER_MSEC); + bell = v; + printf("vgpu3d: SCGPU: the doorbell is %s (%s)\n", vio_name(v), vio_intr_active(v) ? "interrupts" : "POLLED"); + return true; +} + +/* Ring the app (under scgpu_mailbox_lock): a status event, which it reads from its socket. */ +static void +bell_ring(void) +{ + uint32_t len; + + while (vio_queue_get(bell_statusq, &len) != NULL) { + } + if (vio_queue_num_free(bell_statusq) == 0) { + return; /* rings not yet taken: it is awake anyway */ + } + unsigned i = bell_status_next++ % BELL_STATUS; + struct vio_sg sg = { bell_status_pa + i * sizeof(struct bell_event), sizeof(struct bell_event), false }; + + /* EV_LED: crosvm writes every status event but EV_MSC into the socket */ + bell_status[i] = (struct bell_event){ 0x11, 0, (uint32_t)bell_rung }; + vio_queue_add(bell_statusq, &sg, 1, (void *)(uintptr_t)(i + 1)); + vio_queue_kick(bell_statusq); + bell_rung++; +} + /* One command through the mailbox, as hakios_vgpu_ctrl puts one on the control queue. */ static int scgpu_ctrl(const void *cmd, uint32_t cmd_len, void *resp, uint32_t resp_len) @@ -131,8 +218,13 @@ scgpu_ctrl(const void *cmd, uint32_t cmd_len, void *resp, uint32_t resp_len) scgpu->resp_max = resp_len; seq = scgpu->req_seq + 1; __atomic_store_n((uint32_t *)(uintptr_t)&scgpu->req_seq, seq, __ATOMIC_RELEASE); + /* the app asleep on the doorbell is rung (a full barrier first: the other half of its host_sleeping) */ + __atomic_thread_fence(__ATOMIC_SEQ_CST); + if (bell != NULL && __atomic_load_n((uint32_t *)(uintptr_t)&scgpu->host_sleeping, __ATOMIC_RELAXED)) { + bell_ring(); + } now = mach_absolute_time(); - nanoseconds_to_absolutetime(SCGPU_SPIN_US * NSEC_PER_USEC, &spin_until); + nanoseconds_to_absolutetime((bell != NULL ? SCGPU_BELL_SPIN_US : SCGPU_SPIN_US) * NSEC_PER_USEC, &spin_until); nanoseconds_to_absolutetime(SCGPU_TIMEOUT_S * NSEC_PER_SEC, &give_up); spin_until += now; give_up += now; @@ -143,8 +235,30 @@ scgpu_ctrl(const void *cmd, uint32_t cmd_len, void *resp, uint32_t resp_len) err = ETIMEDOUT; break; } - if (now > spin_until) { + if (now <= spin_until) { + continue; + } + if (bell == NULL) { delay_for_interval(SCGPU_NAP_US, NSEC_PER_USEC); + continue; + } + /* asleep until the app rings: say so, then look once more, so one of the two sees the other */ + uint64_t deadline; + clock_interval_to_deadline(SCGPU_BELL_SAFETY_MS, kMillisecondScale, &deadline); + assert_wait_deadline((event_t)&scgpu_reply_chan, THREAD_UNINT, deadline); + __atomic_store_n((uint32_t *)(uintptr_t)&scgpu->guest_waiting, 1, __ATOMIC_RELAXED); + __atomic_thread_fence(__ATOMIC_SEQ_CST); + if (__atomic_load_n((uint32_t *)(uintptr_t)resp_seq, __ATOMIC_ACQUIRE) == seq) { + clear_wait(current_thread(), THREAD_AWAKENED); + } else { + thread_block(THREAD_CONTINUE_NULL); + } + __atomic_store_n((uint32_t *)(uintptr_t)&scgpu->guest_waiting, 0, __ATOMIC_RELAXED); + /* (a ring that found no free status descriptor: the app may still be asleep on it) */ + __atomic_thread_fence(__ATOMIC_SEQ_CST); + if (__atomic_load_n((uint32_t *)(uintptr_t)&scgpu->host_sleeping, __ATOMIC_RELAXED) && + __atomic_load_n((uint32_t *)(uintptr_t)resp_seq, __ATOMIC_ACQUIRE) != seq) { + bell_ring(); } } if (!err && scgpu->status != 0) { diff --git a/xnu/files/bsd/sys/hakios.scgpu.h b/xnu/files/bsd/sys/hakios.scgpu.h index 5f7114cce..ae6b38acb 100644 --- a/xnu/files/bsd/sys/hakios.scgpu.h +++ b/xnu/files/bsd/sys/hakios.scgpu.h @@ -21,7 +21,15 @@ * * One command at a time, as on a control queue: the guest writes the command and req_len, then bumps * req_seq (a release); the app sees it (an acquire), runs it, writes the reply, resp_len and status, - * then sets resp_seq = req_seq (a release). Both sides poll for now. + * then sets resp_seq = req_seq (a release). + * + * Waiting: each side spins a little, then sleeps on a doorbell -- a virtio-input device named + * "scion-scgpu" (SCGPU_BELL_NAME) whose socket is the app's. The guest rings by putting an event on + * its status queue (crosvm writes it into the socket), the app by writing one into the socket (crosvm + * puts it on the event queue: an interrupt). A side rings only when the other has said it sleeps + * (host_sleeping, guest_waiting), and says so before its last look at the sequence it waits for, with + * a full barrier on both sides, so one of the two always sees the other. Without the device (an older + * app) both sides poll as before. * * Commands are virtio-gpu's (virtio 1.2, 5.7.6), with these meanings at the app: * GET_CAPSET_INFO, GET_CAPSET gfxstream's capsets (stream_renderer_get_cap_set/fill_caps) @@ -58,6 +66,9 @@ */ #define SCGPU_G_PRESENTING (1u << 0) +/* the doorbell's virtio-input device (its name, ID_NAME) */ +#define SCGPU_BELL_NAME "scion-scgpu" + /* SCGPU_CMD_TRANSFER: outside virtio-gpu's command numbers */ #define SCGPU_CMD_TRANSFER 0x0f00u @@ -77,7 +88,9 @@ struct scgpu_header { /* counters, for the logs of both sides */ uint64_t commands, presents, host_busy_ns; uint32_t guest_flags; /* SCGPU_G_*, set by the guest */ - uint32_t reserved[15]; + uint32_t host_sleeping; /* the app waits for the doorbell, not polling: ring it */ + uint32_t guest_waiting; /* the guest waits for a reply asleep: ring it */ + uint32_t reserved[13]; }; /* struct virtio_gpu_transfer_host_3d's twin, with the staging buffer instead of the backing */ diff --git a/xnu/patches/1305-scgpu-doorbell.patch b/xnu/patches/1305-scgpu-doorbell.patch new file mode 100644 index 000000000..6ecf7f266 --- /dev/null +++ b/xnu/patches/1305-scgpu-doorbell.patch @@ -0,0 +1,25 @@ +diff --git a/bsd/dev/hakios.virtio_input.c b/bsd/dev/hakios.virtio_input.c +--- a/bsd/dev/hakios.virtio_input.c ++++ b/bsd/dev/hakios.virtio_input.c +@@ -32,6 +32,9 @@ + #include + #if __has_include() + #include ++#include ++ ++extern bool hakios_scgpu_bell_attach(vio_dev_t *v); /* bsd/dev/hakios.vgpu3d_xnu.c */ + #define INPUT_VIO 1 + #endif + +@@ -550,6 +553,11 @@ + size = size < sizeof(name) - 1 ? size : sizeof(name) - 1; + vio_cfg_read(v, CFG_UNION, name, size); + name[size] = 0; ++ /* SCGPU's doorbell, not a touch screen: the SCGPU code takes it (bsd/dev/hakios.vgpu3d_xnu.c) */ ++ if (strcmp(name, SCGPU_BELL_NAME) == 0) { ++ hakios_scgpu_bell_attach(v); ++ return false; ++ } + input_defaults(d); + vio_axis(v, ABS_X, &d->ax); + vio_axis(v, ABS_Y, &d->ay);