diff --git a/kernel/ibdev.c b/kernel/ibdev.c index cb96971..ebb0e02 100644 --- a/kernel/ibdev.c +++ b/kernel/ibdev.c @@ -35,6 +35,9 @@ #include "../proto/apple_tx.h" #include "../proto/native_data.h" #include "../proto/reliability.h" +#include "../proto/tbv_cq_shm.h" +#include +#include #include "tbv.h" #define TBV_IBDEV_ABI_VERSION 1 @@ -249,18 +252,38 @@ struct tbv_pd { struct tbv_state *owner; }; +struct tbv_cq_mmap_entry { + struct rdma_user_mmap_entry rdma_entry; + void *address; +}; + struct tbv_cq { struct ib_cq base; struct tbv_state *owner; spinlock_t lock; + /* + * One vmalloc_user buffer holds the whole CQ: page 0 is the shared + * header the provider maps, then the kernel's ib_wc ring (which carries + * the qp pointer that no uapi format can), then the ib_uverbs_wc ring + * the provider reads. tail and ovf_seen are PRIVATE indices; the only + * shared state is the pair of monotonic totals in the header, so every + * field has exactly one writer. See proto/tbv_cq_shm.h. + */ + struct tbv_cq_shm *shm; struct ib_wc *entries; + struct ib_uverbs_wc *shm_ring; u32 cqe; - u32 head; - u32 tail; - u32 count; + u32 tail; /* next ring slot to write; producer-private */ + u32 ovf_seen; /* ovf_seq this CQ's consumer has already reported */ bool notify_armed; - bool overflowed; + void *mmap_buf; + size_t mmap_len; + struct tbv_cq_mmap_entry *mmap_entry; }; +static inline u32 tbv_cq_pending(const struct tbv_cq *tcq) +{ + return READ_ONCE(tcq->shm->produced) - READ_ONCE(tcq->shm->consumed); +} struct tbv_recv_wqe { u64 wr_id; @@ -615,6 +638,23 @@ struct tbv_gsi_send_ctx { static DEFINE_IDA(tbv_qpn_ida); static atomic_t tbv_mr_key = ATOMIC_INIT(1); +static void tbv_cqe_from_wc(struct ib_uverbs_wc *u, const struct ib_wc *wc) +{ + u->wr_id = wc->wr_id; + u->status = wc->status; + u->opcode = wc->opcode; + u->vendor_err = wc->vendor_err; + u->byte_len = wc->byte_len; + u->qp_num = wc->qp ? wc->qp->qp_num : 0; + u->ex.imm_data = wc->ex.imm_data; + u->src_qp = wc->src_qp; + u->wc_flags = wc->wc_flags; + u->pkey_index = wc->pkey_index; + u->slid = wc->slid; + u->sl = wc->sl; + u->dlid_path_bits = wc->dlid_path_bits; +} + static int tbv_cq_push(struct tbv_cq *tcq, const struct ib_wc *wc); static void tbv_send_ctx_put(struct tbv_send_ctx *send); static bool tbv_send_complete(struct tbv_send_ctx *send, int status); @@ -2459,28 +2499,125 @@ static int tbv_create_cq(struct ib_cq *cq, const struct ib_cq_init_attr *attr, struct uverbs_attr_bundle *attrs) { struct tbv_cq *tcq = container_of(cq, struct tbv_cq, base); + struct ib_udata *udata = &attrs->driver_udata; + size_t len; if (!attr || attr->cqe <= 0 || attr->cqe > TBV_IBDEV_MAX_CQE) return -EINVAL; + /* + * ONLY the header and the uapi wire-format ring are mapped to userspace. + * + * The kernel's own ib_wc ring is kcalloc'd SEPARATELY and is deliberately + * NOT in the mapped buffer: struct ib_wc carries `struct ib_qp *qp`, a live + * kernel pointer, and anything in this buffer is readable by the process + * that created the CQ. Mapping the two rings together -- which is what an + * earlier revision of this patch did -- hands userspace a kernel address + * and with it a KASLR/heap-layout bypass. The uapi ring is safe because + * ib_uverbs_wc carries qp_num (a u32), not a pointer. + */ + len = PAGE_SIZE + PAGE_ALIGN(attr->cqe * sizeof(*tcq->shm_ring)); + tcq->mmap_buf = vmalloc_user(len); + if (!tcq->mmap_buf) + return -ENOMEM; + memset(tcq->mmap_buf, 0, len); + tcq->mmap_len = len; tcq->entries = kcalloc(attr->cqe, sizeof(*tcq->entries), GFP_KERNEL); - if (!tcq->entries) + if (!tcq->entries) { + vfree(tcq->mmap_buf); + tcq->mmap_buf = NULL; return -ENOMEM; + } + tcq->shm = tcq->mmap_buf; + tcq->shm_ring = (struct ib_uverbs_wc *)((char *)tcq->mmap_buf + PAGE_SIZE); + tcq->shm->magic = TBV_CQ_SHM_MAGIC; + tcq->shm->abi = TBV_CQ_SHM_ABI; + tcq->shm->cqe = attr->cqe; + tcq->shm->entry_size = sizeof(struct ib_uverbs_wc); + tcq->shm->ring_offset = PAGE_SIZE; + tcq->shm->map_len = len; spin_lock_init(&tcq->lock); tcq->owner = tbv_ibdev_state(cq->device); tcq->cqe = attr->cqe; + + if (udata && udata->outlen >= sizeof(struct tbv_uresp_create_cq)) { + struct tbv_uresp_create_cq uresp = {}; + struct tbv_ucontext *ctx = + rdma_udata_to_drv_context(udata, struct tbv_ucontext, base); + struct tbv_cq_mmap_entry *me; + + me = kzalloc(sizeof(*me), GFP_KERNEL); + if (me) { + me->address = tcq->mmap_buf; + if (!rdma_user_mmap_entry_insert(&ctx->base, &me->rdma_entry, len)) { + tcq->mmap_entry = me; + uresp.cq_mmap_offset = + rdma_user_mmap_get_offset(&me->rdma_entry); + uresp.cqe = attr->cqe; + uresp.shm_abi = TBV_CQ_SHM_ABI; + /* Published so the provider maps the whole + * buffer in ONE mmap. The first version mapped a + * single page to discover the length and the + * kernel rejected that call outright. + */ + uresp.map_len = len; + if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) { + rdma_user_mmap_entry_remove(&me->rdma_entry); + tcq->mmap_entry = NULL; + } + } else { + kfree(me); + } + } + } atomic_inc(&tcq->owner->verbs_cqs); return 0; } +static int tbv_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) +{ + struct rdma_user_mmap_entry *entry; + struct tbv_cq_mmap_entry *me; + unsigned long len = vma->vm_end - vma->vm_start; + int ret; + + if (vma->vm_start & (PAGE_SIZE - 1)) { + pr_info("tbv: mmap pgoff=%lu len=%lu REJECTED: start %#lx is not page aligned\n", + vma->vm_pgoff, len, vma->vm_start); + return -EINVAL; + } + entry = rdma_user_mmap_entry_get(context, vma); + if (!entry) { + pr_info("tbv: mmap pgoff=%lu len=%lu REJECTED: no mmap entry at that pgoff\n", + vma->vm_pgoff, len); + return -EINVAL; + } + me = container_of(entry, struct tbv_cq_mmap_entry, rdma_entry); + ret = remap_vmalloc_range(vma, me->address, 0); + pr_info("tbv: mmap pgoff=%lu len=%lu -> %d\n", vma->vm_pgoff, len, ret); + rdma_user_mmap_entry_put(entry); + return ret; +} + +static void tbv_mmap_free(struct rdma_user_mmap_entry *entry) +{ + struct tbv_cq_mmap_entry *me = + container_of(entry, struct tbv_cq_mmap_entry, rdma_entry); + + kfree(me); +} + static int tbv_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) { struct tbv_cq *tcq = container_of(cq, struct tbv_cq, base); if (tcq->owner) atomic_dec(&tcq->owner->verbs_cqs); + if (tcq->mmap_entry) + rdma_user_mmap_entry_remove(&tcq->mmap_entry->rdma_entry); kfree(tcq->entries); + vfree(tcq->mmap_buf); return 0; } @@ -5390,12 +5527,23 @@ static int tbv_cq_push(struct tbv_cq *tcq, const struct ib_wc *wc) unsigned long flags; bool notify = false; int ret = 0; + u32 produced, consumed; spin_lock_irqsave(&tcq->lock, flags); - if (tcq->overflowed || tcq->count == tcq->cqe) { - tcq->overflowed = true; + produced = READ_ONCE(tcq->shm->produced); + consumed = READ_ONCE(tcq->shm->consumed); + if (produced - consumed >= tcq->cqe) { + /* + * Advance a sequence, never a latch. A sticky flag made the CQ + * permanently dead, and perftest's `do { } while (ne == 0)` poll + * loop spins on it forever -- which is how one dropped + * completion became an unkillable "poll on Send CQ failed -1". + */ + WRITE_ONCE(tcq->shm->ovf_seq, tcq->shm->ovf_seq + 1); if (tcq->owner) atomic64_inc(&tcq->owner->data_cq_overflow); + pr_info_ratelimited("tbv: CQ overflow cqe=%u produced=%u consumed=%u\n", + tcq->cqe, produced, consumed); if (tcq->notify_armed) { tcq->notify_armed = false; notify = true; @@ -5408,8 +5556,11 @@ static int tbv_cq_push(struct tbv_cq *tcq, const struct ib_wc *wc) } tcq->entries[tcq->tail] = *wc; + if (wc) + tbv_cqe_from_wc(&tcq->shm_ring[tcq->tail], wc); + smp_wmb(); + WRITE_ONCE(tcq->shm->produced, produced + 1); tcq->tail = (tcq->tail + 1) % tcq->cqe; - tcq->count++; if (tcq->notify_armed) { tcq->notify_armed = false; notify = true; @@ -8751,22 +8902,33 @@ static int tbv_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc) struct tbv_cq *tcq = container_of(cq, struct tbv_cq, base); unsigned long flags; int polled = 0; - bool overflowed; + u32 produced, consumed; if (num_entries <= 0 || !wc) return 0; spin_lock_irqsave(&tcq->lock, flags); - while (polled < num_entries && tcq->count) { - wc[polled++] = tcq->entries[tcq->head]; - tcq->head = (tcq->head + 1) % tcq->cqe; - tcq->count--; + /* + * The kernel's own poll is a second CONSUMER of the same ring, so it + * reports overflow on the same terms as the provider's: once per + * event, and without killing the CQ. A client uses one path or the + * other for a given CQ, never both at once. + */ + if (READ_ONCE(tcq->shm->ovf_seq) != tcq->ovf_seen) { + tcq->ovf_seen = READ_ONCE(tcq->shm->ovf_seq); + spin_unlock_irqrestore(&tcq->lock, flags); + return -EIO; } - overflowed = tcq->overflowed; + produced = READ_ONCE(tcq->shm->produced); + consumed = READ_ONCE(tcq->shm->consumed); + while (polled < num_entries && produced - consumed > 0) { + wc[polled++] = tcq->entries[consumed % tcq->cqe]; + consumed++; + } + if (polled) + WRITE_ONCE(tcq->shm->consumed, consumed); spin_unlock_irqrestore(&tcq->lock, flags); - if (!polled && overflowed) - return -EIO; return polled; } @@ -8777,11 +8939,12 @@ static int tbv_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags) int ret; spin_lock_irqsave(&tcq->lock, irq_flags); - if (tcq->overflowed) { + if (READ_ONCE(tcq->shm->ovf_seq) != tcq->ovf_seen) { + tcq->ovf_seen = READ_ONCE(tcq->shm->ovf_seq); ret = -EIO; } else { tcq->notify_armed = true; - ret = tcq->count ? 1 : 0; + ret = tbv_cq_pending(tcq) != 0; } spin_unlock_irqrestore(&tcq->lock, irq_flags); return ret; @@ -9233,6 +9396,8 @@ static const struct ib_device_ops tbv_ibdev_ops = { .destroy_ah = tbv_destroy_ah, .create_cq = tbv_create_cq, .destroy_cq = tbv_destroy_cq, + .mmap = tbv_mmap, + .mmap_free = tbv_mmap_free, .create_qp = tbv_create_qp, .destroy_qp = tbv_destroy_qp, .modify_qp = tbv_modify_qp, diff --git a/proto/tbv_cq_shm.h b/proto/tbv_cq_shm.h new file mode 100644 index 0000000..008d5da --- /dev/null +++ b/proto/tbv_cq_shm.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ +#ifndef TBV_CQ_SHM_H +#define TBV_CQ_SHM_H + +/* + * Shared header for the userspace-mapped completion queue (Phase 1 of the + * userspace-mapped-queues work, 2026-09-12). + * + * WHY THIS EXISTS. Every post_send and every poll_cq on this driver used to be + * a uverbs kernel crossing, and perftest's poll loop made ~96 ioctls per + * pingpong iteration (measured with perf syscall tracepoints). The fix is the + * standard userspace-mapped-queue design: the kernel publishes completions + * into a page shared with the provider, which polls by reading memory instead + * of issuing ioctls. + * + * THE PROTOCOL. Two monotonic totals, one writer each: + * + * Producer (kernel, tcq->lock held): + * if (produced - consumed >= cqe) { ovf_seq++; drop; } + * entries[tail] = wc; shm_ring[tail] = uverbs_wc(wc); + * smp_wmb(); + * WRITE_ONCE(shm->produced, produced + 1); + * tail = (tail + 1) % cqe; + * + * Consumer (one polling thread, or the kernel's ioctl poll -- never both at + * once for a given CQ): + * produced = load_acquire(&shm->produced); + * while (polled < n && produced - consumed > 0) { out[polled++] = ring[head]; head=(head+1)%cqe; consumed++; } + * store_release(&shm->consumed, consumed); + * + * WHY TOTALS AND NOT head/tail. The first version published `head` and `tail` + * and derived occupancy, which is a ring that cannot distinguish full from + * empty and therefore has to reserve a slot. That is merely inconvenient at + * cqe == 512 and BROKEN at cqe == 1, which is what perftest actually asks for + * on its send CQ (measured: `requested=1`): `(tail + 1) % 1 == head` is a + * tautology, so the CQ declared overflow before storing a single completion. + * Occupancy is now `produced - consumed` -- exact, unambiguous, and correct at + * every capacity including one. The subtraction is unsigned, and the ring + * never lets the difference reach 2^31, so wrap-around is not a hazard. + * + * `ovf_seq` is monotonic rather than a sticky flag on purpose. An overflow + * means a completion was dropped; the consumer must learn that, and it must + * learn it ONCE. A latched flag makes the CQ permanently dead -- perftest's + * `do { } while (ne == 0)` spins on it forever -- which is how a single + * dropped completion became an unkillable `poll on Send CQ failed -1`. + * + * Layout: page 0 is this header; the ib_wc ring starts at page 1. The whole + * buffer is one vmalloc_user() allocation, mapped by the driver's .mmap op. + */ + +/* + * Both sides of this ABI are Linux, and both take the fixed-width types from + * the same place: the kernel directly, and the userspace provider because + * rdma-core's includes this header too. + * + * What this replaced was a userspace branch that declared its own `__u32` and + * `__aligned_u64`. That is not a shadow, it is a hard error -- rdma-core + * already defines both (`__aligned_u64` as an aligned `__u64`), so the + * redefinition collides on `__u64` and the FIRST userspace build of this + * header failed outright. A header that hand-rolls names the platform already + * owns cannot be the wire format for a kernel/userspace boundary. + */ +#include + +#define TBV_CQ_SHM_MAGIC 0x54425643u /* "TBVC" */ +#define TBV_CQ_SHM_ABI 2u + +struct tbv_cq_shm { + __u32 magic; /* TBV_CQ_SHM_MAGIC -- provider sanity-checks this */ + __u32 abi; /* TBV_CQ_SHM_ABI */ + __u32 cqe; /* ring capacity in entries */ + __u32 entry_size; /* sizeof(struct ib_uverbs_wc) -- sanity */ + __u32 ring_offset; /* byte offset of the ring within the mapping */ + __u32 produced; /* total completions ever queued (KERNEL writes) */ + __u32 consumed; /* total completions ever taken (CONSUMER writes) */ + __u32 ovf_seq; /* times the kernel dropped an entry (KERNEL writes) */ + __u32 map_len; /* total bytes the consumer must map (kernel-set) */ + __u32 reserved[39]; /* pad to 192 bytes; room to grow without an abi bump */ +}; + +/* Vendor tail of create_cq's response (after the generic ib_uverbs resp). + * + * The kernel fills it only when the provider's outbuf is big enough, so an old + * provider sees nothing (and never mmaps); a new provider on an old kernel + * reads all-zero from its own calloc and takes the fallback. + * + * `shm_abi` is the offer FLAG, and that is deliberate. The first version keyed + * the decision on `cq_mmap_offset != 0`, but 0 is a legitimate offset that + * rdma_user_mmap_entry_insert can hand out (measured: the first CQ on this leg + * was offered offset 0), so a nonzero test silently discards a real offer. + * `shm_abi` is zero in a response the kernel never filled and equal to + * TBV_CQ_SHM_ABI in one it did -- which is exactly the question being asked. + * + * `map_len` is published here so the consumer can map the whole buffer in ONE + * mmap. The first version mapped a single page to discover the length and then + * remapped; the kernel rejected that first mmap (measured: EINVAL, errno + * "Invalid argument") and the feature silently fell back to ioctls. + */ +struct tbv_uresp_create_cq { + __aligned_u64 cq_mmap_offset; /* mmap() offset; may legitimately be 0 */ + __u32 cqe; + __u32 shm_abi; /* TBV_CQ_SHM_ABI = the kernel offered */ + __u32 reserved; + __u32 pad; + __aligned_u64 map_len; /* bytes to map; 0 = nothing offered */ +}; + +#endif /* TBV_CQ_SHM_H */ diff --git a/userspace/usb4_rdma/tbv_cq_shm.h b/userspace/usb4_rdma/tbv_cq_shm.h new file mode 100644 index 0000000..008d5da --- /dev/null +++ b/userspace/usb4_rdma/tbv_cq_shm.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ +#ifndef TBV_CQ_SHM_H +#define TBV_CQ_SHM_H + +/* + * Shared header for the userspace-mapped completion queue (Phase 1 of the + * userspace-mapped-queues work, 2026-09-12). + * + * WHY THIS EXISTS. Every post_send and every poll_cq on this driver used to be + * a uverbs kernel crossing, and perftest's poll loop made ~96 ioctls per + * pingpong iteration (measured with perf syscall tracepoints). The fix is the + * standard userspace-mapped-queue design: the kernel publishes completions + * into a page shared with the provider, which polls by reading memory instead + * of issuing ioctls. + * + * THE PROTOCOL. Two monotonic totals, one writer each: + * + * Producer (kernel, tcq->lock held): + * if (produced - consumed >= cqe) { ovf_seq++; drop; } + * entries[tail] = wc; shm_ring[tail] = uverbs_wc(wc); + * smp_wmb(); + * WRITE_ONCE(shm->produced, produced + 1); + * tail = (tail + 1) % cqe; + * + * Consumer (one polling thread, or the kernel's ioctl poll -- never both at + * once for a given CQ): + * produced = load_acquire(&shm->produced); + * while (polled < n && produced - consumed > 0) { out[polled++] = ring[head]; head=(head+1)%cqe; consumed++; } + * store_release(&shm->consumed, consumed); + * + * WHY TOTALS AND NOT head/tail. The first version published `head` and `tail` + * and derived occupancy, which is a ring that cannot distinguish full from + * empty and therefore has to reserve a slot. That is merely inconvenient at + * cqe == 512 and BROKEN at cqe == 1, which is what perftest actually asks for + * on its send CQ (measured: `requested=1`): `(tail + 1) % 1 == head` is a + * tautology, so the CQ declared overflow before storing a single completion. + * Occupancy is now `produced - consumed` -- exact, unambiguous, and correct at + * every capacity including one. The subtraction is unsigned, and the ring + * never lets the difference reach 2^31, so wrap-around is not a hazard. + * + * `ovf_seq` is monotonic rather than a sticky flag on purpose. An overflow + * means a completion was dropped; the consumer must learn that, and it must + * learn it ONCE. A latched flag makes the CQ permanently dead -- perftest's + * `do { } while (ne == 0)` spins on it forever -- which is how a single + * dropped completion became an unkillable `poll on Send CQ failed -1`. + * + * Layout: page 0 is this header; the ib_wc ring starts at page 1. The whole + * buffer is one vmalloc_user() allocation, mapped by the driver's .mmap op. + */ + +/* + * Both sides of this ABI are Linux, and both take the fixed-width types from + * the same place: the kernel directly, and the userspace provider because + * rdma-core's includes this header too. + * + * What this replaced was a userspace branch that declared its own `__u32` and + * `__aligned_u64`. That is not a shadow, it is a hard error -- rdma-core + * already defines both (`__aligned_u64` as an aligned `__u64`), so the + * redefinition collides on `__u64` and the FIRST userspace build of this + * header failed outright. A header that hand-rolls names the platform already + * owns cannot be the wire format for a kernel/userspace boundary. + */ +#include + +#define TBV_CQ_SHM_MAGIC 0x54425643u /* "TBVC" */ +#define TBV_CQ_SHM_ABI 2u + +struct tbv_cq_shm { + __u32 magic; /* TBV_CQ_SHM_MAGIC -- provider sanity-checks this */ + __u32 abi; /* TBV_CQ_SHM_ABI */ + __u32 cqe; /* ring capacity in entries */ + __u32 entry_size; /* sizeof(struct ib_uverbs_wc) -- sanity */ + __u32 ring_offset; /* byte offset of the ring within the mapping */ + __u32 produced; /* total completions ever queued (KERNEL writes) */ + __u32 consumed; /* total completions ever taken (CONSUMER writes) */ + __u32 ovf_seq; /* times the kernel dropped an entry (KERNEL writes) */ + __u32 map_len; /* total bytes the consumer must map (kernel-set) */ + __u32 reserved[39]; /* pad to 192 bytes; room to grow without an abi bump */ +}; + +/* Vendor tail of create_cq's response (after the generic ib_uverbs resp). + * + * The kernel fills it only when the provider's outbuf is big enough, so an old + * provider sees nothing (and never mmaps); a new provider on an old kernel + * reads all-zero from its own calloc and takes the fallback. + * + * `shm_abi` is the offer FLAG, and that is deliberate. The first version keyed + * the decision on `cq_mmap_offset != 0`, but 0 is a legitimate offset that + * rdma_user_mmap_entry_insert can hand out (measured: the first CQ on this leg + * was offered offset 0), so a nonzero test silently discards a real offer. + * `shm_abi` is zero in a response the kernel never filled and equal to + * TBV_CQ_SHM_ABI in one it did -- which is exactly the question being asked. + * + * `map_len` is published here so the consumer can map the whole buffer in ONE + * mmap. The first version mapped a single page to discover the length and then + * remapped; the kernel rejected that first mmap (measured: EINVAL, errno + * "Invalid argument") and the feature silently fell back to ioctls. + */ +struct tbv_uresp_create_cq { + __aligned_u64 cq_mmap_offset; /* mmap() offset; may legitimately be 0 */ + __u32 cqe; + __u32 shm_abi; /* TBV_CQ_SHM_ABI = the kernel offered */ + __u32 reserved; + __u32 pad; + __aligned_u64 map_len; /* bytes to map; 0 = nothing offered */ +}; + +#endif /* TBV_CQ_SHM_H */ diff --git a/userspace/usb4_rdma/usb4_rdma.c b/userspace/usb4_rdma/usb4_rdma.c index 9c34ff8..4e3a69e 100644 --- a/userspace/usb4_rdma/usb4_rdma.c +++ b/userspace/usb4_rdma/usb4_rdma.c @@ -19,9 +19,11 @@ #include #include #include +#include #include #include "usb4_rdma.h" +#include "tbv_cq_shm.h" #define USB4_RDMA_NODE_GUID 0x0200544256524253ULL #define USB4_APPLE_NODE_GUID 0x0200544256524254ULL @@ -88,33 +90,191 @@ static int usb4_rdma_dealloc_pd(struct ibv_pd *base_pd) /* ----- cq ---------------------------------------------------------- */ +/* + * Userspace-mapped CQ (Phase 1 of the userspace-mapped-queues work). + * + * When the kernel offers it (a vendor resp tail with cq_mmap_offset != 0), + * the provider maps the CQ's shared page and polls it lock-free: an acquire + * load of the tail, read the entries, a release store of the head. That is + * the whole poll -- zero ioctls, where the generic path issues one per call + * (measured: ~96 ioctls per pingpong iteration, 2026-09-12). + * + * When the kernel offers nothing (old kernel) or the map fails, the cq keeps + * hdr == NULL and every call falls through to the generic ibv_cmd_* path -- + * byte-identical with the pre-mmap provider. + */ + +struct usb4_rdma_cq { + struct ibv_cq base_cq; + struct tbv_cq_shm *hdr; + /* The ring carries struct ib_uverbs_wc, the uapi wire format -- NOT the + * kernel's struct ib_wc (which holds pointers) and not userspace's + * struct ibv_wc. Sharing either side's internal struct across the + * boundary is what made the first version return exit 17 with zero + * iterations: the provider read a kernel pointer as qp_num. */ + struct ib_uverbs_wc *ring; + size_t map_len; + /* + * Consumer-private cursor. The shared state is a pair of monotonic + * totals (see proto/tbv_cq_shm.h); this side keeps only where it has + * read up to, so no shared field has two writers. `head` is the ring + * index and `consumed` the total taken, and they advance together. + */ + uint32_t head; + uint32_t consumed; + uint32_t ovf_seen; /* hdr->ovf_seq values already reported */ + /* + * TBV_SHM_DEBUG diagnostics. On this driver the shared-page path is + * invisible when it goes wrong -- a failure arrives at the application + * as a bare negative return from ibv_poll_cq, with nothing to say + * whether the ring was empty, the kernel never published, or the + * kernel published and then declared overflow. These counters and + * dumps exist so the failure can be DESCRIBED rather than guessed at. + */ + unsigned dbg_polls; + unsigned dbg_eio; +}; + +/* Mirror of the kernel's struct tbv_uresp_create_cq, which is written at the + * response base -- i.e. immediately after the generic ib_uverbs_create_cq_resp + * this provider declares first. Every field must stay in step with + * proto/tbv_cq_shm.h or the two sides disagree silently. */ +struct usb4_rdma_create_cq_resp { + struct ib_uverbs_create_cq_resp ibv_resp; + __aligned_u64 cq_mmap_offset; + uint32_t cqe; + uint32_t shm_abi; + uint32_t reserved; + uint32_t pad; + __aligned_u64 map_len; +}; + +static bool usb4_rdma_shm_debug(void) +{ + static int on = -1; + + if (on < 0) + on = getenv("TBV_SHM_DEBUG") != NULL; + return on; +} + +/* Dump the shared header as the CONSUMER sees it. */ +static void usb4_rdma_dump_shm(const char *what, const struct usb4_rdma_cq *cq) +{ + const struct tbv_cq_shm *h = cq->hdr; + + if (!usb4_rdma_shm_debug()) + return; + fprintf(stderr, + "[shm] %s: mapped=%d magic=%#x abi=%u cqe=%u entry_size=%u " + "ring_offset=%u produced=%u consumed=%u ovf_seq=%u map_len=%u " + "[consumer head=%u consumed=%u ovf_seen=%u] " + "dbg_polls=%u dbg_eio=%u\n", + what, h != NULL, + h ? h->magic : 0, h ? h->abi : 0, h ? h->cqe : 0, + h ? h->entry_size : 0, h ? h->ring_offset : 0, + h ? h->produced : 0, h ? h->consumed : 0, + h ? h->ovf_seq : 0, h ? h->map_len : 0, + cq->head, cq->consumed, cq->ovf_seen, + cq->dbg_polls, cq->dbg_eio); + fflush(stderr); +} + static struct ibv_cq *usb4_rdma_create_cq(struct ibv_context *ctx, int num_cqe, struct ibv_comp_channel *channel, int comp_vector) { - struct ib_uverbs_create_cq_resp resp; - struct ibv_create_cq cmd; - struct ibv_cq *cq; + struct usb4_rdma_create_cq_resp resp = {}; + struct ibv_create_cq cmd = {}; + struct usb4_rdma_cq *cq; int rv; cq = calloc(1, sizeof(*cq)); if (!cq) return NULL; - rv = ibv_cmd_create_cq(ctx, num_cqe, channel, comp_vector, cq, - &cmd, sizeof(cmd), &resp, sizeof(resp)); + rv = ibv_cmd_create_cq(ctx, num_cqe, channel, comp_vector, &cq->base_cq, + &cmd, sizeof(cmd), &resp.ibv_resp, sizeof(resp)); if (rv) { free(cq); errno = rv; return NULL; } - return cq; + + if (usb4_rdma_shm_debug()) + fprintf(stderr, + "[shm] create_cq: requested=%d offered_offset=%#llx " + "offered_cqe=%u offered_abi=%u offered_len=%llu\n", + num_cqe, (unsigned long long)resp.cq_mmap_offset, + resp.cqe, resp.shm_abi, + (unsigned long long)resp.map_len); + /* + * The offer test keys on shm_abi, NOT on the offset being nonzero. 0 is + * a legitimate offset -- measured: the first CQ this leg created was + * offered exactly 0 -- so the old `if (resp.cq_mmap_offset && ...)` + * silently discarded a real offer, and one of the two CQs perftest + * creates never used the shared ring at all. + * + * ONE mmap for the whole buffer: the kernel publishes map_len in the + * response. The first version mapped a single page to discover the + * length and then remapped, and the kernel rejected that first call + * outright, so the feature never engaged and every poll was an ioctl. + */ + if (resp.shm_abi == TBV_CQ_SHM_ABI && resp.map_len) { + size_t len = resp.map_len; + void *m = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, + ctx->cmd_fd, resp.cq_mmap_offset); + + if (m == MAP_FAILED) { + if (usb4_rdma_shm_debug()) + fprintf(stderr, + "[shm] create_cq: mmap(len=%zu, off=%#llx) failed: %s\n", + len, + (unsigned long long)resp.cq_mmap_offset, + strerror(errno)); + } else { + struct tbv_cq_shm *hdr = m; + + if (hdr->magic == TBV_CQ_SHM_MAGIC && + hdr->abi == TBV_CQ_SHM_ABI && + hdr->cqe > 0 && + hdr->entry_size == sizeof(struct ib_uverbs_wc) && + hdr->map_len == len && + hdr->ring_offset + + (size_t)hdr->cqe * sizeof(struct ib_uverbs_wc) <= len) { + cq->hdr = hdr; + cq->ring = (struct ib_uverbs_wc *) + ((char *)m + hdr->ring_offset); + cq->map_len = len; + cq->consumed = hdr->consumed; + } else { + if (usb4_rdma_shm_debug()) + fprintf(stderr, + "[shm] create_cq: header REJECTED " + "magic=%#x abi=%u cqe=%u entry_size=%u " + "ring_offset=%u map_len=%u " + "want_len=%zu want_entry=%zu\n", + hdr->magic, hdr->abi, hdr->cqe, + hdr->entry_size, hdr->ring_offset, + hdr->map_len, len, + sizeof(struct ib_uverbs_wc)); + munmap(m, len); + } + } + } + usb4_rdma_dump_shm("create_cq", cq); + return &cq->base_cq; } -static int usb4_rdma_destroy_cq(struct ibv_cq *cq) +static int usb4_rdma_destroy_cq(struct ibv_cq *base_cq) { - int rv = ibv_cmd_destroy_cq(cq); + struct usb4_rdma_cq *cq = + container_of(base_cq, struct usb4_rdma_cq, base_cq); + int rv; + if (cq->hdr) + munmap(cq->hdr, cq->map_len); + rv = ibv_cmd_destroy_cq(base_cq); if (rv) return rv; free(cq); @@ -184,10 +344,78 @@ static int usb4_rdma_post_recv(struct ibv_qp *qp, struct ibv_recv_wr *wr, return ibv_cmd_post_recv(qp, wr, bad_wr); } -static int usb4_rdma_poll_cq(struct ibv_cq *cq, int num_entries, +static int usb4_rdma_poll_cq(struct ibv_cq *base_cq, int num_entries, struct ibv_wc *wc) { - return ibv_cmd_poll_cq(cq, num_entries, wc); + struct usb4_rdma_cq *cq = + container_of(base_cq, struct usb4_rdma_cq, base_cq); + struct tbv_cq_shm *hdr = cq->hdr; + uint32_t produced, ovf; + int polled = 0; + + if (!hdr) + return ibv_cmd_poll_cq(base_cq, num_entries, wc); + + cq->dbg_polls++; + /* + * Overflow first, and once per EVENT. The kernel advances ovf_seq when + * it drops a completion; we remember the values we have already + * reported and surface each new one a single time. Testing before + * consuming keeps a clean error from arriving alongside a partial + * batch, and the latch lives HERE rather than in shared memory so it + * cannot turn one dropped completion into a permanently dead CQ. + */ + ovf = __atomic_load_n(&hdr->ovf_seq, __ATOMIC_ACQUIRE); + if (ovf != cq->ovf_seen) { + cq->ovf_seen = ovf; + cq->dbg_eio++; + if (usb4_rdma_shm_debug()) + usb4_rdma_dump_shm("poll -EIO", cq); + return -EIO; + } + + /* + * Lock-free single-consumer read: the producer publishes the entry and + * then the total (release); we read the total (acquire) and then the + * entries, and publish our own total (release). Occupancy is the + * difference of the two totals -- exact at every capacity, including + * the cqe == 1 perftest asks for, where an index comparison cannot + * distinguish full from empty. We keep our own head and consumed, so + * nothing we read has a second writer. + */ + produced = __atomic_load_n(&hdr->produced, __ATOMIC_ACQUIRE); + while (polled < num_entries && produced - cq->consumed > 0) { + struct ib_uverbs_wc *u = &cq->ring[cq->head]; + + /* ib_uverbs_wc -> ibv_wc, the same field-for-field translation + * ibv_cmd_poll_cq performs on the generic path. */ + memset(&wc[polled], 0, sizeof(wc[polled])); + wc[polled].wr_id = u->wr_id; + wc[polled].status = u->status; + wc[polled].opcode = u->opcode; + wc[polled].vendor_err = u->vendor_err; + wc[polled].byte_len = u->byte_len; + wc[polled].imm_data = u->ex.imm_data; + wc[polled].qp_num = u->qp_num; + wc[polled].src_qp = u->src_qp; + wc[polled].wc_flags = u->wc_flags; + wc[polled].pkey_index = u->pkey_index; + wc[polled].slid = u->slid; + wc[polled].sl = u->sl; + wc[polled].dlid_path_bits = u->dlid_path_bits; + polled++; + cq->head = (cq->head + 1) % hdr->cqe; + cq->consumed++; + } + if (polled) + __atomic_store_n(&hdr->consumed, cq->consumed, __ATOMIC_RELEASE); + + if (usb4_rdma_shm_debug() && cq->dbg_polls <= 8) + fprintf(stderr, + "[shm] poll #%u: n=%d polled=%d produced_read=%u consumed_out=%u\n", + cq->dbg_polls, num_entries, polled, produced, + cq->consumed); + return polled; } static int usb4_rdma_req_notify_cq(struct ibv_cq *cq, int solicited_only)