diff --git a/src/Home/Net/Drivers/Run.ZC b/src/Home/Net/Drivers/Run.ZC index ebc0246c6..b403212b5 100644 --- a/src/Home/Net/Drivers/Run.ZC +++ b/src/Home/Net/Drivers/Run.ZC @@ -76,7 +76,7 @@ U0 NetDriverInit() net_driver = net_driver->next; } -//No supported PCI NIC: fall back to a USB CDC-ECM adapter if one is bound. +//No PCI NIC: fall back to a bound USB CDC-ECM adapter. if (!found && UsbEcmPresent) { found = TRUE; diff --git a/src/Home/Net/Drivers/UsbEcm.ZC b/src/Home/Net/Drivers/UsbEcm.ZC index a1f30ec9a..baea6dddd 100644 --- a/src/Home/Net/Drivers/UsbEcm.ZC +++ b/src/Home/Net/Drivers/UsbEcm.ZC @@ -1,23 +1,15 @@ -/* USB CDC-ECM network driver (Zeal net stack side). - - Bridges the kernel USB CDC-ECM class driver (src/Kernel/Usb/KUsbEcm.ZC) - into the net stack's driver contract. The kernel side owns the xHCI bulk - transfers and a receive frame ring; this file just implements the - Ethernet + NetDriver entry points the stack calls, and runs a task that - drains received frames into the NetQueue (the USB stack is polled - there - is no NIC interrupt to hook). - - Selected by NetDriverInit (Run.ZC) when a USB NIC is bound and no PCI NIC - matched. Works for QEMU `-device usb-net` (its CDC-ECM config) and the - RTL8153 USB-C adapter. +/* USB CDC-ECM network driver. + + Net stack side of the kernel CDC-ECM class driver (Kernel/Usb/KUsbEcm.ZC), + which owns the bulk transfers and the receive ring. Selected by + NetDriverInit when no PCI NIC matched. RX is drained by a task; the USB + stack is polled, so there is no NIC interrupt. */ -#define USB_ECM_TX_BUFFERS 8 //power of two; TX frames in flight +#define USB_ECM_TX_BUFFERS 8 //power of two -//The kernel USB ECM interface (UsbEcmPresent/MacAddr/Send/Recv) is provided by -//public AOT funcs in $LK,"KUsbEcm.ZC",A="FF:::/Kernel/Usb/KUsbEcm.ZC"$; they are -//already in the global symbol table, so call them directly (no extern - an -//extern decl creates an unresolved JIT import instead of binding the public). +//UsbEcm* are public kernel AOT funcs, already in the global symbol table. +//Do not extern them: that makes an unresolved JIT import instead of binding. class CUsbEcmNet { @@ -35,17 +27,17 @@ U8 *EthernetMACGet() } I64 NetDriverPacketBufferGet(I64 de_index) -{//Start of the built TX frame for this descriptor. +{//Built TX frame for this descriptor. return usb_ecm_net.tx_buffers + de_index * ETHERNET_FRAME_SIZE; } I64 NetDriverPacketLengthGet(I64 de_index) -{//Total frame length (header + payload) stored at allocate time. +{//Frame length recorded at allocate time. return usb_ecm_net.tx_lengths[de_index]; } U0 NetDriverTransmitPacketFinish(I64 de_index) -{//Hand the finished frame to the kernel for a bulk-OUT transfer. +{//Hand the frame to the kernel for a bulk-OUT transfer. U8 *frame = usb_ecm_net.tx_buffers + de_index * ETHERNET_FRAME_SIZE; I64 len = usb_ecm_net.tx_lengths[de_index]; @@ -54,9 +46,8 @@ U0 NetDriverTransmitPacketFinish(I64 de_index) I64 EthernetFrameAllocate(U8 **packet_buffer_out, U8 *source_address, U8 *destination_address, U16 ethertype, I64 packet_length) -{/* Build an Ethernet frame header into the next TX buffer and hand back the - payload pointer + descriptor index. Mirrors the PCNet driver: pad the - payload up to the minimum, cap the whole frame to the buffer size. */ +{/* Build a frame header in the next TX buffer, return the payload pointer + and descriptor index. Pads the payload to the minimum, caps to buffer. */ U8 *frame; I64 de_index, total; @@ -84,7 +75,7 @@ I64 EthernetFrameAllocate(U8 **packet_buffer_out, U8 *source_address, } U0 NetStop() -{//Pause RX delivery (the kernel keeps draining USB into its ring regardless). +{//Pause RX delivery; the kernel keeps filling its ring. usb_ecm_net.running = FALSE; } @@ -94,7 +85,7 @@ U0 NetStart() } U0 UsbEcmNetRxTask(I64 dummy=0) -{//Drain received frames from the kernel ECM ring into the NetQueue. +{//Drain the kernel ECM ring into the NetQueue. U8 *buf; I64 len; diff --git a/src/Kernel/Usb/KUsb.HH b/src/Kernel/Usb/KUsb.HH index 146ac5b8a..7d1305f4b 100644 --- a/src/Kernel/Usb/KUsb.HH +++ b/src/Kernel/Usb/KUsb.HH @@ -8,6 +8,7 @@ #define USB_DEVS_MAX 16 #define USB_EPS_MAX 8 #define USB_IFS_MAX 6 +#define USB_HCS_MAX 4 //xHCI controllers driven at once //USB descriptor types #define USB_DESC_DEV 1 @@ -62,7 +63,7 @@ #define USB_MSD_RQ_RESET 0xFF #define USB_MSD_RQ_GET_MAX_LUN 0xFE -//CDC / Ethernet Networking Control Model (ECM) +//CDC / Ethernet Networking Control Model #define USB_CLASS_COMM 0x02 //CDC communications (control interface) #define USB_CLASS_CDC_DATA 0x0A //CDC data interface (bulk endpoints) #define USB_CDC_SUBCLASS_ECM 0x06 //Ethernet Networking Control Model @@ -78,8 +79,7 @@ //Hub class #define USB_DESC_HUB 0x29 //hub descriptor (USB2) #define USB_DESC_SSHUB 0x2A //SuperSpeed hub descriptor -//SuperSpeed hubs must be told their tier before they can route packets -//(USB 3.x 10.14.2.7). Without it, downstream Address Device fails. +//SuperSpeed hubs must be told their tier before they can route (USB 3.x 10.14.2.7). #define USB_HUB_RQ_SET_DEPTH 12 //Hub class feature selectors (port) #define USB_HUB_F_PORT_CONNECTION 0 @@ -253,14 +253,13 @@ class CUsbEPDesc #define XHCI_RING_TRBS 256 //4096 / 16 #define XHCI_EVT_RING_TRBS 256 -//Interrupt/bulk-IN auto-requeue report handler kinds (see CUsbEP.auto_kind). +//Auto-requeue handler kinds (CUsbEP.auto_kind). #define USB_AUTO_NONE 0 #define USB_AUTO_KBD 1 #define USB_AUTO_MOUSE 2 #define USB_AUTO_ECM 3 //CDC-ECM bulk-IN: receive ethernet frame -//Forward decls: report handlers live in UsbHid.ZC / KUsbEcm.ZC (compiled after -//KXhci.ZC), but XhciEventPoll dispatches to them. Single-pass compiler needs these. +//XhciEventPoll dispatches to handlers compiled after it. extern U0 UsbKbdCB(U8 *dev, U8 *ep); extern U0 UsbMouseCB(U8 *dev, U8 *ep); extern U0 UsbEcmRxCB(U8 *dev, U8 *ep); @@ -312,9 +311,35 @@ class CUsbIf ep_count; }; +class CXhciGlobals +{ + Bool present, running; + U8 *mmio, //capability base (uncached alias) + *op, //operational base + *rt; //runtime base + U32 *db; //doorbell array + I64 max_slots, max_ports, ctx_size, + scratch_count; + U64 *dcbaa, + *scratch_arr, + *erst; + CXhciRing cmd_ring; + CXhciTRB *evt_ring; + I64 evt_deq, evt_cycle; + I64 lock, //ring enqueue lock + poll_lock; //event ring consumer lock (boot inline poll vs USB task) +//Command completion, set by event poll + Bool cmd_done; + I64 cmd_code, cmd_slot; + CXhciTRB *cmd_trb; //pending command TRB + Bool port_change; //port status change event seen + I64 pci_bus, pci_dev, pci_fun; +}; + class CUsbDev { Bool present, configured; + CXhciGlobals *hc; //owning controller; slot/port are unique only within it I64 slot, //xHCI slot id port, //root port index (0-based) speed; //USB_SPEED_* @@ -328,8 +353,7 @@ class CUsbDev I64 vendor_id, product_id, dev_class, dev_subclass, dev_proto, config_val; -//Topology: root devices have route 0, depth 0, no parent. Devices behind hubs -//carry an xHCI route string + the parent hub's slot/port (for TT of LS/FS). +//Topology. Root devices: route 0, depth 0, no parent. I64 route, //xHCI route string (20 bits) depth, //hub tier (0 = attached to a root port) parent_slot, //parent hub xHCI slot id (0 = root) @@ -342,39 +366,15 @@ class CUsbDev U8 desc_name[64]; //product string, ASCII }; -class CXhciGlobals -{ - Bool present, running; - U8 *mmio, //capability base (uncached alias) - *op, //operational base - *rt; //runtime base - U32 *db; //doorbell array - I64 max_slots, max_ports, ctx_size, - scratch_count; - U64 *dcbaa, - *scratch_arr, - *erst; - CXhciRing cmd_ring; - CXhciTRB *evt_ring; - I64 evt_deq, evt_cycle; - I64 lock, //ring enqueue lock - poll_lock; //event ring consumer lock (boot inline poll vs USB task) -//Command completion, set by event poll - Bool cmd_done; - I64 cmd_code, cmd_slot; - CXhciTRB *cmd_trb; //pending command TRB - Bool port_change; //port status change event seen - I64 pci_bus, pci_dev, pci_fun; -}; - -//Hub driver (KUsbHub.ZC) enumerates downstream devices then binds them via -//UsbDrvBind (KUsbBoot.ZC, compiled later): forward decls for the single-pass -//compiler. Declared here (after CUsbDev/CUsbIf) so the struct types are known. +//KUsbHub.ZC calls UsbDrvBind from KUsbBoot.ZC, compiled later. Declared after +//CUsbDev/CUsbIf so the types are known. extern U0 UsbDrvBind(CUsbDev *dev); extern U0 UsbHubAttach(CUsbDev *dev, CUsbIf *uif); //Globals defined in $LK,"KXhci.ZC",A="FF:::/Kernel/Usb/KXhci.ZC,usb_xhci"$. -extern CXhciGlobals usb_xhci; +extern CXhciGlobals usb_hcs[USB_HCS_MAX]; +extern I64 usb_hc_count; +extern CXhciGlobals *usb_xhci; //active controller extern CUsbDev usb_devs[USB_DEVS_MAX]; extern CTask *usb_task; extern U8 usb_boot_iso_drv_let; //Drive letter of mounted USB install media, else 0 @@ -383,12 +383,10 @@ public extern U0 UsbRep(); public extern I64 UsbBootInit(); public extern U0 UsbBootDriveFallback(); public extern U0 UsbTaskMain(I64 dummy=0); -public extern Bool XhciInit(I64 start_n=0, Bool any=FALSE); -public extern U0 XhciSwitch(); -public extern U0 XhciFindNic(I64 max_tries=8); +public extern I64 XhciInitAll(); -//CDC-ECM USB NIC interface, consumed by the /System net driver -//$LK,"UsbEcm.ZC",A="FF:::/Home/Net/Drivers/UsbEcm.ZC"$. Operates on the first NIC. +//CDC-ECM NIC interface for $LK,"UsbEcm.ZC",A="FF:::/Home/Net/Drivers/UsbEcm.ZC"$. +//Operates on the first bound NIC. public extern U0 UsbEcmRep(); public extern Bool UsbEcmPresent(); public extern U0 UsbEcmMacAddr(U8 *out6); diff --git a/src/Kernel/Usb/KUsbBoot.ZC b/src/Kernel/Usb/KUsbBoot.ZC index 4e6913760..2b3bb7624 100644 --- a/src/Kernel/Usb/KUsbBoot.ZC +++ b/src/Kernel/Usb/KUsbBoot.ZC @@ -43,31 +43,41 @@ U0 UsbDrvUnbind(CUsbDev *dev) UsbEcmDetach(dev); } -U0 UsbPortScan() -{//Check all root ports for connects/disconnects. USB task only. +U0 UsbPortScan(CXhciGlobals *hc=NULL) +{//Check one controller's root ports for connects/disconnects. USB task only. CUsbDev *dev, *ndev; - I64 port, sc; + I64 port, sc, i; - for (port = 0; port < usb_xhci.max_ports; port++) + if (!hc) hc = usb_xhci; + for (port = 0; port < hc->max_ports; port++) { - sc = XhciPortSC(port); + sc = XhciPortSC(port, hc); if (sc & XHCI_PORTSC_CHANGE_BITS) - XhciPortSCWrite(port, XHCI_PORTSC_PP | (sc & XHCI_PORTSC_CHANGE_BITS)); - dev = UsbDevByPort(port); + XhciPortSCWrite(port, XHCI_PORTSC_PP | (sc & XHCI_PORTSC_CHANGE_BITS), hc); + dev = UsbDevByPort(port, hc); if (sc & XHCI_PORTSC_CCS) { if (!dev) { - if (XhciPortReset(port)) + if (XhciPortReset(port, hc)) { - ndev = UsbDevEnum(port); + ndev = UsbDevEnum(port, hc); if (ndev) UsbDrvBind(ndev); } } } else if (dev) - { + {//Drop the device and anything a hub on this port was carrying; hub + //children share the root port number and have no connect status of + //their own, so nothing else would ever reap them. + for (i = 0; i < USB_DEVS_MAX; i++) + if (usb_devs[i].present && usb_devs[i].hc == hc && + usb_devs[i].port == port && usb_devs[i].depth) + { + UsbDrvUnbind(&usb_devs[i]); + UsbDevDetachCore(&usb_devs[i]); + } UsbDrvUnbind(dev); UsbDevDetachCore(dev); SysLog("USB: port %d: device removed.\n", port); @@ -75,33 +85,45 @@ U0 UsbPortScan() } } +U0 UsbPortScanAll() +{//Scan every running controller. + I64 n; + + for (n = 0; n < usb_hc_count; n++) + UsbPortScan(&usb_hcs[n]); +} + public U0 UsbRep() {//Report attached USB devices. - CUsbDev *dev, *ndev; - CUsbIf *uif; - I64 i, j; + CUsbDev *dev; + CUsbIf *uif; + CXhciGlobals *hc; + I64 i, j, n; - "$$FG,GREEN$$UsbRep BUILD MARKER: ss-hubdepth-5$$FG$$\n"; //bump each rebuild to catch stale kernels - - if (!usb_xhci.present) + if (!usb_hc_count) { "No xHCI controller. Re-probing:\n"; - XhciInit; + XhciInitAll; } - if (!usb_xhci.present) + if (!usb_hc_count) return; - UsbPortScan; //rescan so a just-plugged device shows up - "xHCI: %d ports, %d slots.\n", usb_xhci.max_ports, usb_xhci.max_slots; - for (i = 0; i < usb_xhci.max_ports; i++) - " port %d: PORTSC=0x%08X CCS=%d PED=%d PP=%d\n", i, XhciPortSC(i), - XhciPortSC(i) & 1, XhciPortSC(i) >> 1 & 1, XhciPortSC(i) >> 9 & 1; + UsbPortScanAll; //rescan so a just-plugged device shows up + for (n = 0; n < usb_hc_count; n++) + { + hc = &usb_hcs[n]; + "xHCI #%d: %d ports, %d slots.\n", n, hc->max_ports, hc->max_slots; + for (i = 0; i < hc->max_ports; i++) + " port %d: PORTSC=0x%08X CCS=%d PED=%d PP=%d\n", i, XhciPortSC(i, hc), + XhciPortSC(i, hc) & 1, XhciPortSC(i, hc) >> 1 & 1, + XhciPortSC(i, hc) >> 9 & 1; + } for (i = 0; i < USB_DEVS_MAX; i++) { dev = &usb_devs[i]; if (dev->present) { - "Port %2d: %04X:%04X devcls=%d \"%s\"", dev->port, dev->vendor_id, - dev->product_id, dev->dev_class, dev->desc_name; + "hc%d port %2d: %04X:%04X \"%s\"", XhciHcIdx(dev->hc), dev->port, + dev->vendor_id, dev->product_id, dev->desc_name; for (j = 0; j < dev->if_count; j++) { uif = &dev->ifs[j]; @@ -120,13 +142,6 @@ public U0 UsbRep() " [class %d]", uif->if_class; } "\n"; - for (j = 0; j < dev->if_count; j++) - { - uif = &dev->ifs[j]; - " if%d: cls=%d sub=%d proto=%d alt=%d eps=%d mac_str=%d\n", - uif->if_num, uif->if_class, uif->if_subclass, uif->if_proto, - uif->alt, uif->ep_count, uif->ecm_mac_str_idx; - } } } for (i = 0; i < USB_MSDS_MAX; i++) @@ -135,84 +150,29 @@ public U0 UsbRep() ((usb_msds[i].max_lba + 1) * BLK_SIZE) / 0x100000, usb_msds[i].first_drive_let; -//Diagnose root ports that are connected but have no bound device (enumeration -//gaps, e.g. SuperSpeed). Re-attempt enumeration HERE so the enum trace + direct -//failure prints land in this terminal (usb_task SysLogs are hidden single-user). - for (i = 0; i < usb_xhci.max_ports; i++) - { - if ((XhciPortSC(i) & XHCI_PORTSC_CCS) && !UsbDevByPort(i)) - { - "UNBOUND port %d: PORTSC=0x%08X speed=%d - retrying enum:\n", - i, XhciPortSC(i), (XhciPortSC(i) >> 10) & 0xF; - if (XhciPortReset(i)) - { - ndev = UsbDevEnum(i); - if (ndev) - { - UsbDrvBind(ndev); - " port %d: enumerated OK.\n", i; - } - } - else - " port %d: XhciPortReset did not enable the port.\n", i; - } - } -} - -public U0 XhciSwitch() -{//Force the NEXT xHCI controller and rescan (cycles through them). The boot init -//auto-picks the one with a device plugged in, so this is only for overriding. - static I64 idx = 0; - - idx++; - if (!XhciInit(idx, TRUE)) - { - idx = 0; - XhciInit(idx, TRUE); - } - "XhciSwitch: now on controller #%d.\n", idx; - Sleep(200); - UsbPortScan; - UsbRep; -} - -public U0 XhciFindNic(I64 max_tries=8) -{//Cycle controllers until one has a USB ethernet NIC bound. A USB-C dock puts -//the SuperSpeed NIC on the Thunderbolt controller while HID sits on the PCH one, -//and this stack drives a single controller at a time - this hunts for the NIC. - I64 i; - - for (i = 0; i < max_tries; i++) - { - if (UsbEcmPresent) - { - "XhciFindNic: NIC found.\n"; - UsbEcmRep; - return; - } - XhciSwitch; - } - "XhciFindNic: no USB NIC found after %d controller switches.\n", max_tries; } U0 UsbTaskMain(I64 dummy=0) {//Hotplug + HID poll loop. Spawned from $LK,"KMain",A="FF:::/Kernel/KMain.ZC,UsbTaskMain"$. - I64 tick = 0; + I64 tick = 0, n; no_warn dummy; usb_task = Fs; - if (!usb_xhci.running) + if (!usb_hc_count) return; while (TRUE) { - XhciEventPoll; + for (n = 0; n < usb_hc_count; n++) + XhciEventPoll(&usb_hcs[n]); UsbHidTick; + UsbEcmTick; tick++; - if (usb_xhci.port_change || tick & 0x7F == 0) - { - usb_xhci.port_change = FALSE; - UsbPortScan; - } + for (n = 0; n < usb_hc_count; n++) + if (usb_hcs[n].port_change || tick & 0x7F == 0) + { + usb_hcs[n].port_change = FALSE; + UsbPortScan(&usb_hcs[n]); + } Sleep(2); } } @@ -223,10 +183,10 @@ I64 UsbBootInit() //the kernel config's mount code runs, so USB drives exist at boot. I64 i, count = 0; - if (!XhciInit) + if (!XhciInitAll) return 0; Sleep(100); //let root port connect status settle - UsbPortScan; + UsbPortScanAll; for (i = 0; i < USB_MSDS_MAX; i++) if (usb_msds[i].valid && usb_msds[i].ready && usb_msds[i].bd) count++; diff --git a/src/Kernel/Usb/KUsbCore.ZC b/src/Kernel/Usb/KUsbCore.ZC index 9b629ab03..6dc167053 100644 --- a/src/Kernel/Usb/KUsbCore.ZC +++ b/src/Kernel/Usb/KUsbCore.ZC @@ -65,7 +65,7 @@ CUsbDev *UsbDevAlloc() } CUsbIf *UsbIfByNum(CUsbDev *dev, I64 if_num) -{//Return the recorded (alt-0) interface with this bInterfaceNumber, else NULL. +{//Recorded (alt-0) interface with this bInterfaceNumber, else NULL. I64 i; for (i = 0; i < dev->if_count; i++) @@ -76,15 +76,14 @@ CUsbIf *UsbIfByNum(CUsbDev *dev, I64 if_num) } I64 UsbSetInterface(CUsbDev *dev, I64 if_num, I64 alt) -{//SET_INTERFACE: select an alternate setting to activate its endpoints. +{//Select an alternate setting to activate its endpoints. return UsbCtrl(dev, USB_RT_HOST_TO_DEV | USB_RT_RECIP_IF, USB_RQ_SET_INTERFACE, alt, if_num, NULL, 0); } I64 UsbPickEcmConfig(CUsbDev *dev, I64 num_configs) -{//Scan each configuration for a CDC-ECM control interface; return its config -//descriptor index (for GET/SET_CONFIGURATION), else 0. RTL8153 and QEMU usb-net -//are multi-config and their ECM configuration is not index 0. +{//Config descriptor index holding a CDC-ECM control interface, else 0. +//RTL8153 and QEMU usb-net are multi-config; ECM is not index 0. CUsbConfigDesc *cd; CUsbIfDesc *ifd; U8 *buf; @@ -167,16 +166,15 @@ U0 UsbParseConfig(CUsbDev *dev, U8 *buf, I64 len) cur_if = NULL; } else - {//Alternate setting: attach its endpoints to the matching alt-0 interface - //and remember the alt number (CDC-ECM data endpoints live under alt 1). + {//Alternate setting: its endpoints belong to the matching alt-0 + //interface. CDC-ECM data endpoints live under alt 1. cur_if = UsbIfByNum(dev, ifd->if_num); if (cur_if) cur_if->alt = ifd->alt; } } else if (dtype == USB_DESC_CS_IF) - {//Class-specific interface descriptor. Grab the CDC-ECM Ethernet functional - //descriptor's iMACAddress string index (it follows the comm interface). + {//CDC-ECM Ethernet functional descriptor: iMACAddress string index. if (cur_if && dlen >= 4 && buf[off + 2] == USB_CDC_ST_ETHERNET) cur_if->ecm_mac_str_idx = buf[off + 3]; } @@ -218,8 +216,8 @@ U0 UsbDevDetachCore(CUsbDev *dev) dev->configured = FALSE; if (dev->slot) { - XhciDisableSlot(dev->slot); - usb_xhci.dcbaa[dev->slot] = 0; + XhciDisableSlot(dev->slot, dev->hc); + dev->hc->dcbaa[dev->slot] = 0; } dev->slot = 0; dev->ep0.valid = FALSE; @@ -232,17 +230,18 @@ U0 UsbDevDetachCore(CUsbDev *dev) } CUsbDev *UsbDevEnumEx(I64 root_port, I64 speed, I64 route, I64 depth, - I64 parent_slot, I64 parent_port) -{//Enumerate an already-reset+enabled device. Root-port devices pass route 0 / -//depth 0 / no parent; hub children pass the xHCI route string + parent hub -//slot/port (see KUsbHub.ZC). USB task only. + I64 parent_slot, I64 parent_port, CXhciGlobals *hc=NULL) +{//Enumerate an already-reset+enabled device. Root-port devices pass route 0, +//depth 0, no parent; hub children pass a route string + parent slot/port. +//USB task only. CUsbDev *dev; CUsbDevDesc *dd; CUsbConfigDesc *cd; U8 *buf; I64 slot, code, total, mps0, cfg_idx, port = root_port; - slot = XhciEnableSlot; + if (!hc) hc = usb_xhci; + slot = XhciEnableSlot(hc); if (!slot) { "USB: port %d: Enable Slot failed.\n", port; @@ -251,9 +250,10 @@ CUsbDev *UsbDevEnumEx(I64 root_port, I64 speed, I64 route, I64 depth, dev = UsbDevAlloc; if (!dev) { - XhciDisableSlot(slot); + XhciDisableSlot(slot, hc); return NULL; } + dev->hc = hc; dev->slot = slot; dev->port = root_port; dev->speed = speed; @@ -270,13 +270,9 @@ CUsbDev *UsbDevEnumEx(I64 root_port, I64 speed, I64 route, I64 depth, dev->drv_data = NULL; dev->desc_name[0] = 0; -//Direct print (not SysLog): visible from any task, incl. UsbRep's re-enum. - "USB: enum port %d speed %d route 0x%05X depth %d slot %d\n", - root_port, speed, route, depth, slot; - code = XhciAddressDev(dev); if (code != XHCI_CC_SUCCESS) - {//One retry: a SuperSpeed device behind a hub can be slow to settle after reset. + {//A SuperSpeed device behind a hub can be slow to settle after reset. Sleep(100); code = XhciAddressDev(dev); } @@ -321,14 +317,10 @@ CUsbDev *UsbDevEnumEx(I64 root_port, I64 speed, I64 route, I64 depth, dev->dev_proto = dd->dev_proto; UsbStrGet(dev, dd->i_product, dev->desc_name, 64); -//Pick a configuration. Most devices are single-config (index 0); multi-config -//USB NICs (RTL8153, QEMU usb-net) hide their CDC-ECM config behind a later index, -//so prefer that one when present. +//Multi-config USB NICs hide CDC-ECM behind a later index; prefer it. cfg_idx = 0; if (dd->num_configs > 1) cfg_idx = UsbPickEcmConfig(dev, dd->num_configs); - SysLog("USB: port %d: %04X:%04X num_configs=%d picked cfg_idx=%d\n", - port, dd->vendor_id, dd->product_id, dd->num_configs, cfg_idx); //Config descriptor: header first for total length, then full. code = UsbGetDesc(dev, USB_DESC_CONFIG, cfg_idx, buf, 9); @@ -380,7 +372,7 @@ CUsbDev *UsbDevEnumEx(I64 root_port, I64 speed, I64 route, I64 depth, return dev; } -CUsbDev *UsbDevEnum(I64 port) +CUsbDev *UsbDevEnum(I64 port, CXhciGlobals *hc=NULL) {//Enumerate a device on a root port (already reset + enabled). USB task only. - return UsbDevEnumEx(port, (XhciPortSC(port) >> 10) & 0xF, 0, 0, 0, 0); + return UsbDevEnumEx(port, (XhciPortSC(port, hc) >> 10) & 0xF, 0, 0, 0, 0, hc); } diff --git a/src/Kernel/Usb/KUsbEcm.ZC b/src/Kernel/Usb/KUsbEcm.ZC index b2a9e3b70..97d0b6ccb 100644 --- a/src/Kernel/Usb/KUsbEcm.ZC +++ b/src/Kernel/Usb/KUsbEcm.ZC @@ -1,16 +1,16 @@ -//USB CDC-ECM (Ethernet Networking Control Model) class driver. -//Binds a USB Ethernet NIC (RTL8153 in its ECM config, QEMU usb-net CDC config), -//exposes raw Ethernet frame RX/TX over the data interface's bulk endpoints. -//The /System net driver $LK,"UsbEcm.ZC",A="FF:::/Home/Net/Drivers/UsbEcm.ZC"$ -//drives it via the public UsbEcm* interface (declared in KUsb.HH). +//USB CDC-ECM class driver. Raw ethernet frames over the data interface's bulk +//endpoints. Consumed by the net driver +//$LK,"UsbEcm.ZC",A="FF:::/Home/Net/Drivers/UsbEcm.ZC"$ via the public UsbEcm*. // -//RX is event-driven like HID: a bulk-IN TRB auto-requeues in XhciEventPoll -//(auto_kind == USB_AUTO_ECM), and UsbEcmRxCB pushes each received frame into a -//single-producer/single-consumer ring the net task drains via UsbEcmRecv. +//Both directions are queues drained by the USB task, because the net stack +//transmits from the NetHandler interrupt (TCP ACKs) where blocking on a USB +//transfer is not possible. RX auto-requeues a bulk-IN TRB in XhciEventPoll +//(USB_AUTO_ECM); TX is enqueued by UsbEcmSend and sent from UsbEcmTick. #define USB_ECMS_MAX 2 -#define USB_ECM_FRAME_MAX 1600 //>= max ethernet frame (1514) + slack +#define USB_ECM_FRAME_MAX 2048 //>= ETHERNET_FRAME_SIZE, the net stack's buffer #define USB_ECM_RXQ 16 //RX ring depth (power of two) +#define USB_ECM_TXQ 16 //TX ring depth (power of two) class CUsbEcmFrame { @@ -23,32 +23,25 @@ class CUsbEcm Bool valid, link_up; CUsbDev *dev; CUsbEP *bulk_in, *bulk_out, *int_ep; - I64 data_if_num, comm_if_num, data_alt, - lock; + I64 data_if_num, comm_if_num, data_alt; U8 mac[6]; - U8 *tx_buf; //bulk OUT DMA staging -//RX ring: producer = UsbEcmRxCB (USB task), consumer = UsbEcmRecv (net task). +//RX ring: UsbEcmRxCB produces (USB task), UsbEcmRecv consumes (net task). CUsbEcmFrame *rxq; I64 rxq_head, //consumer index rxq_tail, //producer index rx_drops; +//TX ring: any caller produces (incl. interrupt context), UsbEcmTick consumes. +//A slot's len is written last and cleared when sent, so it doubles as "ready". + CUsbEcmFrame *txq; + I64 txq_head, //consumer index + txq_tail, //producer index + tx_drops; }; CUsbEcm usb_ecms[USB_ECMS_MAX]; -U0 UsbEcmLock(CUsbEcm *ecm) -{ - while (LBts(&ecm->lock, 0)) - Yield; -} - -U0 UsbEcmUnlock(CUsbEcm *ecm) -{ - LBtr(&ecm->lock, 0); -} - CUsbEcm *UsbEcmFirst() -{//First bound NIC, else NULL. v1 supports a single active NIC. +{//First bound NIC, else NULL. I64 i; for (i = 0; i < USB_ECMS_MAX; i++) @@ -91,8 +84,7 @@ Bool UsbEcmMacParse(CUsbDev *dev, I64 str_idx, U8 *mac) } U0 UsbEcmRxCB(U8 *_dev, U8 *_ep) -{//Bulk-IN completion (USB task). Copy the frame into the RX ring and requeue -//happens back in XhciEventPoll. Received length = auto_len - residue. +{//Bulk-IN completion. Length = auto_len - residue; XhciEventPoll requeues. CUsbDev *dev = _dev; CUsbEP *ep = _ep; CUsbEcm *ecm = dev->drv_data; @@ -119,25 +111,67 @@ U0 UsbEcmRxCB(U8 *_dev, U8 *_ep) } public I64 UsbEcmSend(U8 *frame, I64 len) -{//Send one ethernet frame over bulk OUT. 0 = ok, <0 = error. - CUsbEcm *ecm = UsbEcmFirst; - I64 code; +{//Queue one ethernet frame for transmit. Returns 0 if queued, <0 if not. +//Callable from interrupt context (the net stack ACKs from NetHandler), so this +//never blocks: UsbEcmTick performs the transfer. + CUsbEcm *ecm = UsbEcmFirst; + CUsbEcmFrame *slot; + I64 idx, next; if (!ecm || !ecm->valid || len <= 0 || len > USB_ECM_FRAME_MAX) return -1; - UsbEcmLock(ecm); - MemCopy(ecm->tx_buf, frame, len); - code = XhciBulkXfer(ecm->dev, ecm->bulk_out, ecm->tx_buf, len, 1.0); - if (code == XHCI_CC_STALL) + PUSHFD + CLI + idx = ecm->txq_tail; + next = (idx + 1) & (USB_ECM_TXQ - 1); + if (next == ecm->txq_head) { - UsbClearHalt(ecm->dev, ecm->bulk_out); - code = XhciBulkXfer(ecm->dev, ecm->bulk_out, ecm->tx_buf, len, 1.0); + POPFD + ecm->tx_drops++; + return -1; } - UsbEcmUnlock(ecm); - if (code == XHCI_CC_SUCCESS || code == XHCI_CC_SHORT_PKT) - return 0; + ecm->txq_tail = next; + POPFD + slot = &ecm->txq[idx]; + MemCopy(slot->data, frame, len); + slot->len = len; //publish last: the consumer waits on a non-zero len - return -1; + return 0; +} + +U0 UsbEcmTick() +{//Drain the TX ring onto bulk OUT. USB task only. + CUsbEcm *ecm; + CUsbEcmFrame *slot; + I64 i, len, code; + + for (i = 0; i < USB_ECMS_MAX; i++) + { + ecm = &usb_ecms[i]; + if (!ecm->valid) + goto next_ecm; + while (ecm->txq_head != ecm->txq_tail) + { + slot = &ecm->txq[ecm->txq_head]; + len = slot->len; + if (!len) + break; //producer still copying + code = XhciBulkXfer(ecm->dev, ecm->bulk_out, slot->data, len, 1.0); + if (code == XHCI_CC_STALL) + { + UsbClearHalt(ecm->dev, ecm->bulk_out); + code = XhciBulkXfer(ecm->dev, ecm->bulk_out, slot->data, len, 1.0); + } +//A frame that exactly fills the last packet needs a zero-length packet to mark +//the end of the transfer, or the device keeps waiting for more. + if ((code == XHCI_CC_SUCCESS || code == XHCI_CC_SHORT_PKT) && + !(len % ecm->bulk_out->max_packet)) + XhciBulkXfer(ecm->dev, ecm->bulk_out, slot->data, 0, 1.0); + slot->len = 0; + ecm->txq_head = (ecm->txq_head + 1) & (USB_ECM_TXQ - 1); + } +next_ecm:; + } } public I64 UsbEcmRecv(U8 *dst, I64 max) @@ -168,7 +202,7 @@ public Bool UsbEcmPresent() } public U0 UsbEcmRep() -{//Report USB NIC state: presence, MAC, ring stats. On-demand (boot log scrolls). +{//Report NIC state: link, MAC, endpoints, ring stats. CUsbEcm *ecm = UsbEcmFirst; I64 pending; @@ -182,9 +216,11 @@ public U0 UsbEcmRep() ecm->dev->port, ecm->link_up, ecm->mac[0], ecm->mac[1], ecm->mac[2], ecm->mac[3], ecm->mac[4], ecm->mac[5]; - " data_if=%d alt=%d bulk_in=0x%02X bulk_out=0x%02X rxq=%d rx_drops=%d\n", + " data_if=%d alt=%d bulk_in=0x%02X bulk_out=0x%02X\n", ecm->data_if_num, ecm->data_alt, - ecm->bulk_in->addr, ecm->bulk_out->addr, pending, ecm->rx_drops; + ecm->bulk_in->addr, ecm->bulk_out->addr; + " rxq=%d rx_drops=%d txq=%d tx_drops=%d\n", pending, ecm->rx_drops, + (ecm->txq_tail - ecm->txq_head) & (USB_ECM_TXQ - 1), ecm->tx_drops; } public U0 UsbEcmMacAddr(U8 *out6) @@ -198,8 +234,8 @@ public U0 UsbEcmMacAddr(U8 *out6) } U0 UsbEcmDetach(CUsbDev *dev) -{//Release NIC state on disconnect. Keep DMA/ring allocations for reuse. -//Scan our own array by dev (UsbDrvUnbind calls every class detacher). +{//Release NIC state on disconnect; keep allocations. Matched by dev, since +//UsbDrvUnbind calls every class detacher. I64 i; for (i = 0; i < USB_ECMS_MAX; i++) @@ -212,8 +248,8 @@ U0 UsbEcmDetach(CUsbDev *dev) } U0 UsbEcmAttach(CUsbDev *dev, CUsbIf *uif) -{//uif = the CDC-ECM control (comm) interface. Pair it with the CDC-Data -//interface, activate its bulk endpoints, read the MAC, start RX. +{//uif = CDC-ECM control interface. Pairs it with the CDC-Data interface, +//activates its bulk endpoints, reads the MAC, starts RX. CUsbEcm *ecm = NULL; CUsbIf *data_if = NULL; CUsbEP *ep; @@ -231,7 +267,7 @@ U0 UsbEcmAttach(CUsbDev *dev, CUsbIf *uif) return; } -//Locate the CDC-Data interface (class 0x0A) that carries the bulk endpoints. +//CDC-Data interface carries the bulk endpoints. for (i = 0; i < dev->if_count; i++) if (dev->ifs[i].valid && dev->ifs[i].if_class == USB_CLASS_CDC_DATA && dev->ifs[i].ep_count) @@ -245,8 +281,8 @@ U0 UsbEcmAttach(CUsbDev *dev, CUsbIf *uif) return; } -//Clear state but keep persistent DMA + ring allocations. - MemSet(ecm, 0, offset(CUsbEcm.tx_buf)); +//Clear state, keep allocations. + MemSet(ecm, 0, offset(CUsbEcm.rxq)); ecm->dev = dev; ecm->comm_if_num = uif->if_num; ecm->data_if_num = data_if->if_num; @@ -263,7 +299,7 @@ U0 UsbEcmAttach(CUsbDev *dev, CUsbIf *uif) ecm->bulk_out = ep; } } -//Notification endpoint (INT IN) on the comm interface; optional for v1. +//Notification endpoint; unused. for (i = 0; i < uif->ep_count; i++) { ep = &dev->ep_list[uif->ep_idxs[i]]; @@ -276,7 +312,7 @@ U0 UsbEcmAttach(CUsbDev *dev, CUsbIf *uif) return; } -//Activate the data interface's alt setting that exposes the bulk endpoints. +//Bulk endpoints live on a non-zero alt setting. if (ecm->data_alt) { code = UsbSetInterface(dev, ecm->data_if_num, ecm->data_alt); @@ -285,34 +321,36 @@ U0 UsbEcmAttach(CUsbDev *dev, CUsbIf *uif) dev->port, ecm->data_alt, code); } -//MAC from the ECM functional descriptor's iMACAddress string. +//MAC from the ECM functional descriptor. if (!uif->ecm_mac_str_idx || !UsbEcmMacParse(dev, uif->ecm_mac_str_idx, ecm->mac)) MemSet(ecm->mac, 0, 6); -//Tell the device which frames to deliver: directed + broadcast + multicast. +//Frames to deliver: directed + broadcast + multicast. UsbCtrl(dev, USB_RT_HOST_TO_DEV | USB_RT_CLASS | USB_RT_RECIP_IF, USB_CDC_RQ_SET_ETH_PACKET_FILTER, USB_CDC_PACKET_TYPE_DIRECTED | USB_CDC_PACKET_TYPE_BROADCAST | USB_CDC_PACKET_TYPE_ALL_MULTICAST, ecm->comm_if_num, NULL, 0); - if (!ecm->tx_buf) - ecm->tx_buf = CAlloc(USB_ECM_FRAME_MAX, sys_task->code_heap); if (!ecm->rxq) ecm->rxq = CAlloc(sizeof(CUsbEcmFrame) * USB_ECM_RXQ, sys_task->code_heap); + if (!ecm->txq) + ecm->txq = CAlloc(sizeof(CUsbEcmFrame) * USB_ECM_TXQ, sys_task->code_heap); ecm->rxq_head = 0; ecm->rxq_tail = 0; + ecm->txq_head = 0; + ecm->txq_tail = 0; -//Arm the bulk-IN endpoint for auto-requeued receive (like HID interrupt IN). +//Arm bulk-IN for auto-requeued receive. if (!ecm->bulk_in->auto_buf) ecm->bulk_in->auto_buf = CAlloc(USB_ECM_FRAME_MAX, sys_task->code_heap); ecm->bulk_in->auto_len = USB_ECM_FRAME_MAX; ecm->bulk_in->auto_kind = USB_AUTO_ECM; ecm->valid = TRUE; - ecm->link_up = TRUE; //assume up; notification EP can refine later + ecm->link_up = TRUE; //no notification-endpoint link tracking dev->drv_data = ecm; - XhciIntQueue(dev, ecm->bulk_in); //kick off first receive + XhciIntQueue(dev, ecm->bulk_in); //first receive SysLog("USB: ECM port %d: NIC ready, MAC %02X:%02X:%02X:%02X:%02X:%02X\n", dev->port, ecm->mac[0], ecm->mac[1], ecm->mac[2], diff --git a/src/Kernel/Usb/KUsbHub.ZC b/src/Kernel/Usb/KUsbHub.ZC index 819072f6a..754adbf53 100644 --- a/src/Kernel/Usb/KUsbHub.ZC +++ b/src/Kernel/Usb/KUsbHub.ZC @@ -1,11 +1,5 @@ -//USB hub class driver: enumerate devices attached to downstream hub ports. -//Needed because USB NICs (and many devices) sit behind hubs - both QEMU's -//usb-net and the RTL8153 USB-C adapter enumerate through a hub tier. -// -//v1: at attach time we power every downstream port, then reset + enumerate -//whatever is already connected. No status-change-endpoint hotplug polling yet. -//Downstream devices are addressed with an xHCI route string + the parent hub's -//slot/port for TT (see UsbDevEnumEx / XhciAddressDev). +//USB hub class driver. Enumerates devices on downstream ports at attach time; +//no status-change-endpoint hotplug polling. I64 UsbHubGetPortStatus(CUsbDev *hub, I64 port, U32 *_status) {//wPortStatus (low 16) + wPortChange (high 16). @@ -67,14 +61,14 @@ keep_waiting:; } U0 UsbHubAttach(CUsbDev *dev, CUsbIf *uif) -{//Configure the hub, power its ports, enumerate connected downstream devices. +{//Configure the hub, power its ports, enumerate what's connected. CUsbDev *ndev; U8 buf[16]; U32 st; I64 i, code, num_ports, hub_char, desc_type, child_speed, child_route; no_warn uif; -//xHCI route strings are 5 nibbles: a hub deeper than tier 5 can't route children. +//Route strings are 5 nibbles. if (dev->depth >= 5) { SysLog("USB: hub port %d: max hub depth reached, ignoring.\n", dev->port); @@ -97,25 +91,27 @@ U0 UsbHubAttach(CUsbDev *dev, CUsbIf *uif) dev->is_hub = TRUE; dev->hub_ports = num_ports; dev->hub_tt_think = (hub_char >> 5) & 3; - dev->hub_mtt = 0; //v1: single-TT (default hub alt setting) + dev->hub_mtt = 0; //single-TT -//Re-issue Configure Endpoint so the HC learns this slot is a hub (Hub bit + -//port count + TT think time now set in XhciConfigureEPs). Direct-print the -//result: if this fails, downstream Address Device gets a Transaction Error. +//Re-Configure so the HC sees the Hub bit, port count and TT think time. code = XhciConfigureEPs(dev); - "USB: hub port %d: cfg-as-hub code=%d, ports=%d, TTT=%d, depth=%d, speed=%d\n", - dev->port, code, num_ports, dev->hub_tt_think, dev->depth, dev->speed; + if (code != XHCI_CC_SUCCESS) + SysLog("USB: hub port %d: configure-as-hub failed (%d).\n", dev->port, code); -//SuperSpeed hubs route by route string only after being told their tier -//(USB 3.x 10.14.2.7 SetHubDepth). Skipping this makes every downstream -//Address Device fail with a Transaction Error. USB2 hubs don't use it. +//A SuperSpeed hub won't route by route string until told its tier (USB 3.x +//10.14.2.7). Without this every downstream Address Device fails with a +//Transaction Error. USB2 hubs don't use it. if (dev->speed == USB_SPEED_SUPER) { code = UsbCtrl(dev, USB_RT_HOST_TO_DEV | USB_RT_CLASS | USB_RT_RECIP_DEV, USB_HUB_RQ_SET_DEPTH, dev->depth, 0, NULL, 0); - "USB: hub port %d: SET_HUB_DEPTH(%d) code=%d\n", dev->port, dev->depth, code; + if (code != XHCI_CC_SUCCESS) + SysLog("USB: hub port %d: SET_HUB_DEPTH failed (%d).\n", dev->port, code); } + SysLog("USB: hub port %d: %d ports, TTT=%d, depth=%d.\n", + dev->port, num_ports, dev->hub_tt_think, dev->depth); + for (i = 1; i <= num_ports; i++) UsbHubSetPortFeature(dev, i, USB_HUB_F_PORT_POWER); Sleep(100); //power-good settle @@ -131,14 +127,14 @@ U0 UsbHubAttach(CUsbDev *dev, CUsbIf *uif) Sleep(100); //connect debounce if (!UsbHubPortReset(dev, i, &child_speed)) { - "USB: hub port %d.%d: reset failed.\n", dev->port, i; + SysLog("USB: hub port %d.%d: reset failed.\n", dev->port, i); goto next_port; } - Sleep(50); //post-reset recovery (SS link training needs more than 10ms) -//Child route: append this port number at the hub's tier nibble. + Sleep(50); //post-reset recovery; SS link training needs >10ms +//Append this port number at the hub's tier nibble. child_route = dev->route | (i << (4 * dev->depth)); ndev = UsbDevEnumEx(dev->port, child_speed, child_route, dev->depth + 1, - dev->slot, i); + dev->slot, i, dev->hc); if (ndev) UsbDrvBind(ndev); next_port:; diff --git a/src/Kernel/Usb/KXhci.ZC b/src/Kernel/Usb/KXhci.ZC index d9ad24e2a..8ddcd5f89 100644 --- a/src/Kernel/Usb/KXhci.ZC +++ b/src/Kernel/Usb/KXhci.ZC @@ -1,82 +1,108 @@ //xHCI (USB3) host controller driver. Polled, no MSI. //Commands are issued from the USB task, or from the boot task before it exists. -//Sync transfers may be awaited from any task; XhciEventPoll is the single -//event ring consumer, guarded by usb_xhci.poll_lock so the boot task and the -//USB task can never drain it concurrently. - -CXhciGlobals usb_xhci; +//Sync transfers may be awaited from any task; a per-controller poll_lock keeps +//the boot task and the USB task from draining an event ring concurrently. +// +//All controllers run at once: a machine has several (Thunderbolt/USB4 plus PCH) +//and a USB-C dock can land an ethernet NIC on one while HID sits on another. +//usb_xhci is only the default for calls that don't name a controller. +CXhciGlobals usb_hcs[USB_HCS_MAX]; +I64 usb_hc_count = 0; +CXhciGlobals *usb_xhci = &usb_hcs[0]; CUsbDev usb_devs[USB_DEVS_MAX]; CTask *usb_task = NULL; U8 usb_boot_iso_drv_let = 0; -U32 XhciCapR32(I64 off) +//Controller-scoped calls take an hc; NULL means the active controller. Code +//holding a CUsbDev passes dev->hc, since slot ids and port numbers are only +//unique within one controller. +U32 XhciCapR32(I64 off, CXhciGlobals *hc=NULL) { - U32 *p = usb_xhci.mmio + off; + U32 *p; + + if (!hc) hc = usb_xhci; + p = hc->mmio + off; return *p; } -U32 XhciOpR32(I64 off) +U32 XhciOpR32(I64 off, CXhciGlobals *hc=NULL) { - U32 *p = usb_xhci.op + off; + U32 *p; + + if (!hc) hc = usb_xhci; + p = hc->op + off; return *p; } -U0 XhciOpW32(I64 off, I64 val) +U0 XhciOpW32(I64 off, I64 val, CXhciGlobals *hc=NULL) { - U32 *p = usb_xhci.op + off; + U32 *p; + if (!hc) hc = usb_xhci; + p = hc->op + off; *p = val; } -U0 XhciOpW64(I64 off, I64 val) +U0 XhciOpW64(I64 off, I64 val, CXhciGlobals *hc=NULL) { - U64 *p = usb_xhci.op + off; + U64 *p; + if (!hc) hc = usb_xhci; + p = hc->op + off; *p = val; } -U32 XhciRtR32(I64 off) +U32 XhciRtR32(I64 off, CXhciGlobals *hc=NULL) { - U32 *p = usb_xhci.rt + off; + U32 *p; + + if (!hc) hc = usb_xhci; + p = hc->rt + off; return *p; } -U0 XhciRtW32(I64 off, I64 val) +U0 XhciRtW32(I64 off, I64 val, CXhciGlobals *hc=NULL) { - U32 *p = usb_xhci.rt + off; + U32 *p; + if (!hc) hc = usb_xhci; + p = hc->rt + off; *p = val; } -U0 XhciRtW64(I64 off, I64 val) +U0 XhciRtW64(I64 off, I64 val, CXhciGlobals *hc=NULL) { - U64 *p = usb_xhci.rt + off; + U64 *p; + if (!hc) hc = usb_xhci; + p = hc->rt + off; *p = val; } -U32 XhciPortSC(I64 port) +U32 XhciPortSC(I64 port, CXhciGlobals *hc=NULL) { - return XhciOpR32(XHCI_OP_PORTSC_BASE + 0x10 * port); + return XhciOpR32(XHCI_OP_PORTSC_BASE + 0x10 * port, hc); } -U0 XhciPortSCWrite(I64 port, I64 val) +U0 XhciPortSCWrite(I64 port, I64 val, CXhciGlobals *hc=NULL) { - XhciOpW32(XHCI_OP_PORTSC_BASE + 0x10 * port, val); + XhciOpW32(XHCI_OP_PORTSC_BASE + 0x10 * port, val, hc); } -U0 XhciLock() +U0 XhciLock(CXhciGlobals *hc=NULL) { - while (LBts(&usb_xhci.lock, 0)) + if (!hc) hc = usb_xhci; + while (LBts(&hc->lock, 0)) Yield; } -U0 XhciUnlock() +U0 XhciUnlock(CXhciGlobals *hc=NULL) { - LBtr(&usb_xhci.lock, 0); + if (!hc) hc = usb_xhci; + LBtr(&hc->lock, 0); } U0 XhciRingInit(CXhciRing *ring) @@ -123,23 +149,37 @@ CXhciTRB *XhciRingEnq(CXhciRing *ring, I64 lo, I64 hi, I64 status, I64 control) return trb; } -CUsbDev *UsbDevBySlot(I64 slot) +I64 XhciHcIdx(CXhciGlobals *hc) +{//Index of a controller in usb_hcs, or -1. + I64 n; + + for (n = 0; n < usb_hc_count; n++) + if (&usb_hcs[n] == hc) + return n; + + return -1; +} + +CUsbDev *UsbDevBySlot(I64 slot, CXhciGlobals *hc=NULL) { I64 i; + if (!hc) hc = usb_xhci; for (i = 0; i < USB_DEVS_MAX; i++) - if (usb_devs[i].present && usb_devs[i].slot == slot) + if (usb_devs[i].present && usb_devs[i].hc == hc && usb_devs[i].slot == slot) return &usb_devs[i]; return NULL; } -CUsbDev *UsbDevByPort(I64 port) -{ +CUsbDev *UsbDevByPort(I64 port, CXhciGlobals *hc=NULL) +{//Root-port devices only; hub children share their root port number. I64 i; + if (!hc) hc = usb_xhci; for (i = 0; i < USB_DEVS_MAX; i++) - if (usb_devs[i].present && usb_devs[i].port == port) + if (usb_devs[i].present && usb_devs[i].hc == hc && + usb_devs[i].port == port && !usb_devs[i].depth) return &usb_devs[i]; return NULL; @@ -159,57 +199,58 @@ CUsbEP *UsbDevEPByDCI(CUsbDev *dev, I64 dci) } U0 XhciIntQueue(CUsbDev *dev, CUsbEP *ep) -{//Queue one interrupt IN TRB on ep, report dispatched via ep->auto_kind. - XhciLock; +{//Queue one interrupt IN TRB on ep; report dispatched via ep->auto_kind. + CXhciGlobals *hc = dev->hc; + + XhciLock(hc); ep->pending_trb = XhciRingEnq(ep->ring, ep->auto_buf, 0, ep->auto_len, (XHCI_TRB_NORMAL << 10) | XHCI_TRBF_IOC | XHCI_TRBF_ISP); - usb_xhci.db[dev->slot] = ep->dci; - XhciUnlock; + hc->db[dev->slot] = ep->dci; + XhciUnlock(hc); } -U0 XhciEventPoll() -{//Drain event ring. Called ONLY in USB task context. +U0 XhciEventPoll(CXhciGlobals *hc=NULL) +{//Drain one controller's event ring. USB task, or an inline waiter. CXhciTRB *e; CUsbDev *dev; CUsbEP *ep; I64 type, code, slot, dci, addr, work = FALSE; U64 *erdp; - if (!usb_xhci.running) + if (!hc) hc = usb_xhci; + if (!hc->running) return; - while (LBts(&usb_xhci.poll_lock, 0)) + while (LBts(&hc->poll_lock, 0)) Yield; while (TRUE) { - e = &usb_xhci.evt_ring[usb_xhci.evt_deq]; - if ((e->control & 1) != usb_xhci.evt_cycle) + e = &hc->evt_ring[hc->evt_deq]; + if ((e->control & 1) != hc->evt_cycle) break; work = TRUE; type = (e->control >> 10) & 0x3F; switch (type) { case XHCI_TRB_EVT_CMD_DONE: - usb_xhci.cmd_code = (e->status >> 24) & 0xFF; - usb_xhci.cmd_slot = (e->control >> 24) & 0xFF; - addr = usb_xhci.cmd_trb; - if (usb_xhci.cmd_trb && e->lo == addr & 0xFFFFFFFF) - usb_xhci.cmd_done = TRUE; + hc->cmd_code = (e->status >> 24) & 0xFF; + hc->cmd_slot = (e->control >> 24) & 0xFF; + addr = hc->cmd_trb; + if (hc->cmd_trb && e->lo == addr & 0xFFFFFFFF) + hc->cmd_done = TRUE; break; case XHCI_TRB_EVT_TRANSFER: slot = (e->control >> 24) & 0xFF; dci = (e->control >> 16) & 0x1F; code = (e->status >> 24) & 0xFF; - dev = UsbDevBySlot(slot); + dev = UsbDevBySlot(slot, hc); if (dev) { ep = UsbDevEPByDCI(dev, dci); if (ep) { if (ep->auto_kind) - {//HID interrupt IN / ECM bulk IN: deliver + requeue. - //Stash residue so byte-count-sensitive handlers (ECM) can - //compute the received length (auto_len - residue). + {//Deliver + requeue. Residue lets ECM derive the length. ep->xfer_residue = e->status & 0xFFFFFF; if (code == XHCI_CC_SUCCESS || code == XHCI_CC_SHORT_PKT) { @@ -239,57 +280,59 @@ U0 XhciEventPoll() break; case XHCI_TRB_EVT_PORT_SC: - usb_xhci.port_change = TRUE; + hc->port_change = TRUE; break; } - usb_xhci.evt_deq++; - if (usb_xhci.evt_deq == XHCI_EVT_RING_TRBS) + hc->evt_deq++; + if (hc->evt_deq == XHCI_EVT_RING_TRBS) { - usb_xhci.evt_deq = 0; - usb_xhci.evt_cycle ^= 1; + hc->evt_deq = 0; + hc->evt_cycle ^= 1; } } if (work) { - addr = usb_xhci.evt_ring + usb_xhci.evt_deq; - erdp = usb_xhci.rt + XHCI_RT_ERDP0; + addr = hc->evt_ring + hc->evt_deq; + erdp = hc->rt + XHCI_RT_ERDP0; *erdp = addr | 8; //8 = clear Event Handler Busy } - LBtr(&usb_xhci.poll_lock, 0); + LBtr(&hc->poll_lock, 0); } -I64 XhciCmd(I64 lo, I64 hi, I64 status, I64 control, I64 *_slot=NULL) +I64 XhciCmd(I64 lo, I64 hi, I64 status, I64 control, I64 *_slot=NULL, + CXhciGlobals *hc=NULL) {//Issue one command TRB, wait completion. USB task only. F64 timeout; - XhciLock; - usb_xhci.cmd_done = FALSE; - usb_xhci.cmd_code = XHCI_CC_TIMEOUT; - usb_xhci.cmd_trb = XhciRingEnq(&usb_xhci.cmd_ring, lo, hi, status, control); - usb_xhci.db[0] = 0; - XhciUnlock; + if (!hc) hc = usb_xhci; + XhciLock(hc); + hc->cmd_done = FALSE; + hc->cmd_code = XHCI_CC_TIMEOUT; + hc->cmd_trb = XhciRingEnq(&hc->cmd_ring, lo, hi, status, control); + hc->db[0] = 0; + XhciUnlock(hc); timeout = tS + 1.0; - while (!usb_xhci.cmd_done && tS < timeout) + while (!hc->cmd_done && tS < timeout) { - XhciEventPoll; + XhciEventPoll(hc); Yield; } if (_slot) - *_slot = usb_xhci.cmd_slot; - if (!usb_xhci.cmd_done) + *_slot = hc->cmd_slot; + if (!hc->cmd_done) return XHCI_CC_TIMEOUT; - return usb_xhci.cmd_code; + return hc->cmd_code; } -I64 XhciXferWait(CUsbEP *ep, F64 timeout_s) +I64 XhciXferWait(CUsbEP *ep, F64 timeout_s, CXhciGlobals *hc=NULL) { F64 timeout = tS + timeout_s; while (!ep->xfer_done && tS < timeout) { if (Fs == usb_task || !usb_task) - XhciEventPoll; + XhciEventPoll(hc); Yield; } if (!ep->xfer_done) @@ -300,8 +343,9 @@ I64 XhciXferWait(CUsbEP *ep, F64 timeout_s) I64 XhciCtrlXfer(CUsbDev *dev, I64 rt, I64 rq, I64 val, I64 idx, U8 *buf, I64 len) {//Control transfer on EP0. Returns completion code. - CUsbEP *ep = &dev->ep0; - I64 trt, ctl, addr; + CUsbEP *ep = &dev->ep0; + CXhciGlobals *hc = dev->hc; + I64 trt, ctl, addr; if (len) { @@ -312,7 +356,7 @@ I64 XhciCtrlXfer(CUsbDev *dev, I64 rt, I64 rq, I64 val, I64 idx, U8 *buf, I64 le } else trt = XHCI_TRT_NO_DATA; - XhciLock; + XhciLock(hc); ep->xfer_done = FALSE; ep->xfer_code = XHCI_CC_TIMEOUT; ep->xfer_residue = 0; @@ -330,25 +374,26 @@ I64 XhciCtrlXfer(CUsbDev *dev, I64 rt, I64 rq, I64 val, I64 idx, U8 *buf, I64 le if (!len || !(rt & USB_RT_DEV_TO_HOST)) ctl |= XHCI_TRBF_DIR_IN; //status stage opposite dir; IN if no data XhciRingEnq(ep->ring, 0, 0, 0, ctl); - usb_xhci.db[dev->slot] = 1; - XhciUnlock; + hc->db[dev->slot] = 1; + XhciUnlock(hc); - return XhciXferWait(ep, 1.0); + return XhciXferWait(ep, 1.0, hc); } I64 XhciBulkXfer(CUsbDev *dev, CUsbEP *ep, U8 *buf, I64 len, F64 timeout_s, I64 *_residue=NULL) {//Single-TRB bulk transfer (max 64KB). Returns completion code. - I64 addr = buf, code; + CXhciGlobals *hc = dev->hc; + I64 addr = buf, code; - XhciLock; + XhciLock(hc); ep->xfer_done = FALSE; ep->xfer_code = XHCI_CC_TIMEOUT; ep->xfer_residue = 0; XhciRingEnq(ep->ring, addr & 0xFFFFFFFF, addr >> 32, len, (XHCI_TRB_NORMAL << 10) | XHCI_TRBF_IOC | XHCI_TRBF_ISP); - usb_xhci.db[dev->slot] = ep->dci; - XhciUnlock; - code = XhciXferWait(ep, timeout_s); + hc->db[dev->slot] = ep->dci; + XhciUnlock(hc); + code = XhciXferWait(ep, timeout_s, hc); if (_residue) *_residue = ep->xfer_residue; @@ -357,14 +402,14 @@ I64 XhciBulkXfer(CUsbDev *dev, CUsbEP *ep, U8 *buf, I64 len, F64 timeout_s, I64 U32 *XhciInputCtx(CUsbDev *dev, I64 idx) {//idx: 0=input control ctx, 1=slot ctx, 1+dci=EP ctx - U32 *p = dev->in_ctx + idx * usb_xhci.ctx_size; + U32 *p = dev->in_ctx + idx * dev->hc->ctx_size; return p; } U32 *XhciDevCtxGet(CUsbDev *dev, I64 idx) {//idx: 0=slot ctx, dci=EP ctx - U32 *p = dev->dev_ctx + idx * usb_xhci.ctx_size; + U32 *p = dev->dev_ctx + idx * dev->hc->ctx_size; return p; } @@ -417,35 +462,36 @@ I64 XhciEPCtxType(CUsbEP *ep) return XHCI_EP_TYPE_CTRL; } -I64 XhciEnableSlot() +I64 XhciEnableSlot(CXhciGlobals *hc=NULL) {//Returns slot id, or 0 on failure. I64 slot = 0, code; - code = XhciCmd(0, 0, 0, XHCI_TRB_ENABLE_SLOT << 10, &slot); + code = XhciCmd(0, 0, 0, XHCI_TRB_ENABLE_SLOT << 10, &slot, hc); if (code != XHCI_CC_SUCCESS) return 0; return slot; } -U0 XhciDisableSlot(I64 slot) +U0 XhciDisableSlot(I64 slot, CXhciGlobals *hc=NULL) { - XhciCmd(0, 0, 0, (XHCI_TRB_DISABLE_SLOT << 10) | (slot << 24)); + XhciCmd(0, 0, 0, (XHCI_TRB_DISABLE_SLOT << 10) | (slot << 24),, hc); } I64 XhciAddressDev(CUsbDev *dev) {//Alloc contexts + EP0 ring, issue Address Device. Returns completion code. - U32 *icc, *slot_ctx, *ep0_ctx; - I64 mps, addr; + CXhciGlobals *hc = dev->hc; + U32 *icc, *slot_ctx, *ep0_ctx; + I64 mps, addr; if (!dev->dev_ctx) - dev->dev_ctx = CAllocAligned(usb_xhci.ctx_size * 32, 0x1000, sys_task->code_heap); + dev->dev_ctx = CAllocAligned(hc->ctx_size * 32, 0x1000, sys_task->code_heap); else - MemSet(dev->dev_ctx, 0, usb_xhci.ctx_size * 32); + MemSet(dev->dev_ctx, 0, hc->ctx_size * 32); if (!dev->in_ctx) - dev->in_ctx = CAllocAligned(usb_xhci.ctx_size * 33, 0x1000, sys_task->code_heap); + dev->in_ctx = CAllocAligned(hc->ctx_size * 33, 0x1000, sys_task->code_heap); else - MemSet(dev->in_ctx, 0, usb_xhci.ctx_size * 33); + MemSet(dev->in_ctx, 0, hc->ctx_size * 33); XhciEPRingInit(&dev->ep0); dev->ep0.dci = 1; dev->ep0.valid = TRUE; @@ -483,39 +529,41 @@ I64 XhciAddressDev(CUsbDev *dev) ep0_ctx[3] = addr >> 32; ep0_ctx[4] = 8; - usb_xhci.dcbaa[dev->slot] = dev->dev_ctx; + hc->dcbaa[dev->slot] = dev->dev_ctx; addr = dev->in_ctx; return XhciCmd(addr & 0xFFFFFFFF, addr >> 32, 0, - (XHCI_TRB_ADDRESS_DEV << 10) | (dev->slot << 24)); + (XHCI_TRB_ADDRESS_DEV << 10) | (dev->slot << 24),, hc); } I64 XhciEvalCtxMPS0(CUsbDev *dev, I64 mps) {//Update EP0 max packet size after reading device descriptor. - U32 *icc, *ep0_ctx; - I64 addr; + CXhciGlobals *hc = dev->hc; + U32 *icc, *ep0_ctx; + I64 addr; - MemSet(dev->in_ctx, 0, usb_xhci.ctx_size * 33); + MemSet(dev->in_ctx, 0, hc->ctx_size * 33); icc = XhciInputCtx(dev, 0); icc[0] = 0; icc[1] = 2; //A1 = EP0 ep0_ctx = XhciInputCtx(dev, 2); - MemCopy(ep0_ctx, XhciDevCtxGet(dev, 1), usb_xhci.ctx_size); + MemCopy(ep0_ctx, XhciDevCtxGet(dev, 1), hc->ctx_size); ep0_ctx[1] = (ep0_ctx[1] & 0xFFFF) | (mps << 16); dev->ep0.max_packet = mps; addr = dev->in_ctx; return XhciCmd(addr & 0xFFFFFFFF, addr >> 32, 0, - (XHCI_TRB_EVAL_CTX << 10) | (dev->slot << 24)); + (XHCI_TRB_EVAL_CTX << 10) | (dev->slot << 24),, hc); } I64 XhciConfigureEPs(CUsbDev *dev) {//Build input context with all parsed EPs, issue Configure Endpoint. - U32 *icc, *slot_ctx, *ep_ctx; - CUsbEP *ep; - I64 i, addr, max_dci = 1, type, ival; + CXhciGlobals *hc = dev->hc; + U32 *icc, *slot_ctx, *ep_ctx; + CUsbEP *ep; + I64 i, addr, max_dci = 1, type, ival; - MemSet(dev->in_ctx, 0, usb_xhci.ctx_size * 33); + MemSet(dev->in_ctx, 0, hc->ctx_size * 33); icc = XhciInputCtx(dev, 0); icc[0] = 0; icc[1] = 1; //A0 = slot @@ -543,7 +591,7 @@ I64 XhciConfigureEPs(CUsbDev *dev) next_ep:; } slot_ctx = XhciInputCtx(dev, 1); - MemCopy(slot_ctx, XhciDevCtxGet(dev, 0), usb_xhci.ctx_size); + MemCopy(slot_ctx, XhciDevCtxGet(dev, 0), hc->ctx_size); slot_ctx[0] = (slot_ctx[0] & 0x07FFFFFF) | (max_dci << 27); //Hub: advertise it to the HC so it routes traffic to downstream ports. if (dev->is_hub) @@ -557,32 +605,36 @@ next_ep:; addr = dev->in_ctx; return XhciCmd(addr & 0xFFFFFFFF, addr >> 32, 0, - (XHCI_TRB_CONFIG_EP << 10) | (dev->slot << 24)); + (XHCI_TRB_CONFIG_EP << 10) | (dev->slot << 24),, hc); } I64 XhciResetEP(CUsbDev *dev, CUsbEP *ep) {//Recover a halted endpoint: Reset EP + Set TR Dequeue to fresh ring. - I64 code, addr; + CXhciGlobals *hc = dev->hc; + I64 code, addr; - code = XhciCmd(0, 0, 0, (XHCI_TRB_RESET_EP << 10) | (ep->dci << 16) | (dev->slot << 24)); + code = XhciCmd(0, 0, 0, + (XHCI_TRB_RESET_EP << 10) | (ep->dci << 16) | (dev->slot << 24),, hc); XhciEPRingInit(ep); addr = ep->ring->trbs; code = XhciCmd((addr & 0xFFFFFFFF) | 1, addr >> 32, 0, - (XHCI_TRB_SET_TR_DEQ << 10) | (ep->dci << 16) | (dev->slot << 24)); + (XHCI_TRB_SET_TR_DEQ << 10) | (ep->dci << 16) | (dev->slot << 24),, hc); return code; } -U0 XhciBiosHandoff() +U0 XhciBiosHandoff(CXhciGlobals *hc=NULL) { - I64 hcc = XhciCapR32(XHCI_CAP_HCCPARAMS1), xecp, id, next; + I64 hcc, xecp, id, next; U32 *cap; F64 timeout; + if (!hc) hc = usb_xhci; + hcc = XhciCapR32(XHCI_CAP_HCCPARAMS1, hc); xecp = ((hcc >> 16) & 0xFFFF) * 4; while (xecp) { - cap = usb_xhci.mmio + xecp; + cap = hc->mmio + xecp; id = *cap & 0xFF; if (id == 1) {//USB Legacy Support: claim OS ownership @@ -599,32 +651,31 @@ U0 XhciBiosHandoff() } } -Bool XhciPortReset(I64 port) +Bool XhciPortReset(I64 port, CXhciGlobals *hc=NULL) {//Reset root port, wait for enabled. Returns TRUE if port enabled. I64 sc; F64 timeout; - XhciPortSCWrite(port, XHCI_PORTSC_PP | XHCI_PORTSC_PR); + XhciPortSCWrite(port, XHCI_PORTSC_PP | XHCI_PORTSC_PR, hc); timeout = tS + 0.5; while (tS < timeout) { - sc = XhciPortSC(port); + sc = XhciPortSC(port, hc); if (sc & XHCI_PORTSC_PED) break; Sleep(10); } //Clear change bits - XhciPortSCWrite(port, XHCI_PORTSC_PP | XHCI_PORTSC_CHANGE_BITS); - sc = XhciPortSC(port); + XhciPortSCWrite(port, XHCI_PORTSC_PP | XHCI_PORTSC_CHANGE_BITS, hc); + sc = XhciPortSC(port, hc); return ToBool(sc & XHCI_PORTSC_PED); } U8 *XhciMmioMap(I64 phys) {//UEFI parks xHCI BARs above 4GB, past the low-4GB uncached alias. ZealBooter -//stretches mem_physical_space to cover them, so the BAR is in the identity map -- -//just flip its page to uncached (PCD) and return the identity pointer. No flush: -//nothing has touched this address, so there's no stale TLB entry. +//stretches mem_physical_space to cover them, so the BAR is already identity +//mapped: flip its page to uncached. No TLB flush; the page is untouched. I64 *pte, base = phys & ~(mem_page_size - 1); pte = MemPageTable(base); @@ -632,152 +683,158 @@ U8 *XhciMmioMap(I64 phys) return phys; } -public Bool XhciInit(I64 start_n=0, Bool any=FALSE) -{//Init a usable xHCI controller at/after index start_n. A machine has several (a -//Thunderbolt/USB4 one and the PCH one wired to the real ports), so prefer one -//with a device already connected, else fall back to the first that runs. Pass a -//start_n to force a specific controller (e.g. XhciInit(1)). TRUE on success. - I64 n, bdf, bar_lo, bar_hi, addr, i, fallback = -1; - I64 hcs1, hcs2, hcc, cmd; - F64 timeout; - U8 *page; - Bool connected; +Bool XhciHcStart(CXhciGlobals *hc, I64 bdf) +{//Bring one controller up: map BAR, reset, alloc rings/contexts, run, power +//ports. TRUE if it is running. + I64 bar_lo, bar_hi, addr, i, hcs1, hcs2, hcc, cmd; + F64 timeout; + U8 *page; - for (n = start_n; TRUE; n++) - { - bdf = PCIClassFind(XHCI_PCI_CLASS, n); - if (bdf == -1) - break; - usb_xhci.pci_bus = bdf.u8[2]; - usb_xhci.pci_dev = bdf.u8[1]; - usb_xhci.pci_fun = bdf.u8[0]; - - bar_lo = PCIReadU32(usb_xhci.pci_bus, usb_xhci.pci_dev, usb_xhci.pci_fun, 0x10); + MemSet(hc, 0, sizeof(CXhciGlobals)); + hc->pci_bus = bdf.u8[2]; + hc->pci_dev = bdf.u8[1]; + hc->pci_fun = bdf.u8[0]; + + bar_lo = PCIReadU32(hc->pci_bus, hc->pci_dev, hc->pci_fun, 0x10); bar_hi = 0; if ((bar_lo & 6) == 4) //64-bit BAR - bar_hi = PCIReadU32(usb_xhci.pci_bus, usb_xhci.pci_dev, usb_xhci.pci_fun, 0x14); + bar_hi = PCIReadU32(hc->pci_bus, hc->pci_dev, hc->pci_fun, 0x14); addr = bar_hi << 32 | bar_lo & ~0xF; if (!addr) - goto xi_next; //BAR not assigned + return FALSE; //BAR not assigned //Enable memory space + bus master - cmd = PCIReadU16(usb_xhci.pci_bus, usb_xhci.pci_dev, usb_xhci.pci_fun, 4); - PCIWriteU16(usb_xhci.pci_bus, usb_xhci.pci_dev, usb_xhci.pci_fun, 4, cmd | 6); + cmd = PCIReadU16(hc->pci_bus, hc->pci_dev, hc->pci_fun, 4); + PCIWriteU16(hc->pci_bus, hc->pci_dev, hc->pci_fun, 4, cmd | 6); if (bar_hi) - usb_xhci.mmio = XhciMmioMap(addr); //BAR above 4GB gets its own mapping + hc->mmio = XhciMmioMap(addr); //BAR above 4GB gets its own mapping else - usb_xhci.mmio = dev.uncached_alias + addr; - usb_xhci.op = usb_xhci.mmio + (XhciCapR32(XHCI_CAP_CAPLENGTH) & 0xFF); - usb_xhci.rt = usb_xhci.mmio + (XhciCapR32(XHCI_CAP_RTSOFF) & ~0x1F); - usb_xhci.db = usb_xhci.mmio + (XhciCapR32(XHCI_CAP_DBOFF) & ~3); - - hcs1 = XhciCapR32(XHCI_CAP_HCSPARAMS1); - hcs2 = XhciCapR32(XHCI_CAP_HCSPARAMS2); - hcc = XhciCapR32(XHCI_CAP_HCCPARAMS1); - usb_xhci.max_slots = hcs1 & 0xFF; - usb_xhci.max_ports = (hcs1 >> 24) & 0xFF; + hc->mmio = dev.uncached_alias + addr; + hc->op = hc->mmio + (XhciCapR32(XHCI_CAP_CAPLENGTH, hc) & 0xFF); + hc->rt = hc->mmio + (XhciCapR32(XHCI_CAP_RTSOFF, hc) & ~0x1F); + hc->db = hc->mmio + (XhciCapR32(XHCI_CAP_DBOFF, hc) & ~3); + + hcs1 = XhciCapR32(XHCI_CAP_HCSPARAMS1, hc); + hcs2 = XhciCapR32(XHCI_CAP_HCSPARAMS2, hc); + hcc = XhciCapR32(XHCI_CAP_HCCPARAMS1, hc); + hc->max_slots = hcs1 & 0xFF; + hc->max_ports = (hcs1 >> 24) & 0xFF; if (hcc & 4) - usb_xhci.ctx_size = 64; + hc->ctx_size = 64; else - usb_xhci.ctx_size = 32; + hc->ctx_size = 32; - XhciBiosHandoff; + XhciBiosHandoff(hc); //Stop + reset controller - XhciOpW32(XHCI_OP_USBCMD, XhciOpR32(XHCI_OP_USBCMD) & ~XHCI_CMD_RS); + XhciOpW32(XHCI_OP_USBCMD, XhciOpR32(XHCI_OP_USBCMD, hc) & ~XHCI_CMD_RS, hc); timeout = tS + 1.0; - while (!(XhciOpR32(XHCI_OP_USBSTS) & XHCI_STS_HCH) && tS < timeout) + while (!(XhciOpR32(XHCI_OP_USBSTS, hc) & XHCI_STS_HCH) && tS < timeout) Sleep(1); - XhciOpW32(XHCI_OP_USBCMD, XHCI_CMD_HCRST); + XhciOpW32(XHCI_OP_USBCMD, XHCI_CMD_HCRST, hc); timeout = tS + 1.0; - while ((XhciOpR32(XHCI_OP_USBCMD) & XHCI_CMD_HCRST) && tS < timeout) + while ((XhciOpR32(XHCI_OP_USBCMD, hc) & XHCI_CMD_HCRST) && tS < timeout) Sleep(1); timeout = tS + 1.0; - while ((XhciOpR32(XHCI_OP_USBSTS) & XHCI_STS_CNR) && tS < timeout) + while ((XhciOpR32(XHCI_OP_USBSTS, hc) & XHCI_STS_CNR) && tS < timeout) Sleep(1); - XhciOpW32(XHCI_OP_CONFIG, usb_xhci.max_slots); + XhciOpW32(XHCI_OP_CONFIG, hc->max_slots, hc); //Device context base address array - if (!usb_xhci.dcbaa) - usb_xhci.dcbaa = CAllocAligned((usb_xhci.max_slots + 1) * 8, 0x1000, sys_task->code_heap); - else - MemSet(usb_xhci.dcbaa, 0, (usb_xhci.max_slots + 1) * 8); + hc->dcbaa = CAllocAligned((hc->max_slots + 1) * 8, 0x1000, sys_task->code_heap); //Scratchpad buffers - usb_xhci.scratch_count = (((hcs2 >> 21) & 0x1F) << 5) | ((hcs2 >> 27) & 0x1F); - if (usb_xhci.scratch_count) + hc->scratch_count = (((hcs2 >> 21) & 0x1F) << 5) | ((hcs2 >> 27) & 0x1F); + if (hc->scratch_count) { - usb_xhci.scratch_arr = CAllocAligned(usb_xhci.scratch_count * 8, 0x1000, sys_task->code_heap); - for (i = 0; i < usb_xhci.scratch_count; i++) + hc->scratch_arr = CAllocAligned(hc->scratch_count * 8, 0x1000, sys_task->code_heap); + for (i = 0; i < hc->scratch_count; i++) { page = CAllocAligned(0x1000, 0x1000, sys_task->code_heap); - usb_xhci.scratch_arr[i] = page; + hc->scratch_arr[i] = page; } - usb_xhci.dcbaa[0] = usb_xhci.scratch_arr; + hc->dcbaa[0] = hc->scratch_arr; } - XhciOpW64(XHCI_OP_DCBAAP, usb_xhci.dcbaa); + XhciOpW64(XHCI_OP_DCBAAP, hc->dcbaa, hc); //Command ring - XhciRingInit(&usb_xhci.cmd_ring); - addr = usb_xhci.cmd_ring.trbs; - XhciOpW64(XHCI_OP_CRCR, addr | 1); + XhciRingInit(&hc->cmd_ring); + addr = hc->cmd_ring.trbs; + XhciOpW64(XHCI_OP_CRCR, addr | 1, hc); //Event ring, single segment, interrupter 0 - if (!usb_xhci.evt_ring) - usb_xhci.evt_ring = CAllocAligned(XHCI_EVT_RING_TRBS * sizeof(CXhciTRB), 0x1000, sys_task->code_heap); - else - MemSet(usb_xhci.evt_ring, 0, XHCI_EVT_RING_TRBS * sizeof(CXhciTRB)); - usb_xhci.evt_deq = 0; - usb_xhci.evt_cycle = 1; - if (!usb_xhci.erst) - usb_xhci.erst = CAllocAligned(64, 64, sys_task->code_heap); - addr = usb_xhci.evt_ring; - usb_xhci.erst[0] = addr; - usb_xhci.erst[1] = XHCI_EVT_RING_TRBS; - XhciRtW32(XHCI_RT_ERSTSZ0, 1); - XhciRtW64(XHCI_RT_ERDP0, addr); - addr = usb_xhci.erst; - XhciRtW64(XHCI_RT_ERSTBA0, addr); + hc->evt_ring = CAllocAligned(XHCI_EVT_RING_TRBS * sizeof(CXhciTRB), 0x1000, sys_task->code_heap); + hc->evt_deq = 0; + hc->evt_cycle = 1; + hc->erst = CAllocAligned(64, 64, sys_task->code_heap); + addr = hc->evt_ring; + hc->erst[0] = addr; + hc->erst[1] = XHCI_EVT_RING_TRBS; + XhciRtW32(XHCI_RT_ERSTSZ0, 1, hc); + XhciRtW64(XHCI_RT_ERDP0, addr, hc); + addr = hc->erst; + XhciRtW64(XHCI_RT_ERSTBA0, addr, hc); //Interrupts stay disabled; we poll. //Run - XhciOpW32(XHCI_OP_USBCMD, XhciOpR32(XHCI_OP_USBCMD) | XHCI_CMD_RS); + XhciOpW32(XHCI_OP_USBCMD, XhciOpR32(XHCI_OP_USBCMD, hc) | XHCI_CMD_RS, hc); timeout = tS + 1.0; - while ((XhciOpR32(XHCI_OP_USBSTS) & XHCI_STS_HCH) && tS < timeout) + while ((XhciOpR32(XHCI_OP_USBSTS, hc) & XHCI_STS_HCH) && tS < timeout) Sleep(1); - if (XhciOpR32(XHCI_OP_USBSTS) & XHCI_STS_HCH) - goto xi_next; //never left halt -- try the next controller - - usb_xhci.present = TRUE; - usb_xhci.running = TRUE; - -//Power the root ports. They come up off, so a device never reports connected -//(CCS) until PP is set -- QEMU powers them for us, real hardware does not. - for (i = 0; i < usb_xhci.max_ports; i++) - XhciPortSCWrite(i, XHCI_PORTSC_PP); - Sleep(20); - -//Prefer a controller with something already plugged in. If nothing is connected -//and we weren't told to take any, remember it and keep looking. - connected = FALSE; - for (i = 0; i < usb_xhci.max_ports; i++) - if (XhciPortSC(i) & XHCI_PORTSC_CCS) - connected = TRUE; - if (connected || any) + if (XhciOpR32(XHCI_OP_USBSTS, hc) & XHCI_STS_HCH) + return FALSE; //never left halt + + hc->present = TRUE; + hc->running = TRUE; + +//Ports come up unpowered on real hardware; no CCS until PP is set. + for (i = 0; i < hc->max_ports; i++) + XhciPortSCWrite(i, XHCI_PORTSC_PP, hc); + + return TRUE; +} + +public I64 XhciInitAll() +{//Bring up every xHCI controller. Devices are spread across them: a USB-C dock +//can put an ethernet NIC on the Thunderbolt controller while HID sits on the +//PCH one. Returns the number running. + CXhciGlobals *hc; + I64 n, i, bdf; + + usb_hc_count = 0; + for (n = 0; usb_hc_count < USB_HCS_MAX; n++) { - "xHCI #%d at %02X:%02X:%01X: %d ports, %d slots.\n", - n, usb_xhci.pci_bus, usb_xhci.pci_dev, usb_xhci.pci_fun, - usb_xhci.max_ports, usb_xhci.max_slots; - return TRUE; + bdf = PCIClassFind(XHCI_PCI_CLASS, n); + if (bdf == -1) + break; + hc = &usb_hcs[usb_hc_count]; + if (XhciHcStart(hc, bdf)) + { + "xHCI #%d at %02X:%02X:%01X: %d ports, %d slots.\n", + usb_hc_count, hc->pci_bus, hc->pci_dev, hc->pci_fun, + hc->max_ports, hc->max_slots; + usb_hc_count++; + } } - if (fallback == -1) - fallback = n; -xi_next:; + if (!usb_hc_count) + { + "xHCI: no usable controller found.\n"; + usb_xhci = &usb_hcs[0]; + return 0; } -//Nothing had a device connected; use the first controller that ran. - if (!any && fallback >= 0) - return XhciInit(fallback, TRUE); - "xHCI: no usable controller found.\n"; - return FALSE; + Sleep(20); //port power settle +//Default target for calls that don't name a controller: prefer one with a +//device connected. + usb_xhci = &usb_hcs[0]; + for (n = 0; n < usb_hc_count; n++) + for (i = 0; i < usb_hcs[n].max_ports; i++) + if (XhciPortSC(i, &usb_hcs[n]) & XHCI_PORTSC_CCS) + { + usb_xhci = &usb_hcs[n]; + return usb_hc_count; + } + + return usb_hc_count; } +