Implement libhv-based asyncio event loop (M1–M4)

.github/workflows/build.yml

@@ -1,79 +1,161 @@
-name: Build and upload to PyPI
+name: Build and Test
 
 on:
+  push:
+    branches: [main, develop]
+  pull_request:
+    branches: [main]
   release:
     types: [published]
+  workflow_dispatch:
+
+# Least privilege: nothing here needs to write to the repo. The upload_pypi
+# job re-declares its own permissions (id-token) for trusted publishing.
+permissions:
+  contents: read
+
+# Cancel superseded runs of the same branch/PR (a stuck queued job would
+# otherwise hold a stale run open for hours).
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
-  build_wheels:
-    name: Build whells on ${{ matrix.os }}
+  # --------------------------------------------------------------------------
+  # Source build + FULL pytest (incl. the ASGI/uvicorn/TLS integration suite)
+  # on the three platforms (tech design section 12.4 gate).
+  # --------------------------------------------------------------------------
+  test:
+    name: "Source build & pytest (${{ matrix.os }}, py${{ matrix.python-version }})"
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
-        os: [ubuntu-18.04, windows-2019, macos-10.15]
-
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.10", "3.14"]  # oldest + newest supported
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
-            submodules: true
-      - uses: actions/setup-python@v2
+          submodules: recursive
+          persist-credentials: false
+
+      - uses: actions/setup-python@v5
         with:
-            python-version: "3.8"
-      - name: Install dependencies
-        run: |
-          pip install -U pip wheel
-          pip install cibuildwheel
+          python-version: ${{ matrix.python-version }}
 
-      - name: Build wheels
+      - name: Build and install (with test extras)
         run: |
-          cibuildwheel --output-dir wheelhouse
+          python -m pip install -U pip
+          python -m pip install -e ".[test]" -v
+
+      - name: Run full test suite
+        run: python -m pytest tests/ -v
+
+  # --------------------------------------------------------------------------
+  # Wheels: cibuildwheel matrix (tech design section 11).
+  #   CPython 3.10-3.14 x manylinux2014 x86_64/aarch64 + musllinux x86_64
+  #   + macOS arm64 & x86_64 + Windows AMD64.
+  # Version selection / images / test command live in pyproject.toml
+  # ([tool.cibuildwheel]); this matrix only picks runner+arch. Linux aarch64
+  # uses the native arm64 runner (no QEMU), so wheel tests run everywhere.
+  # --------------------------------------------------------------------------
+  build_wheels:
+    name: "Wheels: ${{ matrix.name }}"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: linux-x86_64        # manylinux2014 + musllinux x86_64
+            os: ubuntu-latest
+            archs: x86_64
+          - name: linux-aarch64       # manylinux2014 aarch64 (native runner)
+            os: ubuntu-24.04-arm
+            archs: aarch64
+          - name: macos-x86_64
+            # Intel (macos-13) runners are retired — jobs queue forever. Build
+            # x86_64 wheels by cross-compiling on the arm64 runner instead;
+            # cibuildwheel runs their tests under Rosetta 2.
+            os: macos-latest
+            archs: x86_64
+          - name: macos-arm64
+            os: macos-latest
+            archs: arm64
+          - name: windows-amd64
+            os: windows-latest
+            archs: AMD64
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          persist-credentials: false
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install cibuildwheel
+        run: python -m pip install "cibuildwheel>=3.2,<4"
+
+      - name: Build and test wheels
+        run: python -m cibuildwheel --output-dir wheelhouse
         env:
-          CIBW_BUILD: cp3?-*
-          CIBW_SKIP: "*-win32 *-manylinux_i686 cp35-*"
-          CIBW_BEFORE_BUILD: "pip install -U cython cmake"
+          CIBW_ARCHS: ${{ matrix.archs }}
 
-      - name: Show files
+      - name: Show wheels
         run: ls -lh wheelhouse
         shell: bash
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         with:
-          path: ./wheelhouse/*.whl
+          name: wheels-${{ matrix.name }}
+          path: wheelhouse/*.whl
 
+  # --------------------------------------------------------------------------
+  # sdist (built once, on Linux) + import smoke from the sdist itself.
+  # --------------------------------------------------------------------------
   build_sdist:
-    name: Build source distribution
-    runs-on: ubuntu-18.04
+    name: Build sdist
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
-          submodules: true
+          submodules: recursive
+          persist-credentials: false
 
-      - uses: actions/setup-python@v2
-        name: Install Python
+      - uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: "3.12"
 
       - name: Build sdist
-        run: python setup.py sdist
+        run: pipx run build --sdist
 
-      - uses: actions/upload-artifact@v2
+      - name: Smoke-test sdist (build + import)
+        run: |
+          python -m pip install dist/*.tar.gz -v
+          python -c "import hvloop; loop = hvloop.new_event_loop(); loop.close(); print('hvloop', hvloop.__version__, 'ok')"
+
+      - uses: actions/upload-artifact@v4
         with:
+          name: sdist
           path: dist/*.tar.gz
 
+  # --------------------------------------------------------------------------
+  # PyPI upload on GitHub release (trusted publishing; configure the project
+  # as a trusted publisher on PyPI before the first release).
+  # --------------------------------------------------------------------------
   upload_pypi:
-    needs: [ build_wheels, build_sdist ]
-    runs-on: ubuntu-18.04
-    # upload to PyPI on every tag starting with 'v'
-#    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
-    # alternatively, to publish when a GitHub Release is created, use the following rule:
-    # if: github.event_name == 'release' && github.event.action == 'published'
+    name: Upload to PyPI
+    needs: [test, build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'release' && github.event.action == 'published'
+    environment: pypi
+    permissions:
+      id-token: write
     steps:
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v4
         with:
-          name: artifact
           path: dist
+          merge-multiple: true
 
-      - uses: pypa/gh-action-pypi-publish@master
-        with:
-          user: __token__
-          password: ${{ secrets.pypi_password }}
+      - uses: pypa/gh-action-pypi-publish@release/v1

.gitignore

@@ -1,5 +1,8 @@
 # Created by .ignore support plugin (hsz.mobi)
 ### Python template
+# macOS
+.DS_Store
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

CLAUDE.md

@@ -0,0 +1,160 @@
+# CLAUDE.md
+
+Guidance for working in the hvloop repository.
+
+## What this is
+
+hvloop is a drop-in asyncio event loop (like uvloop) implemented as a **Cython
+extension** on top of the vendored C library **libhv** (`vendor/libhv`, a git
+submodule). Target use case: web servers — it runs FastAPI/ASGI under uvicorn,
+including WebSocket and TLS. Cross-platform: Linux (epoll), macOS (kqueue),
+Windows (wepoll).
+
+The authoritative design is `docs/plans/2026-06-13-hvloop-tech-design.md`. Read
+it before making non-trivial changes — it explains *why* the loop is driven the
+way it is.
+
+## Layout
+
+- `src/hvloop/_core.pyx` — **everything native lives here in one compilation
+  unit** (the loop, `TCPTransport`, `Server`, `_TCPListener`, `_FDWatcher`,
+  TLS wiring, `sock_*`, signals, and all libhv C callbacks). ~3000 lines.
+- `src/hvloop/includes/hv.pxd` — libhv C API declarations (subset we use).
+  Truth source for the C API is `vendor/libhv/event/hloop.h`.
+- `src/hvloop/hvloop_shim.h` — small `static inline` C helpers for things the
+  public libhv API doesn't expose (reaching private struct fields, etc.).
+- `src/hvloop/__init__.py` — Python-level public API: `Loop`,
+  `EventLoopPolicy`, `install()`/`uninstall()`, `new_event_loop()`, `run()`.
+- `tests/` — pytest suite (synchronous test functions that build their own
+  loop; `tests/certs/` holds committed self-signed certs for TLS tests).
+- `benchmarks/`, `examples/fastapi_app.py` — runnable perf scripts and a demo.
+- `CMakeLists.txt` + `pyproject.toml` — scikit-build-core + cython-cmake build;
+  libhv is compiled as a static lib with only its core event engine enabled.
+
+## Build & test
+
+```shell
+# Editable install (rebuilds the extension). Run after any .pyx/.pxd/.h change.
+uv pip install -e .
+
+# Full suite (expect 149 passing).
+.venv/bin/python -m pytest tests/ -q
+
+# Faster inner loop without reinstalling: build + install the target directly.
+cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DSKBUILD_PROJECT_NAME=hvloop \
+  -DSKBUILD_PROJECT_VERSION=0.2.0 -DPython_EXECUTABLE=$PWD/.venv/bin/python
+cmake --build build --target _core -j
+cmake --install build --component python_modules --prefix src
+PYTHONPATH=src .venv/bin/python -m pytest
+
+# Wheel/sdist.
+uv build
+```
+
+Environment here: macOS arm64, venv at `.venv` (Python 3.14). Editing a `.pyx`
+and re-running pytest **without rebuilding tests stale code** — always rebuild
+first.
+
+## Core invariants — do not break these
+
+These were hard-won during development; violating them causes crashes, leaks,
+or busy-spins that tests may not immediately catch.
+
+1. **The loop is self-driven; we do NOT call `hloop_run`.** `run_forever()`
+   loops manually: run a snapshot of the `_ready` deque, then
+   `hloop_process_events(timeout)` with `timeout=0` when ready is non-empty
+   else a large cap. libhv clamps the poll to the nearest timer internally.
+
+2. **libhv's loop status must be forced to RUNNING while we drive it.**
+   `hloop_process_events` returns early (skipping pending-event dispatch) when
+   status is `STOP`, which is the initial value since we never call
+   `hloop_run`. `run_forever()` calls `hvloop_set_status_running()` on entry and
+   `hvloop_set_status_stop()` on exit (in `hvloop_shim.h`). Without this the
+   wakeup fd never drains and the loop busy-spins at 100% CPU.
+
+3. **Create the wakeup eventfd before the first poll.** `run_forever()` calls
+   `hloop_wakeup()` once up front; otherwise libhv sleeps in an un-wakeable
+   `hv_msleep` when there are no ios and `call_soon_threadsafe` can't interrupt.
+
+4. **`hloop_new(0)`** — pass 0 explicitly so `HLOOP_FLAG_AUTO_FREE` is off,
+   else libhv double-frees against our `hloop_free()` in `close()`.
+
+5. **Every libhv C callback is `noexcept nogil` + `with gil` + full try/except.**
+   No Python exception may propagate back into C. KeyboardInterrupt/SystemExit
+   raised inside a callback are stashed on `loop._pending_exc` and re-raised by
+   `run_forever` after the poll returns (they can't cross the `noexcept` frame).
+
+6. **Reference discipline for anything registered with libhv.** Objects handed
+   to libhv (timers, transports, listeners, fd watchers) do `Py_INCREF` at
+   registration and are tracked in a loop-level set (`_timer_handles`,
+   `_hio_objs`, `_fd_watchers`). There is exactly **one** release path per
+   object — the close/cancel dispatch and `loop.close()` teardown are mutually
+   exclusive and each nulls the C pointer before `Py_DECREF`. `loop.close()`
+   must tear all of these down *before* `hloop_free`. Getting this wrong = UAF
+   or leak (both happened and were fixed; regression tests guard them).
+
+7. **libhv's read buffer is loop-level shared memory.** In a read callback,
+   copy the bytes immediately (`PyBytes_FromStringAndSize`) before handing to
+   the protocol — the buffer is reused as soon as the callback returns.
+
+8. **fd ownership.** fds we create (`create_connection`, host/port
+   `create_server`) are handed to libhv via `sock.detach()` and libhv closes
+   them. Caller-owned fds (`create_server(sock=)`, `add_reader/writer`, signal
+   socketpair read end) are never closed by us — teardown uses
+   `hvloop_hio_release_external` (resets io type so `hio_close` skips the fd).
+
+## Gotchas / decisions worth knowing
+
+- **uvicorn ≥ 0.36 ignores the asyncio policy.** `loop="asyncio"` maps straight
+  to `SelectorEventLoop`; `hvloop.install()` has no effect there. Wire uvicorn
+  with `loop="hvloop:new_event_loop"` (recommended) or `loop="none"` +
+  `hvloop.install()`. See the README for all three wirings.
+- **TLS reuses stdlib `asyncio.sslproto.SSLProtocol`** (MemoryBIO), *not*
+  libhv's OpenSSL. This keeps any `ssl.SSLContext` working and avoids an
+  OpenSSL link/distribution dependency. libhv is built with `WITH_OPENSSL=OFF`.
+  Code is version-gated (`_PY311`) for the 3.10 vs 3.11+ sslproto differences.
+- **`add_reader`/`add_writer` replace libhv's io callback.** We `hio_add` our
+  own raw callback (libhv then won't read/write the fd itself) and translate
+  `hio_revents` into the reader/writer handles, clearing revents afterward
+  (mirrors `nio.c`'s `hio_handle_events`). A watched fd must not also be a
+  transport, and high-level `hio_read`/`hio_write` must not be used on it.
+- **No half-close.** libhv closes on EOF, so `eof_received()` returning True
+  can't keep the transport open; `connection_lost` always follows. Fine for
+  HTTP/WebSocket. Documented deviation from asyncio.
+- **Signals (Unix only):** `set_wakeup_fd` + a socketpair registered as an
+  internal reader. Windows raises `NotImplementedError` (uvicorn falls back to
+  `signal.signal`, same as asyncio's Proactor loop).
+
+## Known Windows limitations
+
+CI runs the full matrix (Linux/macOS/Windows × py3.10/3.14). Windows-specific
+notes:
+
+- **Signals are Unix-only** — `add_signal_handler`/`remove_signal_handler` raise
+  `NotImplementedError`; those tests are skipped on Windows (uvicorn falls back
+  to `signal.signal`, same as asyncio's Proactor loop).
+- **Write-buffer backpressure is unverified on Windows.** Two flow-control
+  tests (`test_write_flow_control`, `test_close_flushes_pending_writes`) assert
+  that a large `write()` to a paused/slow peer leaves data in libhv's write
+  queue (`get_write_buffer_size() > 0`). On Windows loopback the payload is
+  absorbed by the larger default socket buffers (and/or libhv's Windows
+  write-queue accounting differs), so the buffer reads 0 and `pause_writing`
+  doesn't fire in the test. These are marked `xfail(strict=False)` on Windows.
+  **Open question for a Windows maintainer:** does watermark flow control
+  actually engage under real backpressure on Windows, or is this a genuine gap?
+  Needs verification on real hardware (shrinking SO_SNDBUF/SO_RCVBUF in the test
+  is the likely fix if it's just buffer sizing).
+- `os.fstat()` does **not** work on Windows socket handles (they aren't CRT
+  fds) — use `sock.getsockname()` for "is this socket still open" checks in
+  tests.
+
+## Conventions
+
+- `vendor/libhv/` is **read-only** — never modify it. It's a submodule pinned
+  to a specific commit. Reach private internals via `hvloop_shim.h`.
+- Don't edit `docs/plans/` design docs unless changing the design deliberately.
+- Match the existing code style in `_core.pyx` (dense comments explaining *why*
+  at every non-obvious libhv interaction; module-level aliases for hot-path
+  attribute lookups).
+- New native features generally include their own pytest coverage; the test
+  suite is the acceptance bar.