diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d155265c..f371b90a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,12 +1,14 @@ name: Release # A tag `vX.Y.Z` on master builds `kern` and `kern-serve` for Linux x86_64 -# and aarch64 on Ubuntu 24.04, proves each binary starts there without a -# CUDA toolkit, and publishes a GitHub release with the archives, one -# SHA256SUMS, the installer, and build provenance. The target is a Linux -# CUDA box, nothing older than the runner's glibc. A pull request that -# touches this pipeline runs the build and packaging, never the publish. -# The GPU gate is manual and comes after: docs/release.md. +# and aarch64 on Ubuntu 24.04 and the `kern-llm` wheel (abi3, manylinux +# 2.28) for both, proves each starts there without a CUDA toolkit, and +# publishes a GitHub release with the archives, the wheels, one SHA256SUMS, +# the installer and build provenance, then the wheels to PyPI by trusted +# publishing: no token, PyPI trusts this workflow in the `pypi` +# environment. A pull request that touches this pipeline or kern-py runs +# the builds and checks, never the publish. The GPU gate is manual and +# comes after: docs/release.md. on: push: @@ -15,6 +17,7 @@ on: paths: - ".github/workflows/release.yml" - "scripts/**" + - "crates/kern-py/**" workflow_dispatch: permissions: @@ -70,10 +73,52 @@ jobs: if-no-files-found: error retention-days: 3 + wheel: + name: Wheel ${{ matrix.arch }} + strategy: + fail-fast: false + matrix: + include: + - arch: x86_64 + runner: ubuntu-24.04 + - arch: aarch64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Build + uses: PyO3/maturin-action@v1 + with: + working-directory: crates/kern-py + manylinux: "2_28" + maturin-version: v1.14.1 + args: --release --locked --out ../../wheels + + - name: Check + run: | + set -euo pipefail + version=$(grep -m1 '^version = ' Cargo.toml | cut -d'"' -f2) + if [ "$GITHUB_REF_TYPE" = tag ] && [ "$GITHUB_REF_NAME" != "v$version" ]; then + echo "::error::tag $GITHUB_REF_NAME does not match workspace version $version" + exit 1 + fi + scripts/check_wheel.sh "$version" wheels/*.whl + + - uses: actions/upload-artifact@v4 + with: + name: wheel-${{ matrix.arch }} + path: wheels/*.whl + if-no-files-found: error + retention-days: 3 + publish: name: Publish if: github.ref_type == 'tag' - needs: build + needs: [build, wheel] runs-on: ubuntu-24.04 timeout-minutes: 15 permissions: @@ -87,7 +132,7 @@ jobs: - uses: actions/download-artifact@v4 with: - pattern: kern-* + pattern: "*" path: dist merge-multiple: true @@ -97,13 +142,16 @@ jobs: cd dist cat ./*.sha256 >SHA256SUMS rm ./*.sha256 + sha256sum *.whl >>SHA256SUMS cp ../scripts/install.sh install.sh sha256sum -c SHA256SUMS ls -la - uses: actions/attest-build-provenance@v3 with: - subject-path: dist/*.tar.gz + subject-path: | + dist/*.tar.gz + dist/*.whl - name: Create release env: @@ -123,6 +171,10 @@ jobs: Install the latest release, or this one with \`KERN_VERSION=$GITHUB_REF_NAME\`: curl -fsSL https://kern-baa.pages.dev/install.sh | sh + + The Python runtime and the vLLM and SGLang plugins, for Python 3.10 or newer: + + pip install kern-llm==$VERSION EOF gh release create "$GITHUB_REF_NAME" dist/* \ --repo "$GITHUB_REPOSITORY" \ @@ -130,3 +182,23 @@ jobs: --title "kern $VERSION" \ --notes-file notes.md \ --generate-notes + + pypi: + name: PyPI + if: github.ref_type == 'tag' + needs: publish + runs-on: ubuntu-24.04 + timeout-minutes: 15 + environment: pypi + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v4 + with: + pattern: wheel-* + path: wheels + merge-multiple: true + + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: wheels/ diff --git a/README.md b/README.md index 63512eac..4f9b6c66 100644 --- a/README.md +++ b/README.md @@ -263,14 +263,13 @@ cargo build --release Inside vLLM, the manifest is the model and vLLM keeps everything else: ```bash -(cd crates/kern-py && maturin build --release) && pip install target/wheels/kern-*.whl -pip install python/kern_vllm +pip install kern-llm hf download Pegainfer/kern-qwen38-sm103 --local-dir kern-qwen38 KERN_MANIFEST=kern-qwen38/manifests/qwen3.8-27b-vllm.json KERN_KERNELS=kern-qwen38/cubins \ vllm serve Qwen/Qwen3.8-27B --hf-overrides '{"architectures": ["KernForCausalLM"]}' ``` -SGLang works the same way through `python/kern_sglang`: [docs/sglang.md](docs/sglang.md). +SGLang works the same way through `kern_sglang`, in the same package: [docs/sglang.md](docs/sglang.md). `kern --help` lists the flags; `crates/kern-run/src/config.rs` documents `kern.toml`. @@ -298,7 +297,7 @@ golden-checked in CI: | `crates/kern-test` | The `kern test` harness: diff, tap, noise floor, fuzz, perf, verdict (pure, no CUDA) | | `crates/kern-run` | The `kern` binary: `run`, `test`, `bench`, `server`, `verify`, `cut`, `kernels` | | `crates/kern-serve` | Continuous batching and the OpenAI-compatible endpoint | -| `crates/kern-py`, `python/` | `import kern`, and the vLLM and SGLang plugins | +| `crates/kern-py` | `pip install kern-llm`: `import kern`, and the vLLM and SGLang plugins | | `tools/` | Capture, kernel mining, manifest generators: all model knowledge lives here | | `examples/` | Generated manifests, the artifact a provider ships | diff --git a/crates/kern-py/README.md b/crates/kern-py/README.md new file mode 100644 index 00000000..77b50274 --- /dev/null +++ b/crates/kern-py/README.md @@ -0,0 +1,23 @@ +# kern-llm + +[kern](https://github.com/pegainfer-project/kern) runs a model as a manifest: a typed declaration of +buffers, states, ops and programs, compiled kernels, and weights. This package is kern for Python: + +- `kern`: the runtime binding (`kern.Runtime`), the readers of a manifest's host contract + (`kern.host`) and the rank group over `torch.distributed` (`kern.group`); +- `kern_vllm`: a vLLM plugin that runs a manifest as the model; +- `kern_sglang`: the same for SGLang. + +```bash +pip install kern-llm +hf download Pegainfer/kern-qwen38-sm103 --local-dir kern-qwen38 +KERN_MANIFEST=kern-qwen38/manifests/qwen3.8-27b-vllm.json KERN_KERNELS=kern-qwen38/cubins \ +vllm serve Qwen/Qwen3.8-27B --hf-overrides '{"architectures": ["KernForCausalLM"]}' \ + --max-num-seqs 128 --max-num-batched-tokens 8192 --enable-prefix-caching \ + -cc '{"mode": 0, "cudagraph_mode": "FULL_DECODE_ONLY"}' +``` + +Linux x86_64 and aarch64, Python 3.10 or newer. The runtime loads the NVIDIA driver (CUDA 13, r580 or +newer) and cuBLAS 13 when it first runs; torch comes from the engine. The plugins do nothing unless +`KERN_MANIFEST` is set. Guides: [vLLM](https://github.com/pegainfer-project/kern/blob/master/docs/vllm.md), +[SGLang](https://github.com/pegainfer-project/kern/blob/master/docs/sglang.md). diff --git a/crates/kern-py/pyproject.toml b/crates/kern-py/pyproject.toml index 1941501a..745392c1 100644 --- a/crates/kern-py/pyproject.toml +++ b/crates/kern-py/pyproject.toml @@ -3,10 +3,30 @@ requires = ["maturin>=1.8,<2"] build-backend = "maturin" [project] -name = "kern" +name = "kern-llm" +description = "The kern runtime for Python, and the plugins that run a kern manifest as a vLLM or SGLang model" +readme = "README.md" requires-python = ">=3.10" +license = "Apache-2.0" dynamic = ["version"] +classifiers = [ + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Operating System :: POSIX :: Linux", + "Environment :: GPU :: NVIDIA CUDA :: 13", +] + +[project.urls] +Source = "https://github.com/pegainfer-project/kern" +Documentation = "https://kern-baa.pages.dev" + +[project.entry-points."vllm.general_plugins"] +kern = "kern_vllm:register" + +[project.entry-points."sglang.srt.plugins"] +kern = "kern_sglang:register" [tool.maturin] module-name = "kern._kern" python-source = "python" +python-packages = ["kern_vllm", "kern_sglang"] diff --git a/python/kern_sglang/kern_sglang/__init__.py b/crates/kern-py/python/kern_sglang/__init__.py similarity index 97% rename from python/kern_sglang/kern_sglang/__init__.py rename to crates/kern-py/python/kern_sglang/__init__.py index 04074d0f..deb42bd4 100644 --- a/python/kern_sglang/kern_sglang/__init__.py +++ b/crates/kern-py/python/kern_sglang/__init__.py @@ -5,7 +5,7 @@ its forward is one or more programs of a manifest whose states are host states, bound to the tensors SGLang allocated for each layer. - pip install kern kern-sglang + pip install kern-llm KERN_MANIFEST= KERN_KERNELS= \\ sglang serve --model-path --page-size 64 --attention-backend trtllm_mha \\ --mamba-radix-cache-strategy extra_buffer --disable-prefill-cuda-graph ... diff --git a/python/kern_sglang/kern_sglang/model.py b/crates/kern-py/python/kern_sglang/model.py similarity index 100% rename from python/kern_sglang/kern_sglang/model.py rename to crates/kern-py/python/kern_sglang/model.py diff --git a/python/kern_vllm/kern_vllm/__init__.py b/crates/kern-py/python/kern_vllm/__init__.py similarity index 96% rename from python/kern_vllm/kern_vllm/__init__.py rename to crates/kern-py/python/kern_vllm/__init__.py index 3c4d3506..b82763f2 100644 --- a/python/kern_vllm/kern_vllm/__init__.py +++ b/crates/kern-py/python/kern_vllm/__init__.py @@ -5,7 +5,7 @@ the model: its forward is one or more programs of a manifest whose states are host states, bound to the pages vLLM allocated for each layer. - pip install kern kern-vllm + pip install kern-llm KERN_MANIFEST= KERN_KERNELS= \\ vllm serve --hf-overrides '{"architectures": ["KernForCausalLM"]}' ... diff --git a/python/kern_vllm/kern_vllm/backend.py b/crates/kern-py/python/kern_vllm/backend.py similarity index 100% rename from python/kern_vllm/kern_vllm/backend.py rename to crates/kern-py/python/kern_vllm/backend.py diff --git a/python/kern_vllm/kern_vllm/model.py b/crates/kern-py/python/kern_vllm/model.py similarity index 100% rename from python/kern_vllm/kern_vllm/model.py rename to crates/kern-py/python/kern_vllm/model.py diff --git a/python/kern_vllm/kern_vllm/trace.py b/crates/kern-py/python/kern_vllm/trace.py similarity index 100% rename from python/kern_vllm/kern_vllm/trace.py rename to crates/kern-py/python/kern_vllm/trace.py diff --git a/python/kern_vllm/kern_vllm/workload.py b/crates/kern-py/python/kern_vllm/workload.py similarity index 100% rename from python/kern_vllm/kern_vllm/workload.py rename to crates/kern-py/python/kern_vllm/workload.py diff --git a/crates/kern-py/src/lib.rs b/crates/kern-py/src/lib.rs index 619eccb0..6656f46f 100644 --- a/crates/kern-py/src/lib.rs +++ b/crates/kern-py/src/lib.rs @@ -4,7 +4,7 @@ //! buffers, host states, programs, vars — so it is the same for every host. //! What a particular engine calls its KV cache, its block table or its //! stream lives in that engine's adapter, in Python, next to the engine it -//! tracks (`python/kern_vllm` for vLLM): an engine that changes its +//! tracks (`kern_vllm` for vLLM): an engine that changes its //! internals changes the adapter, never this crate. //! //! The host reads the verified manifest as JSON ([`Runtime::manifest`]) to diff --git a/docs/release.md b/docs/release.md index 9d92d7b0..42446632 100644 --- a/docs/release.md +++ b/docs/release.md @@ -1,17 +1,25 @@ # Release:怎么发一个 kern binary,什么时候算发完 -kern 的 release 是两个 binary:`kern` 与 `kern-serve`,Linux x86_64 与 aarch64 -各一份,不含 CUDA 库(runtime 在首次使用时 dlopen driver 与 cuBLAS),不含 Python, -不含模型。目标机器就是一台 Linux CUDA 机:Ubuntu 24.04 或更新。装完能跑什么由 manifest 决定,不在本页范围。 +kern 的 release 是两样东西,Linux x86_64 与 aarch64 各一份,都不含 CUDA 库(runtime +在首次使用时 dlopen driver 与 cuBLAS),不含模型: -流水线在 `.github/workflows/release.yml`,打包与自检在 -`scripts/package_release.sh`,安装脚本在 `scripts/install.sh`。 +- 两个 binary:`kern` 与 `kern-serve`,不含 Python;目标机器是 Ubuntu 24.04 或更新。 +- PyPI 包 `kern-llm`:一个 abi3 wheel(Python 3.10+,manylinux 2.28),装的是 + `import kern` 与 vLLM、SGLang 两个插件(`kern_vllm`、`kern_sglang`,entry point 注册, + 不设 `KERN_MANIFEST` 时不做事)。torch 由引擎带,wheel 不依赖它。 + +装完能跑什么由 manifest 决定,不在本页范围。 + +流水线在 `.github/workflows/release.yml`,binary 的打包与自检在 +`scripts/package_release.sh`,wheel 的自检在 `scripts/check_wheel.sh`,安装脚本在 +`scripts/install.sh`。 ## 用户看到的 ```bash curl -fsSL https://kern-baa.pages.dev/install.sh | sh -kern --version # kern 0.2.0 (, cuda 13.0) +kern --version # kern 0.3.0 (, cuda 13.0) +pip install kern-llm # import kern; vllm / sglang serve 时插件自动加载 ``` - 网站的 `/install.sh` 是一条 302,指向最新 release 里的 `install.sh`; @@ -34,8 +42,8 @@ kern --version # kern 0.2.0 (, cuda 13.0) ## 流水线做什么 -1. **触发**:push 一个 `v*` tag。改到 workflow 或 `scripts/` 的 PR 也跑 build 与 - 打包(不 publish),所以流水线本身的改动在合并前就被 CI 验过。 +1. **触发**:push 一个 `v*` tag。改到 workflow、`scripts/` 或 `crates/kern-py` 的 PR + 也跑 build、打包与 wheel(不 publish),所以流水线本身的改动在合并前就被 CI 验过。 2. **构建**:两个 target 各自在 Ubuntu 24.04 runner 上原生编(aarch64 用 GitHub 的 arm runner,不交叉),apt 装 protoc 与 libssl-dev,`--locked` 一次编出 kern-run 与 kern-serve。cudarc 绑定的 CUDA 版本由 Cargo.toml 的 @@ -45,19 +53,27 @@ kern --version # kern 0.2.0 (, cuda 13.0) tag;`NEEDED` 里没有任何 CUDA 库;`kern verify examples/qwen3-4b.json` 与 `kern-serve --help` 跑通(无 GPU,证明 loader 满足)。 任一条不过,release 不出。 -4. **发布**:合并两份 `.sha256` 成一份 `SHA256SUMS`,附上 `install.sh`, - 对两个 tar.gz 做 GitHub build provenance attestation,`gh release create - --verify-tag`,notes 是固定的一段要求说明 + 自动生成的 commit 列表。 +4. **wheel**:两个 arch 各在原生 runner 上用 maturin-action 的 manylinux 2.28 容器编 + `crates/kern-py`;`check_wheel.sh` 装进干净 venv,查 `kern.__version__` = tag、 + 两个插件包在、两个 entry point 在、扩展模块 `NEEDED` 里没有 CUDA 库。 +5. **发布**:合并两份 `.sha256` 与 wheel 的 sha256 成一份 `SHA256SUMS`,附上 + `install.sh`,对 tar.gz 与 wheel 做 GitHub build provenance attestation, + `gh release create --verify-tag`,notes 是固定的一段要求说明 + 自动生成的 commit 列表。 同名 release 已存在则拒绝,不覆盖资产。 +6. **PyPI**:GitHub release 出了之后,同一批 wheel 用 trusted publishing 上传 + `kern-llm`:没有 token,PyPI 认的是 `pegainfer-project/kern` 的 `release.yml` + 在 `pypi` environment 里跑出的 OIDC 身份(pypi.org 项目页 Publishing 里登记)。 + 单独一个 job:这步失败只红它自己,修好后 `gh run rerun --failed`, + release 不用重发。PyPI 不允许覆盖同版本。 ## 怎么发 ```bash # 1. bump:workspace version 在根 Cargo.toml 的 [workspace.package] -git commit -am "release: v0.1.0" +git commit -am "release: v0.3.0" git push origin master # 2. tag -git tag -a v0.1.0 -m "kern v0.1.0" && git push origin v0.1.0 +git tag -a v0.3.0 -m "kern v0.3.0" && git push origin v0.3.0 # 3. 看 Actions 的 Release 跑绿 gh run watch ``` @@ -70,16 +86,21 @@ GitHub runner 没有 GPU,流水线只能证明 binary 起得来。release 发 ```bash curl -fsSL https://kern-baa.pages.dev/install.sh | sh kern --version # commit 与 tag 一致,cuda 13.0 -cd ~/kern-1 && kern test qwen3-4b # 最后一行 PASS +cd && kern test qwen3-4b # 最后一行 PASS +pip install kern-llm==0.3.0 && python -c 'import kern; print(kern.__version__)' # 引擎的 venv 里 ``` -三行都过,release 才算发完。不过就删掉 release 与 tag 重发,0.x 不做 hotfix 版本: +再在引擎里过一遍 conc1:[vllm.md](vllm.md) 与 [sglang.md](sglang.md) 开头那条命令起 +Qwen3.8-27B,greedy 输出与 `kern run` 逐 token 相同。都过,release 才算发完。 + +不过时:PyPI 上的版本号用过就不能再传,所以不是删了重发,而是 yank 掉 PyPI 上那一版、 +删掉 GitHub release 与 tag,修好后 bump 一个 patch 版本再发: ```bash -gh release delete v0.1.0 --yes --cleanup-tag +gh release delete v0.3.0 --yes --cleanup-tag # PyPI 那版在 pypi.org 的项目页 yank ``` ## 不做的 -macOS、Windows、musl(静态 musl 的 dlopen 不能用)、老 glibc、Homebrew、PyPI、 +macOS、Windows、musl(静态 musl 的 dlopen 不能用)、老 glibc、Homebrew、sdist、 docker 镜像。 diff --git a/docs/roadmap.md b/docs/roadmap.md index efa66b18..60716f1a 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -79,8 +79,8 @@ kern 在内部 RSI 里是 kernel agent 的反馈环境(`kern test` 裁判、`k | V4 ✅ | serving 协议进 manifest(schema v4,设计 v4-design.md):buffer 的 `fill`、program 的 `batch` / `once`,`spec` 块删除;`Verified` newtype + `Protocol::check` 投影,kern-run / kern-serve 不读 JSON、不认名字(CI grep);投机轮统一成 `round` program(dspark 也补了:splice / 计数 / prefill 收编 head 与 precompute 全在设备上),`--spec` → `--rows`;`kern verify` 打印协议事实 | **2026-09-03 tray03 通过**:`kern test` 位一致;qwen3-4b / qwen3.8-27b / dspark / dflash2 四份 conc1 与 `kern run` 逐字同;dflash2 与 v3 `--spec` 逐字节同;dspark 7 行 round 在一个 0.125 的 bf16 平局上与 v3 的 8 行 verify 分叉(核噪声);conc32 接受率 34% / 24% 不塌,5930 / 1709 tok/s。记录 v4-design.md §9、serve.md | | V5.1 ✅ | 轴写在 var 上(`axis: rows / groups / tray`),`Protocol` 不再从 `slot` / `seq_len` fill 反推行轴与组轴、不再从"第三个 var"猜 tray 轴、不读 `cut`;`token` / `slot` / `seq_len` 从每个 forward 必有的角色变成有就用(流水线首段之外的段没有它们) | **2026-10-08**:kern-manifest / kern-run / kern-test / kern-serve 测试过;qwen3-4b 2 段 chain / serve gate 位一致;K3 PP8 的下游段(无 token、无 seq_len)不特判通过 `Protocol` | | W1 ✅ | 权重就是 HF checkpoint:weight buffer 的 `bind` 列出拼成它的张量段(qkv / gate_up 拼接、draft fc 列块、tied lm_head),runtime 只读 safetensors header、按段 memcpy / 2D copy;派生表(Gemma +1、A_log f32、rope、kv_scales、chunk 索引)是 `carry`,由 `load` once program 用 `weight_prep.cu` 算;`tools/export_*.py` 与自定义 .safetensors 全删,`--weights` 给 snapshot 目录(K3 的 bind 未写,examples 仍是逐 buffer 同名的平凡 bind) | **2026-09-08 tray03 通过**:qwen3.8-27b / dflash2 / qwen3-4b-dspark 的每种 bind 形态(拼接、列块、按层 cat、tied)与每种派生表的 dump 对旧导出逐字节相同(qwen3-4b 的 rope 表 2078/5.2M 元素差 1 bf16 ulp:旧表是 CPU torch 算的);`kern test qwen3-4b` 位一致,`kern run` 文本同;qwen3.8-27b plain 与 dflash2 8 行 round(`qwen38_compare.py`,5 条 prompt × 400 token)与旧导出在同一 binary 上逐 token 相同,对 vLLM 参考的分叉点两者一样、全在 margin ≤ 0.125 的近平局上 | -| H1 ✅ | 宿主模式:manifest 的 `host` state(宿主分配、kern 验 layout)、`bind_host`(绑定时编译,可重绑)/ `enqueue_after`(event 挂宿主 stream,可进宿主的图)/ `region`;`crates/kern-py`(`import kern`);`python/kern_vllm` 把 manifest 装成 vLLM MRv2 的模型,KV / 调度 / graph / 采样归 vLLM(vllm.md) | **2026-09-25 单卡 GB300 过**:Qwen3.8-27B 在 vLLM 里 eager 与 FULL decode graph 下 32 个 greedy token 与 `kern run` 逐个同;32 条并发对串行分叉点 margin 全 ≤ 0.25;在线 A/B 与 TTFT 扫描在 vllm.md | -| H2 ✅ | SGLang 宿主:`python/kern_sglang`(`sglang.srt.plugins` 入口,沿用 checkpoint 的 architecture 名,SGLang 按 config 建池、kern 适配池);`tools/qwen38_sglang.py` 把 K/V/conv/ssm 拆成每层四族 host state,碰 conv state 的核另编 `-DCONV_DIM_MAJOR` 版(源码与 cubin 在 HF artifact repo,不带宏编译字节不变);examples 与 vLLM RSI 优化版两种 base 都能转;prefix cache 用 `extra_buffer`:extend 在快照点切成两次 `prefill`,decode 在图内用 SGLang 的 masked copy 存快照(sglang.md) | **2026-10-05 单卡 GB300(tray04)过**:SGLang nightly `f70e8c68` 里 greedy 与 `kern run` 一致到参考自身 top-2 精确平局的一步;prefill 快照与 decode 快照命中后的输出与 flush 后冷算逐 token 同,8 条并发分叉点 margin 全 0.125;RSI 版 `kern test` 对其 vLLM 版每个 span 位一致、greedy 48/48 同 `kern run`;AgentX 一小时对同参数(page 64、trtllm_mha)的 SGLang 多服务 10.5% 请求,表在 sglang.md | +| H1 ✅ | 宿主模式:manifest 的 `host` state(宿主分配、kern 验 layout)、`bind_host`(绑定时编译,可重绑)/ `enqueue_after`(event 挂宿主 stream,可进宿主的图)/ `region`;`crates/kern-py`(`import kern`);`kern_vllm` 把 manifest 装成 vLLM MRv2 的模型,KV / 调度 / graph / 采样归 vLLM(vllm.md) | **2026-09-25 单卡 GB300 过**:Qwen3.8-27B 在 vLLM 里 eager 与 FULL decode graph 下 32 个 greedy token 与 `kern run` 逐个同;32 条并发对串行分叉点 margin 全 ≤ 0.25;在线 A/B 与 TTFT 扫描在 vllm.md | +| H2 ✅ | SGLang 宿主:`kern_sglang`(`sglang.srt.plugins` 入口,沿用 checkpoint 的 architecture 名,SGLang 按 config 建池、kern 适配池);`tools/qwen38_sglang.py` 把 K/V/conv/ssm 拆成每层四族 host state,碰 conv state 的核另编 `-DCONV_DIM_MAJOR` 版(源码与 cubin 在 HF artifact repo,不带宏编译字节不变);examples 与 vLLM RSI 优化版两种 base 都能转;prefix cache 用 `extra_buffer`:extend 在快照点切成两次 `prefill`,decode 在图内用 SGLang 的 masked copy 存快照(sglang.md) | **2026-10-05 单卡 GB300(tray04)过**:SGLang nightly `f70e8c68` 里 greedy 与 `kern run` 一致到参考自身 top-2 精确平局的一步;prefill 快照与 decode 快照命中后的输出与 flush 后冷算逐 token 同,8 条并发分叉点 margin 全 0.125;RSI 版 `kern test` 对其 vLLM 版每个 span 位一致、greedy 48/48 同 `kern run`;AgentX 一小时对同参数(page 64、trtllm_mha)的 SGLang 多服务 10.5% 请求,表在 sglang.md | | H3 | 多卡宿主:`kern.Runtime(..., topology)` 每进程装一个 rank,`kern.group` 在引擎的 gloo TP 组上交换 offer、跑 `once` 再 barrier;`kern_vllm` 一个 TP rank 一个 kern rank,MLA 走 `MLAAttentionSpec`、`seq_lens` 取 DCP 本地长度、logits 的 vocab 分片在 TP 组上 all-gather;K3 的宿主 transform 在 kern-k3-pp-prefill(`k3_hosted.py` 程序边界、`k3_vllm.py` 内存布局);`kern_sglang` 同样一个 TP rank 一个 kern rank,在 SGLang 建好池、warmup 之前的 `prepare_before_cuda_graph_capture` 里绑定并启动,K3 的 transform 是 `k3_sglang.py`(MLA latent 按 64 行页、KDA 的 ssm / conv 就是 SGLang 自己的数组,`kda_core` 换成读 `[slot][tap][q\|k\|v]`、跳过 slot −1 的 CONV_TAP_MAJOR + LINE_GUARD 版) | **2026-10-13 两台 4×GB300 过 4 层 probe**:vLLM `12b005eef` `-tp 8 -dcp 8`,block 768 / KDA page 884,736 B / kernel block 64 与 transform 所设一致;eager 与 FULL decode graph 每步 error word 0,logits `[n, 163840]`;64 条单 token prompt × 32 greedy 各跑两遍,四遍两两分叉点 margin 全 ≤ 0.24(除一处外 ≤ 0.06),eager 对 graph 全同的请求数(44、41 / 64)与 eager 对 eager(47)相当(vllm.md)。**93 层**:同配置上卡,vLLM 的 greedy token 喂 `k3_step` teacher-forced top-1 一致 97.3%(graph)/ 96.9%(eager),分歧处 k3_step 一侧 margin ≤ 0.47;decode A/B(假 KV、闭合 batch、TPOT p50 对原生较好的布局)1 / 8 并发 kern 慢 26.8% / 7.1%,32 / 64 并发快 4.7% / 6.9%,32 × 32k 持平,128 并发 D 不支持(≤ 64 行)(vllm.md)。**2026-10-13 SGLang `e6072935` 同两台过 93 层**:`--tp-size 8 --dcp-size 8`,每 rank 绑定 162 个 host state,`kda_core` 变体对 D 构建逐位相同;普通 server 上同一组 64 × 32 一 token prompt 每模式两遍,eager 每步 error word 0,重跑 / eager 对 graph 分叉 margin 全 ≤ 0.87(中位 0.07–0.15),256 条里 149 条与 vLLM 托管的某一遍逐 token 同;fake-PD decode A/B 对 SGLang 自己的 K3(MegaMoE EP8 与 MoE-TP 较好者)TPOT p50 1 / 8 / 32 / 64 并发与 32 × 32k 分别快 19.2% / 14.6% / 10.4% / 13.8% / 19.4%,kern 步时与在 vLLM 里差 ≤ 1 ms,差在 SGLang 的 K3 比 vLLM 的慢(sglang.md) | ## K5 规划(2026-09-03) diff --git a/docs/sglang.md b/docs/sglang.md index 045ba5e4..1045b560 100644 --- a/docs/sglang.md +++ b/docs/sglang.md @@ -9,8 +9,7 @@ radix cache, the KV and recurrent-state pools, CUDA graphs, sampling and the API. kern runs the forward pass on the tensors SGLang allocates. ```bash -(cd crates/kern-py && maturin build --release) && pip install target/wheels/kern-*.whl -pip install python/kern_sglang +pip install kern-llm hf download Pegainfer/kern-qwen38-sm103 --local-dir kern-qwen38 KERN_MANIFEST=kern-qwen38/manifests/qwen3.8-27b-rsi-sglang.json KERN_KERNELS=kern-qwen38/cubins \ sglang serve --model-path Qwen/Qwen3.8-27B \ @@ -66,7 +65,7 @@ function can do anything; kern's puts its class into the model registry under the checkpoint's architecture: ```python -# python/kern_sglang/kern_sglang/__init__.py +# crates/kern-py/python/kern_sglang/__init__.py def register() -> None: if not os.environ.get("KERN_MANIFEST"): return @@ -187,7 +186,7 @@ repo [`Pegainfer/kern-qwen38-sm103`](https://huggingface.co/Pegainfer/kern-qwen3 `qwen3.8-27b-sglang.json` from the examples base, `qwen3.8-27b-rsi-sglang.json` from the loop's, next to its vLLM twin `qwen3.8-27b-rsi-vllm.json`. -### The plugin: `python/kern_sglang` +### The plugin: `kern_sglang` The model class does five things: @@ -340,5 +339,5 @@ Scripts, logs and the gate and A/B tables: metadata and `track_mamba_states_all_layers` from SGLang's modules, so an SGLang bump is checked against the stub probe and the gates above. - **Named contract.** As under vLLM, the adapter finds inputs and programs - by name, documented at the top of `python/kern_sglang/kern_sglang/model.py`. + by name, documented at the top of `crates/kern-py/python/kern_sglang/model.py`. Another model needs a transform that produces the same contract. diff --git a/docs/vllm.md b/docs/vllm.md index e70e6e36..977cdb8e 100644 --- a/docs/vllm.md +++ b/docs/vllm.md @@ -9,8 +9,7 @@ and the profiler endpoint. kern runs the forward pass on memory that vLLM allocates. ```bash -(cd crates/kern-py && maturin build --release) && pip install target/wheels/kern-*.whl -pip install python/kern_vllm +pip install kern-llm hf download Pegainfer/kern-qwen38-sm103 --local-dir kern-qwen38 KERN_MANIFEST=kern-qwen38/manifests/qwen3.8-27b-vllm.json KERN_KERNELS=kern-qwen38/cubins \ vllm serve Qwen/Qwen3.8-27B --hf-overrides '{"architectures": ["KernForCausalLM"]}' \ @@ -335,7 +334,7 @@ are published with the model's other artifacts in the HF repo strides, the conv kernel, the gemm16 issue) are in [qwen38-vllm.md](qwen38-vllm.md). -### The vLLM plugin: `python/kern_vllm` +### The vLLM plugin: `kern_vllm` A `vllm.general_plugins` entry point registers the architecture `KernForCausalLM`, selected with @@ -544,5 +543,5 @@ Scripts, logs and both tables: `~/bench_results/2026-10-13-kern-vllm-k3-ab`. `positions`, `slot_mapping`, `seq_lens`, `cu_seqlens_q`, `block_table`, `hidden`, `head_in`, `logits`; the line table by its domain) and programs likewise (`prefill`, `decode_batch`, `head`), as documented at the top of - `python/kern_vllm/kern_vllm/model.py`. Another model needs a transform that + `crates/kern-py/python/kern_vllm/model.py`. Another model needs a transform that produces the same contract. diff --git a/python/kern_sglang/pyproject.toml b/python/kern_sglang/pyproject.toml deleted file mode 100644 index cdb1dfb6..00000000 --- a/python/kern_sglang/pyproject.toml +++ /dev/null @@ -1,16 +0,0 @@ -[build-system] -requires = ["setuptools>=68"] -build-backend = "setuptools.build_meta" - -[project] -name = "kern-sglang" -version = "0.1.0" -description = "Run a kern manifest as an SGLang model" -requires-python = ">=3.10" -dependencies = ["kern"] - -[project.entry-points."sglang.srt.plugins"] -kern = "kern_sglang:register" - -[tool.setuptools] -packages = ["kern_sglang"] diff --git a/python/kern_vllm/pyproject.toml b/python/kern_vllm/pyproject.toml deleted file mode 100644 index 26e67670..00000000 --- a/python/kern_vllm/pyproject.toml +++ /dev/null @@ -1,16 +0,0 @@ -[build-system] -requires = ["setuptools>=68"] -build-backend = "setuptools.build_meta" - -[project] -name = "kern-vllm" -version = "0.1.0" -description = "Run a kern manifest as a vLLM model" -requires-python = ">=3.10" -dependencies = ["kern"] - -[project.entry-points."vllm.general_plugins"] -kern = "kern_vllm:register" - -[tool.setuptools] -packages = ["kern_vllm"] diff --git a/scripts/check_wheel.sh b/scripts/check_wheel.sh new file mode 100755 index 00000000..c8c9de0b --- /dev/null +++ b/scripts/check_wheel.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Prove a built kern-llm wheel installs into a clean Python without a GPU: +# the version the tag says, both engine plugins and their entry points, no +# CUDA library linked (the runtime dlopens them). +# +# scripts/check_wheel.sh VERSION WHEEL +set -euo pipefail + +[[ $# -eq 2 ]] || { echo "usage: $0 VERSION WHEEL" >&2; exit 2; } +version=${1#v} +wheel=$(realpath "$2") + +fail() { echo "wheel: $*" >&2; exit 1; } + +venv=$(mktemp -d) +trap 'rm -rf -- "$venv"' EXIT +python3 -m venv "$venv" +"$venv/bin/pip" install --quiet "$wheel" + +cd "$venv" +reported=$("$venv/bin/python" -c 'import kern; print(kern.__version__)') +[[ $reported == "$version" ]] || fail "kern reports $reported, release is $version" + +points=$("$venv/bin/python" -c ' +import importlib.metadata as m +print(" ".join(sorted(f"{e.group}={e.value}" for e in m.distribution("kern-llm").entry_points)))') +[[ $points == "sglang.srt.plugins=kern_sglang:register vllm.general_plugins=kern_vllm:register" ]] || + fail "entry points are: $points" + +for plugin in kern_vllm kern_sglang; do + "$venv/bin/python" -c "import importlib.util, sys; sys.exit(importlib.util.find_spec('$plugin') is None)" || + fail "the wheel has no $plugin" +done + +ext=$(find "$venv" -name '_kern*.so') +needed=$(objdump -p "$ext" | awk '$1 == "NEEDED" {print $2}') +if grep -Eq 'libcu|libnv' <<<"$needed"; then + fail "$ext links a CUDA library:"$'\n'"$needed" +fi +echo "$wheel" diff --git a/tools/qwen38_sglang.py b/tools/qwen38_sglang.py index cecc45c2..0e79f2f6 100644 --- a/tools/qwen38_sglang.py +++ b/tools/qwen38_sglang.py @@ -1,7 +1,7 @@ """The Qwen3.8-27B manifest on SGLang's memory. Rewrites `examples/qwen3.8-27b.json`, or a manifest derived from it (the -vLLM optimization loop's), into a manifest an SGLang model runner drives (`python/kern_sglang`), as `qwen38_vllm.py` does for vLLM: +vLLM optimization loop's), into a manifest an SGLang model runner drives (`kern_sglang`), as `qwen38_vllm.py` does for vLLM: the same programs (prefill, decode_batch, head, stopping at the hidden states), over SGLang's pools instead of vLLM's pages. diff --git a/tools/qwen38_vllm.py b/tools/qwen38_vllm.py index 1a09fad6..39f6be83 100644 --- a/tools/qwen38_vllm.py +++ b/tools/qwen38_vllm.py @@ -1,7 +1,7 @@ """The Qwen3.8-27B manifest on vLLM's memory. Rewrites `examples/qwen3.8-27b.json` into a manifest a vLLM model runner -drives (`python/kern_vllm`): the KV cache and the GDN states become host +drives (`kern_vllm`): the KV cache and the GDN states become host states, one per layer, laid out the way vLLM's hybrid allocator lays them out, and the programs stop at the hidden states, since vLLM samples. The tables index them as vLLM's ids do: 64-token blocks of every attention