diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea9cdf4..30d318c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,10 +23,10 @@ jobs: toolchain: '1.95.0' components: clippy,rustfmt - uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2 - - name: Install release signing executable + - name: Install signing and I/O test dependencies run: | sudo apt-get update - sudo apt-get install --no-install-recommends -y minisign + sudo apt-get install --no-install-recommends -y minisign python3-dev liburing-dev - run: make lint - run: make test - run: make test-client diff --git a/.github/workflows/spark-release.yml b/.github/workflows/spark-release.yml index adb9f94..58af97c 100644 --- a/.github/workflows/spark-release.yml +++ b/.github/workflows/spark-release.yml @@ -22,10 +22,10 @@ jobs: - uses: taiki-e/install-action@94c31af3204a9f15ab40b35ad084410b905bbc73 # v2 with: tool: cargo-deny - - name: Install release signing executable + - name: Install signing and I/O test dependencies run: | sudo apt-get update - sudo apt-get install --no-install-recommends -y minisign + sudo apt-get install --no-install-recommends -y minisign python3-dev liburing-dev - run: cargo deny --no-default-features --features appliance check - run: make lint-spark - run: make test diff --git a/Cargo.lock b/Cargo.lock index 6047b80..c4a6b94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -557,7 +557,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -880,7 +880,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2685,7 +2685,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2743,7 +2743,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3287,7 +3287,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4024,7 +4024,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4640,27 +4640,27 @@ checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" [[package]] name = "zstd" -version = "0.13.3" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.2.4" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.1.2+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index f4a9902..fd29ba5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,7 +66,7 @@ base64 = "0.22" minisign-verify = "0.2" nix = { version = "0.29", features = ["fs", "signal", "process"] } uuid = { version = "1", features = ["v4"] } -zstd = "0.13" +zstd = "0.14.3" walkdir = "2" sd-notify = "0.5" libc = "0.2" diff --git a/README.md b/README.md index ee2c765..3dfeaf5 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ cargo build --locked --release Use the pinned Rust 1.95.0 toolchain (selected automatically by rustup), a C/C++ toolchain, CMake, pkg-config and Python 3.11+. Install the Minisign executable for the encrypted release-signing tests. +Appliance I/O tests also need Python development headers and liburing development +files (`python3-dev` and `liburing-dev` on Debian/Ubuntu). The client runs on Linux x86-64/ARM64; the appliance targets Linux ARM64 with NVIDIA GB10, Docker and the NVIDIA container toolkit already installed. diff --git a/configs/sparkplane/engines/checks/qwen38_fp4_checkpoint.py b/configs/sparkplane/engines/checks/qwen38_fp4_checkpoint.py new file mode 100644 index 0000000..1fcbb67 --- /dev/null +++ b/configs/sparkplane/engines/checks/qwen38_fp4_checkpoint.py @@ -0,0 +1,161 @@ +"""Exercise the actual B12X checkpoint reader with CPU CUDA/uring test doubles. + +Only CUDA transport is simulated. File reads use real aligned O_DIRECT I/O; +the installed reader supplies all slot ownership, striding and conversion code. +""" + +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + + +URING = r""" +#pragma once +#include +struct io_uring { int unused; }; +struct io_uring_sqe { uint64_t data; }; +struct io_uring_cqe { uint64_t data; int res; }; +static int uring_calls; +static inline int io_uring_queue_init(unsigned n, struct io_uring *r, unsigned f) { + (void)n; (void)r; (void)f; uring_calls++; return -EPERM; +} +static inline void io_uring_queue_exit(struct io_uring *r) { (void)r; uring_calls++; } +static inline int io_uring_register_buffers(struct io_uring *r, const struct iovec *v, unsigned n) { + (void)r; (void)v; (void)n; uring_calls++; return -EPERM; +} +static inline struct io_uring_sqe *io_uring_get_sqe(struct io_uring *r) { (void)r; uring_calls++; return NULL; } +static inline void io_uring_prep_read_fixed(struct io_uring_sqe *s, int f, void *b, unsigned n, int64_t o, int i) { + (void)s; (void)f; (void)b; (void)n; (void)o; (void)i; uring_calls++; +} +static inline void io_uring_sqe_set_data64(struct io_uring_sqe *s, uint64_t d) { s->data=d; } +static inline int io_uring_submit(struct io_uring *r) { (void)r; uring_calls++; return -EPERM; } +static inline int io_uring_wait_cqe(struct io_uring *r, struct io_uring_cqe **c) { (void)r; (void)c; uring_calls++; return -EPERM; } +static inline uint64_t io_uring_cqe_get_data64(struct io_uring_cqe *c) { return c->data; } +static inline void io_uring_cqe_seen(struct io_uring *r, struct io_uring_cqe *c) { (void)r; (void)c; uring_calls++; } +""" + +PREFIX = r""" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define IO_ALIGNMENT 4096 +typedef struct { char message[1024]; } failure_t; +typedef void *cudaStream_t; +enum { cudaHostRegisterDefault=0, cudaStreamNonBlocking=1, cudaMemcpyHostToDevice=1 }; +static atomic_int fail_copy; +static int pread_fault; +static bool cuda_ok(int code, const char *where, failure_t *f) { + if (code && !f->message[0]) snprintf(f->message,sizeof(f->message),"%s: CUDA failure",where); + return !code; +} +static void system_error(failure_t *f, const char *where) { snprintf(f->message,sizeof(f->message),"%s: %s",where,strerror(errno)); } +static int cudaGetDevice(int *d) { *d=0; return 0; } +static int cudaSetDevice(int d) { (void)d; return 0; } +static int cudaHostRegister(void *p,size_t n,int flags) { (void)p;(void)n;(void)flags;return 0; } +static int cudaHostUnregister(void *p) { (void)p;return 0; } +static int cudaStreamCreateWithFlags(cudaStream_t *s,int f) { (void)f;*s=(void*)1;return 0; } +static int cudaStreamDestroy(cudaStream_t s) { (void)s;return 0; } +static int cudaStreamSynchronize(cudaStream_t s) { (void)s;return 0; } +static int cudaMemcpyAsync(void *d,const void *s,size_t n,int k,cudaStream_t stream) { + (void)k;(void)stream;if(atomic_load(&fail_copy))return 1;memcpy(d,s,n);return 0; +} +static int cudaMemcpy2DAsync(void *d,size_t dp,const void *s,size_t sp,size_t w,size_t h,int k,cudaStream_t stream) { + (void)k;(void)stream;if(atomic_load(&fail_copy))return 1; + for(size_t r=0;rring_ready);return r; +} +int main(int argc,char **argv) { + assert(argc==2);assert(setenv("B12X_CHECKPOINT_PREAD","1",1)==0); + const size_t size=21*1024*1024;unsigned char *source=malloc(size);assert(source); + uint32_t seed=7319;for(size_t i=0;i>24;} + char path[4096];assert(snprintf(path,sizeof(path),"%s/checkpoint.bin",argv[1])>0); + int output=open(path,O_WRONLY|O_CREAT|O_EXCL,0600);assert(output>=0); + size_t done=0;while(done0);done+=(size_t)n;}assert(close(output)==0); + int fd=open(path,O_RDONLY|O_DIRECT);assert(fd>=0); + size_t length=20*1024*1024+79;unsigned char *dest=malloc(length+64);assert(dest);memset(dest,0xa5,length+64); + bounce_ring_t *r=reader();failure_t f={{0}}; + read_job_t job={.fd=fd,.offset=113,.bytes=(int64_t)length,.destination=(char*)dest,.rows=1}; + pread_fault=1;assert(bounce_execute(r,&job,1,&f));assert(!f.message[0]);assert(memcmp(dest,source+113,length)==0); + for(size_t i=length;ireads>16 && r->submits==0);bounce_release(r); + /* Strided rows retain untouched destination gaps. */ + memset(dest,0xa5,length+64);r=reader();memset(&f,0,sizeof(f)); + job=(read_job_t){.fd=fd,.offset=277,.bytes=32,.destination=(char*)dest,.rows=4096,.source_stride=80,.destination_stride=64}; + assert(bounce_execute(r,&job,1,&f)); + for(size_t row=0;row<4096;row++){assert(memcmp(dest+row*64,source+277+row*80,32)==0);for(size_t j=32;j<64;j++)assert(dest[row*64+j]==0xa5);} + bounce_release(r); + /* BF16 expansion preserves the exact source bits and destination pitch. */ + memset(dest,0xa5,length+64);r=reader();memset(&f,0,sizeof(f)); + job=(read_job_t){.fd=fd,.offset=512,.bytes=128,.destination=(char*)dest,.expand_bf16=true,.rows=512,.source_stride=256,.destination_stride=512}; + assert(bounce_execute(r,&job,1,&f)); + for(size_t row=0;row<512;row++)for(size_t col=0;col<64;col++){uint16_t b;uint32_t v;memcpy(&b,source+512+row*256+col*2,2);memcpy(&v,dest+row*512+col*4,4);assert(v==((uint32_t)b<<16));} + bounce_release(r); + r=reader();memset(&f,0,sizeof(f));memset(dest,0xa5,length+64); + job=(read_job_t){.offset=(int64_t)(uintptr_t)source,.bytes=512,.destination=(char*)dest,.host_copy=true,.rows=1}; + assert(bounce_execute(r,&job,1,&f));assert(memcmp(dest,source,512)==0);bounce_release(r); + /* Failed reads poison the executor and never publish stale slot bytes. */ + for(int fault=2;fault<=3;fault++){ + r=reader();memset(&f,0,sizeof(f));memset(dest,0xa5,length+64);pread_fault=fault; + job=(read_job_t){.fd=fd,.offset=0,.bytes=512,.destination=(char*)dest,.rows=1}; + assert(!bounce_execute(r,&job,1,&f));assert(f.message[0] && r->poisoned); + for(size_t i=0;i<512;i++)assert(dest[i]==0xa5); + pread_fault=0;memset(&f,0,sizeof(f));assert(!bounce_execute(r,&job,1,&f));bounce_release(r); + } + r=reader();memset(&f,0,sizeof(f));atomic_store(&fail_copy,1); + assert(!bounce_execute(r,&job,1,&f));assert(r->poisoned);bounce_release(r);atomic_store(&fail_copy,0); + assert(uring_calls==0); + /* The existing mode still fails closed when io_uring is unavailable. */ + assert(unsetenv("B12X_CHECKPOINT_PREAD")==0);memset(&f,0,sizeof(f)); + assert(!bounce_create(0,8,&f));assert(uring_calls==1 && f.message[0]); + assert(close(fd)==0);free(dest);free(source); + puts("{\"pread_checkpoint_cases\":8,\"passed\":true}");return 0; +} +""" + + +def main(): + root = Path(sys.argv[1]) + batch = (root / "_batch.c").read_text() + job = batch[batch.index("typedef struct {"):batch.index("} read_job_t;") + len("} read_job_t;")] + direct = (root / "_direct.c").read_text() + validate = direct[direct.index("static bool validate_direct_range("):direct.index("static bool direct_read_range(")] + compiler = shutil.which("cc") or shutil.which("gcc") + if not compiler: + raise SystemExit("C compiler required for checkpoint-reader checks") + with tempfile.TemporaryDirectory(prefix="spark-fp4-reader-") as temporary: + directory = Path(temporary) + (directory / "liburing.h").write_text(URING) + (directory / "_bounce.c").write_bytes((root / "_bounce.c").read_bytes()) + (directory / "check.c").write_text(PREFIX + job + validate + '\n#define pread checked_pread\n#include "_bounce.c"\n' + CASES) + subprocess.run([compiler, "-std=c11", "-D_GNU_SOURCE", "-DB12X_HAVE_LIBURING=1", "-O2", "-Wall", "-Wextra", "-Werror", "-pthread", "-I", str(directory), str(directory / "check.c"), "-o", str(directory / "check")], check=True, timeout=60) + subprocess.run([str(directory / "check"), str(directory)], check=True, timeout=30) + + +if __name__ == "__main__": + main() diff --git a/configs/sparkplane/engines/checks/qwen38_fp4_ple.py b/configs/sparkplane/engines/checks/qwen38_fp4_ple.py new file mode 100644 index 0000000..1ab7236 --- /dev/null +++ b/configs/sparkplane/engines/checks/qwen38_fp4_ple.py @@ -0,0 +1,163 @@ +"""Compile and exercise the installed PLE reader and planner with real O_DIRECT I/O. + +No GPU is needed: PLE scatters into host buffers. The Python capsule API, planner, +file validation, worker ownership and error handling are the actual shipped code. +""" +import array +from concurrent.futures import ThreadPoolExecutor +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import sysconfig +import tempfile + +PREFIX = r''' +#define _GNU_SOURCE +#define PY_SSIZE_T_CLEAN +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef struct { char message[512]; } failure_t; +static atomic_int fault, interrupted, active, peak, launches; +static int uring_calls; +static void system_error(failure_t* f,const char* op) { + snprintf(f->message,sizeof(f->message),"%s: %s",op,strerror(errno)); +} +static int denied_init(unsigned n,struct io_uring* r,unsigned f) { + (void)n;(void)r;(void)f;uring_calls++;return -EPERM; +} +static inline ssize_t checked_pread(int fd,void* b,size_t n,off_t o) { + int running=atomic_fetch_add(&active,1)+1; + int old=atomic_load(&peak); + while(running>old && !atomic_compare_exchange_weak(&peak,&old,running)) {} + ssize_t result; + int mode=atomic_load(&fault); + if(mode==1 && !atomic_exchange(&interrupted,1)){errno=EINTR;result=-1;} + else if(mode==2)result=0; + else if(mode==3){errno=EIO;result=-1;} + else result=pread(fd,b,n,o); + atomic_fetch_sub(&active,1);return result; +} +static inline int checked_create(pthread_t* t,const pthread_attr_t* a,void*(*f)(void*),void* p) { + if(atomic_load(&fault)==4 && atomic_fetch_add(&launches,1)==2)return EAGAIN; + return pthread_create(t,a,f,p); +} +#define io_uring_queue_init denied_init +#define pread checked_pread +#define pthread_create checked_create +#include "_ple_reader.c" +static PyObject* set_fault(PyObject* self,PyObject* arg) { + (void)self;long value=PyLong_AsLong(arg);if(PyErr_Occurred())return NULL; + atomic_store(&fault,(int)value);atomic_store(&interrupted,0);atomic_store(&launches,0);Py_RETURN_NONE; +} +static PyObject* counters(PyObject* self,PyObject* args) { + (void)self;(void)args;return Py_BuildValue("iii",uring_calls,atomic_load(&active),atomic_load(&peak)); +} +static PyMethodDef methods[]={ + {"reader",py_ple_reader,METH_VARARGS,NULL}, + {"add",py_ple_reader_add,METH_VARARGS,NULL}, + {"run",py_ple_reader_run,METH_VARARGS,NULL}, + {"stats",py_ple_reader_stats,METH_O,NULL}, + {"fault",set_fault,METH_O,NULL},{"counters",counters,METH_NOARGS,NULL},{NULL,NULL,0,NULL}}; +static PyModuleDef module={PyModuleDef_HEAD_INIT,.m_name="_spark_ple",.m_size=-1,.m_methods=methods}; +PyMODINIT_FUNC PyInit__spark_ple(void){return PyModule_Create(&module);} +''' + + +def main(): + root = Path(sys.argv[1]) + compiler = shutil.which("cc") or shutil.which("gcc") + if not compiler: + raise SystemExit("C compiler required for PLE checks") + with tempfile.TemporaryDirectory(prefix="spark-fp4-ple-") as temporary: + directory = Path(temporary) + for name in ("_ple_reader.c", "_row_plan.h"): + shutil.copyfile(root / name, directory / name) + (directory / "check.c").write_text(PREFIX) + library = directory / ("_spark_ple" + sysconfig.get_config_var("EXT_SUFFIX")) + subprocess.run([compiler, "-std=c11", "-DB12X_HAVE_LIBURING=1", "-O2", "-Wall", "-Wextra", "-Werror", "-shared", "-fPIC", "-pthread", "-I", sysconfig.get_path("include"), str(directory / "check.c"), "-luring", "-o", str(library)], check=True, timeout=60) + spec = importlib.util.spec_from_file_location("_spark_ple", library) + native = importlib.util.module_from_spec(spec) + spec.loader.exec_module(native) + os.environ["B12X_DISK_BACKEND"] = "pread" + rows, width, scale_width = 512, 8193, 17 + weights = bytes((i * 131 + i // 251) % 256 for i in range(rows * width)) + scales = bytes((i * 47 + i // 61) % 256 for i in range(rows * scale_width)) + sources = [] + for shard in range(2): + for is_scale, data, stride, offset in ((False, weights, width, 113), (True, scales, scale_width, 73)): + path = directory / f"source-{shard}-{is_scale}" + path.write_bytes(bytes(offset) + data[shard*256*stride:(shard+1)*256*stride]) + sources.append((shard, str(path), offset, is_scale)) + + def reader(): + handle = native.reader(256, rows, 0, rows, width, scale_width, 128, 1024) + for source in sources: + native.add(handle, *source) + assert native.stats(handle)["staging_bytes"] == 16 * 65536 + return handle + + ids = [((i*71) % rows) for i in range(100)] + [3, 3, -1, rows] + + def check(handle): + out = bytearray([165]) * (len(ids)*width + 16) + scale = bytearray([165]) * (len(ids)*scale_width + 16) + native.run(handle, array.array("q", ids), out, scale, len(ids)) + for i, row in enumerate(ids): + assert out[i*width:(i+1)*width] == (weights[row*width:(row+1)*width] if 0 <= row < rows else bytes(width)) + assert scale[i*scale_width:(i+1)*scale_width] == (scales[row*scale_width:(row+1)*scale_width] if 0 <= row < rows else bytes(scale_width)) + assert out[-16:] == bytes([165])*16 and scale[-16:] == bytes([165])*16 + assert native.stats(handle)["submit_calls"] == 0 + + handle = reader() + native.fault(1) + check(handle) # EINTR, unaligned sources, multiple waves, shards and planes. + native.fault(0) + with ThreadPoolExecutor(max_workers=2) as pool: + list(pool.map(lambda _: check(handle), range(4))) + # Failed waves must drain, publish no stale bytes, and reject reader reuse. + for mode in (2, 3, 4): + broken = reader() + native.fault(mode) + out = bytearray([165]) * (len(ids)*width) + scale = bytearray([165]) * (len(ids)*scale_width) + try: + native.run(broken, array.array("q", ids[:-2]), out, scale, len(ids)-2) + raise AssertionError("injected failure was accepted") + except RuntimeError: + pass + assert out == bytes([165])*len(out) + assert native.counters()[1] == 0 + native.fault(0) + try: + native.run(broken, array.array("q", [0]), out, scale, 1) + raise AssertionError("poisoned reader reused") + except RuntimeError: + pass + assert native.counters()[0] == 0 and native.counters()[2] <= 16 + os.environ["B12X_DISK_BACKEND"] = "io_uring" + try: + native.reader(256, rows, 0, rows, width, scale_width, 128, 16) + raise AssertionError("denied io_uring accepted") + except RuntimeError: + pass + assert native.counters()[0] == 1 + print(json.dumps({"pread_ple_cases": 9, "passed": True})) + + +if __name__ == "__main__": + main() diff --git a/configs/sparkplane/engines/checks/qwen38_redhat_config.py b/configs/sparkplane/engines/checks/qwen38_redhat_config.py new file mode 100644 index 0000000..6eae919 --- /dev/null +++ b/configs/sparkplane/engines/checks/qwen38_redhat_config.py @@ -0,0 +1,66 @@ +"""Check serialized QSA compatibility against the image's real vLLM config.""" +import copy +import unittest + +from vllm.models.qwen4_exp.config import Qwen4ExpConfig, Qwen4ExpTextConfig + + +def checkpoint_config(): + # RedHatAI c8f2fb1b9869f686b214782036123b10ff96d14a text geometry. + return { + "num_hidden_layers": 48, + "hidden_size": 2560, + "head_dim": 256, + "partial_rotary_factor": 0.25, + "layer_types": ["linear_attention"] * 3 + ["qwen_sparse_attention"], + "indexer_n_heads": 4, + "indexer_kv_heads": 1, + "indexer_head_dim": 128, + "indexer_budget": 2048, + "indexer_compress_ratio": 4, + "ple_layer_ids": [2], + "ple_embed_dim": 2560, + } + + +class SerializedQSATests(unittest.TestCase): + def options(self): + options = checkpoint_config() + options["layer_types"] *= 12 + return options + + def test_serialized_sparse_attention_retains_qsa_geometry(self): + options = self.options() + original = copy.deepcopy(options) + config = Qwen4ExpConfig(text_config=options).get_text_config() + self.assertEqual(options, original) + self.assertEqual(config.layer_types, (["linear_attention"] * 3 + ["full_attention"]) * 12) + # The pinned model chooses QSA for precisely these layers when its + # indexer fields are present; no indexer or attention geometry changes. + self.assertEqual([i for i, kind in enumerate(config.layer_types) if kind == "full_attention"], list(range(3, 48, 4))) + for field in ["indexer_n_heads", "indexer_kv_heads", "indexer_head_dim", "indexer_budget", "indexer_compress_ratio", "head_dim", "partial_rotary_factor", "hidden_size", "ple_embed_dim", "ple_layer_ids"]: + self.assertEqual(getattr(config, field), original[field]) + self.assertEqual(config.layers_block_type.count("attention"), 12) + self.assertEqual(config.layers_block_type.count("linear_attention"), 36) + + def test_legacy_layer_labels_are_unchanged(self): + options = self.options() + options["layer_types"] = (["linear_attention"] * 3 + ["full_attention"]) * 12 + config = Qwen4ExpTextConfig(**options) + self.assertEqual(config.layer_types, options["layer_types"]) + + def test_unknown_layer_label_is_still_rejected(self): + options = self.options() + options["layer_types"][3] = "unknown_attention" + with self.assertRaises(ValueError): + Qwen4ExpTextConfig(**options) + + def test_invalid_qsa_geometry_is_still_rejected(self): + options = self.options() + options["indexer_budget"] = 3 + with self.assertRaises(ValueError): + Qwen4ExpTextConfig(**options) + + +if __name__ == "__main__": + unittest.main() diff --git a/configs/sparkplane/engines/patches/sha256-33d669fc7b0e3761f6ef213b11789196fca52801c7ceb991bb4b271489aac1bc.py b/configs/sparkplane/engines/patches/sha256-33d669fc7b0e3761f6ef213b11789196fca52801c7ceb991bb4b271489aac1bc.py new file mode 100644 index 0000000..6c63ae2 --- /dev/null +++ b/configs/sparkplane/engines/patches/sha256-33d669fc7b0e3761f6ef213b11789196fca52801c7ceb991bb4b271489aac1bc.py @@ -0,0 +1,56 @@ +"""Add confined bounded direct pread waves to the pinned disk PLE reader.""" +import hashlib +from pathlib import Path +import sys + +root = Path(sys.argv[1]) +path = root / '_ple_reader.c' +if hashlib.sha256(path.read_bytes()).hexdigest() != '36bcce65d3e131a02749a21105eda1771f80ca0e55686f84c576240fa783f585': + raise SystemExit("unsupported B12X PLE source; refusing to patch") +source = path.read_text() +old, new = ' bool poisoned;\n', ' bool poisoned;\n bool pread_mode;\n' +if source.count(old) != 1: + raise SystemExit("ambiguous PLE source anchor") +source = source.replace(old, new, 1) +old, new = ' reader->slots_count = (unsigned)depth;', ' const char* backend = getenv("B12X_DISK_BACKEND");\n reader->pread_mode = backend && strcmp(backend, "pread") == 0;\n /* A finite worker/64 KiB staging slot per concurrent direct read. */\n reader->slots_count = reader->pread_mode && depth > 16 ? 16 : (unsigned)depth;' +if source.count(old) != 1: + raise SystemExit("ambiguous PLE source anchor") +source = source.replace(old, new, 1) +old, new = ' error = io_uring_queue_init(reader->slots_count, &reader->ring, 0);\n if (error < 0) {\n ple_release(reader);\n return PyErr_Format(\n PyExc_RuntimeError,\n "io_uring initialization failed: %s; enable io_uring in the kernel/container policy",\n strerror(-error));\n }\n reader->ring_ready = true;\n reader->iovecs = calloc(reader->slots_count, sizeof(*reader->iovecs));\n reader->free_slots = calloc(reader->slots_count, sizeof(*reader->free_slots));\n if (!reader->iovecs || !reader->free_slots) {\n ple_release(reader);\n return PyErr_NoMemory();\n }\n for (unsigned i = 0; i < reader->slots_count; i++) {\n reader->iovecs[i].iov_base = reader->slots[i].buffer;\n reader->iovecs[i].iov_len = PLE_READ_MAX;\n }\n error = io_uring_register_buffers(&reader->ring, reader->iovecs, reader->slots_count);\n if (error < 0) {\n ple_release(reader);\n return PyErr_Format(PyExc_RuntimeError,\n "io_uring buffer registration failed: %s; raise the locked-memory limit or "\n "reduce queue_depth",\n strerror(-error));\n }\n', ' if (!reader->pread_mode) {\n error = io_uring_queue_init(reader->slots_count, &reader->ring, 0);\n if (error < 0) {\n ple_release(reader);\n return PyErr_Format(\n PyExc_RuntimeError,\n "io_uring initialization failed: %s; enable io_uring in the kernel/container policy",\n strerror(-error));\n }\n reader->ring_ready = true;\n reader->iovecs = calloc(reader->slots_count, sizeof(*reader->iovecs));\n reader->free_slots = calloc(reader->slots_count, sizeof(*reader->free_slots));\n if (!reader->iovecs || !reader->free_slots) {\n ple_release(reader);\n return PyErr_NoMemory();\n }\n for (unsigned i = 0; i < reader->slots_count; i++) {\n reader->iovecs[i].iov_base = reader->slots[i].buffer;\n reader->iovecs[i].iov_len = PLE_READ_MAX;\n }\n error = io_uring_register_buffers(&reader->ring, reader->iovecs, reader->slots_count);\n if (error < 0) {\n ple_release(reader);\n return PyErr_Format(PyExc_RuntimeError,\n "io_uring buffer registration failed: %s; raise the locked-memory limit or "\n "reduce queue_depth",\n strerror(-error));\n }\n }\n' +if source.count(old) != 1: + raise SystemExit("ambiguous PLE source anchor") +source = source.replace(old, new, 1) +old, new = ' ple_reader_t* reader = owner;\n', ' ple_reader_t* reader = owner;\n if (reader->pread_mode) return 0;\n' +if source.count(old) != 1: + raise SystemExit("ambiguous PLE source anchor") +source = source.replace(old, new, 1) +old, new = 'static void ple_uring_run(ple_reader_t* reader) {', '/* Each wave owns disjoint aligned slots. Join every issued read before scatter,\n * failure return, slot reuse or reader destruction; no detached I/O survives. */\ntypedef struct {\n int fd;\n char* buffer;\n unsigned length;\n int64_t offset;\n int result;\n} ple_pread_job_t;\n\nstatic void* ple_pread_worker(void* opaque) {\n ple_pread_job_t* job = opaque;\n ssize_t result;\n do {\n result = pread(job->fd, job->buffer, job->length, job->offset);\n } while (result < 0 && errno == EINTR);\n job->result = result < 0 ? -errno : (int)result;\n return NULL;\n}\n\nstatic void ple_pread_run(ple_reader_t* reader) {\n pthread_t workers[16];\n ple_pread_job_t reads[16];\n for (size_t next = 0; next < reader->plan.job_count;) {\n size_t left = reader->plan.job_count - next;\n unsigned count = left < reader->slots_count ? (unsigned)left : reader->slots_count;\n unsigned started = 0;\n for (unsigned i = 0; i < count; i++) {\n ple_job_t* job = &reader->plan.jobs[next + i];\n reads[i] = (ple_pread_job_t){.fd = reader->plan.files[job->file].fd,\n .buffer = reader->slots[i].buffer,\n .length = job->length, .offset = job->offset};\n if (count == 1) {\n ple_pread_worker(&reads[i]);\n } else {\n int error = pthread_create(&workers[i], NULL, ple_pread_worker, &reads[i]);\n if (error) {\n ple_error(reader, "PLE pread worker", error);\n reader->poisoned = true;\n break;\n }\n }\n started++;\n }\n for (unsigned i = 0; i < started; i++) {\n if (count > 1) pthread_join(workers[i], NULL);\n }\n /* All reads have finished. A failed wave never publishes stale slot data. */\n for (unsigned i = 0; i < started; i++) {\n ple_job_t* job = &reader->plan.jobs[next + i];\n if (reads[i].result < 0 || (unsigned)reads[i].result != job->expected) {\n ple_error(reader, "PLE pread", reads[i].result < 0 ? -reads[i].result : EIO);\n reader->poisoned = true;\n }\n }\n if (reader->poisoned) return;\n for (unsigned i = 0; i < started; i++)\n if (!ple_scatter(reader, next + i, reads[i].buffer, reads[i].result)) {\n reader->poisoned = true;\n return;\n }\n next += started;\n }\n}\n\nstatic void ple_uring_run(ple_reader_t* reader) {' +if source.count(old) != 1: + raise SystemExit("ambiguous PLE source anchor") +source = source.replace(old, new, 1) +old, new = ' if (planned && reader->plan.job_count) ple_uring_run(reader);', ' if (planned && reader->plan.job_count) {\n if (reader->pread_mode) ple_pread_run(reader);\n else ple_uring_run(reader);\n }' +if source.count(old) != 1: + raise SystemExit("ambiguous PLE source anchor") +source = source.replace(old, new, 1) +old, new = 'PLE io_uring reader is unusable after a submission/completion failure', 'PLE reader is unusable after an I/O failure' +if source.count(old) != 1: + raise SystemExit("ambiguous PLE source anchor") +source = source.replace(old, new, 1) +path.write_text(source) +path = root / '../sequence/_shared/disk_table.py' +if hashlib.sha256(path.read_bytes()).hexdigest() != '601348d52d488057911b5a0b22d36f0e922d80cb956c2658901bee56e63e7c9a': + raise SystemExit("unsupported B12X PLE source; refusing to patch") +source = path.read_text() +old, new = 'self._backend not in ("io_uring", "gds")', 'self._backend not in ("io_uring", "pread", "gds")' +if source.count(old) != 1: + raise SystemExit("ambiguous PLE source anchor") +source = source.replace(old, new, 1) +old, new = 'B12X_DISK_BACKEND must be io_uring or gds', 'B12X_DISK_BACKEND must be io_uring, pread or gds' +if source.count(old) != 1: + raise SystemExit("ambiguous PLE source anchor") +source = source.replace(old, new, 1) +old, new = 'self._backend == "io_uring"', 'self._backend in ("io_uring", "pread")' +if source.count(old) != 1: + raise SystemExit("ambiguous PLE source anchor") +source = source.replace(old, new, 1) +path.write_text(source) diff --git a/configs/sparkplane/engines/patches/sha256-7dc4adb2a6d401da7d92bb45016a1dc68bdfad6b3b4d06b0c9920151d12152f9.py b/configs/sparkplane/engines/patches/sha256-7dc4adb2a6d401da7d92bb45016a1dc68bdfad6b3b4d06b0c9920151d12152f9.py new file mode 100644 index 0000000..aa5d405 --- /dev/null +++ b/configs/sparkplane/engines/patches/sha256-7dc4adb2a6d401da7d92bb45016a1dc68bdfad6b3b4d06b0c9920151d12152f9.py @@ -0,0 +1,65 @@ +"""Add an explicit bounded pread transport to the pinned B12X bounce reader.""" +import hashlib +from pathlib import Path +import sys + +path = Path(sys.argv[1]) / '_bounce.c' +original = path.read_text() +if hashlib.sha256(path.read_bytes()).hexdigest() != '45235c9008d3b2c775dac65bdd38116cd266db881470ea8bdbd9e74964d9d3db': + raise SystemExit('unsupported B12X checkpoint reader; refusing to patch') + +def replace(old, new): + global original + if original.count(old) != 1: + raise SystemExit('checkpoint reader patch anchor is ambiguous') + original = original.replace(old, new, 1) + +replace(' bool registered;\n', ' bool registered;\n bool pread_mode;\n') +replace(' ring->depth = depth;\n', ''' ring->depth = depth; + const char* pread_mode = getenv("B12X_CHECKPOINT_PREAD"); + ring->pread_mode = pread_mode && strcmp(pread_mode, "1") == 0; +''') +start = original.index(' error = io_uring_queue_init(BOUNCE_SLOTS, &ring->ring, 0);') +end = original.index(' error = pthread_create(&ring->copier, NULL, bounce_copy_worker, ring);', start) +block = original[start:end] +replace(block, ''' if (ring->pread_mode) { + for (unsigned i = 0; i < BOUNCE_SLOTS; i++) ring->free_slots[ring->free_count++] = i; + } else { +''' + ''.join(' '+line+'\n' for line in block.splitlines()) + ' }\n') +start = original.index(' struct io_uring_sqe* sqe = io_uring_get_sqe(&ring->ring);') +end = original.index(' within += slot->job.bytes;', start) +block = original[start:end] +replace(block, ''' if (ring->pread_mode) { + /* The same fixed slot stays owned until its CUDA copy completes. */ + ssize_t received; + do { + received = pread(job->fd, ring->buffers + (size_t)index * BOUNCE_SLOT_BYTES, + length, aligned); + } while (received < 0 && errno == EINTR); + ring->reads++; + if (received > 0) ring->physical_bytes += received; + if (received < 0 || (size_t)received < slot->required) { + snprintf(failure->message, sizeof(failure->message), "checkpoint pread: %s", + received < 0 ? strerror(errno) : "short O_DIRECT read"); + pthread_mutex_lock(&ring->mutex); + ring->free_slots[ring->free_count++] = index; + pthread_mutex_unlock(&ring->mutex); + ring->poisoned = true; + break; + } + if (!ring->max_inflight) ring->max_inflight = 1; + pthread_mutex_lock(&ring->mutex); + ring->ready_slots[ring->tail++ % BOUNCE_SLOTS] = index; + ring->ready_count++; + ring->copying++; + pthread_cond_signal(&ring->ready); + pthread_mutex_unlock(&ring->mutex); + } else { +''' + ''.join(' '+line+'\n' for line in block.splitlines()) + ' }\n') +replace('"io_uring bounce executor is unusable after a submission/completion failure"', + '"checkpoint bounce executor is unusable after a read/copy failure"') +replace(''' if (ring->poisoned) { + /* Drain submitted reads above; never submit SQEs left by a failed submit. */''', + ''' if (ring->poisoned && ring->ring_ready) { + /* Drain submitted reads above; never submit SQEs left by a failed submit. */''') +path.write_text(original) diff --git a/configs/sparkplane/engines/patches/sha256-abfa66016b36ea4035777f9b02a5c9a6c3badc49d48e12c3320f016ba005efe4.py b/configs/sparkplane/engines/patches/sha256-abfa66016b36ea4035777f9b02a5c9a6c3badc49d48e12c3320f016ba005efe4.py new file mode 100644 index 0000000..35cfe4f --- /dev/null +++ b/configs/sparkplane/engines/patches/sha256-abfa66016b36ea4035777f9b02a5c9a6c3badc49d48e12c3320f016ba005efe4.py @@ -0,0 +1,22 @@ +"""Normalize the serialized QSA layer label in the pinned Qwen4Exp runtime.""" +import hashlib +from pathlib import Path +import sys + +path = Path(sys.argv[1]) +source = path.read_bytes() +if hashlib.sha256(source).hexdigest() != "71230ef5e53e016416f54ae96b9f8b4e2c10a6311d8e6035ac063ac48fea2471": + raise SystemExit("unsupported Qwen4Exp config; refusing to patch") +old = b" super().__init__(layer_types=layer_types, **kwargs)\n" +new = b""" # This runtime selects QSA from indexer_n_heads on full_attention layers. + # Normalize only the checkpoint label; retain every QSA geometry field. + if layer_types is not None: + layer_types = [ + "full_attention" if kind == "qwen_sparse_attention" else kind + for kind in layer_types + ] + super().__init__(layer_types=layer_types, **kwargs) +""" +if source.count(old) != 1: + raise SystemExit("Qwen4Exp config patch anchor is ambiguous") +path.write_bytes(source.replace(old, new, 1)) diff --git a/configs/sparkplane/engines/vllm-qwen38-fp4-local.Dockerfile b/configs/sparkplane/engines/vllm-qwen38-fp4-local.Dockerfile new file mode 100644 index 0000000..e6393fc --- /dev/null +++ b/configs/sparkplane/engines/vllm-qwen38-fp4-local.Dockerfile @@ -0,0 +1,22 @@ +FROM eugr/spark-vllm-b12x@sha256:03a956544e9f9b4b7ba91690469fbe7848bdbef29a7d2e0f5788368467bb3595 + +ARG LOADER=/usr/local/lib/python3.12/dist-packages/flashinfer/experimental/b12x/loader +COPY patches/sha256-7dc4adb2a6d401da7d92bb45016a1dc68bdfad6b3b4d06b0c9920151d12152f9.py /opt/llm/patches/checkpoint_pread.py +RUN python3 /opt/llm/patches/checkpoint_pread.py ${LOADER} +COPY checks/qwen38_fp4_checkpoint.py /opt/llm/checks/qwen38_fp4_checkpoint.py +COPY patches/sha256-33d669fc7b0e3761f6ef213b11789196fca52801c7ceb991bb4b271489aac1bc.py /opt/llm/patches/ple_pread.py +RUN python3 /opt/llm/patches/ple_pread.py ${LOADER} +COPY checks/qwen38_fp4_ple.py /opt/llm/checks/qwen38_fp4_ple.py +RUN chmod -R a+rX /opt/llm +USER 65534:65534 +RUN HOME=/tmp python3 /opt/llm/checks/qwen38_fp4_checkpoint.py ${LOADER} +RUN HOME=/tmp python3 /opt/llm/checks/qwen38_fp4_ple.py ${LOADER} +USER root + +LABEL org.opencontainers.image.source="https://github.com/Sumatoshi-tech/sparkplane" \ + org.opencontainers.image.revision="b12x-8244d1c44-pread-v2" \ + sparkplane.base-manifest="sha256:03a956544e9f9b4b7ba91690469fbe7848bdbef29a7d2e0f5788368467bb3595" \ + sparkplane.cuda-architecture="12.1a" \ + sparkplane.ple-transport="bounded-pread" \ + sparkplane.checkpoint-transport="bounded-pread" \ + sparkplane.checkpoint-self-test="qwen38_fp4_checkpoint.py" diff --git a/configs/sparkplane/engines/vllm-qwen38-fp4-local.toml b/configs/sparkplane/engines/vllm-qwen38-fp4-local.toml new file mode 100644 index 0000000..ff3a450 --- /dev/null +++ b/configs/sparkplane/engines/vllm-qwen38-fp4-local.toml @@ -0,0 +1,137 @@ +schema = "sparkplane.engine/v3" +id = "vllm-qwen38-fp4-local-arm64" +priority = 200 +family = "vllm" +version = "b12x-8244d1c44-20261015-pread-v2" +image_transport = "local" +image_repository = "sparkplane/vllm-qwen38-fp4-local" +image_digest = "sha256:ef7c260eafbca2ca1d7330c79f5599314838191f93c87953202834a097471bf2" +image_architecture = "arm64" +entrypoint = ["vllm", "serve"] +arguments = [ + "{model_snapshot}", + "--served-model-name", "{served_model}", + "--host", "0.0.0.0", + "--port", "{port}", +] +# Disk-backed PLE uses bounded pread and retains the default seccomp policy. +# Executable JIT temporaries stay in the managed, per-engine compile cache. +executable_cache_environment = ["TMPDIR"] +environment = [ + "HOME=/compile-cache", + "TMPDIR=/compile-cache", + "HF_HUB_OFFLINE=1", + "TRANSFORMERS_OFFLINE=1", + "HF_DATASETS_OFFLINE=1", + "VLLM_NO_USAGE_STATS=1", + "DO_NOT_TRACK=1", + "VLLM_PLE_TABLE_MEMORY=disk", + "B12X_CHECKPOINT_PREAD=1", + "B12X_DISK_BACKEND=pread", + "CUTE_DSL_ARCH=sm_121a", + "SAFETENSORS_FAST_GPU=1", + "VLLM_WORKER_MULTIPROC_METHOD=spawn", + "VLLM_SSM_CONV_STATE_LAYOUT=DS", + "VLLM_USE_AOT_COMPILE=1", + "VLLM_USE_MEGA_AOT_ARTIFACT=1", + "VLLM_USE_V2_MODEL_RUNNER=1", + "B12X_POLICY_MODE=auto", + "VLLM_MXFP8_LM_HEAD=1", + "VLLM_B12X_MOE_FP4_LAYER_MAX_INPUT_SCALE=w13", + "VLLM_B12X_MOE_FP4_FORCE_A16=1", + "B12X_W4A16_A4_PREFILL=1", + "B12X_WEIGHTS_COMPILE_WORKERS=1", + "PYTORCH_ALLOC_CONF=expandable_segments:True,garbage_collection_threshold:0.6,per_process_memory_fraction:0.82,throw_on_cudamalloc_oom:True", + "VLLM_CACHE_ROOT=/compile-cache/vllm", + "TORCHINDUCTOR_CACHE_DIR=/compile-cache/torchinductor", + "TRITON_CACHE_DIR=/compile-cache/triton", + "CUDA_CACHE_PATH=/compile-cache/cuda", + "XDG_CACHE_HOME=/compile-cache/xdg", + "UMASK=0027", +] +default_profile = "qwen3.8-flash-next-fp4-local" +model_cache_root = "/var/lib/sparkplane/huggingface" +compile_cache_root = "/var/lib/sparkplane/compile-cache" +network = "sparkplane-internal" +port = 8000 +run_as_uid = 65534 +pid_limit = 2048 +seccomp = "default" +ipc_mode = "private" +shm_size_bytes = 17179869184 +tmpfs = ["/tmp"] +startup_deadline_seconds = 1800 +health_method = "GET" +health_path = "/health" +semantic_prompt = "Generate one completion token." +semantic_max_tokens = 1 +model_mount = "snapshot" + +[matcher] +formats = ["safetensors"] +quantizations = ["NVFP4"] +engine_profiles = ["qwen3.8-flash-next-fp4-local"] +required_capabilities = ["text_generation"] + +[artifact_arguments] +primary = [] +bindings = [] +ignored_roles = ["weight_shard"] + +[resources] +image_bytes = 30000000000 +startup_peak_bytes = 114000000000 +steady_peak_bytes = 114000000000 +compile_cache_bytes = 8589934592 + +[[routes]] +method = "GET" +path = "/v1/models" + +[[routes]] +method = "POST" +path = "/v1/completions" + +[[routes]] +method = "POST" +path = "/v1/chat/completions" + +[[routes]] +method = "POST" +path = "/tokenize" + +[[profiles]] +id = "qwen3.8-flash-next-fp4-local" +model_types = ["qwen4_exp"] +context_window = 262144 +stream_idle_timeout_seconds = 600 +arguments = [ + "--tensor-parallel-size", "1", + "--pipeline-parallel-size", "1", + "--mamba-cache-mode", "align", + "--enable-prefix-caching", + "--enable-chunked-prefill", + "--dtype", "bfloat16", + "--kv-cache-dtype", "fp8", + "--kv-cache-memory", "6442450944", + "--quantization", "modelopt_mixed", + "--block-size", "16", + "--load-format", "b12x", + "--max-model-len", "262144", + "--max-num-seqs", "8", + "--max-num-batched-tokens", "4096", + "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":4}", + "--gdn-decode-kernel", "b12x", + "--linear-backend", "b12x", + "--moe-backend", "b12x", + "--no-enable-flashinfer-autotune", + "--mm-encoder-tp-mode", "data", + "--reasoning-parser", "qwen3", + "--tool-call-parser", "qwen3_xml", + "--enable-auto-tool-choice", + "--compilation-config", "{\"pass_config\":{\"fuse_act_quant\":true}}", +] +capabilities = ["text_generation", "tool_calling", "vision"] +vision = { processor_sha256 = "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516", media_types = ["image/jpeg", "image/png", "image/webp"], max_bytes = 524288, max_total_bytes = 524288, max_count = 16, max_width = 4096, max_height = 4096, health_media_type = "image/png", health_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAOAAAADgAQAAAAAyVldMAAAAHUlEQVRYw+3BMQEAAADCoPVPbQZ/oAAAAAAAAHgMGWAAAct2tnAAAAAASUVORK5CYII=", health_image_sha256 = "1b8ab2918e9fb346d8ff5b7579372510b0d8b1566d590dac31aea311fff15fc6", health_prompt = "Name the single dominant color using one lowercase word.", health_expected_text = "black", health_max_tokens = 32, health_disable_thinking = true } +startup_protocol_probe = true +sampling = { temperature = 1.0, top_p = 0.95, top_k = 20, min_p = 0.0, presence_penalty = 0.0, repetition_penalty = 1.0, thinking_token_budget = 8192, reasoning_effort_map = { low = "low", medium = "medium", high = "xhigh", max = "xhigh" } } diff --git a/configs/sparkplane/engines/vllm-qwen38-redhat.Dockerfile b/configs/sparkplane/engines/vllm-qwen38-redhat.Dockerfile new file mode 100644 index 0000000..5019370 --- /dev/null +++ b/configs/sparkplane/engines/vllm-qwen38-redhat.Dockerfile @@ -0,0 +1,33 @@ +FROM eugr/spark-vllm-b12x@sha256:03a956544e9f9b4b7ba91690469fbe7848bdbef29a7d2e0f5788368467bb3595 + +ARG LOADER=/usr/local/lib/python3.12/dist-packages/flashinfer/experimental/b12x/loader +COPY patches/sha256-7dc4adb2a6d401da7d92bb45016a1dc68bdfad6b3b4d06b0c9920151d12152f9.py /opt/llm/patches/checkpoint_pread.py +RUN python3 /opt/llm/patches/checkpoint_pread.py ${LOADER} +COPY checks/qwen38_fp4_checkpoint.py /opt/llm/checks/qwen38_fp4_checkpoint.py +COPY patches/sha256-33d669fc7b0e3761f6ef213b11789196fca52801c7ceb991bb4b271489aac1bc.py /opt/llm/patches/ple_pread.py +RUN python3 /opt/llm/patches/ple_pread.py ${LOADER} +COPY checks/qwen38_fp4_ple.py /opt/llm/checks/qwen38_fp4_ple.py +RUN chmod -R a+rX /opt/llm +USER 65534:65534 +RUN HOME=/tmp python3 /opt/llm/checks/qwen38_fp4_checkpoint.py ${LOADER} +RUN HOME=/tmp python3 /opt/llm/checks/qwen38_fp4_ple.py ${LOADER} +USER root + +LABEL org.opencontainers.image.source="https://github.com/Sumatoshi-tech/sparkplane" \ + org.opencontainers.image.revision="b12x-8244d1c44-pread-v2" \ + sparkplane.base-manifest="sha256:03a956544e9f9b4b7ba91690469fbe7848bdbef29a7d2e0f5788368467bb3595" \ + sparkplane.cuda-architecture="12.1a" \ + sparkplane.ple-transport="bounded-pread" \ + sparkplane.checkpoint-transport="bounded-pread" \ + sparkplane.checkpoint-self-test="qwen38_fp4_checkpoint.py" + +# Keep the existing pread inputs and add only serialized QSA label compatibility. +ARG QWEN_CONFIG=/usr/local/lib/python3.12/dist-packages/vllm/models/qwen4_exp/config.py +COPY patches/sha256-abfa66016b36ea4035777f9b02a5c9a6c3badc49d48e12c3320f016ba005efe4.py /opt/llm/patches/qwen38_redhat_config.py +RUN python3 /opt/llm/patches/qwen38_redhat_config.py ${QWEN_CONFIG} +COPY checks/qwen38_redhat_config.py /opt/llm/checks/qwen38_redhat_config.py +USER 65534:65534 +RUN HOME=/tmp HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 CUDA_VISIBLE_DEVICES=-1 python3 /opt/llm/checks/qwen38_redhat_config.py +USER root +LABEL org.opencontainers.image.revision="b12x-8244d1c44-redhat-v1" \ + sparkplane.redhat-config-self-test="qwen38_redhat_config.py" diff --git a/configs/sparkplane/engines/vllm-qwen38-redhat.toml b/configs/sparkplane/engines/vllm-qwen38-redhat.toml new file mode 100644 index 0000000..7b71fce --- /dev/null +++ b/configs/sparkplane/engines/vllm-qwen38-redhat.toml @@ -0,0 +1,143 @@ +schema = "sparkplane.engine/v3" +id = "vllm-qwen38-redhat-arm64" +priority = 200 +family = "vllm" +version = "b12x-8244d1c44-20261015-redhat-v1" +image_transport = "local" +image_repository = "sparkplane/vllm-qwen38-redhat" +image_digest = "sha256:5cfc1fe12892c42654abd337fae28be0dea8e67f575e669ba9c547c2b9fb0a62" +image_architecture = "arm64" +entrypoint = ["vllm", "serve"] +arguments = [ + "{model_snapshot}", + "--served-model-name", "{served_model}", + "--host", "0.0.0.0", + "--port", "{port}", +] +# Disk-backed PLE uses bounded pread and retains the default seccomp policy. +# Executable JIT temporaries stay in the managed, per-engine compile cache. +executable_cache_environment = ["TMPDIR"] +environment = [ + "HOME=/compile-cache", + "TMPDIR=/compile-cache", + "HF_HUB_OFFLINE=1", + "TRANSFORMERS_OFFLINE=1", + "HF_DATASETS_OFFLINE=1", + "VLLM_NO_USAGE_STATS=1", + "DO_NOT_TRACK=1", + "VLLM_PLE_TABLE_MEMORY=disk", + "B12X_CHECKPOINT_PREAD=1", + "B12X_DISK_BACKEND=pread", + "CUTE_DSL_ARCH=sm_121a", + "SAFETENSORS_FAST_GPU=1", + "VLLM_WORKER_MULTIPROC_METHOD=spawn", + "VLLM_SSM_CONV_STATE_LAYOUT=DS", + "VLLM_USE_AOT_COMPILE=1", + "VLLM_USE_MEGA_AOT_ARTIFACT=1", + "VLLM_USE_V2_MODEL_RUNNER=1", + "B12X_POLICY_MODE=auto", + "B12X_WEIGHTS_COMPILE_WORKERS=1", + "B12X_STATE_COMPILE_WORKERS=2", + "B12X_BIND_COMPILE_WORKERS=2", + "PYTORCH_ALLOC_CONF=expandable_segments:True,garbage_collection_threshold:0.6,per_process_memory_fraction:0.82,throw_on_cudamalloc_oom:True", + "VLLM_CACHE_ROOT=/compile-cache/vllm", + "TORCHINDUCTOR_CACHE_DIR=/compile-cache/torchinductor", + "TRITON_CACHE_DIR=/compile-cache/triton", + "CUDA_CACHE_PATH=/compile-cache/cuda", + "XDG_CACHE_HOME=/compile-cache/xdg", + "UMASK=0027", +] +default_profile = "qwen3.8-flash-next-redhat-nvfp4" +model_cache_root = "/var/lib/sparkplane/huggingface" +compile_cache_root = "/var/lib/sparkplane/compile-cache" +network = "sparkplane-internal" +port = 8000 +run_as_uid = 65534 +pid_limit = 2048 +seccomp = "default" +ipc_mode = "private" +shm_size_bytes = 17179869184 +tmpfs = ["/tmp"] +startup_deadline_seconds = 1800 +health_method = "GET" +health_path = "/health" +semantic_prompt = "Generate one completion token." +semantic_max_tokens = 1 +model_mount = "snapshot" + +[matcher] +formats = ["safetensors"] +quantizations = ["NVFP4"] +engine_profiles = ["qwen3.8-flash-next-redhat-nvfp4"] +required_capabilities = ["text_generation"] + +[artifact_arguments] +primary = [] +bindings = [] +ignored_roles = ["weight_shard"] + +[resources] +image_bytes = 30000000000 +startup_peak_bytes = 114000000000 +steady_peak_bytes = 114000000000 +compile_cache_bytes = 8589934592 + +[[routes]] +method = "GET" +path = "/v1/models" + +[[routes]] +method = "POST" +path = "/v1/completions" + +[[routes]] +method = "POST" +path = "/v1/chat/completions" + +[[routes]] +method = "POST" +path = "/tokenize" + +[[profiles]] +id = "qwen3.8-flash-next-redhat-nvfp4" +model_types = ["qwen4_exp"] +context_window = 262144 +stream_idle_timeout_seconds = 600 +arguments = [ + "--tensor-parallel-size", "1", + "--pipeline-parallel-size", "1", + "--mamba-cache-mode", "align", + "--enable-prefix-caching", + "--enable-chunked-prefill", + "--dtype", "bfloat16", + "--kv-cache-dtype", "fp8", + "--kv-cache-memory", "6442450944", + "--quantization", "compressed-tensors", + "--block-size", "16", + "--load-format", "b12x", + "--max-model-len", "262144", + "--max-num-seqs", "8", + "--max-num-batched-tokens", "4096", + "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":4}", + "--gdn-decode-kernel", "b12x", + "--linear-backend", "b12x", + "--moe-backend", "b12x", + "--no-enable-flashinfer-autotune", + "--mm-encoder-tp-mode", "data", + "--reasoning-parser", "qwen3", + "--tool-call-parser", "qwen3_xml", + "--enable-auto-tool-choice", + "--compilation-config", "{\"pass_config\":{\"fuse_act_quant\":true}}", +] +capabilities = ["text_generation", "tool_calling", "vision"] +vision = { processor_sha256 = "bfbc24af59a3e73a9cd0653b8d4ae758dfaec4e3e6c15dfb1c6c8ee8d5683c85", media_types = ["image/jpeg", "image/png", "image/webp"], max_bytes = 524288, max_total_bytes = 524288, max_count = 16, max_width = 4096, max_height = 4096, health_media_type = "image/png", health_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAOAAAADgAQAAAAAyVldMAAAAHUlEQVRYw+3BMQEAAADCoPVPbQZ/oAAAAAAAAHgMGWAAAct2tnAAAAAASUVORK5CYII=", health_image_sha256 = "1b8ab2918e9fb346d8ff5b7579372510b0d8b1566d590dac31aea311fff15fc6", health_prompt = "Name the single dominant color using one lowercase word.", health_expected_text = "black", health_max_tokens = 32, health_disable_thinking = true } +startup_protocol_probe = true +parallel_tool_calls = false +verbosity = false +sampling = { temperature = 1.0, top_p = 0.95, top_k = 20, min_p = 0.0, presence_penalty = 0.0, repetition_penalty = 1.0, thinking_token_budget = 8192, reasoning_effort_map = { low = "low", medium = "medium", high = "xhigh", max = "xhigh" } } + +# Public client features are qualified with this exact engine generation. +[profiles.reasoning] +supported_efforts = ["none", "low", "medium", "high", "xhigh"] +default_effort = "xhigh" +summaries = true diff --git a/configs/sparkplane/models.toml b/configs/sparkplane/models.toml index ba5e2f6..8020e90 100644 --- a/configs/sparkplane/models.toml +++ b/configs/sparkplane/models.toml @@ -612,7 +612,7 @@ bytes = 3413864960 sha256 = "9758875fc55e49561165f4a44342b654c123c3e25c6811b34abb83553fb1a164" [[models]] -aliases = ["qwen3.8:flash-next", "qwen3.8:flash-next-nvfp4"] +aliases = ["qwen3.8:flash-next-radixark", "qwen3.8:flash-next-radixark-nvfp4"] repository = "RadixArk/Qwen3.8-Flash-Next-NVFP4" revision = "7b719225242aacd3dbd3f9407468c2ee9a9d2594" @@ -1862,3 +1862,198 @@ role = "weight_shard" path = "model-plefp8-00009.safetensors" bytes = 4400023163 sha256 = "61de98b89bb79f386795787d7a76827a26f1e292c26edbb0a1b613da455f5a9c" + +# Standalone FP4-Local derivative; the compressed scales require the B12X runtime. +[[models]] +aliases = ["qwen3.8:flash-next-abliterated", "qwen3.8:flash-next-abliterated-nvfp4"] +repository = "DevelopingDad/Qwen3.8-Flash-Next-FP4-Local-Abliterated" +revision = "fe8c47ff55d0692e28dbcad2c53df2f51294be46" + +[models.artifact] +format = "safetensors" +quantization = "NVFP4" +capabilities = ["reasoning", "text_generation", "tool_calling", "vision"] +engine_profile = "qwen3.8-flash-next-fp4-local" + +[models.artifact.primary] +path = "model.safetensors.index.json" +bytes = 46529994 +sha256 = "96e5ea03565556c588b8d7b29feabeec7858c275d6c33b36b231bf92b1941e03" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "tensors/model-00001-of-00021.safetensors" +bytes = 5000643504 +sha256 = "a3c8c67934589154119f3eb4fff6bde91114496b04a98858f3a59494257da7b4" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "tensors/model-00002-of-00021.safetensors" +bytes = 5002792096 +sha256 = "39fcd6ac464fcf9db002f13ddb7fd2e8c6f6fc78f1f1615eaac966d43d4b8b99" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "tensors/model-00003-of-00021.safetensors" +bytes = 5002927400 +sha256 = "6a3bf036982c524faeaea05d8ce8b442280c18263b3c6fd14801b8019e5b4289" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "tensors/model-00004-of-00021.safetensors" +bytes = 5003194672 +sha256 = "b2083c08d0438cd030bee4bce8c55a3740c565f42bab14fa6a5d52049230f6de" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "tensors/model-00005-of-00021.safetensors" +bytes = 5003151448 +sha256 = "c0419be6776d0dd85605d2ea99671cfe96349923eefdc35f2986cca4ac837fff" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "tensors/model-00006-of-00021.safetensors" +bytes = 5002892148 +sha256 = "25e35e23e622cd655104f126d739420645d60dcca442a90c02dcb8d139df2c8a" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "tensors/model-00007-of-00021.safetensors" +bytes = 5002907708 +sha256 = "af3ef717daaf20d306d76ec4b191909f8dd02a2b90a1eb59fcb86a046333c116" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "tensors/model-00008-of-00021.safetensors" +bytes = 5003103756 +sha256 = "fc7ae6bbaf2dea684a07f68c749daf4580df0d75e7da1d6e33e401e804fe4e3a" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "tensors/model-00009-of-00021.safetensors" +bytes = 5002976316 +sha256 = "e69f58486070c9e133421d251fee4f0d71ba59ea5ac23473e117069abca9fc71" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "tensors/model-00010-of-00021.safetensors" +bytes = 5002834136 +sha256 = "d48514fb5d28d5ce1f8cc2dad83a12ffe5d1b17c60357d0cb79fe14e469edafb" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "tensors/model-00011-of-00021.safetensors" +bytes = 5002685828 +sha256 = "2a5562f77811ae444051086084ddb26afb6aee9ccf539af9c23cd306f6d4b2be" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "tensors/model-00012-of-00021.safetensors" +bytes = 5002912920 +sha256 = "aa1749594dff70b62e17aac65755a7ab6f51061e0d59215ca4301581d1782c6a" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "tensors/model-00013-of-00021.safetensors" +bytes = 5003237060 +sha256 = "0068f9b62a261d84e8c692cff9d4eadde052526076d141fdb970724c06187745" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "tensors/model-00014-of-00021.safetensors" +bytes = 4845006312 +sha256 = "7d471564f621088cd4f9f95f9f17913bb13118a82d153d0f047d1da5c41efe5b" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "tensors/model-00015-of-00021.safetensors" +bytes = 4860574920 +sha256 = "ed33b835f0f2cd5fc14012bd64f8efcb84f268964983fc5406e57bbadab37f72" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "tensors/model-00016-of-00021.safetensors" +bytes = 4950031028 +sha256 = "635e97fbad78fa988647d407e2b58e55d551ef94d798c9b6fb5550de25e86429" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "tensors/model-00017-of-00021.safetensors" +bytes = 4950030896 +sha256 = "bf60d564d89a805b67f8be5f2847a1b1567a70dbc23d19754395c127568474ad" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "tensors/model-00018-of-00021.safetensors" +bytes = 4950030896 +sha256 = "c5bde22f9364e82eb25973456795592fdcb4b158bc2095898979a2566bf4c562" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "tensors/model-00019-of-00021.safetensors" +bytes = 4950030904 +sha256 = "da3dc3b4f428f5b9dd53362691043d0242f48d03adca9b7b553bd9f3a1e96350" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "tensors/model-00020-of-00021.safetensors" +bytes = 4950030904 +sha256 = "4978bb6b8480cfc02d3c8bc84ba86202889f5f10d090bb17c21fee663cea7683" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "tensors/model-00021-of-00021.safetensors" +bytes = 2700016864 +sha256 = "48203838f2c76921bddbf1cc51d20a23858c128821a21add55fa1778e5e25283" + +[[models]] +aliases = ["qwen3.8:flash-next", "qwen3.8:flash-next-nvfp4", "qwen3.8:flash-next-redhat", "qwen3.8:flash-next-redhat-nvfp4"] +repository = "RedHatAI/Qwen3.8-Flash-Next-NVFP4" +revision = "c8f2fb1b9869f686b214782036123b10ff96d14a" + +[models.artifact] +format = "safetensors" +quantization = "NVFP4" +capabilities = ["reasoning", "text_generation", "tool_calling", "vision"] +engine_profile = "qwen3.8-flash-next-redhat-nvfp4" + +[models.artifact.primary] +path = "model.safetensors.index.json" +bytes = 34196882 +sha256 = "30b79f475ef97e449349e2eca8bbc26287353a31f27ddab308748be17c1d1ba7" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "model-00001-of-00005.safetensors" +bytes = 102400512288 +sha256 = "64836044b0d06b54a3a1f3fd3cf4ce06ba005fb6319abb100479b7a151f7f44c" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "model-00002-of-00005.safetensors" +bytes = 20009465096 +sha256 = "6a5845e3cf6be70e345c4cb25a4d06d20b398ff7a29576fc78da82272578f531" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "model-00003-of-00005.safetensors" +bytes = 20010984128 +sha256 = "4fdc4e2d027527cf1bac49ee14d6df469a13baa8b856f3bdef8fc9b157596c90" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "model-00004-of-00005.safetensors" +bytes = 20011112280 +sha256 = "4ec199fa49917691056485e0c61b4eff77432c5ee55e63d28819234f5843a140" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "model-00005-of-00005.safetensors" +bytes = 18751777152 +sha256 = "89c6b9272faa31fdc208fbf538efcf5e7947baff2923a99338ac7ac540a1029f" + +[[models.artifact.auxiliary]] +role = "weight_shard" +path = "model_mtp.safetensors" +bytes = 5214305424 +sha256 = "f346d471136314ebe66af5193c0a01fd00ffdc22a4a8c9f86bd15948bc9a8212" diff --git a/docs/how-to/develop-spark.md b/docs/how-to/develop-spark.md index 9449adf..95ad9fa 100644 --- a/docs/how-to/develop-spark.md +++ b/docs/how-to/develop-spark.md @@ -355,6 +355,30 @@ Certificate rotation is separate from a software release: `sparkplane dgx-spark cert rotate --dry-run --json`, followed by `--yes` after review. Use `--ca` only when replacing the certificate authority. +### Declare client features in the recipe + +An engine profile can declare `parallel_tool_calls` and `verbosity` (both default +to false), plus an optional reasoning contract: + +```toml +[profiles.reasoning] +supported_efforts = ["none", "low", "medium", "high", "xhigh"] +default_effort = "xhigh" +summaries = true +``` + +The default must be a supported effort, levels must be unique, reasoning requires +text generation and startup protocol probes, and parallel tools require tool +calling. Verbosity is allowed only on native Responses profiles that preserve +that control. A reasoning-enabled engine must pass a bounded probe with both +reasoning and a completed visible answer before its route is published. + +The published generation carries these declarations through executor IPC and +reconciliation to the API. Client adapters consume those API values; do not +hardcode model names or blanket capability defaults in a launcher. Changes to a +recipe's declarations change its signed fingerprint and require a new managed +generation. Existing qualified generations and historical evidence are retained. + ### Activate a capability change on an existing instance A control-plane-only upgrade can preserve a running engine. Changing an engine diff --git a/docs/how-to/serve-a-model-on-spark.md b/docs/how-to/serve-a-model-on-spark.md index 831c524..aad5ffd 100644 --- a/docs/how-to/serve-a-model-on-spark.md +++ b/docs/how-to/serve-a-model-on-spark.md @@ -131,11 +131,138 @@ length-limited generations are not retried by this recovery path. For Ornith, reasoning streams independently from final answer text as OpenAI `reasoning_content`, Responses reasoning-summary events, or Anthropic thinking -blocks. `sparkplane ... launch codex` advertises that capability in its private -model catalog, so Codex renders the Responses reasoning stream. Requests that +blocks. A recipe with an explicit, qualified reasoning declaration advertises that +capability through the API and generated Codex catalog. Requests that omit sampling settings receive Ornith's precise-coding defaults; explicit client settings remain unchanged. +## Default Qwen3.8 Flash Next: Red Hat AI NVFP4 + +`qwen3.8:flash-next` and `qwen3.8:flash-next-nvfp4` now resolve to +`RedHatAI/Qwen3.8-Flash-Next-NVFP4`, pinned to revision +`c8f2fb1b9869f686b214782036123b10ff96d14a`. The explicit +`qwen3.8:flash-next-redhat` alias selects the same checkpoint. The former default +remains available as `qwen3.8:flash-next-radixark`; its qualified engine is +unchanged. The abliterated alias is separate. + +Red Hat's routed experts use NVFP4 with dynamic FP4 activations in the +`compressed-tensors` format. The embedding tables, attention, shared experts, +vision tower and MTP remain BF16. All six weight files, including the separate +MTP checkpoint, are pinned by size and SHA-256. The approximately 186 GB snapshot +requires more disk space than FP4-Local. + +The dedicated profile uses the pinned B12X runtime built by +`vllm-qwen38-redhat.Dockerfile`, with bounded `pread` transports for checkpoint +loading and disk-backed BF16 PLE lookup. It retains default seccomp, AppArmor, +private IPC/network, dropped capabilities, a non-root UID and read-only model +mounts. It preserves 262,144-token context, eight sequence slots, FP8 KV with an +explicit 6 GiB allocation, MTP4, prefix caching, reasoning, tool calling and +vision. It does not enable the FP4-Local recipe's optional LM-head quantization +or forced A16 expert mode. Compiler concurrency is bounded within the existing +114 GB resource envelope and 30-minute startup deadline. +The hash-guarded compatibility patch normalizes Red Hat's serialized +`qwen_sparse_attention` label to this runtime's `full_attention` label. The +unchanged indexer geometry still selects Qwen sparse attention; no checkpoint +weights or attention algorithm are changed. Image-build tests use the real +vLLM configuration classes and retain rejection of invalid layer labels and QSA +geometry. + +```bash +# --update-alias explicitly moves an already-installed former default. +sparkplane dgx-spark download qwen3.8:flash-next --update-alias --dry-run --json +sparkplane dgx-spark download qwen3.8:flash-next --update-alias --detach --json +sparkplane dgx-spark serve qwen3.8:flash-next --name qwen38-redhat --dry-run --json +sparkplane dgx-spark serve qwen3.8:flash-next --name qwen38-redhat --detach --json +``` + +Wait for the immutable download to finish, confirm the current instance has no +active requests, and use managed `stop` before serving another large model. +Retain a tested model for rollback. A cold startup may spend several minutes +compiling and tuning kernels; use the operation and engine logs to follow it. +Run public HTTPS checks for reasoning, tools, image input, cancellation and +near-limit context before using the new profile for coding sessions. + +If a transfer fails, retry the same immutable download. Admission credits only +space already allocated by regular, singly linked partial files in the managed +cache; sparse holes and unsafe paths receive no credit. The 100 GiB disk reserve +and independent verification of every completed file still apply. + +The Red Hat recipe declares reasoning levels through `xhigh`, with `xhigh` as +its Codex default. The control API and public model metadata advertise the +qualified generation's capabilities. Launch normally, or resume the same thread: + +```bash +sparkplane dgx-spark launch codex --model qwen3.8:flash-next -- resume +``` + +Explicit per-launch effort choices remain available with Codex's +`-c 'model_reasoning_effort="low"'` after `--`. + +[Red Hat's checkpoint and evaluation](https://huggingface.co/RedHatAI/Qwen3.8-Flash-Next-NVFP4). + +## Qwen3.8 Flash Next FP4-Local Abliterated + +The separate alias `qwen3.8:flash-next-abliterated` selects +`DevelopingDad/Qwen3.8-Flash-Next-FP4-Local-Abliterated` at immutable revision +`fe8c47ff55d0692e28dbcad2c53df2f51294be46`. Its 21 weight shards use NVIDIA's +FP4-Local layout with compressed NVFP4 scales. The B12X engine profile pins +an owned image derived from the publisher's pinned ARM64 runtime. The explicit +`qwen3.8:flash-next-radixark` alias retains the RadixArk checkpoint and its +qualified runtime. The default non-abliterated alias selects Red Hat AI. + +```bash +sparkplane dgx-spark download qwen3.8:flash-next-abliterated --dry-run --json +sparkplane dgx-spark download qwen3.8:flash-next-abliterated --detach --json +sparkplane dgx-spark serve qwen3.8:flash-next-abliterated --name qwen38-abliterated --dry-run --json +sparkplane dgx-spark serve qwen3.8:flash-next-abliterated --name qwen38-abliterated --detach --json +sparkplane dgx-spark launch codex --model qwen3.8:flash-next-abliterated +``` + +Wait for the download operation to finish before serving. Run one large model +at a time: save the current instance identity, confirm it has no active requests, +and use managed `stop` before starting the new instance. Retain the previous +model and engine for rollback. Both download and serve enforce the normal disk +and memory reserves. + +The profile retains 262,144-token context, FP8 KV, four-token MTP, prefix caching, +thinking and tool calling. It uses the publisher's disk-backed PLE embeddings +and an explicit 6 GiB KV allocation within a 114 GB resource envelope. Bounded +`pread` transports retain Docker's default seccomp policy, AppArmor, a private +network, a non-root UID and read-only model mounts. Executable JIT temporaries are confined to the managed compile cache; +the general temporary mount remains non-executable. Qualify memory headroom, +long-context reasoning, tool continuation, image input and cancellation on the +actual Spark before relying on this profile for coding sessions. + +The checkpoint loader also needs a confined transport: its GB10 bounce reader +normally initializes `io_uring` during weight loading. The owned Dockerfile +applies a hash-guarded patch that selects `pread` with +`B12X_CHECKPOINT_PREAD=1`. It retains the original 16 MiB fixed-slot pool, CUDA +copy worker, descriptor validation, source identity checks, strides and BF16 +expansion. Interrupted reads retry; short reads and failed copies fail closed +and drain owned slots before cleanup. The build runs eight real-file I/O cases +with CUDA transport simulated, as UID 65534. These checks also run against the +frozen upstream source fixtures in `make test`; hardware inference remains a +separate qualification step. + +Allow time for a cold start: the publisher reports about 23 minutes for first-run +compilation and autotuning, and about five minutes with a populated cache. The +managed compile cache persists across starts; monitor the durable operation and +engine logs until semantic health succeeds. + +Disk PLE uses `B12X_DISK_BACKEND=pread`: at most 16 workers read aligned 64 KiB +slots, with the upstream row planner, source validation, weights and scale +scatter preserved. Each wave joins all workers before publishing results or +reusing slots. Short reads, I/O errors and worker creation failures poison the +reader and drain outstanding work. Nine real-file cases exercise the capsule +API, repeated and out-of-range IDs, both planes, unaligned sources, concurrent +calls, bounded staging and failure cleanup. They run in the image build as UID +65534 and against the frozen fixtures in `make test`. + +Publisher provenance and runtime recipe: +[model repository](https://huggingface.co/DevelopingDad/Qwen3.8-Flash-Next-FP4-Local-Abliterated/tree/fe8c47ff55d0692e28dbcad2c53df2f51294be46). +The model remains subject to its NVIDIA Open Model Agreement and upstream Qwen +Community License; abliteration does not change those terms. + ## See also - [How to install the Spark agent](install-spark.md) diff --git a/docs/reference/spark.md b/docs/reference/spark.md index 7e91930..4646d68 100644 --- a/docs/reference/spark.md +++ b/docs/reference/spark.md @@ -184,9 +184,30 @@ The gateway preserves Ornith's parsed reasoning as a distinct channel. OpenAI Chat streams `reasoning_content`; Responses streams reasoning summary item events and includes the same item in non-stream documents; Anthropic streams a thinking block and accepts Sparkplane's integrity-checked block back in later turns. -The generated coding-client catalog currently disables requests for reasoning -summaries. Gateway protocol support and the coding client's requested display -mode are separate contracts; changing the catalog requires continuation tests. +Translated Responses, Chat and Anthropic streams send SSE comment heartbeats +every ten seconds while inference is quiet, including long prompt prefill. +Comments carry no model output, sequence numbers or token usage, and stop when +the stream ends. The private launch adapter forwards them unchanged. Its HTTP +and Linux TCP unacknowledged-data deadlines both use the finite 1,800-second +inference budget; a library's shorter TCP default cannot override that budget. +Connection establishment remains bounded at ten seconds, and engine idle +deadlines, cancellation and admission limits still apply. Adapter transport +diagnostics identify the request/body stage and timeout classification without +logging credentials or conversation content. + +Each healthy instance publishes `capabilities` through the control API and its +public `/v1/models` metadata. The values come from the exact generation's verified +gateway profile: tool calling, parallel tool calls, verbosity, and optional +reasoning settings (`supported_efforts`, `default_effort`, `summaries`). Unknown +or legacy reasoning support stays unadvertised. A warming, unhealthy, or +mismatched generation never advertises another generation's capabilities. + +The Codex launcher derives its model catalog and invocation defaults from this +API contract. Reasoning-capable recipes expose the effort picker and reasoning +stream, and the launcher overrides stale generated-profile preferences such as +`model_reasoning_effort = "none"`. Explicit Codex `-c` settings after Sparkplane's +`--` still win. Red Hat's default is `xhigh`; no reasoning workaround flags are +needed. Other clients can consume the same API capabilities. Anthropic `display: "omitted"` emits the block and signature without thinking deltas, while `type: "disabled"` disables thinking in the Ornith template. diff --git a/specs/openapi/sparkplane-control-v1.json b/specs/openapi/sparkplane-control-v1.json index ba97ea8..d4629ae 100644 --- a/specs/openapi/sparkplane-control-v1.json +++ b/specs/openapi/sparkplane-control-v1.json @@ -47,6 +47,7 @@ "ExecutorHealth", "ExecutorSnapshot", "HostResourceSnapshot", + "InferenceCapabilitiesDocument", "InstanceDesiredState", "InstanceDocument", "InstanceListDocument", @@ -83,6 +84,7 @@ "QualificationResultDocument", "QualificationSubmission", "QualificationTarget", + "ReasoningCapabilitiesDocument", "RecipeResourceEnvelopeDocument", "RemovalPlanDocument", "RemoveModelRequest", diff --git a/src/spark/agent.rs b/src/spark/agent.rs index 43e6524..5c92edc 100644 --- a/src/spark/agent.rs +++ b/src/spark/agent.rs @@ -19,7 +19,10 @@ use axum::{ }, http::{HeaderMap, HeaderValue, Method, StatusCode, header}, middleware::{self, Next}, - response::{IntoResponse, Response, Sse, sse::Event}, + response::{ + IntoResponse, Response, Sse, + sse::{Event, KeepAlive}, + }, routing::{any, delete, get}, }; use governor::{Quota, RateLimiter, clock::DefaultClock, state::keyed::DashMapStateStore}; @@ -74,6 +77,7 @@ type HmacSha256 = Hmac; type TokenLimiter = RateLimiter, DefaultClock>; type InferenceSlots = BTreeMap)>; const DATABASE_QUEUE_CAPACITY: usize = 64; +const INFERENCE_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(10); #[path = "web.rs"] mod web; @@ -410,6 +414,8 @@ impl Cidr { ServeRequest, StopRequest, InstanceDocument, + super::wire::InferenceCapabilitiesDocument, + super::wire::ReasoningCapabilitiesDocument, InstanceListDocument, EngineLogDocument, AnthropicTokenCountDocument, @@ -998,7 +1004,13 @@ fn anthropic_sse( } }, ); - Sse::new(stream).into_response() + Sse::new(stream) + .keep_alive( + KeepAlive::new() + .interval(INFERENCE_HEARTBEAT_INTERVAL) + .text("keep-alive"), + ) + .into_response() } fn anthropic_warming() -> Response { @@ -1645,7 +1657,13 @@ fn chat_sse( )) }, ); - Sse::new(stream).into_response() + Sse::new(stream) + .keep_alive( + KeepAlive::new() + .interval(INFERENCE_HEARTBEAT_INTERVAL) + .text("keep-alive"), + ) + .into_response() } async fn gateway_responses_stream( @@ -1839,7 +1857,13 @@ fn responses_sse( )) }, ); - Sse::new(stream).into_response() + Sse::new(stream) + .keep_alive( + KeepAlive::new() + .interval(INFERENCE_HEARTBEAT_INTERVAL) + .text("keep-alive"), + ) + .into_response() } fn gateway_warming() -> Response { @@ -2189,6 +2213,7 @@ async fn serve_instance( context_window, input_modalities: vec!["text".into()], default_reasoning_effort, + capabilities: Default::default(), generation: 0, desired: InstanceDesiredState::Running, observed: InstanceObservedState::Creating, @@ -2498,7 +2523,7 @@ async fn run_serve( return; } let endpoint = format!("/openai/{}/v1", instance.name); - let healthy_instance = match database + let mut healthy_instance = match database .set_instance_observed( &instance.id, instance.generation, @@ -2532,6 +2557,7 @@ async fn run_serve( route, ); } + project_route_health(&routes, std::slice::from_mut(&mut healthy_instance)); let _ = database .transition( &operation_id, @@ -2905,7 +2931,7 @@ async fn complete_recovered_operation( }) else { return; }; - let Ok(current) = database.instance(&instance.id).await else { + let Ok(mut current) = database.instance(&instance.id).await else { return; }; let route_ready = routes.is_some_and(|routes| { @@ -2933,6 +2959,9 @@ async fn complete_recovered_operation( if !recovered { return; } + if let Some(routes) = routes { + project_route_health(routes, std::slice::from_mut(&mut current)); + } let result = serde_json::to_value(current).ok(); if operation.state == super::wire::OperationState::Accepted { let _ = database @@ -3173,7 +3202,25 @@ async fn semantic_probe_result( route .protocol_probe(&observed.served_model, require_tools, remaining) .await - .map_err(|error| error.diagnostic()) + .map_err(|error| error.diagnostic())?; + if let Some(reasoning) = &observed.gateway_profile.reasoning { + let effort = observed + .gateway_profile + .sampling + .reasoning_effort_map + .get(&reasoning.default_effort) + .unwrap_or(&reasoning.default_effort); + route + .reasoning_probe( + &observed.served_model, + effort, + reasoning.summaries, + timeout.saturating_sub(started.elapsed()), + ) + .await + .map_err(|error| error.diagnostic())?; + } + Ok(()) } fn semantic_failure_detail(reason: &'static str) -> String { @@ -3275,12 +3322,14 @@ async fn list_instances(State(state): State, RawQuery(query): RawQue fn project_route_health(routes: &RouteRegistry, instances: &mut [InstanceDocument]) { for instance in instances { instance.input_modalities = vec!["text".into()]; + instance.capabilities = Default::default(); if !instance.healthy { continue; } if let RouteLookup::Healthy(route) = routes.lookup(&instance.name) && route.generation == instance.generation { + instance.capabilities = route.profile.inference_capabilities(); if route.profile.capabilities.contains("vision") && route.profile.vision.is_some() { instance.input_modalities.push("image".into()); } @@ -6756,6 +6805,150 @@ mod tests { engine_server.abort(); } + #[tokio::test] + async fn quiet_prefill_heartbeats_cross_pinned_https_and_the_private_adapter() { + use axum::{ + response::{Sse, sse::Event}, + routing::post, + }; + use futures_util::StreamExt; + use sha2::{Digest, Sha256}; + use std::sync::atomic::{AtomicBool, Ordering}; + let release = Arc::new(tokio::sync::Notify::new()); + let generated = Arc::new(AtomicBool::new(false)); + let ready = release.clone(); + let engine_generated = generated.clone(); + let engine = axum::Router::new().route( + "/v1/chat/completions", + post(move || { + let ready = ready.clone(); + let generated = engine_generated.clone(); + async move { + let stream = futures_util::stream::once(async move { + ready.notified().await; + generated.store(true, Ordering::SeqCst); + Ok::<_, std::convert::Infallible>(Event::default().data( + r#"{"choices":[{"delta":{"content":"READY"},"finish_reason":null}]}"#, + )) + }) + .chain(futures_util::stream::iter([ + Ok(Event::default() + .data(r#"{"choices":[{"delta":{},"finish_reason":"stop"}]}"#)), + Ok(Event::default().data( + r#"{"choices":[],"usage":{"prompt_tokens":3,"completion_tokens":1}}"#, + )), + Ok(Event::default().data("[DONE]")), + ])); + Sse::new(stream) + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let engine_address = listener.local_addr().unwrap(); + let engine_server = tokio::spawn(async move { + axum::serve(listener, engine).await.unwrap(); + }); + let state = AgentState::new(TOKEN, Vec::new(), Vec::new()); + state.routes.publish( + "fixture", + "public-model".into(), + "served-model".into(), + crate::spark::upstream::ObservedRoute::new( + "i_11111111111111111111111111111111", + 1, + engine_address.ip(), + engine_address.port(), + [("POST", "/v1/chat/completions")], + ) + .unwrap(), + ); + let rcgen::CertifiedKey { cert, signing_key } = + rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let certificate = cert.pem(); + let tls = super::tls13_config( + certificate.as_bytes().to_vec(), + signing_key.serialize_pem().into_bytes(), + ) + .await + .unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let address = listener.local_addr().unwrap(); + let handle = axum_server::Handle::new(); + let server_handle = handle.clone(); + let server = tokio::spawn(async move { + axum_server::from_tcp_rustls(listener, tls) + .unwrap() + .handle(server_handle) + .serve(router(state).into_make_service_with_connect_info::()) + .await + .unwrap(); + }); + let config = tempfile::tempdir().unwrap(); + std::fs::create_dir(config.path().join("spark")).unwrap(); + std::fs::write(config.path().join("spark/fixture.ca.pem"), &certificate).unwrap(); + let pin = format!("sha256:{:x}", Sha256::digest(certificate.as_bytes())); + std::fs::write(config.path().join("spark.toml"), format!("[hosts.fixture]\nurl='https://localhost:{}'\nca_cert_sha256='{pin}'\ncredential='unused'\nrequest_timeout_seconds=1\n", address.port())).unwrap(); + let adapter = crate::spark::inference_adapter::Adapter::start( + config.path(), + "fixture", + "fixture", + TOKEN, + ) + .unwrap(); + let mut response = reqwest::Client::new() + .post(format!("{}/openai/v1/responses", adapter.base_url)) + .bearer_auth(adapter.token()) + .json(&serde_json::json!({"input":"work","stream":true})) + .send() + .await + .unwrap(); + assert_eq!(response.status(), reqwest::StatusCode::OK); + let mut text = String::new(); + tokio::time::timeout(Duration::from_secs(12), async { + while !text.contains(": keep-alive\n\n") { + text.push_str( + std::str::from_utf8( + &response + .chunk() + .await + .unwrap() + .expect("stream remains open during prefill"), + ) + .unwrap(), + ); + } + }) + .await + .expect("quiet inference must emit a transport heartbeat"); + assert!(!generated.load(Ordering::SeqCst)); + assert!(!text.contains("READY") && !text.contains("response.completed")); + release.notify_one(); + tokio::time::timeout(Duration::from_secs(3), async { + while let Some(chunk) = response.chunk().await.unwrap() { + text.push_str(std::str::from_utf8(&chunk).unwrap()); + } + }) + .await + .unwrap(); + let events: Vec = text + .lines() + .filter_map(|line| line.strip_prefix("data: ")) + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + let terminal: Vec<_> = events + .iter() + .filter(|event| event["type"] == "response.completed") + .collect(); + assert_eq!(terminal.len(), 1); + assert_eq!(terminal[0]["response"]["usage"]["total_tokens"], 4); + assert!(text.contains("READY")); + drop(adapter); + handle.shutdown(); + server.await.unwrap(); + engine_server.abort(); + } + #[tokio::test] async fn upstream_client_rejections_keep_safe_status_codes_and_context_errors() { use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -8056,6 +8249,7 @@ mod tests { context_window: 65_536, input_modalities: vec!["text".into()], default_reasoning_effort: None, + capabilities: Default::default(), generation: 0, desired: crate::spark::wire::InstanceDesiredState::Running, observed: crate::spark::wire::InstanceObservedState::Creating, @@ -8190,7 +8384,7 @@ mod tests { .build() .unwrap(); let vision = crate::spark::engine::EnginePolicy::parse(include_str!( - "../../configs/sparkplane/engines/llama-cpp.toml" + "../../configs/sparkplane/engines/vllm-qwen38-redhat.toml" )) .unwrap() .gateway_profile(None); @@ -8214,6 +8408,8 @@ mod tests { [("POST", "/v1/chat/completions")], ) .unwrap(); + let advertised_reasoning = + generation == instance.generation && profile.reasoning.is_some(); routes.publish_with_profile( &instance.name, instance.model.clone(), @@ -8237,6 +8433,36 @@ mod tests { .unwrap(); assert_eq!(list.instances[0].input_modalities, expected); assert_eq!(list.instances[0].healthy, generation == instance.generation); + assert_eq!( + list.instances[0].capabilities.reasoning.is_some(), + advertised_reasoning + ); + let catalog = crate::spark::launch::codex_catalog(&model, &list.instances[0]); + assert_eq!( + catalog["models"][0]["supports_reasoning_summaries"], + advertised_reasoning + ); + if advertised_reasoning { + assert_eq!(catalog["models"][0]["default_reasoning_level"], "xhigh"); + let metadata: serde_json::Value = client + .get(format!( + "https://localhost:{}/openai/ornith/v1/models", + address.port() + )) + .bearer_auth(TOKEN) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json() + .await + .unwrap(); + assert_eq!( + metadata["data"][0]["capabilities"], + serde_json::to_value(&list.instances[0].capabilities).unwrap() + ); + } } handle.graceful_shutdown(None); server.await.unwrap(); @@ -8793,6 +9019,7 @@ mod tests { context_window: 65_536, input_modalities: vec!["text".into()], default_reasoning_effort: None, + capabilities: Default::default(), generation: 0, desired: crate::spark::wire::InstanceDesiredState::Running, observed: crate::spark::wire::InstanceObservedState::Creating, diff --git a/src/spark/cli.rs b/src/spark/cli.rs index 45a9f02..ded03b0 100644 --- a/src/spark/cli.rs +++ b/src/spark/cli.rs @@ -2131,6 +2131,7 @@ mod tests { context_window: 65_536, input_modalities: vec!["text".into()], default_reasoning_effort: None, + capabilities: Default::default(), generation: 1, desired: InstanceDesiredState::Running, observed: InstanceObservedState::Healthy, diff --git a/src/spark/client.rs b/src/spark/client.rs index cdc9274..201d9f7 100644 --- a/src/spark/client.rs +++ b/src/spark/client.rs @@ -27,6 +27,7 @@ use super::{ const MAX_SAFE_READ_ATTEMPTS: usize = 3; const OPERATION_POLL_INTERVAL: Duration = Duration::from_millis(200); +const INFERENCE_TIMEOUT: Duration = Duration::from_secs(1800); #[derive(Debug, Clone, Deserialize)] #[serde(deny_unknown_fields)] @@ -94,13 +95,19 @@ pub(super) fn inference_transport( } let certificate = reqwest::Certificate::from_pem(&ca) .map_err(|_| unreachable("pinned Spark CA certificate is invalid"))?; - let http = reqwest::Client::builder() + let builder = reqwest::Client::builder() .tls_built_in_root_certs(false) .add_root_certificate(certificate) .https_only(base.scheme() == "https") .redirect(reqwest::redirect::Policy::none()) .connect_timeout(Duration::from_secs(10)) - .timeout(Duration::from_secs(1800)) + .timeout(INFERENCE_TIMEOUT); + // reqwest's Linux TCP_USER_TIMEOUT default is 30 seconds, independently of + // its HTTP deadline. Transient unacknowledged data on the appliance LAN must + // share the finite inference budget rather than silently shortening it. + #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] + let builder = builder.tcp_user_timeout(INFERENCE_TIMEOUT); + let http = builder .build() .map_err(|_| unreachable("could not construct pinned inference transport"))?; Ok((http, base)) @@ -1029,6 +1036,58 @@ fn write_private_atomic(_: &Path, _: &[u8]) -> Result<(), ClientError> { #[cfg(test)] mod tests { + #[cfg(all(target_os = "linux", feature = "appliance"))] + #[tokio::test] + async fn inference_survives_tcp_backpressure_beyond_the_library_default() { + use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let size = 8 * 1024 * 1024; + let server = tokio::spawn(async move { + let (socket, _) = listener.accept().await.unwrap(); + // Stop acknowledging more upload data once the kernel receive window + // fills. No packet filter, privilege or host network mutation is used. + tokio::time::sleep(Duration::from_secs(35)).await; + let mut socket = tokio::io::BufReader::new(socket); + loop { + let mut line = String::new(); + socket.read_line(&mut line).await.unwrap(); + if line == "\r\n" { + break; + } + assert!( + !line.is_empty(), + "inference TCP connection closed before recovery" + ); + } + let mut body = vec![0; size]; + socket.read_exact(&mut body).await.unwrap(); + assert!(body.iter().all(|byte| *byte == b'x')); + socket + .get_mut() + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nOK") + .await + .unwrap(); + }); + let root = tempfile::tempdir().unwrap(); + fs::create_dir(root.path().join("spark")).unwrap(); + let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]) + .unwrap() + .cert + .pem(); + fs::write(root.path().join("spark/fixture.ca.pem"), &cert).unwrap(); + fs::write(root.path().join("spark.toml"), format!("[hosts.fixture]\nurl='http://{address}'\nca_cert_sha256='sha256:{:x}'\ncredential='unused'\nrequest_timeout_seconds=1\n", sha2::Sha256::digest(cert.as_bytes()))).unwrap(); + let (http, base) = super::inference_transport(root.path(), "fixture").unwrap(); + let response = tokio::time::timeout( + Duration::from_secs(55), + http.post(base).body(vec![b'x'; size]).send(), + ) + .await + .unwrap() + .expect("a short library TCP default must not override the inference budget"); + assert_eq!(response.text().await.unwrap(), "OK"); + server.await.unwrap(); + } use super::SparkClient; use crate::spark::wire::{ OPERATION_SCHEMA, OperationDocument, OperationProgress, OperationState, diff --git a/src/spark/engine.rs b/src/spark/engine.rs index 20c1e8a..6d73f55 100644 --- a/src/spark/engine.rs +++ b/src/spark/engine.rs @@ -172,6 +172,12 @@ pub struct EngineProfile { pub capabilities: Vec, pub startup_protocol_probe: bool, #[serde(default)] + pub reasoning: Option, + #[serde(default)] + pub parallel_tool_calls: bool, + #[serde(default)] + pub verbosity: bool, + #[serde(default)] pub resources: Option, #[serde(default)] pub vision: Option, @@ -391,6 +397,9 @@ impl EnginePolicy { vision: profile.vision.clone(), embeddings: None, startup_protocol_probe: profile.startup_protocol_probe, + reasoning: profile.reasoning.clone(), + parallel_tool_calls: profile.parallel_tool_calls, + verbosity: profile.verbosity, native_responses: profile.native_responses, native_response_timeout_seconds: profile.native_response_timeout_seconds, stream_idle_timeout_seconds: profile.stream_idle_timeout_seconds, @@ -434,6 +443,9 @@ impl EnginePolicy { vision: supports_vision.then(|| profile.vision.clone()).flatten(), embeddings, startup_protocol_probe: profile.startup_protocol_probe, + reasoning: profile.reasoning.clone(), + parallel_tool_calls: profile.parallel_tool_calls, + verbosity: profile.verbosity, native_responses: profile.native_responses, native_response_timeout_seconds: profile.native_response_timeout_seconds, stream_idle_timeout_seconds: profile.stream_idle_timeout_seconds, @@ -762,6 +774,27 @@ fn validate_profiles(config: &EngineConfig) -> Result<(), String> { { return Err("engine profile is invalid or ambiguous".into()); } + if profile.reasoning.as_ref().is_some_and(|reasoning| { + !reasoning.valid() + || !profile + .capabilities + .iter() + .any(|value| value == "text_generation") + || !profile.startup_protocol_probe + || profile + .sampling + .default_reasoning_effort + .as_ref() + .is_some_and(|value| value != &reasoning.default_effort) + }) || profile.parallel_tool_calls + && !profile + .capabilities + .iter() + .any(|value| value == "tool_calling") + || profile.verbosity && !profile.native_responses + { + return Err("engine client capabilities disagree with the qualified profile".into()); + } if let Some(vision) = &profile.vision { let processor = format!("sha256:{}", vision.processor_sha256); if !valid_digest(&processor) || super::gateway::vision_health_image(vision).is_err() { @@ -918,20 +951,26 @@ mod tests { #[test] fn declared_executable_caches_must_avoid_noexec_tmpfs() { - let path = std::fs::read_dir("configs/sparkplane/engines") + for entry in std::fs::read_dir("configs/sparkplane/engines") .unwrap() .filter_map(Result::ok) - .map(|entry| entry.path()) - .find(|path| path.extension().and_then(|value| value.to_str()) == Some("toml")) - .unwrap(); - let text = std::fs::read_to_string(path).unwrap(); - let invalid = text.replacen( - "environment = [", - "executable_cache_environment = [\"JIT_CACHE_DIR\"]\nenvironment = [\n \"JIT_CACHE_DIR=/tmp/jit\",", - 1, - ); - let error = EnginePolicy::parse(&invalid).err().unwrap(); - assert!(error.contains("executable cache path must be under /compile-cache")); + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "toml")) + { + let text = std::fs::read_to_string(entry.path()).unwrap(); + let mut invalid: toml::Value = toml::from_str(&text).unwrap(); + invalid.as_table_mut().unwrap().insert( + "executable_cache_environment".into(), + toml::Value::try_from(vec!["JIT_CACHE_DIR"]).unwrap(), + ); + invalid["environment"] + .as_array_mut() + .unwrap() + .push("JIT_CACHE_DIR=/tmp/jit".into()); + let error = EnginePolicy::parse(&toml::to_string(&invalid).unwrap()) + .err() + .unwrap(); + assert!(error.contains("executable cache path must be under /compile-cache")); + } } #[test] @@ -1136,12 +1175,12 @@ mod tests { } #[test] - fn recommended_nvfp4_model_selects_the_patched_engine_from_config() { + fn retained_radixark_model_selects_the_patched_engine_from_config() { let models = super::super::model_catalog::ModelCatalog::parse(include_str!( "../../configs/sparkplane/models.toml" )) .unwrap(); - let model = models.resolve("qwen3.8:flash-next-nvfp4").unwrap(); + let model = models.resolve("qwen3.8:flash-next-radixark").unwrap(); assert_eq!( catalog().select(model.artifacts()).unwrap().config().id, diff --git a/src/spark/gateway.rs b/src/spark/gateway.rs index 46958f8..3394879 100644 --- a/src/spark/gateway.rs +++ b/src/spark/gateway.rs @@ -75,6 +75,12 @@ pub struct GatewayProfile { pub vision: Option, pub embeddings: Option, pub startup_protocol_probe: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning: Option, + #[serde(default, skip_serializing_if = "is_false")] + pub parallel_tool_calls: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub verbosity: bool, #[serde(default)] pub native_responses: bool, #[serde(default)] @@ -85,7 +91,25 @@ pub struct GatewayProfile { pub sampling: SamplingPolicy, } +fn is_false(value: &bool) -> bool { + !value +} + impl GatewayProfile { + pub fn inference_capabilities(&self) -> super::wire::InferenceCapabilitiesDocument { + let generation = self.capabilities.contains("text_generation"); + let tools = generation && self.capabilities.contains("tool_calling"); + super::wire::InferenceCapabilitiesDocument { + tool_calling: tools, + parallel_tool_calls: tools && self.parallel_tool_calls, + verbosity: generation && self.verbosity, + reasoning: generation + .then(|| self.reasoning.clone()) + .flatten() + .filter(|reasoning| reasoning.valid()), + } + } + pub fn stream_idle_timeout(&self) -> std::time::Duration { if self.stream_idle_timeout_seconds == 0 { DEFAULT_STREAM_IDLE_TIMEOUT @@ -100,6 +124,9 @@ impl GatewayProfile { vision: None, embeddings: None, startup_protocol_probe: true, + reasoning: None, + parallel_tool_calls: false, + verbosity: false, native_responses: false, native_response_timeout_seconds: 0, stream_idle_timeout_seconds: 0, @@ -126,6 +153,9 @@ impl GatewayProfile { normalization_tolerance_ppm, }), startup_protocol_probe: true, + reasoning: None, + parallel_tool_calls: false, + verbosity: false, native_responses: false, native_response_timeout_seconds: 0, stream_idle_timeout_seconds: 0, @@ -1416,7 +1446,8 @@ pub fn models_document(route: &HealthyRoute) -> Value { "data": [{ "id": route.public_model, "object": "model", - "owned_by": "sparkplane" + "owned_by": "sparkplane", + "capabilities": route.profile.inference_capabilities() }] }) } @@ -2837,6 +2868,17 @@ fn response_tool_choice(value: Option<&Value>) -> Result { mod tests { use super::*; + #[test] + fn legacy_gateway_profile_serialization_preserves_protected_identity() { + let profile = GatewayProfile::text(); + let document = serde_json::to_value(&profile).unwrap(); + for key in ["reasoning", "parallel_tool_calls", "verbosity"] { + assert!(document.get(key).is_none()); + } + let restored: GatewayProfile = serde_json::from_value(document.clone()).unwrap(); + assert_eq!(serde_json::to_value(restored).unwrap(), document); + } + fn configured_sampling_profile() -> GatewayProfile { let mut profile = GatewayProfile::text(); profile.sampling.defaults = BTreeMap::from([ diff --git a/src/spark/inference_adapter.rs b/src/spark/inference_adapter.rs index 8dbd367..f86e9d1 100644 --- a/src/spark/inference_adapter.rs +++ b/src/spark/inference_adapter.rs @@ -460,7 +460,10 @@ async fn handle(request: Request, state: Arc) -> Response } let response = match upstream.send().await { Ok(r) => r, - Err(_) => return error(502, "pinned Spark inference transport is unavailable"), + Err(error) => { + transport_failure(&error, "request"); + return self::error(502, "pinned Spark inference transport is unavailable"); + } }; let mut builder = Response::builder().status(response.status()); for name in ["content-type", "retry-after"] { @@ -472,7 +475,10 @@ async fn handle(request: Request, state: Arc) -> Response match response.chunk().await { Ok(Some(chunk)) => Ok(Some((Frame::data(chunk), response))), Ok(None) => Ok(None), - Err(_) => Err(io::Error::other("inference response stream disconnected")), + Err(error) => { + transport_failure(&error, "body"); + Err(io::Error::other("inference response stream disconnected")) + } } }); builder @@ -480,6 +486,18 @@ async fn handle(request: Request, state: Arc) -> Response .expect("upstream status and headers") } +fn transport_failure(error: &reqwest::Error, stage: &'static str) { + // No URL, headers, credentials or request/response content enters diagnostics. + tracing::warn!( + event_code = "spark.inference.transport-failed", + stage, + timeout = error.is_timeout(), + connect = error.is_connect(), + body = error.is_body(), + decode = error.is_decode(), + ); +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/spark/launch.rs b/src/spark/launch.rs index 44a06f8..c8b2d82 100644 --- a/src/spark/launch.rs +++ b/src/spark/launch.rs @@ -830,7 +830,7 @@ fn configure_integration( .map_err(|error| usage(format!("Codex provider registration: {error:#}"))) } -fn codex_catalog(model: &ModelDocument, instance: &InstanceDocument) -> Value { +pub(crate) fn codex_catalog(model: &ModelDocument, instance: &InstanceDocument) -> Value { serde_json::json!({ "owned_by": OWNED_MARKER, "models": [{ @@ -849,12 +849,13 @@ fn codex_catalog(model: &ModelDocument, instance: &InstanceDocument) -> Value { "truncation_policy": { "mode": "bytes", "limit": 10000 }, "input_modalities": instance.input_modalities, "base_instructions": "", - "default_reasoning_summary": "none", - "support_verbosity": true, + "default_reasoning_summary": if instance.capabilities.reasoning.as_ref().is_some_and(|reasoning| reasoning.summaries) { "auto" } else { "none" }, + "default_reasoning_level": instance.capabilities.reasoning.as_ref().map(|reasoning| &reasoning.default_effort), + "support_verbosity": instance.capabilities.verbosity, "default_verbosity": "low", - "supports_parallel_tool_calls": false, - "supports_reasoning_summaries": false, - "supported_reasoning_levels": [], + "supports_parallel_tool_calls": instance.capabilities.parallel_tool_calls, + "supports_reasoning_summaries": instance.capabilities.reasoning.as_ref().is_some_and(|reasoning| reasoning.summaries), + "supported_reasoning_levels": instance.capabilities.reasoning.as_ref().map(|reasoning| reasoning.supported_efforts.iter().map(|effort| serde_json::json!({"effort": effort, "description": format!("{effort} reasoning effort")})).collect::>()).unwrap_or_default(), "experimental_supported_tools": [], "supports_search_tool": false, "web_search_tool_type": "text", @@ -1140,7 +1141,36 @@ fn configure_codex_command( instance.context_window * 3 / 4 ), ]; - if let Some(effort) = &instance.default_reasoning_effort { + // Explicit invocation overrides beat stale preferences in an older generated + // profile, including Codex's persisted model_reasoning_effort="none". + overrides.push(format!( + "model_reasoning_summary={:?}", + if instance + .capabilities + .reasoning + .as_ref() + .is_some_and(|reasoning| reasoning.summaries) + { + "auto" + } else { + "none" + } + )); + overrides.push(format!( + "model_supports_reasoning_summaries={}", + instance + .capabilities + .reasoning + .as_ref() + .is_some_and(|reasoning| reasoning.summaries) + )); + if let Some(effort) = instance + .capabilities + .reasoning + .as_ref() + .map(|reasoning| &reasoning.default_effort) + .or(instance.default_reasoning_effort.as_ref()) + { overrides.push(format!("model_reasoning_effort={effort:?}")); } for value in overrides { @@ -1637,6 +1667,7 @@ mod tests { context_window: 65_536, input_modalities: vec!["text".into()], default_reasoning_effort: None, + capabilities: Default::default(), generation: 1, desired: InstanceDesiredState::Running, observed: InstanceObservedState::Healthy, @@ -2049,8 +2080,71 @@ mod tests { let instance = instance("vision", "m_one"); let mut document = serde_json::to_value(instance).unwrap(); document.as_object_mut().unwrap().remove("input_modalities"); + document.as_object_mut().unwrap().remove("capabilities"); let instance: InstanceDocument = serde_json::from_value(document).unwrap(); assert_eq!(instance.input_modalities, ["text"]); + assert_eq!(instance.capabilities, Default::default()); + } + + #[test] + fn codex_uses_advertised_generation_features_instead_of_blanket_defaults() { + let model = model("m_one", "redhat"); + let mut document = serde_json::to_value(instance("redhat", &model.id)).unwrap(); + document["capabilities"] = serde_json::json!({ + "tool_calling": true, "parallel_tool_calls": false, "verbosity": false, + "reasoning": {"supported_efforts":["none","low","medium","high","xhigh"], + "default_effort":"xhigh", "summaries":true} + }); + let instance: InstanceDocument = serde_json::from_value(document).unwrap(); + let catalog = codex_catalog(&model, &instance); + let root = tempfile::tempdir().unwrap(); + client_config(root.path()); + let mut command = Command::new("codex"); + configure_codex_command( + &mut command, + root.path(), + "dgx", + &instance, + &model, + &LaunchSecret("inference-secret".into()), + &["-c".into(), "model_reasoning_effort=\"low\"".into()], + ) + .unwrap(); + let args = command + .get_args() + .map(|value| value.to_string_lossy().into_owned()) + .collect::>(); + assert!( + args.windows(2) + .any(|pair| pair == ["-c", "model_supports_reasoning_summaries=true"]) + ); + assert!( + args.windows(2) + .any(|pair| pair == ["-c", "model_reasoning_summary=\"auto\""]) + ); + let default = args + .iter() + .position(|value| value == "model_reasoning_effort=\"xhigh\"") + .unwrap(); + let explicit = args + .iter() + .position(|value| value == "model_reasoning_effort=\"low\"") + .unwrap(); + assert!( + default < explicit, + "explicit user settings must override recipe defaults" + ); + assert_eq!(catalog["models"][0]["supports_reasoning_summaries"], true); + assert_eq!(catalog["models"][0]["default_reasoning_level"], "xhigh"); + assert_eq!(catalog["models"][0]["default_reasoning_summary"], "auto"); + assert_eq!(catalog["models"][0]["support_verbosity"], false); + assert!( + catalog["models"][0]["supported_reasoning_levels"] + .as_array() + .unwrap() + .iter() + .any(|level| level["effort"] == "xhigh") + ); } #[test] diff --git a/src/spark/model.rs b/src/spark/model.rs index b77ef79..403368d 100644 --- a/src/spark/model.rs +++ b/src/spark/model.rs @@ -20,6 +20,57 @@ use super::wire::{ const MODEL_PROGRESS_QUEUE_CAPACITY: usize = 64; +// Both Hub transports write the same content-addressed `.incomplete` file and +// rename it on success. Its existing allocation is already absent from statvfs +// available space; charging it again prevents a bounded retry of a large shard. +// This is disk accounting only: partial contents still require full verification. +fn partial_blob_allocation(cache: &Path, repository: &Repository, key: &str, size: u64) -> u64 { + use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; + + if !matches!(key.len(), 40 | 64) || !key.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return 0; + } + let Ok(cache_metadata) = fs::symlink_metadata(cache) else { + return 0; + }; + let repository_path = cache.join(repository_cache_name(repository)); + let blobs = repository_path.join("blobs"); + // Refuse symlinked directories as well as symlinked partial files. + if !cache_metadata.is_dir() + || [&repository_path, &blobs] + .iter() + .any(|path| !fs::symlink_metadata(path).is_ok_and(|metadata| metadata.is_dir())) + { + return 0; + } + let Ok(file) = fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(blobs.join(format!("{key}.incomplete"))) + else { + return 0; + }; + let Ok(metadata) = file.metadata() else { + return 0; + }; + if !metadata.is_file() + || metadata.nlink() != 1 + || metadata.uid() != cache_metadata.uid() + || metadata.dev() != cache_metadata.dev() + || metadata.len() > size + { + return 0; + } + // Linux st_blocks counts 512-byte units, including on sparse files. Never + // credit logical holes, block rounding, or bytes beyond the pinned size. + metadata + .blocks() + .checked_mul(512) + .unwrap_or_default() + .min(metadata.len()) + .min(size) +} + #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct Repository(String); @@ -975,6 +1026,7 @@ impl HubAcquirer { .join(repository_cache_name(&repository)) .join("blobs"); let mut unique_bytes = 0_u64; + let mut unfinished_bytes = 0_u64; for file in &expected { let path = file.path.to_string_lossy(); let resolved = tree @@ -989,7 +1041,20 @@ impl HubAcquirer { resolved.size, ) && !blobs.join(&resolved.blob_key).is_file() { - unique_bytes = unique_bytes.checked_add(resolved.size).ok_or_else(|| { + unfinished_bytes = + unfinished_bytes.checked_add(resolved.size).ok_or_else(|| { + acquisition_error( + TransferFailure::Policy, + "model size overflows byte accounting", + ) + })?; + let remaining = resolved.size.saturating_sub(partial_blob_allocation( + &self.cache, + &repository, + &resolved.blob_key, + resolved.size, + )); + unique_bytes = unique_bytes.checked_add(remaining).ok_or_else(|| { acquisition_error( TransferFailure::Policy, "model size overflows byte accounting", @@ -997,7 +1062,9 @@ impl HubAcquirer { })?; } } - let temporary_bytes = unique_bytes.min(8 * 1024 * 1024 * 1024); + // A fully allocated partial still needs transport scratch space. Credit + // its final allocation without reducing the existing temporary budget. + let temporary_bytes = unfinished_bytes.min(8 * 1024 * 1024 * 1024); self.enforce_disk_reserve(unique_bytes, temporary_bytes)?; let license = info .tags @@ -1347,6 +1414,92 @@ mod tests { } } + #[test] + fn partial_cache_accounting_does_not_credit_sparse_holes() { + use std::io::Write; + let root = tempfile::tempdir().unwrap(); + let repository = Repository::parse("owner/model").unwrap(); + let blobs = root.path().join("models--owner--model/blobs"); + fs::create_dir_all(&blobs).unwrap(); + let key = "a".repeat(64); + let path = blobs.join(format!("{key}.incomplete")); + let mut file = fs::File::create(&path).unwrap(); + file.write_all(&[42; 4096]).unwrap(); + file.set_len(1024 * 1024).unwrap(); + file.sync_all().unwrap(); + let allocation = + super::partial_blob_allocation(root.path(), &repository, &key, 2 * 1024 * 1024); + assert!(allocation > 0); + assert!(allocation < file.metadata().unwrap().len()); + assert!(allocation <= file.metadata().unwrap().blocks() * 512); + } + + #[test] + fn partial_cache_accounting_rejects_links_directories_and_oversized_files() { + let root = tempfile::tempdir().unwrap(); + let repository = Repository::parse("owner/model").unwrap(); + let blobs = root.path().join("models--owner--model/blobs"); + fs::create_dir_all(&blobs).unwrap(); + let key = "b".repeat(64); + let partial = blobs.join(format!("{key}.incomplete")); + let source = root.path().join("other-file"); + fs::write(&source, [42; 8192]).unwrap(); + symlink(&source, &partial).unwrap(); + assert_eq!( + super::partial_blob_allocation(root.path(), &repository, &key, 8192), + 0 + ); + fs::remove_file(&partial).unwrap(); + fs::hard_link(&source, &partial).unwrap(); + assert_eq!( + super::partial_blob_allocation(root.path(), &repository, &key, 8192), + 0 + ); + fs::remove_file(&partial).unwrap(); + fs::create_dir(&partial).unwrap(); + assert_eq!( + super::partial_blob_allocation(root.path(), &repository, &key, 8192), + 0 + ); + fs::remove_dir(&partial).unwrap(); + fs::write(&partial, [42; 8193]).unwrap(); + assert_eq!( + super::partial_blob_allocation(root.path(), &repository, &key, 8192), + 0 + ); + assert_eq!( + super::partial_blob_allocation(root.path(), &repository, "../other-file", 8192), + 0 + ); + } + + #[test] + fn partial_cache_accounting_rejects_symlinked_cache_directories() { + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let repository = Repository::parse("owner/model").unwrap(); + let key = "c".repeat(64); + fs::create_dir(outside.path().join("blobs")).unwrap(); + fs::write( + outside.path().join(format!("blobs/{key}.incomplete")), + [42; 8192], + ) + .unwrap(); + let repository_path = root.path().join("models--owner--model"); + symlink(outside.path(), &repository_path).unwrap(); + assert_eq!( + super::partial_blob_allocation(root.path(), &repository, &key, 16384), + 0 + ); + fs::remove_file(&repository_path).unwrap(); + fs::create_dir(&repository_path).unwrap(); + symlink(outside.path().join("blobs"), repository_path.join("blobs")).unwrap(); + assert_eq!( + super::partial_blob_allocation(root.path(), &repository, &key, 16384), + 0 + ); + } + #[test] fn repository_revision_and_alias_validation_resists_traversal() { assert!(Repository::parse("ornith-ai/Ornith-1.5-9B").is_ok()); @@ -1714,9 +1867,10 @@ mod tests { } async fn file() -> Response { let mut response = Response::new(Body::from(CONTENT)); - response - .headers_mut() - .insert(header::ETAG, HeaderValue::from_static("\"hermetic-etag\"")); + response.headers_mut().insert( + header::ETAG, + HeaderValue::from_str(&format!("\"{:x}\"", sha2::Sha256::digest(CONTENT))).unwrap(), + ); response .headers_mut() .insert("x-repo-commit", HeaderValue::from_static(COMMIT)); @@ -1751,6 +1905,24 @@ mod tests { None, ) .unwrap(); + let blobs = root.path().join("models--owner--model/blobs"); + fs::create_dir_all(&blobs).unwrap(); + let partial = blobs.join(format!("{:x}.incomplete", sha2::Sha256::digest(CONTENT))); + // Allocation is reusable space, not evidence that these bytes are valid. + fs::write(&partial, b"garbage").unwrap(); + fs::File::open(&partial).unwrap().sync_all().unwrap(); + let plan = acquirer + .resolve_selected( + Repository::parse("owner/model").unwrap(), + Revision::parse("main").unwrap(), + super::ArtifactSelection::generic("model.gguf", Vec::new()).unwrap(), + ) + .await + .unwrap(); + assert_eq!(plan.unique_bytes, (CONTENT.len() - 7) as u64); + assert_eq!(plan.temporary_bytes, CONTENT.len() as u64); + fs::write(&partial, [42; CONTENT.len()]).unwrap(); + fs::File::open(&partial).unwrap().sync_all().unwrap(); let plan = acquirer .resolve_selected( Repository::parse("owner/model").unwrap(), @@ -1759,8 +1931,11 @@ mod tests { ) .await .unwrap(); + assert_eq!(plan.unique_bytes, 0); + assert_eq!(plan.temporary_bytes, CONTENT.len() as u64); let (send, _receive) = super::progress_channel(); let model = acquirer.acquire(&plan, None, send).await.unwrap(); + assert!(!partial.exists()); assert_eq!( (model.commit.as_str(), model.logical_bytes), (COMMIT, CONTENT.len() as u64) @@ -1783,6 +1958,7 @@ mod tests { .await .unwrap(); assert_eq!(resumed.unique_bytes, 0); + assert_eq!(resumed.temporary_bytes, 0); server.abort(); } } diff --git a/src/spark/model_catalog.rs b/src/spark/model_catalog.rs index 6d6fc36..5b5fe4c 100644 --- a/src/spark/model_catalog.rs +++ b/src/spark/model_catalog.rs @@ -316,12 +316,19 @@ mod tests { 2, ), ( - "qwen3.8:flash-next-nvfp4", + "qwen3.8:flash-next-radixark", "RadixArk/Qwen3.8-Flash-Next-NVFP4", "7b719225242aacd3dbd3f9407468c2ee9a9d2594", "model.safetensors.index.json", 206, ), + ( + "qwen3.8:flash-next-nvfp4", + "RedHatAI/Qwen3.8-Flash-Next-NVFP4", + "c8f2fb1b9869f686b214782036123b10ff96d14a", + "model.safetensors.index.json", + 6, + ), ( "qwen3.6:35b-a3b-nvfp4", "nvidia/Qwen3.6-35B-A3B-NVFP4", diff --git a/src/spark/recipe.rs b/src/spark/recipe.rs index 40e7929..0985f04 100644 --- a/src/spark/recipe.rs +++ b/src/spark/recipe.rs @@ -234,6 +234,9 @@ impl Gateway { vision: self.vision.clone(), embeddings: self.embeddings.clone(), startup_protocol_probe: true, + reasoning: None, + parallel_tool_calls: false, + verbosity: false, native_responses: false, native_response_timeout_seconds: 0, stream_idle_timeout_seconds: 0, diff --git a/src/spark/state.rs b/src/spark/state.rs index 766b184..21cd055 100644 --- a/src/spark/state.rs +++ b/src/spark/state.rs @@ -242,7 +242,7 @@ enum Command { reply: oneshot::Sender>, }, BeginServe { - instance: InstanceDocument, + instance: Box, reply: oneshot::Sender>, }, ListInstances(oneshot::Sender, StateError>>), @@ -529,8 +529,11 @@ impl DbActor { instance: InstanceDocument, ) -> Result { validate_instance(&instance)?; - self.submit(|reply| Command::BeginServe { instance, reply }) - .await + self.submit(|reply| Command::BeginServe { + instance: Box::new(instance), + reply, + }) + .await } pub async fn list_instances(&self) -> Result, StateError> { @@ -947,7 +950,7 @@ fn actor_loop( let _ = reply.send(remove_model(&mut connection, &id)); } Command::BeginServe { instance, reply } => { - let _ = reply.send(begin_serve(&mut connection, instance)); + let _ = reply.send(begin_serve(&mut connection, *instance)); } Command::ListInstances(reply) => { let _ = reply.send(list_instances(&connection)); @@ -2582,6 +2585,7 @@ mod tests { context_window: 65_536, input_modalities: vec!["text".into()], default_reasoning_effort: None, + capabilities: Default::default(), generation: 0, desired: InstanceDesiredState::Running, observed: InstanceObservedState::Creating, diff --git a/src/spark/upstream.rs b/src/spark/upstream.rs index bfac4b3..5843f18 100644 --- a/src/spark/upstream.rs +++ b/src/spark/upstream.rs @@ -697,7 +697,7 @@ impl ObservedRoute { "stream_options":{"include_usage":true},"chat_template_kwargs":{"enable_thinking":false,"reasoning_strength":"low"} })) .map_err(|_| UpstreamError::new("protocol probe cannot be encoded"))?; - validate_protocol_stream(self.chat_stream(&body).await?, false).await?; + validate_protocol_stream(self.chat_stream(&body).await?, false, false).await?; if require_tools { let body = serde_json::to_vec(&serde_json::json!({ "model":served_model, @@ -710,11 +710,28 @@ impl ObservedRoute { "temperature":0,"stream":true,"stream_options":{"include_usage":true},"chat_template_kwargs":{"enable_thinking":false,"reasoning_strength":"low"} })) .map_err(|_| UpstreamError::new("tool protocol probe cannot be encoded"))?; - validate_protocol_stream(self.chat_stream(&body).await?, true).await?; + validate_protocol_stream(self.chat_stream(&body).await?, true, false).await?; } Ok(()) } + pub async fn reasoning_probe( + &self, + served_model: &str, + effort: &str, + summaries: bool, + timeout: Duration, + ) -> Result<(), UpstreamError> { + tokio::time::timeout(timeout, async { + let body = serde_json::to_vec(&serde_json::json!({ + "model":served_model, "messages":[{"role":"user","content":"Briefly reason about 17 times 19, then give the answer."}], + "max_tokens":1024, "temperature":0, "stream":true, "stream_options":{"include_usage":true}, + "reasoning_effort":effort, "chat_template_kwargs":{"enable_thinking":true} + })).map_err(|_| UpstreamError::new("reasoning probe cannot be encoded"))?; + validate_protocol_stream(self.chat_stream(&body).await?, false, summaries).await + }).await.map_err(|_| UpstreamError::new("engine reasoning probe timed out"))? + } + pub async fn completion_stream(&self, body: &[u8]) -> Result { self.generation_stream("/v1/completions", body, DEFAULT_STREAM_IDLE_TIMEOUT) .await @@ -943,14 +960,22 @@ async fn rejected_stream_response( async fn validate_protocol_stream( mut stream: CompletionStream, require_tool: bool, + require_reasoning: bool, ) -> Result<(), UpstreamError> { let mut output = false; + let mut reasoning = false; let mut tools = std::collections::BTreeMap::::new(); let mut finish_reason = None; let mut usage = false; while let Some(event) = stream.next().await { match event? { - GenerationEvent::ReasoningDelta { text } | GenerationEvent::TextDelta { text } => { + GenerationEvent::ReasoningDelta { text } => { + if finish_reason.is_some() || text.is_empty() { + return Err(UpstreamError::new("engine protocol event order is invalid")); + } + reasoning = true; + } + GenerationEvent::TextDelta { text } => { if finish_reason.is_some() || text.is_empty() { return Err(UpstreamError::new("engine protocol event order is invalid")); } @@ -1007,7 +1032,10 @@ async fn validate_protocol_stream( } else { matches!(finish_reason.as_deref(), Some("stop" | "length")) }; - if finish_valid && usage && (require_tool && tools_valid || !require_tool && output) + if finish_valid + && usage + && (!require_reasoning || reasoning && finish_reason.as_deref() == Some("stop")) + && (require_tool && tools_valid || !require_tool && output) { return Ok(()); } @@ -1592,6 +1620,49 @@ mod tests { server.await.unwrap(); } + #[tokio::test] + async fn reasoning_probe_requires_thinking_and_a_completed_answer_over_http() { + use axum::{Json, Router, routing::post}; + for (thinking, answer, finish, accepted) in [ + (true, true, "stop", true), + (false, true, "stop", false), + (true, false, "stop", false), + (true, true, "length", false), + ] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + axum::serve(listener, Router::new().route("/v1/chat/completions", post(move |Json(request): Json| async move { + assert_eq!(request["reasoning_effort"], "xhigh"); + assert_eq!(request["chat_template_kwargs"]["enable_thinking"], true); + let mut chunks = Vec::new(); + if thinking { chunks.push(serde_json::json!({"choices":[{"delta":{"reasoning_content":"Multiply briefly."},"finish_reason":null}]})); } + if answer { chunks.push(serde_json::json!({"choices":[{"delta":{"content":"323"},"finish_reason":null}]})); } + chunks.push(serde_json::json!({"choices":[{"delta":{},"finish_reason":finish}]})); + chunks.push(serde_json::json!({"choices":[],"usage":{"prompt_tokens":10,"completion_tokens":5}})); + let body = chunks.into_iter().map(|chunk| format!("data: {chunk}\n\n")).collect::()+"data: [DONE]\n\n"; + ([("content-type","text/event-stream")],body) + }))).await.unwrap(); + }); + let route = ObservedRoute::new( + "i_11111111111111111111111111111111", + 1, + address.ip(), + address.port(), + [("POST", "/v1/chat/completions")], + ) + .unwrap(); + assert_eq!( + route + .reasoning_probe("fixture-model", "xhigh", true, Duration::from_secs(2)) + .await + .is_ok(), + accepted + ); + server.abort(); + } + } + #[tokio::test] async fn llama_cpp_protocol_probe_accepts_complete_streamed_chat_and_parallel_tools() { let (address, server) = fake_llama_protocol().await; diff --git a/src/spark/wire.rs b/src/spark/wire.rs index 29e5e6b..228b616 100644 --- a/src/spark/wire.rs +++ b/src/spark/wire.rs @@ -562,6 +562,46 @@ pub enum InstanceObservedState { Failed, } +/// Features of one qualified, published inference route. Missing metadata is +/// deliberately conservative for clients talking to an older agent. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "appliance", derive(utoipa::ToSchema))] +#[serde(deny_unknown_fields)] +pub struct InferenceCapabilitiesDocument { + pub tool_calling: bool, + pub parallel_tool_calls: bool, + pub verbosity: bool, + pub reasoning: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "appliance", derive(utoipa::ToSchema))] +#[serde(deny_unknown_fields)] +pub struct ReasoningCapabilitiesDocument { + pub supported_efforts: Vec, + pub default_effort: String, + pub summaries: bool, +} + +impl ReasoningCapabilitiesDocument { + pub fn valid(&self) -> bool { + !self.supported_efforts.is_empty() + && self.supported_efforts.iter().all(|value| { + matches!( + value.as_str(), + "none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max" + ) + }) + && self + .supported_efforts + .iter() + .collect::>() + .len() + == self.supported_efforts.len() + && self.supported_efforts.contains(&self.default_effort) + } +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[cfg_attr(feature = "appliance", derive(utoipa::ToSchema))] #[serde(deny_unknown_fields)] @@ -584,7 +624,10 @@ pub struct InstanceDocument { #[serde(default = "default_instance_input_modalities")] pub input_modalities: Vec, #[serde(default)] + /// Legacy sampling intent; client features and defaults are in capabilities. pub default_reasoning_effort: Option, + #[serde(default)] + pub capabilities: InferenceCapabilitiesDocument, pub generation: u64, pub desired: InstanceDesiredState, pub observed: InstanceObservedState, diff --git a/tests/fixtures/spark-fp4-loader/LICENSE b/tests/fixtures/spark-fp4-loader/LICENSE new file mode 100644 index 0000000..ba557f2 --- /dev/null +++ b/tests/fixtures/spark-fp4-loader/LICENSE @@ -0,0 +1,229 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------------------------------------------------------------------------------------- +Some of the code in this project are adapted from other open-source projects with different +licenses. This product also bundles some third-party components under other open source licenses. +This section summarizes those components and their licenses. +See licenses/ for text of these licenses. + +BSD 3-Clause License +-------------------- + +include/flashinfer/attention/hopper/epilogue.cuh +include/flashinfer/attention/hopper/mainloop.cuh +include/flashinfer/attention/hopper/kernel_traits.cuh +include/flashinfer/attention/hopper/named_barrier.cuh +include/flashinfer/attention/hopper/tile_scheduler.cuh +include/flashinfer/attention/hopper/utils.cuh + +BSD 3-Clause "New" License +-------------------------- + +3rdparty/cutlass +include/flashinfer/attention/hopper/block_sparse_gather.cuh + +MIT License +----------- + +3rdparty/spdlog +3rdparty/spdlog/include/spdlog/fmt/bundled (fmt library) diff --git a/tests/fixtures/spark-fp4-loader/NOTICE b/tests/fixtures/spark-fp4-loader/NOTICE new file mode 100644 index 0000000..03d1214 --- /dev/null +++ b/tests/fixtures/spark-fp4-loader/NOTICE @@ -0,0 +1,4 @@ +FlashInfer B12X checkpoint and disk PLE reader source fixtures. +Source: flashinfer-ai/flashinfer, Apache-2.0 (see LICENSE). +Frozen from eugr/spark-vllm-b12x@sha256:03a956544e9f9b4b7ba91690469fbe7848bdbef29a7d2e0f5788368467bb3595. +These files are upstream test inputs; apply owned patches only to temporary copies. diff --git a/tests/fixtures/spark-fp4-loader/_batch.c b/tests/fixtures/spark-fp4-loader/_batch.c new file mode 100644 index 0000000..fa9ef68 --- /dev/null +++ b/tests/fixtures/spark-fp4-loader/_batch.c @@ -0,0 +1,218 @@ +/* Descriptor destinations are ordinary CUDA tensors retained by Python. */ +#include +#include + +#include "_cuda_range.h" + +#define BATCH_CHUNK_BYTES (64 << 20) +typedef struct { + int fd; + int64_t offset, bytes; + char* destination; + bool expand_bf16; + bool host_copy; + int64_t rows, source_stride, destination_stride; +} read_job_t; + +#include "_bounce.c" + +typedef struct { + pthread_mutex_t mutex; + bounce_ring_t* bounce; + int device; + uint64_t batches, descriptors; + double execution_seconds; +} batch_executor_t; + +static void delete_batch(PyObject* capsule) { + batch_executor_t* executor = PyCapsule_GetPointer(capsule, "b12x.batch_executor"); + if (!executor) return; + bounce_release(executor->bounce); + pthread_mutex_destroy(&executor->mutex); + free(executor); +} + +static PyObject* py_batch_executor(PyObject* self, PyObject* args) { + (void)self; + int device, workers; + if (!PyArg_ParseTuple(args, "ii", &device, &workers)) return NULL; + if (workers < 1 || workers > 16) + return PyErr_Format(PyExc_ValueError, "io_threads must be between 1 and 16"); + batch_executor_t* executor = calloc(1, sizeof(*executor)); + if (!executor) return PyErr_NoMemory(); + executor->device = device; + pthread_mutex_init(&executor->mutex, NULL); + failure_t failure = {{0}}; + executor->bounce = bounce_create(device, (unsigned)workers, &failure); + if (!executor->bounce) { + pthread_mutex_destroy(&executor->mutex); + free(executor); + return PyErr_Format(PyExc_RuntimeError, "%s", failure.message); + } + PyObject* capsule = PyCapsule_New(executor, "b12x.batch_executor", delete_batch); + if (!capsule) { + bounce_release(executor->bounce); + pthread_mutex_destroy(&executor->mutex); + free(executor); + } + return capsule; +} + +static int job_destination_order(const void* a, const void* b) { + uintptr_t x = (uintptr_t)((const read_job_t*)a)->destination; + uintptr_t y = (uintptr_t)((const read_job_t*)b)->destination; + return (x > y) - (x < y); +} + +static int job_source_order(const void* a, const void* b) { + const read_job_t *x = a, *y = b; + if (x->fd != y->fd) return (x->fd > y->fd) - (x->fd < y->fd); + return (x->offset > y->offset) - (x->offset < y->offset); +} + +static PyObject* py_batch_execute(PyObject* self, PyObject* args) { + (void)self; + PyObject* capsule; + Py_buffer records; + unsigned long long stream; + if (!PyArg_ParseTuple(args, "Oy*K", &capsule, &records, &stream)) return NULL; + batch_executor_t* executor = PyCapsule_GetPointer(capsule, "b12x.batch_executor"); + if (!executor) { + PyBuffer_Release(&records); + return NULL; + } + failure_t failure = {{0}}; + read_job_t* jobs = NULL; + size_t count = 0, capacity = 0; + if (records.len % (8 * sizeof(uint64_t))) { + snprintf(failure.message, sizeof(failure.message), "invalid read descriptor byte size"); + goto done; + } + size_t descriptors = records.len / (8 * sizeof(uint64_t)); + for (size_t i = 0; i < descriptors; i++) { + uint64_t record[8]; + memcpy(record, (char*)records.buf + i * sizeof(record), sizeof(record)); + uint64_t fd = record[0], offset = record[1], bytes = record[2]; + uint64_t pointer = record[3], transform = record[4]; + uint64_t rows = record[5], source_stride = record[6], destination_stride = record[7]; + bool expand_bf16 = transform == 1; + if (fd > INT_MAX || transform > 2 || offset > INT64_MAX || + bytes > (uint64_t)INT64_MAX - offset || + (expand_bf16 && (bytes % 2 || bytes > INT64_MAX / 2)) || + (transform == 2 && ((!offset && bytes) || rows != 1)) || !rows || rows > INT64_MAX || + source_stride > INT64_MAX || destination_stride > INT64_MAX || + (source_stride && rows - 1 > ((uint64_t)INT64_MAX - offset - bytes) / source_stride) || + (destination_stride && + rows - 1 > ((uint64_t)INT64_MAX - bytes * (1 + expand_bf16)) / destination_stride)) { + snprintf(failure.message, sizeof(failure.message), "invalid read descriptor"); + goto done; + } + if (rows > 1 && destination_stride < bytes * (1 + expand_bf16)) { + snprintf(failure.message, sizeof(failure.message), + "overlapping batch destinations need an explicit dependency"); + goto done; + } + uint64_t extent = (rows - 1) * destination_stride + bytes * (1 + expand_bf16); + if (!device_range(pointer, extent, executor->device)) { + snprintf(failure.message, sizeof(failure.message), + "batch destination exceeds CUDA device allocation"); + goto done; + } + char* destination = (char*)(uintptr_t)pointer; + while (rows && bytes) { + if (count == capacity) { + size_t next = capacity ? capacity * 2 : 1024; + read_job_t* grown = realloc(jobs, next * sizeof(*jobs)); + if (!grown) { + snprintf(failure.message, sizeof(failure.message), "read descriptor allocation failed"); + goto done; + } + jobs = grown; + capacity = next; + } + int64_t chunk = bytes > BATCH_CHUNK_BYTES ? BATCH_CHUNK_BYTES : (int64_t)bytes; + uint64_t chunk_rows = 1; + if (bytes <= BATCH_CHUNK_BYTES && destination_stride == bytes * (1 + expand_bf16)) { + uint64_t stride = source_stride > destination_stride ? source_stride : destination_stride; + chunk_rows = stride ? BATCH_CHUNK_BYTES / stride : 1; + if (!chunk_rows) chunk_rows = 1; + if (chunk_rows > rows) chunk_rows = rows; + } + jobs[count++] = (read_job_t){(int)fd, + (int64_t)offset, + chunk, + destination, + expand_bf16, + transform == 2, + (int64_t)chunk_rows, + (int64_t)source_stride, + (int64_t)destination_stride}; + if (chunk < (int64_t)bytes) { + /* Large contiguous rows can still be divided into independent jobs. */ + if (rows != 1) { + snprintf(failure.message, sizeof(failure.message), + "strided rows exceed batch chunk size"); + goto done; + } + offset += chunk; + bytes -= chunk; + destination += chunk * (1 + expand_bf16); + } else { + rows -= chunk_rows; + if (rows) { + offset += chunk_rows * source_stride; + destination += chunk_rows * destination_stride; + } + } + } + } + qsort(jobs, count, sizeof(*jobs), job_destination_order); + for (size_t i = 1; i < count; i++) { + read_job_t* previous = &jobs[i - 1]; + if ((uintptr_t)previous->destination + previous->bytes * (1 + previous->expand_bf16) + + (previous->rows - 1) * previous->destination_stride > + (uintptr_t)jobs[i].destination) { + snprintf(failure.message, sizeof(failure.message), + "overlapping batch destinations need an explicit dependency"); + goto done; + } + } + qsort(jobs, count, sizeof(*jobs), job_source_order); + Py_BEGIN_ALLOW_THREADS pthread_mutex_lock(&executor->mutex); + if (cuda_ok(cudaSetDevice(executor->device), "select batch device", &failure) && + cuda_ok(cudaStreamSynchronize((cudaStream_t)(uintptr_t)stream), + "synchronize batch destinations", &failure)) { + struct timespec start, stop; + clock_gettime(CLOCK_MONOTONIC, &start); + bounce_execute(executor->bounce, jobs, count, &failure); + clock_gettime(CLOCK_MONOTONIC, &stop); + executor->execution_seconds += + stop.tv_sec - start.tv_sec + (stop.tv_nsec - start.tv_nsec) * 1e-9; + executor->batches++; + executor->descriptors += descriptors; + } + pthread_mutex_unlock(&executor->mutex); + Py_END_ALLOW_THREADS done : free(jobs); + PyBuffer_Release(&records); + if (failure.message[0]) return PyErr_Format(PyExc_RuntimeError, "%s", failure.message); + Py_RETURN_NONE; +} + +static PyObject* py_batch_stats(PyObject* self, PyObject* capsule) { + (void)self; + batch_executor_t* executor = PyCapsule_GetPointer(capsule, "b12x.batch_executor"); + if (!executor) return NULL; + pthread_mutex_lock(&executor->mutex); + bounce_ring_t* ring = executor->bounce; + PyObject* result = Py_BuildValue( + "{s:K,s:K,s:K,s:i,s:K,s:K,s:d,s:i,s:K,s:K,s:I}", "physical_bytes", + (unsigned long long)ring->physical_bytes, "strided_copy_bytes", + (unsigned long long)ring->strided_bytes, "reads", (unsigned long long)ring->reads, + "scratch_bytes", BOUNCE_BYTES, "batches", (unsigned long long)executor->batches, + "descriptors", (unsigned long long)executor->descriptors, "execution_seconds", + executor->execution_seconds, "bounce_buffer_bytes", BOUNCE_BYTES, "bounced_bytes", + (unsigned long long)ring->bounced_bytes, "io_uring_submits", + (unsigned long long)ring->submits, "max_inflight_reads", ring->max_inflight); + pthread_mutex_unlock(&executor->mutex); + return result; +} diff --git a/tests/fixtures/spark-fp4-loader/_bounce.c b/tests/fixtures/spark-fp4-loader/_bounce.c new file mode 100644 index 0000000..01e9eaa --- /dev/null +++ b/tests/fixtures/spark-fp4-loader/_bounce.c @@ -0,0 +1,340 @@ +/* Included by _batch.c. Fixed read slots stay owned until their CUDA copy finishes. */ +#define BOUNCE_SLOTS 16 +#define BOUNCE_SLOT_BYTES (1 << 20) +#define BOUNCE_BYTES (BOUNCE_SLOTS * BOUNCE_SLOT_BYTES) +#ifdef B12X_HAVE_LIBURING +#include +#endif + +typedef struct { + read_job_t job; + size_t delta, required; +} bounce_slot_t; + +typedef struct { + char* buffers; + int device; + cudaStream_t stream; + bool registered; + failure_t copy_failure; + bounce_slot_t slots[BOUNCE_SLOTS]; + pthread_mutex_t mutex; + pthread_cond_t ready, available, drained; + pthread_t copier; + bool copier_started, stopping, poisoned; + unsigned free_slots[BOUNCE_SLOTS], free_count; + unsigned ready_slots[BOUNCE_SLOTS], head, tail, ready_count, copying; + unsigned depth, max_inflight; + uint64_t reads, physical_bytes, bounced_bytes, strided_bytes, submits; +#ifdef B12X_HAVE_LIBURING + struct io_uring ring; + bool ring_ready; +#endif +} bounce_ring_t; + +static void bounce_release(bounce_ring_t* ring) { + if (!ring) return; + if (ring->copier_started) { + pthread_mutex_lock(&ring->mutex); + ring->stopping = true; + pthread_cond_signal(&ring->ready); + pthread_mutex_unlock(&ring->mutex); + pthread_join(ring->copier, NULL); + } +#ifdef B12X_HAVE_LIBURING + if (ring->ring_ready) io_uring_queue_exit(&ring->ring); +#endif + int previous = ring->device; + cudaGetDevice(&previous); + cudaSetDevice(ring->device); + if (ring->stream) { + cudaStreamSynchronize(ring->stream); + cudaStreamDestroy(ring->stream); + } + if (ring->registered) cudaHostUnregister(ring->buffers); + free(ring->buffers); + cudaSetDevice(previous); + pthread_cond_destroy(&ring->ready); + pthread_cond_destroy(&ring->available); + pthread_cond_destroy(&ring->drained); + pthread_mutex_destroy(&ring->mutex); + free(ring); +} + +#ifdef B12X_HAVE_LIBURING +static void expand_bf16_inplace(char* destination, int64_t bytes) { + for (int64_t i = bytes / 2; i > 0;) { + --i; + uint16_t bf16; + memcpy(&bf16, destination + 2 * i, sizeof(bf16)); + uint32_t fp32 = (uint32_t)bf16 << 16; + memcpy(destination + 4 * i, &fp32, sizeof(fp32)); + } +} + +static void* bounce_copy_worker(void* opaque) { + bounce_ring_t* ring = opaque; + failure_t failure = {{0}}; + cuda_ok(cudaSetDevice(ring->device), "select ring copy device", &failure); + pthread_mutex_lock(&ring->mutex); + for (;;) { + while (!ring->ready_count && !ring->stopping) pthread_cond_wait(&ring->ready, &ring->mutex); + if (!ring->ready_count) break; + unsigned index = ring->ready_slots[ring->head++ % BOUNCE_SLOTS]; + ring->ready_count--; + bounce_slot_t slot = ring->slots[index]; + pthread_mutex_unlock(&ring->mutex); + char* buffer = ring->buffers + (size_t)index * BOUNCE_SLOT_BYTES; + char* source = buffer + slot.delta; + size_t width = slot.job.bytes; + size_t pitch = slot.job.rows > 1 ? (size_t)slot.job.source_stride : width; + if (slot.job.expand_bf16) { + for (int64_t row = 0; row < slot.job.rows; row++) + memmove(buffer + row * width, source + row * pitch, width); + expand_bf16_inplace(buffer, slot.job.rows * width); + source = buffer; + width *= 2; + pitch = width; + } + if (!failure.message[0]) + cuda_ok( + cudaMemcpy2DAsync(slot.job.destination, + slot.job.rows > 1 ? (size_t)slot.job.destination_stride : width, source, + pitch, width, slot.job.rows, cudaMemcpyHostToDevice, ring->stream), + "copy checkpoint ring to CUDA weights", &failure); + /* Reads into other slots remain in flight while this slot is on the GPU. */ + cuda_ok(cudaStreamSynchronize(ring->stream), "complete checkpoint ring copy", &failure); + pthread_mutex_lock(&ring->mutex); + if (failure.message[0]) ring->copy_failure = failure; + ring->bounced_bytes += slot.job.rows * slot.job.bytes; + if (slot.job.rows > 1) ring->strided_bytes += slot.job.rows * slot.job.bytes; + ring->free_slots[ring->free_count++] = index; + ring->copying--; + pthread_cond_signal(&ring->available); + if (!ring->copying) pthread_cond_signal(&ring->drained); + } + pthread_mutex_unlock(&ring->mutex); + return NULL; +} +#endif + +static bounce_ring_t* bounce_create(int device, unsigned depth, failure_t* failure) { +#ifndef B12X_HAVE_LIBURING + (void)device; + (void)depth; + snprintf(failure->message, sizeof(failure->message), + "io_uring bounce support is unavailable: install liburing development headers and " + "pkg-config"); + return NULL; +#else + bounce_ring_t* ring = calloc(1, sizeof(*ring)); + if (!ring) { + snprintf(failure->message, sizeof(failure->message), "bounce ring allocation failed"); + return NULL; + } + pthread_mutex_init(&ring->mutex, NULL); + pthread_cond_init(&ring->ready, NULL); + pthread_cond_init(&ring->available, NULL); + pthread_cond_init(&ring->drained, NULL); + ring->device = device; + ring->depth = depth; + int error = posix_memalign((void**)&ring->buffers, IO_ALIGNMENT, BOUNCE_BYTES); + if (error) goto failed; + if (!cuda_ok(cudaSetDevice(device), "select ring device", failure) || + !cuda_ok(cudaHostRegister(ring->buffers, BOUNCE_BYTES, cudaHostRegisterDefault), + "pin checkpoint ring for CUDA copies", failure)) + goto failed; + ring->registered = true; + if (!cuda_ok(cudaStreamCreateWithFlags(&ring->stream, cudaStreamNonBlocking), + "create checkpoint copy stream", failure)) + goto failed; + error = io_uring_queue_init(BOUNCE_SLOTS, &ring->ring, 0); + if (error < 0) { + error = -error; + goto failed; + } + ring->ring_ready = true; + struct iovec buffers[BOUNCE_SLOTS]; + for (unsigned i = 0; i < BOUNCE_SLOTS; i++) { + buffers[i] = (struct iovec){ring->buffers + (size_t)i * BOUNCE_SLOT_BYTES, BOUNCE_SLOT_BYTES}; + ring->free_slots[ring->free_count++] = i; + } + error = io_uring_register_buffers(&ring->ring, buffers, BOUNCE_SLOTS); + if (error < 0) { + error = -error; + goto failed; + } + error = pthread_create(&ring->copier, NULL, bounce_copy_worker, ring); + if (error) goto failed; + ring->copier_started = true; + return ring; +failed: + if (!failure->message[0]) + snprintf(failure->message, sizeof(failure->message), + "io_uring bounce initialization failed: %s; check liburing, kernel/container policy " + "and locked-memory limits", + strerror(error)); + bounce_release(ring); + return NULL; +#endif +} + +static bool bounce_execute(bounce_ring_t* ring, read_job_t* jobs, size_t count, + failure_t* failure) { +#ifndef B12X_HAVE_LIBURING + (void)ring; + (void)jobs; + (void)count; + snprintf(failure->message, sizeof(failure->message), "io_uring bounce support is unavailable"); + return false; +#else + if (ring->poisoned) { + snprintf(failure->message, sizeof(failure->message), + "io_uring bounce executor is unusable after a submission/completion failure"); + return false; + } + for (size_t i = 0; i < count; i++) { + read_job_t* job = &jobs[i]; + if (!job->host_copy && + !validate_direct_range(job->fd, job->offset, + (job->rows - 1) * job->source_stride + job->bytes, failure)) + return false; + } + size_t next = 0; + int64_t row = 0, within = 0; + unsigned outstanding = 0; + while ((!failure->message[0] && next < count) || outstanding) { + unsigned pending = 0; + while (!failure->message[0] && next < count && outstanding + pending < ring->depth) { + read_job_t* job = &jobs[next]; + if (job->host_copy) { + cuda_ok(cudaMemcpyAsync(job->destination, (void*)(uintptr_t)job->offset, job->bytes, + cudaMemcpyHostToDevice, ring->stream), + "copy checkpoint metadata", failure); + next++; + continue; + } + pthread_mutex_lock(&ring->mutex); + if (ring->copy_failure.message[0]) { + *failure = ring->copy_failure; + pthread_mutex_unlock(&ring->mutex); + break; + } + if (!ring->free_count) { + pthread_mutex_unlock(&ring->mutex); + break; + } + unsigned index = ring->free_slots[--ring->free_count]; + pthread_mutex_unlock(&ring->mutex); + bounce_slot_t* slot = &ring->slots[index]; + slot->job = *job; + int64_t offset = job->offset + row * job->source_stride + within; + int64_t aligned = offset & ~(int64_t)(IO_ALIGNMENT - 1); + slot->delta = offset - aligned; + slot->job.destination += row * job->destination_stride + within * (1 + job->expand_bf16); + slot->job.bytes = job->bytes - within; + if (slot->job.bytes > BOUNCE_SLOT_BYTES - (int64_t)slot->delta) + slot->job.bytes = BOUNCE_SLOT_BYTES - slot->delta; + if (job->expand_bf16) { + if (slot->job.bytes > BOUNCE_SLOT_BYTES / 2) slot->job.bytes = BOUNCE_SLOT_BYTES / 2; + slot->job.bytes &= ~(int64_t)1; + } + slot->job.rows = 1; + if (!within && job->bytes < IO_ALIGNMENT && job->source_stride >= job->bytes) { + slot->job.rows += + (BOUNCE_SLOT_BYTES - (int64_t)slot->delta - job->bytes) / job->source_stride; + if (slot->job.rows > job->rows - row) slot->job.rows = job->rows - row; + } + if (job->expand_bf16 && slot->job.rows > BOUNCE_SLOT_BYTES / (2 * slot->job.bytes)) + slot->job.rows = BOUNCE_SLOT_BYTES / (2 * slot->job.bytes); + slot->required = slot->delta + (slot->job.rows - 1) * job->source_stride + slot->job.bytes; + unsigned length = (slot->required + IO_ALIGNMENT - 1) & ~(size_t)(IO_ALIGNMENT - 1); + struct io_uring_sqe* sqe = io_uring_get_sqe(&ring->ring); + if (!sqe) { + snprintf(failure->message, sizeof(failure->message), + "io_uring bounce SQ capacity exhausted"); + ring->poisoned = true; + break; + } + io_uring_prep_read_fixed(sqe, job->fd, ring->buffers + (size_t)index * BOUNCE_SLOT_BYTES, + length, aligned, index); + io_uring_sqe_set_data64(sqe, index); + pending++; + within += slot->job.bytes; + if (within == job->bytes) { + row += slot->job.rows; + within = 0; + } + if (row == job->rows) { + next++; + row = 0; + } + } + while (pending && !failure->message[0]) { + ring->submits++; + int submitted = io_uring_submit(&ring->ring); + if (submitted == -EINTR) continue; + if (submitted <= 0) { + snprintf(failure->message, sizeof(failure->message), "io_uring bounce submit: %s", + strerror(submitted < 0 ? -submitted : EIO)); + ring->poisoned = true; + break; + } + outstanding += submitted; + pending -= submitted; + if (outstanding > ring->max_inflight) ring->max_inflight = outstanding; + } + if (outstanding) { + struct io_uring_cqe* cqe = NULL; + int status = io_uring_wait_cqe(&ring->ring, &cqe); + if (status < 0) { + if (status != -EINTR && status != -EAGAIN) { + if (!failure->message[0]) + snprintf(failure->message, sizeof(failure->message), "io_uring bounce wait: %s", + strerror(-status)); + ring->poisoned = true; + } + continue; + } + unsigned index = (unsigned)io_uring_cqe_get_data64(cqe); + status = cqe->res; + io_uring_cqe_seen(&ring->ring, cqe); + outstanding--; + ring->reads++; + if (status > 0) ring->physical_bytes += status; + bool success = status >= 0 && (size_t)status >= ring->slots[index].required; + if (!success && !failure->message[0]) + snprintf(failure->message, sizeof(failure->message), "io_uring bounce read: %s", + status < 0 ? strerror(-status) : "short O_DIRECT read"); + pthread_mutex_lock(&ring->mutex); + if (success) { + ring->ready_slots[ring->tail++ % BOUNCE_SLOTS] = index; + ring->ready_count++; + ring->copying++; + pthread_cond_signal(&ring->ready); + } else + ring->free_slots[ring->free_count++] = index; + pthread_mutex_unlock(&ring->mutex); + } else if (!failure->message[0] && next < count) { + pthread_mutex_lock(&ring->mutex); + while (!ring->free_count) pthread_cond_wait(&ring->available, &ring->mutex); + pthread_mutex_unlock(&ring->mutex); + } + } + pthread_mutex_lock(&ring->mutex); + while (ring->copying) pthread_cond_wait(&ring->drained, &ring->mutex); + if (ring->copy_failure.message[0]) { + if (!failure->message[0]) *failure = ring->copy_failure; + ring->poisoned = true; + } + pthread_mutex_unlock(&ring->mutex); + if (!cuda_ok(cudaStreamSynchronize(ring->stream), "complete checkpoint metadata copies", failure)) + ring->poisoned = true; + if (ring->poisoned) { + /* Drain submitted reads above; never submit SQEs left by a failed submit. */ + io_uring_queue_exit(&ring->ring); + ring->ring_ready = false; + } + return !failure->message[0]; +#endif +} diff --git a/tests/fixtures/spark-fp4-loader/_direct.c b/tests/fixtures/spark-fp4-loader/_direct.c new file mode 100644 index 0000000..d424556 --- /dev/null +++ b/tests/fixtures/spark-fp4-loader/_direct.c @@ -0,0 +1,125 @@ +/* Small O_DIRECT reads for safetensors headers and CPU metadata. */ +#include +#define IO_ALIGNMENT 4096 +#define IO_SCRATCH_BYTES (64 << 10) +typedef struct { + void* scratch; + int scratch_bytes; + pthread_mutex_t mutex; + uint64_t physical_bytes, reads; +} direct_reader_t; + +static void delete_reader(PyObject* capsule) { + direct_reader_t* reader = PyCapsule_GetPointer(capsule, "b12x.direct_reader"); + if (!reader) return; + free(reader->scratch); + pthread_mutex_destroy(&reader->mutex); + free(reader); +} +static PyObject* py_direct_reader(PyObject* self, PyObject* args) { + (void)self; + int device; + if (!PyArg_ParseTuple(args, "i", &device)) return NULL; + direct_reader_t* reader = calloc(1, sizeof(*reader)); + if (!reader) return PyErr_NoMemory(); + reader->scratch_bytes = IO_SCRATCH_BYTES; + int error = posix_memalign(&reader->scratch, IO_ALIGNMENT, IO_SCRATCH_BYTES); + if (error) { + free(reader); + return PyErr_Format(PyExc_RuntimeError, "metadata scratch: %s", strerror(error)); + } + pthread_mutex_init(&reader->mutex, NULL); + PyObject* capsule = PyCapsule_New(reader, "b12x.direct_reader", delete_reader); + if (!capsule) { + free(reader->scratch); + pthread_mutex_destroy(&reader->mutex); + free(reader); + } + return capsule; +} + +static bool validate_direct_range(int fd, int64_t offset, int64_t bytes, failure_t* failure) { + struct stat status; + int flags = fcntl(fd, F_GETFL); + if (flags < 0 || !(flags & O_DIRECT)) { + snprintf(failure->message, sizeof(failure->message), "direct reader requires O_DIRECT"); + return false; + } + if (fstat(fd, &status) != 0) { + system_error(failure, "fstat direct input"); + return false; + } + if (offset < 0 || bytes < 0 || offset > INT64_MAX - bytes || !S_ISREG(status.st_mode) || + offset + bytes > status.st_size) { + snprintf(failure->message, sizeof(failure->message), "invalid direct input file range"); + return false; + } + return true; +} + +static bool direct_read_range(direct_reader_t* reader, int fd, int64_t offset, int64_t bytes, + char* destination, bool allow_direct, failure_t* failure) { + if (!validate_direct_range(fd, offset, bytes, failure)) return false; + while (bytes) { + int64_t aligned_offset = offset & ~(int64_t)(IO_ALIGNMENT - 1); + size_t delta = offset - aligned_offset; + (void)allow_direct; + size_t payload = bytes < reader->scratch_bytes - (int64_t)delta ? (size_t)bytes + : reader->scratch_bytes - delta; + size_t length = (payload + delta + IO_ALIGNMENT - 1) & ~(size_t)(IO_ALIGNMENT - 1); + void* buffer = reader->scratch; + ssize_t count; + do { + count = pread(fd, buffer, length, aligned_offset); + } while (count < 0 && errno == EINTR); + if (count < 0) { + system_error(failure, "O_DIRECT pread (no buffered fallback)"); + return false; + } + if ((size_t)count < delta + payload) { + snprintf(failure->message, sizeof(failure->message), "short O_DIRECT read"); + return false; + } + reader->physical_bytes += count; + reader->reads++; + memcpy(destination, (char*)buffer + delta, payload); + destination += payload; + offset += payload; + bytes -= payload; + } + return true; +} + +static PyObject* py_direct_bytes(PyObject* self, PyObject* args) { + (void)self; + PyObject* capsule; + int fd; + long long offset, bytes; + if (!PyArg_ParseTuple(args, "OiLL", &capsule, &fd, &offset, &bytes)) return NULL; + direct_reader_t* reader = PyCapsule_GetPointer(capsule, "b12x.direct_reader"); + if (!reader) return NULL; + if (bytes < 0 || bytes > 100 * (1 << 20)) + return PyErr_Format(PyExc_ValueError, "header/metadata read exceeds 100 MiB"); + PyObject* result = PyBytes_FromStringAndSize(NULL, bytes); + if (!result) return NULL; + bool success; + failure_t failure = {{0}}; + char* destination = PyBytes_AS_STRING(result); + Py_BEGIN_ALLOW_THREADS pthread_mutex_lock(&reader->mutex); + success = direct_read_range(reader, fd, offset, bytes, destination, false, &failure); + pthread_mutex_unlock(&reader->mutex); + Py_END_ALLOW_THREADS if (!success) { + Py_DECREF(result); + return PyErr_Format(PyExc_RuntimeError, "%s", failure.message); + } + return result; +} + +static PyObject* py_direct_stats(PyObject* self, PyObject* capsule) { + (void)self; + direct_reader_t* reader = PyCapsule_GetPointer(capsule, "b12x.direct_reader"); + if (!reader) return NULL; + return Py_BuildValue("{s:K,s:K,s:i}", "physical_bytes", + (unsigned long long)reader->physical_bytes, "reads", + (unsigned long long)reader->reads, "scratch_bytes", reader->scratch_bytes); +} diff --git a/tests/fixtures/spark-fp4-loader/_ple_reader.c b/tests/fixtures/spark-fp4-loader/_ple_reader.c new file mode 100644 index 0000000..681b993 --- /dev/null +++ b/tests/fixtures/spark-fp4-loader/_ple_reader.c @@ -0,0 +1,437 @@ +/* Included by _storage.c. File payloads are never mapped or retained between runs. */ +#include +#include +#ifdef B12X_HAVE_LIBURING +#include +#endif + +#include "_row_plan.h" +#define PLE_CAPSULE "b12x.ple_reader" + +typedef struct ple_reader ple_reader_t; +typedef struct { + char* buffer; + size_t job; +} ple_slot_t; + +struct ple_reader { + ple_plan_t plan; + bool poisoned; + pthread_mutex_t api_mutex; + unsigned slots_count; + ple_slot_t* slots; + char* buffers; + char *weights, *scales; +#ifdef B12X_HAVE_LIBURING + struct io_uring ring; + bool ring_ready; + bool files_registered; + int* registered_fds; + struct iovec* iovecs; + unsigned* free_slots; +#endif +}; + +#ifdef B12X_HAVE_LIBURING +static void ple_error(ple_reader_t* reader, const char* operation, int error) { + if (!reader->plan.failure.message[0]) + snprintf(reader->plan.failure.message, sizeof(reader->plan.failure.message), "%s: %s", + operation, strerror(error)); +} + +static bool ple_scatter(ple_reader_t* reader, size_t index, char* buffer, int result) { + ple_job_t* job = &reader->plan.jobs[index]; + reader->plan.read_calls++; + if (result > 0) reader->plan.read_bytes += (unsigned)result; + if (result < 0) { + ple_error(reader, "io_uring READ_FIXED", -result); + return false; + } + if ((unsigned)result != job->expected) { + if (!reader->plan.failure.message[0]) + snprintf(reader->plan.failure.message, sizeof(reader->plan.failure.message), + "short PLE read at offset %lld: expected %u bytes, received %d; source changed or " + "was truncated", + (long long)job->offset, job->expected, result); + return false; + } + for (size_t i = job->begin; i < job->end; i++) { + ple_fragment_t* fragment = &reader->plan.fragments[i]; + memcpy((fragment->scale ? reader->scales : reader->weights) + fragment->destination, + buffer + (fragment->offset - job->offset), fragment->length); + } + return true; +} + +static void ple_release(ple_reader_t* reader) { + if (!reader) return; + if (reader->ring_ready) io_uring_queue_exit(&reader->ring); + free(reader->iovecs); + free(reader->free_slots); + free(reader->registered_fds); + for (size_t i = 0; i < reader->plan.file_count; i++) close(reader->plan.files[i].fd); + free(reader->plan.files); + free(reader->plan.sources); + free(reader->buffers); + free(reader->slots); + free(reader->plan.fragments); + free(reader->plan.jobs); + pthread_mutex_destroy(&reader->api_mutex); + free(reader); +} + +static void ple_delete(PyObject* capsule) { + ple_reader_t* reader = PyCapsule_GetPointer(capsule, PLE_CAPSULE); + if (reader) { + Py_BEGIN_ALLOW_THREADS ple_release(reader); + Py_END_ALLOW_THREADS + } +} +#endif + +static PyObject* py_ple_reader(PyObject* self, PyObject* args) { + (void)self; + long long shard_rows, padded_rows, tp_start, tp_end, weight_bytes, scale_bytes, max_lookups; + int depth; + if (!PyArg_ParseTuple(args, "LLLLLLLi", &shard_rows, &padded_rows, &tp_start, &tp_end, + &weight_bytes, &scale_bytes, &max_lookups, &depth)) + return NULL; +#ifndef B12X_HAVE_LIBURING + return PyErr_Format(PyExc_RuntimeError, + "io_uring PLE support is unavailable: install liburing development headers " + "and pkg-config, then rebuild the b12x native loader"); +#else + if (shard_rows <= 0 || padded_rows <= 0 || tp_start < 0 || tp_end < tp_start || + tp_end > padded_rows || weight_bytes <= 0 || scale_bytes < 0 || max_lookups <= 0 || + depth <= 0 || depth > 32768 || weight_bytes > PY_SSIZE_T_MAX / max_lookups || + scale_bytes > PY_SSIZE_T_MAX / max_lookups || max_lookups > PY_SSIZE_T_MAX / 8) + return PyErr_Format(PyExc_ValueError, "invalid PLE reader geometry or I/O capacity"); + /* At most ceil((row_bytes + block - 1) / block) fragments per plane. */ + size_t per_row = (size_t)weight_bytes / PLE_BLOCK + 2; + if (scale_bytes) per_row += (size_t)scale_bytes / PLE_BLOCK + 2; + if ((size_t)max_lookups > SIZE_MAX / per_row || + (size_t)max_lookups * per_row > SIZE_MAX / sizeof(ple_fragment_t) || + (size_t)max_lookups * per_row > SIZE_MAX / sizeof(ple_job_t)) + return PyErr_Format(PyExc_OverflowError, "PLE batch metadata is too large"); + ple_reader_t* reader = calloc(1, sizeof(*reader)); + if (!reader) return PyErr_NoMemory(); + /* Only an initialized API mutex reaches ple_release. */ + int error = pthread_mutex_init(&reader->api_mutex, NULL); + if (error) { + free(reader); + return PyErr_Format(PyExc_RuntimeError, "PLE mutex: %s", strerror(error)); + } + reader->plan.shard_rows = shard_rows; + reader->plan.padded_rows = padded_rows; + reader->plan.tp_start = tp_start; + reader->plan.tp_end = tp_end; + reader->plan.weight_bytes = weight_bytes; + reader->plan.scale_bytes = scale_bytes; + reader->plan.max_lookups = max_lookups; + reader->plan.capacity = (size_t)max_lookups * per_row; + reader->slots_count = (unsigned)depth; + reader->slots = calloc(reader->slots_count, sizeof(*reader->slots)); + reader->plan.fragments = calloc(reader->plan.capacity, sizeof(*reader->plan.fragments)); + reader->plan.jobs = calloc(reader->plan.capacity, sizeof(*reader->plan.jobs)); + error = posix_memalign((void**)&reader->buffers, PLE_BLOCK, + (size_t)reader->slots_count * PLE_READ_MAX); + if (!reader->slots || !reader->plan.fragments || !reader->plan.jobs || error) { + ple_release(reader); + return PyErr_NoMemory(); + } + for (unsigned i = 0; i < reader->slots_count; i++) { + reader->slots[i].buffer = reader->buffers + (size_t)i * PLE_READ_MAX; + } + error = io_uring_queue_init(reader->slots_count, &reader->ring, 0); + if (error < 0) { + ple_release(reader); + return PyErr_Format( + PyExc_RuntimeError, + "io_uring initialization failed: %s; enable io_uring in the kernel/container policy", + strerror(-error)); + } + reader->ring_ready = true; + reader->iovecs = calloc(reader->slots_count, sizeof(*reader->iovecs)); + reader->free_slots = calloc(reader->slots_count, sizeof(*reader->free_slots)); + if (!reader->iovecs || !reader->free_slots) { + ple_release(reader); + return PyErr_NoMemory(); + } + for (unsigned i = 0; i < reader->slots_count; i++) { + reader->iovecs[i].iov_base = reader->slots[i].buffer; + reader->iovecs[i].iov_len = PLE_READ_MAX; + } + error = io_uring_register_buffers(&reader->ring, reader->iovecs, reader->slots_count); + if (error < 0) { + ple_release(reader); + return PyErr_Format(PyExc_RuntimeError, + "io_uring buffer registration failed: %s; raise the locked-memory limit or " + "reduce queue_depth", + strerror(-error)); + } + PyObject* capsule = PyCapsule_New(reader, PLE_CAPSULE, ple_delete); + if (!capsule) ple_release(reader); + return capsule; +#endif +} + +static int ple_register_files(ple_plan_t* plan, void* owner) { + (void)plan; + ple_reader_t* reader = owner; +#ifdef B12X_HAVE_LIBURING + int* fds = realloc(reader->registered_fds, reader->plan.file_count * sizeof(*fds)); + if (!fds) return -ENOMEM; + reader->registered_fds = fds; + for (size_t i = 0; i < reader->plan.file_count; i++) fds[i] = reader->plan.files[i].fd; + if (reader->files_registered) { + int error = io_uring_unregister_files(&reader->ring); + if (error < 0) { + reader->poisoned = true; + return error; + } + reader->files_registered = false; + } + int error = io_uring_register_files(&reader->ring, fds, (unsigned)reader->plan.file_count); + if (!error) + reader->files_registered = true; + else + reader->poisoned = true; + return error; +#else + (void)reader; + return 0; +#endif +} + +static PyObject* py_ple_reader_add(PyObject* self, PyObject* args) { + (void)self; + PyObject* capsule; + const char* path; + long long shard, offset; + int scale; + if (!PyArg_ParseTuple(args, "OLsLp", &capsule, &shard, &path, &offset, &scale)) return NULL; + ple_reader_t* reader = PyCapsule_GetPointer(capsule, PLE_CAPSULE); + if (!reader) return NULL; + failure_t failure = {{0}}; + int error; + Py_BEGIN_ALLOW_THREADS pthread_mutex_lock(&reader->api_mutex); + reader->plan.failure.message[0] = 0; + error = + ple_add_source(&reader->plan, shard, path, offset, scale != 0, ple_register_files, reader); + failure = reader->plan.failure; + pthread_mutex_unlock(&reader->api_mutex); + Py_END_ALLOW_THREADS if (error == EINVAL) return PyErr_Format( + PyExc_ValueError, "invalid PLE source shard, offset, or scale plane"); + if (error == EOVERFLOW) + return PyErr_Format(PyExc_OverflowError, "PLE source byte range overflows int64"); + if (error) + return PyErr_Format( + PyExc_RuntimeError, "%s", + failure.message[0] ? failure.message : "invalid PLE source geometry or byte range"); + Py_RETURN_NONE; +} + +#ifdef B12X_HAVE_LIBURING +/* Prepare a whole free-slot wave before entering the kernel. A short positive + * submit accounts for exactly that many SQEs; retry only the remaining entries. + * A failed submit may leave pending SQEs, so stop issuing, drain every submitted + * CQE, and retire the ring without submitting those pending entries. */ +static void ple_submit_wave(ple_reader_t* reader, unsigned free_count, size_t* next, + unsigned* outstanding) { + size_t remaining = reader->plan.job_count - *next; + unsigned count = remaining < free_count ? (unsigned)remaining : free_count; + for (unsigned i = 0; i < count; i++) { + unsigned slot = reader->free_slots[i]; + struct io_uring_sqe* sqe = io_uring_get_sqe(&reader->ring); + if (!sqe) { + ple_error(reader, "io_uring SQ capacity", ENOSPC); + reader->poisoned = true; + return; + } + ple_job_t* job = &reader->plan.jobs[*next]; + reader->slots[slot].job = (*next)++; + io_uring_prep_read_fixed(sqe, (int)job->file, reader->slots[slot].buffer, job->length, + job->offset, (int)slot); + sqe->flags |= IOSQE_FIXED_FILE; + io_uring_sqe_set_data64(sqe, slot); + } + unsigned pending = count; + while (pending) { + reader->plan.submit_calls++; + int result = io_uring_submit(&reader->ring); + if (result == -EINTR) continue; + if (result <= 0) { + ple_error(reader, "io_uring submit", result < 0 ? -result : EIO); + reader->poisoned = true; + return; + } + *outstanding += (unsigned)result; + pending -= (unsigned)result; + } +} + +static void ple_uring_run(ple_reader_t* reader) { + unsigned outstanding = 0; + size_t next = 0; + for (unsigned i = 0; i < reader->slots_count; i++) reader->free_slots[i] = i; + ple_submit_wave(reader, reader->slots_count, &next, &outstanding); + while (outstanding) { + struct io_uring_cqe* cqe = NULL; + /* wait/peek do not submit pending SQEs when draining a failed submit. */ + int result = io_uring_wait_cqe(&reader->ring, &cqe); + if (result < 0) { + if (result != -EINTR && result != -EAGAIN) { + ple_error(reader, "io_uring completion wait", -result); + reader->poisoned = true; + } + continue; + } + unsigned free_count = 0, head; + io_uring_for_each_cqe(&reader->ring, head, cqe) { + unsigned slot = (unsigned)io_uring_cqe_get_data64(cqe); + int status = cqe->res; + outstanding--; + ple_scatter(reader, reader->slots[slot].job, reader->slots[slot].buffer, status); + reader->free_slots[free_count++] = slot; + if (!outstanding) break; + } + io_uring_cq_advance(&reader->ring, free_count); + /* Do not prepare refill SQEs until every collected completion has been + * checked. An I/O error therefore leaves no speculative refill entries. */ + if (!reader->plan.failure.message[0] && next < reader->plan.job_count) + ple_submit_wave(reader, free_count, &next, &outstanding); + } + if (reader->poisoned) { + io_uring_queue_exit(&reader->ring); + reader->ring_ready = false; + } +} +#endif + +static bool ple_overlap(const Py_buffer* a, size_t a_bytes, const Py_buffer* b, size_t b_bytes) { + uintptr_t x = (uintptr_t)a->buf, y = (uintptr_t)b->buf; + return a_bytes && b_bytes && (x <= y ? y - x < a_bytes : x - y < b_bytes); +} + +static PyObject* py_ple_reader_run(PyObject* self, PyObject* args) { + (void)self; + PyObject *capsule, *ids_object, *weights_object, *scales_object; + long long count; + if (!PyArg_ParseTuple(args, "OOOOL", &capsule, &ids_object, &weights_object, &scales_object, + &count)) + return NULL; + ple_reader_t* reader = PyCapsule_GetPointer(capsule, PLE_CAPSULE); + if (!reader) return NULL; + if (count < 0 || (uint64_t)count > reader->plan.max_lookups) + return PyErr_Format(PyExc_ValueError, "PLE lookup count exceeds batch capacity"); + Py_buffer ids = {0}, weights = {0}, scales = {0}; + if (PyObject_GetBuffer(ids_object, &ids, PyBUF_CONTIG_RO) < 0) return NULL; + if (PyObject_GetBuffer(weights_object, &weights, PyBUF_CONTIG) < 0) goto failed; + if (reader->plan.scale_bytes && PyObject_GetBuffer(scales_object, &scales, PyBUF_CONTIG) < 0) + goto failed; + if (!reader->plan.scale_bytes && scales_object != Py_None) { + PyErr_SetString(PyExc_ValueError, "PLE reader has no scale plane; pass None"); + goto failed; + } + size_t id_bytes = (size_t)count * 8; + size_t weight_bytes = (size_t)count * reader->plan.weight_bytes; + size_t scale_bytes = (size_t)count * reader->plan.scale_bytes; + if ((size_t)ids.len < id_bytes || (size_t)weights.len < weight_bytes || + (size_t)scales.len < scale_bytes || !PyBuffer_IsContiguous(&ids, 'C') || + !PyBuffer_IsContiguous(&weights, 'C') || + (reader->plan.scale_bytes && !PyBuffer_IsContiguous(&scales, 'C'))) { + PyErr_SetString(PyExc_ValueError, + "PLE buffers must be C-contiguous and cover count rows (IDs are native signed " + "int64 bytes)"); + goto failed; + } + if (ple_overlap(&ids, id_bytes, &weights, weight_bytes) || + ple_overlap(&ids, id_bytes, &scales, scale_bytes) || + ple_overlap(&weights, weight_bytes, &scales, scale_bytes)) { + PyErr_SetString(PyExc_ValueError, "PLE IDs and destination byte ranges must not overlap"); + goto failed; + } + failure_t failure = {{0}}; + Py_BEGIN_ALLOW_THREADS pthread_mutex_lock(&reader->api_mutex); + struct timespec start, planned_at, end; + clock_gettime(CLOCK_MONOTONIC, &start); + reader->plan.failure.message[0] = 0; + reader->plan.lookups = count; + reader->plan.requested_bytes = reader->plan.read_bytes = reader->plan.read_calls = 0; + reader->plan.unique_blocks = reader->plan.coalesced_reads = reader->plan.submit_calls = 0; + reader->plan.planning_seconds = 0; + if (reader->poisoned) + snprintf(reader->plan.failure.message, sizeof(reader->plan.failure.message), + "PLE io_uring reader is unusable after a submission/completion failure; create a new " + "reader"); + else { + bool planned = ple_plan(&reader->plan, ids.buf, weights.buf, scales.buf, count); + clock_gettime(CLOCK_MONOTONIC, &planned_at); + reader->plan.planning_seconds = (double)(planned_at.tv_sec - start.tv_sec) + + (double)(planned_at.tv_nsec - start.tv_nsec) * 1e-9; +#ifdef B12X_HAVE_LIBURING + reader->weights = weights.buf; + reader->scales = scales.buf; + if (planned && reader->plan.job_count) ple_uring_run(reader); +#else + (void)planned; +#endif + } + clock_gettime(CLOCK_MONOTONIC, &end); + reader->plan.execution_seconds = + (double)(end.tv_sec - start.tv_sec) + (double)(end.tv_nsec - start.tv_nsec) * 1e-9; + failure = reader->plan.failure; + pthread_mutex_unlock(&reader->api_mutex); + Py_END_ALLOW_THREADS PyBuffer_Release(&ids); + PyBuffer_Release(&weights); + if (scales.obj) PyBuffer_Release(&scales); + if (failure.message[0]) return PyErr_Format(PyExc_RuntimeError, "%s", failure.message); + Py_RETURN_NONE; +failed: + PyBuffer_Release(&ids); + if (weights.obj) PyBuffer_Release(&weights); + if (scales.obj) PyBuffer_Release(&scales); + return NULL; +} + +static PyObject* py_ple_reader_stats(PyObject* self, PyObject* capsule) { + (void)self; + ple_reader_t* reader = PyCapsule_GetPointer(capsule, PLE_CAPSULE); + if (!reader) return NULL; + uint64_t lookups, requested, bytes, calls, blocks, coalesced, submits, staging, metadata; + double seconds, planning; + Py_BEGIN_ALLOW_THREADS pthread_mutex_lock(&reader->api_mutex); + lookups = reader->plan.lookups; + requested = reader->plan.requested_bytes; + bytes = reader->plan.read_bytes; + calls = reader->plan.read_calls; + blocks = reader->plan.unique_blocks; + coalesced = reader->plan.coalesced_reads; + submits = reader->plan.submit_calls; + seconds = reader->plan.execution_seconds; + planning = reader->plan.planning_seconds; + staging = (uint64_t)reader->slots_count * PLE_READ_MAX; + metadata = + sizeof(*reader) + + reader->plan.capacity * (sizeof(*reader->plan.fragments) + sizeof(*reader->plan.jobs)) + + reader->slots_count * sizeof(*reader->slots) + + reader->plan.source_count * sizeof(*reader->plan.sources) + + reader->plan.file_count * sizeof(*reader->plan.files); +#ifdef B12X_HAVE_LIBURING + metadata += reader->slots_count * (sizeof(*reader->iovecs) + sizeof(*reader->free_slots)); + metadata += reader->plan.file_count * sizeof(*reader->registered_fds); + if (reader->ring_ready) { + metadata += reader->ring.sq.ring_sz; + if (reader->ring.cq.ring_ptr != reader->ring.sq.ring_ptr) metadata += reader->ring.cq.ring_sz; + metadata += reader->ring.sq.ring_entries * sizeof(struct io_uring_sqe); + } +#endif + pthread_mutex_unlock(&reader->api_mutex); + Py_END_ALLOW_THREADS return Py_BuildValue( + "{s:K,s:K,s:K,s:K,s:K,s:K,s:K,s:K,s:K,s:d,s:d}", "lookups", (unsigned long long)lookups, + "requested_bytes", (unsigned long long)requested, "read_bytes", (unsigned long long)bytes, + "read_calls", (unsigned long long)calls, "unique_blocks", (unsigned long long)blocks, + "coalesced_reads", (unsigned long long)coalesced, "submit_calls", (unsigned long long)submits, + "staging_bytes", (unsigned long long)staging, "metadata_bytes", (unsigned long long)metadata, + "execution_seconds", seconds, "planning_seconds", planning); +} diff --git a/tests/fixtures/spark-fp4-loader/_row_plan.h b/tests/fixtures/spark-fp4-loader/_row_plan.h new file mode 100644 index 0000000..582184b --- /dev/null +++ b/tests/fixtures/spark-fp4-loader/_row_plan.h @@ -0,0 +1,309 @@ +/* Shared immutable row-source validation and allocation-free block planning. */ +#ifndef B12X_ROW_PLAN_H +#define B12X_ROW_PLAN_H +#include +#include +#define PLE_BLOCK 4096u +#define PLE_READ_MAX 65536u + +typedef struct { + int fd; + dev_t device; + ino_t inode; + int64_t bytes; +} ple_file_t; + +typedef struct { + int64_t shard; + int64_t offset; + size_t file; + bool scale; +} ple_source_t; + +typedef struct { + size_t file; + int64_t offset; + unsigned length; + size_t destination; + bool scale; +} ple_fragment_t; + +typedef struct { + size_t file, begin, end; + int64_t offset; + unsigned length, expected; +} ple_job_t; + +typedef struct { + int64_t shard_rows, padded_rows, tp_start, tp_end; + size_t weight_bytes, scale_bytes, max_lookups, capacity; + ple_fragment_t* fragments; + ple_job_t* jobs; + size_t fragment_count, job_count; + ple_file_t* files; + ple_source_t* sources; + size_t file_count, source_count; + failure_t failure; + uint64_t lookups, requested_bytes, read_bytes, read_calls; + uint64_t unique_blocks, coalesced_reads, submit_calls; + double execution_seconds, planning_seconds; +} ple_plan_t; + +static inline int ple_source_compare(const void* left, const void* right) { + const ple_source_t *a = left, *b = right; + if (a->shard != b->shard) return a->shard < b->shard ? -1 : 1; + return (int)a->scale - (int)b->scale; +} + +static inline void ple_plan_release(ple_plan_t* reader) { + for (size_t i = 0; i < reader->file_count; i++) close(reader->files[i].fd); + free(reader->files); + free(reader->sources); + free(reader->fragments); + free(reader->jobs); +} + +static inline int ple_plan_init(ple_plan_t* reader, int64_t shard_rows, int64_t padded_rows, + int64_t tp_start, int64_t tp_end, int64_t weight_bytes, + int64_t scale_bytes, int64_t max_lookups) { + if (shard_rows <= 0 || padded_rows <= 0 || tp_start < 0 || tp_end < tp_start || + tp_end > padded_rows || weight_bytes <= 0 || scale_bytes < 0 || max_lookups <= 0 || + weight_bytes > PY_SSIZE_T_MAX / max_lookups || scale_bytes > PY_SSIZE_T_MAX / max_lookups || + max_lookups > PY_SSIZE_T_MAX / 8) + return EINVAL; + size_t per_row = (size_t)weight_bytes / PLE_BLOCK + 2; + if (scale_bytes) per_row += (size_t)scale_bytes / PLE_BLOCK + 2; + if ((size_t)max_lookups > SIZE_MAX / per_row || + (size_t)max_lookups * per_row > SIZE_MAX / sizeof(ple_fragment_t) || + (size_t)max_lookups * per_row > SIZE_MAX / sizeof(ple_job_t)) + return EOVERFLOW; + reader->shard_rows = shard_rows; + reader->padded_rows = padded_rows; + reader->tp_start = tp_start; + reader->tp_end = tp_end; + reader->weight_bytes = weight_bytes; + reader->scale_bytes = scale_bytes; + reader->max_lookups = max_lookups; + reader->capacity = (size_t)max_lookups * per_row; + reader->fragments = calloc(reader->capacity, sizeof(*reader->fragments)); + reader->jobs = calloc(reader->capacity, sizeof(*reader->jobs)); + return reader->fragments && reader->jobs ? 0 : ENOMEM; +} + +static inline int ple_add_source(ple_plan_t* reader, int64_t shard, const char* path, + int64_t offset, bool scale, + int (*register_files)(ple_plan_t*, void*), void* owner) { + if (shard < 0 || shard > (reader->padded_rows - 1) / reader->shard_rows || offset < 0 || + (scale && !reader->scale_bytes)) + return EINVAL; + int64_t first = shard * reader->shard_rows; + int64_t rows = reader->padded_rows - first; + if (rows > reader->shard_rows) rows = reader->shard_rows; + size_t row_bytes = scale ? reader->scale_bytes : reader->weight_bytes; + if (rows > (INT64_MAX - offset) / (int64_t)row_bytes) return EOVERFLOW; + if (first >= reader->tp_end || first + rows <= reader->tp_start) return 0; + failure_t failure = {{0}}; + ple_source_t key = {.shard = shard, .scale = scale}; + if (reader->source_count && + bsearch(&key, reader->sources, reader->source_count, sizeof(key), ple_source_compare)) { + snprintf(failure.message, sizeof(failure.message), + "PLE source shard/plane is already registered"); + } else { + /* Reject FIFOs via fstat without blocking waiting for a writer. */ + int fd = open(path, O_RDONLY | O_CLOEXEC | O_NONBLOCK | O_DIRECT); + struct stat status; + if (fd < 0) + system_error(&failure, "open PLE source"); + else if (fstat(fd, &status) != 0) + system_error(&failure, "fstat PLE source"); + else if (!S_ISREG(status.st_mode) || offset + rows * (int64_t)row_bytes > status.st_size) + snprintf(failure.message, sizeof(failure.message), + "PLE source range exceeds a regular file's size"); + else { + size_t file = 0; + while (file < reader->file_count && (reader->files[file].device != status.st_dev || + reader->files[file].inode != status.st_ino)) + file++; + ple_source_t* sources = + realloc(reader->sources, (reader->source_count + 1) * sizeof(*sources)); + if (!sources) + snprintf(failure.message, sizeof(failure.message), + "could not allocate PLE source descriptor"); + else { + reader->sources = sources; + if (file == reader->file_count) { + ple_file_t* files = realloc(reader->files, (file + 1) * sizeof(*files)); + if (!files) + snprintf(failure.message, sizeof(failure.message), + "could not allocate PLE file descriptor"); + else { + reader->files = files; + reader->files[file] = (ple_file_t){fd, status.st_dev, status.st_ino, status.st_size}; + reader->file_count++; + fd = -1; + int error = register_files ? register_files(reader, owner) : 0; + if (error < 0) { + snprintf(failure.message, sizeof(failure.message), "row file registration failed: %s", + strerror(-error)); + } + } + } else if (reader->files[file].bytes != status.st_size) { + snprintf(failure.message, sizeof(failure.message), + "PLE source file size changed during registration"); + } + if (!failure.message[0]) { + reader->sources[reader->source_count++] = (ple_source_t){shard, offset, file, scale != 0}; + qsort(reader->sources, reader->source_count, sizeof(*sources), ple_source_compare); + } + } + } + if (fd >= 0) close(fd); + } + if (failure.message[0] && !reader->failure.message[0]) reader->failure = failure; + return failure.message[0] ? EIO : 0; +} + +static inline int ple_fragment_compare(const void* left, const void* right) { + const ple_fragment_t *a = left, *b = right; + if (a->file != b->file) return a->file < b->file ? -1 : 1; + return a->offset < b->offset ? -1 : a->offset > b->offset; +} + +/* libc qsort may allocate a merge buffer on every run. Heap-sort in place to + * keep all batch metadata persistent, including for duplicate-heavy batches. */ +static inline void ple_sift(ple_fragment_t* items, size_t root, size_t count) { + ple_fragment_t value = items[root]; + while (root < count / 2) { + size_t child = root * 2 + 1; + if (child + 1 < count && ple_fragment_compare(&items[child], &items[child + 1]) < 0) child++; + if (ple_fragment_compare(&value, &items[child]) >= 0) break; + items[root] = items[child]; + root = child; + } + items[root] = value; +} + +static inline void ple_sort(ple_plan_t* reader) { + ple_fragment_t* items = reader->fragments; + size_t count = reader->fragment_count; + if (count < 1024) { + for (size_t i = count / 2; i > 0; i--) ple_sift(items, i - 1, count); + for (size_t end = count; end > 1; end--) { + ple_fragment_t value = items[end - 1]; + items[end - 1] = items[0]; + items[0] = value; + ple_sift(items, 0, end - 1); + } + return; + } + /* Jobs are built only after sorting. Reuse that fixed-capacity allocation + * as radix scratch instead of allocating another prefill-sized buffer. */ + _Static_assert(sizeof(ple_job_t) >= sizeof(ple_fragment_t), + "job allocation must cover fragment sort scratch"); + ple_fragment_t* scratch = (ple_fragment_t*)reader->jobs; + ple_fragment_t *source = items, *target = scratch; + uint64_t offset_variation = 0, file_variation = 0; + for (size_t i = 1; i < count; i++) { + offset_variation |= (uint64_t)(items[i].offset ^ items[0].offset); + file_variation |= (uint64_t)(items[i].file ^ items[0].file); + } + /* Stable LSD passes sort offset first, then the primary file key. Skip + * constant bytes, including the file key for a single-container table. */ + for (int file = 0; file < 2; file++) { + uint64_t variation = file ? file_variation : offset_variation; + for (unsigned shift = 0; shift < 64; shift += 8) { + if (((variation >> shift) & 255) == 0) continue; + size_t positions[256] = {0}; + for (size_t i = 0; i < count; i++) { + uint64_t key = file ? (uint64_t)source[i].file : (uint64_t)source[i].offset; + positions[(key >> shift) & 255]++; + } + size_t prefix = 0; + for (unsigned i = 0; i < 256; i++) { + size_t size = positions[i]; + positions[i] = prefix; + prefix += size; + } + for (size_t i = 0; i < count; i++) { + uint64_t key = file ? (uint64_t)source[i].file : (uint64_t)source[i].offset; + target[positions[(key >> shift) & 255]++] = source[i]; + } + ple_fragment_t* swap = source; + source = target; + target = swap; + } + } + if (source != items) memcpy(items, source, count * sizeof(*items)); +} + +static inline bool ple_plan(ple_plan_t* reader, const char* ids, char* weights, char* scales, + size_t count) { + reader->fragment_count = reader->job_count = 0; + for (size_t i = 0; i < count; i++) { + int64_t id; + memcpy(&id, ids + i * sizeof(id), sizeof(id)); + if (id < reader->tp_start || id >= reader->tp_end) { + if (weights) memset(weights + i * reader->weight_bytes, 0, reader->weight_bytes); + if (scales && reader->scale_bytes) + memset(scales + i * reader->scale_bytes, 0, reader->scale_bytes); + continue; + } + for (int plane = 0; plane < (reader->scale_bytes ? 2 : 1); plane++) { + ple_source_t key = {.shard = id / reader->shard_rows, .scale = plane != 0}; + ple_source_t* source = reader->source_count + ? bsearch(&key, reader->sources, reader->source_count, sizeof(key), + ple_source_compare) + : NULL; + if (!source) { + snprintf(reader->failure.message, sizeof(reader->failure.message), + "missing PLE %s source for shard %lld", plane ? "scale" : "weight", + (long long)key.shard); + return false; + } + size_t bytes = plane ? reader->scale_bytes : reader->weight_bytes; + size_t destination = i * bytes; + int64_t offset = source->offset + (id % reader->shard_rows) * (int64_t)bytes; + reader->requested_bytes += bytes; + while (bytes) { + unsigned length = PLE_BLOCK - (uint64_t)offset % PLE_BLOCK; + if (bytes < length) length = (unsigned)bytes; + reader->fragments[reader->fragment_count++] = + (ple_fragment_t){source->file, offset, length, destination, plane != 0}; + offset += length; + destination += length; + bytes -= length; + } + } + } + ple_sort(reader); + size_t cursor = 0; + while (cursor < reader->fragment_count) { + size_t begin = cursor; + ple_fragment_t* fragment = &reader->fragments[cursor]; + size_t file = fragment->file; + int64_t first = fragment->offset & ~(int64_t)(PLE_BLOCK - 1); + int64_t last = first; + reader->unique_blocks++; + cursor++; + while (cursor < reader->fragment_count) { + fragment = &reader->fragments[cursor]; + int64_t block = fragment->offset & ~(int64_t)(PLE_BLOCK - 1); + if (fragment->file != file || + (block != last && (block - last != PLE_BLOCK || block - first >= PLE_READ_MAX))) + break; + if (block != last) { + reader->unique_blocks++; + last = block; + } + cursor++; + } + unsigned aligned = (unsigned)(last - first) + PLE_BLOCK; + int64_t available = reader->files[file].bytes - first; + unsigned expected = available < aligned ? (unsigned)available : aligned; + reader->jobs[reader->job_count++] = (ple_job_t){file, begin, cursor, first, aligned, expected}; + if (last != first) reader->coalesced_reads++; + } + return true; +} + +#endif diff --git a/tests/fixtures/spark-fp4-loader/disk_table.py b/tests/fixtures/spark-fp4-loader/disk_table.py new file mode 100644 index 0000000..1251d7e --- /dev/null +++ b/tests/fixtures/spark-fp4-loader/disk_table.py @@ -0,0 +1,411 @@ +"""Bounded disk row staging shared by PLE and Engram. + +The native reader owns bounded request planning and I/O. This module owns +storage, source registration and the host/GPU transaction boundary; hashing +and numerical decoding belong to each embedding operation. +""" + +from __future__ import annotations + +import math +import operator +import os +import sys +import threading +from contextlib import contextmanager, suppress +from typing import TYPE_CHECKING, Iterator + +import torch + +if TYPE_CHECKING: + from cuda.bindings import runtime as cudart + + +def _check_cuda(error: cudart.cudaError_t, operation: str) -> None: + from cuda.bindings import runtime as cudart + + if error != cudart.cudaError_t.cudaSuccess: + raise RuntimeError(f"{operation} failed: {error}") + + +def _contiguous_strides(shape: tuple[int, ...]) -> tuple[int, ...]: + stride = 1 + result = [] + for extent in reversed(shape): + result.append(stride) + stride *= int(extent) + return tuple(reversed(result)) + + +def _tensor_from_pointer( + pointer: int, + *, + shape: tuple[int, ...], + dtype: torch.dtype, + device: torch.device, + nbytes: int, +) -> torch.Tensor: + constructor = getattr(torch._C, "_construct_storage_from_data_pointer", None) + if constructor is None: + raise RuntimeError( + "mapped-host storage requires torch._C._construct_storage_from_data_pointer" + ) + storage = constructor(int(pointer), device, int(nbytes)) + return torch.empty(0, dtype=dtype, device=device).set_( + storage, 0, shape, _contiguous_strides(shape) + ) + + +class MappedHostAllocation: + """Own a mapped page-locked allocation and its CPU/CUDA tensor aliases.""" + + def __init__( + self, shape: tuple[int, ...], dtype: torch.dtype, device: torch.device + ) -> None: + from cuda.bindings import runtime as cudart + + self._host_pointer = 0 + self._closed = True + if device.type != "cuda" or device.index is None: + raise ValueError( + f"mapped-host storage requires an indexed CUDA device, got {device}" + ) + nbytes = math.prod(shape) * dtype.itemsize + if nbytes <= 0: + raise ValueError( + f"mapped-host allocation size must be positive, got {nbytes}" + ) + self.device = device + self.nbytes = nbytes + with torch.cuda.device(device): + error, pointer = cudart.cudaHostAlloc( + nbytes, cudart.cudaHostAllocMapped | cudart.cudaHostAllocWriteCombined + ) + _check_cuda(error, "cudaHostAlloc") + self._host_pointer = int(pointer) + self._closed = False + try: + error, device_pointer = cudart.cudaHostGetDevicePointer(pointer, 0) + _check_cuda(error, "cudaHostGetDevicePointer") + self.host_view = _tensor_from_pointer( + self._host_pointer, + shape=shape, + dtype=dtype, + device=torch.device("cpu"), + nbytes=nbytes, + ) + self.device_view = _tensor_from_pointer( + int(device_pointer), + shape=shape, + dtype=dtype, + device=device, + nbytes=nbytes, + ) + except Exception: + cudart.cudaFreeHost(pointer) + self._host_pointer = 0 + self._closed = True + raise + + def close(self) -> None: + from cuda.bindings import runtime as cudart + + if self._closed: + return + torch.cuda.synchronize(self.device) + _check_cuda(cudart.cudaFreeHost(self._host_pointer)[0], "cudaFreeHost") + self._host_pointer = 0 + self._closed = True + + def __del__(self) -> None: + if ( + self._closed + or self._host_pointer == 0 + or sys is None + or sys.is_finalizing() + ): + return + with suppress(Exception): + self.close() + + +class DiskRowCache: + """Read immutable row planes into a reusable, batch-sized cache. + + Source rows have one weight byte plane and an optional independent scale + byte plane. Neither file data nor full table allocations are retained. + Hold ``transaction`` across ID production, ``read_rows`` and GPU decoding. + Downstream graphs consume the decoder's fixed device output, not disk I/O. + """ + + def __init__( + self, + *, + device: torch.device | str, + max_lookups: int, + table_rows: int, + shard_start: int, + shard_end: int, + shard_rows: int, + weight_row_bytes: int, + scale_row_bytes: int = 0, + queue_depth: int = 64, + ) -> None: + from b12x.loader._native import load + + device = torch.device(device) + if device.type != "cuda": + raise ValueError("disk row staging requires CUDA") + if device.index is None: + device = torch.device("cuda", torch.cuda.current_device()) + self.device = device + self.max_lookups = operator.index(max_lookups) + self.table_rows = operator.index(table_rows) + self.shard_start = operator.index(shard_start) + self.shard_end = operator.index(shard_end) + self.shard_rows = operator.index(shard_rows) + self.weight_row_bytes = operator.index(weight_row_bytes) + self.scale_row_bytes = operator.index(scale_row_bytes) + queue_depth = operator.index(queue_depth) + if not 0 < self.shard_rows <= (1 << 63) - 1: + raise ValueError("shard_rows must be a positive signed int64") + self._backend = os.environ.get("B12X_DISK_BACKEND", "io_uring") + if self._backend not in ("io_uring", "gds"): + raise ValueError("B12X_DISK_BACKEND must be io_uring or gds") + self._gds = None + self._native = None + self._reader = None + if self._backend == "io_uring": + self._native = load() + self._reader = self._native.ple_reader( + self.shard_rows, + self.table_rows, + self.shard_start, + self.shard_end, + self.weight_row_bytes, + self.scale_row_bytes, + self.max_lookups, + queue_depth, + ) + self.shard_count = (self.table_rows + self.shard_rows - 1) // self.shard_rows + self.ids_host = torch.empty( + (self.max_lookups,), dtype=torch.int64, device="cpu", pin_memory=True + ) + self._ids_buffer = memoryview(self.ids_host.numpy()) + self._weight_allocation = self._scale_allocation = None + self.weight_host = self.scale_host = None + self._weight_buffer = self._scale_buffer = None + if self._backend == "gds": + from ._gds import GdsRows + + self._gds = GdsRows(self, queue_depth) + self.weight, self.scale = self._gds.weight, self._gds.scale + else: + self._weight_allocation = MappedHostAllocation( + (self.max_lookups, self.weight_row_bytes), torch.uint8, device + ) + self.weight = self._weight_allocation.device_view + self.weight_host = self._weight_allocation.host_view + self._weight_buffer = memoryview(self.weight_host.numpy()) + self._scale_allocation = None + self.scale = None + self.scale_host = None + self._scale_buffer = None + if self.scale_row_bytes: + self._scale_allocation = MappedHostAllocation( + (self.max_lookups, self.scale_row_bytes), torch.uint8, device + ) + self.scale = self._scale_allocation.device_view + self.scale_host = self._scale_allocation.host_view + self._scale_buffer = memoryview(self.scale_host.numpy()) + self._sources: set[tuple[bool, int]] = set() + self._frozen = False + self._lock = threading.RLock() + self._transaction_thread: int | None = None + self._transaction_stream: torch.cuda.Stream | None = None + self._ids_ready = torch.cuda.Event() + self._cache_done = torch.cuda.Event() + self._cache_used = False + self._closed = False + + def _require_open(self): + if self._closed: + raise RuntimeError("disk row cache is closed") + + def __del__(self): + if not getattr(self, "_closed", True): + with suppress(Exception): + self.close() + + def close(self): + with self._lock: + if self._closed: + return + if self._transaction_thread is not None: + raise RuntimeError("cannot close a disk row cache during a transaction") + if self._cache_used: + with torch.cuda.device(self.device): + self._cache_done.synchronize() + if self._gds is not None: + self._gds.close() + for allocation in (self._scale_allocation, self._weight_allocation): + if allocation is not None: + allocation.close() + self._reader = self._native = None + self._closed = True + + def add_shard( + self, shard_index: int, path: str, offset: int, *, scale: bool = False + ) -> None: + self._require_open() + with self._lock: + if self._frozen: + raise RuntimeError("cannot change disk shards after binding") + shard_index = operator.index(shard_index) + offset = operator.index(offset) + if not 0 <= shard_index < self.shard_count: + raise ValueError("checkpoint shard index is out of range") + if offset < 0: + raise ValueError("checkpoint file offset must be nonnegative") + if scale and not self.scale_row_bytes: + raise ValueError("disk table has no row scale plane") + key = (scale, shard_index) + if key in self._sources: + raise ValueError("checkpoint shard is already registered") + start = shard_index * self.shard_rows + end = min(start + self.shard_rows, self.table_rows) + if end <= self.shard_start or start >= self.shard_end: + return + if self._gds is not None: + self._gds.native.add( + self._gds.reader, shard_index, os.fspath(path), offset, scale + ) + else: + self._native.ple_reader_add( + self._reader, shard_index, os.fspath(path), offset, scale + ) + self._sources.add(key) + + def require_complete(self) -> None: + self._require_open() + with self._lock: + first = self.shard_start // self.shard_rows + last = (self.shard_end + self.shard_rows - 1) // self.shard_rows + for shard in range(first, last): + if (False, shard) not in self._sources: + raise ValueError(f"missing disk weight shard {shard}") + if self.scale_row_bytes and (True, shard) not in self._sources: + raise ValueError(f"missing disk scale shard {shard}") + + def freeze(self) -> None: + with self._lock: + self.require_complete() + self._frozen = True + + @contextmanager + def transaction(self) -> Iterator[DiskRowCache]: + self._require_open() + if torch.compiler.is_compiling(): + raise RuntimeError("disk table preparation cannot run under torch.compile") + with torch.cuda.device(self.device): + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError( + "disk table preparation must run outside CUDA graph capture" + ) + with self._lock: + if self._transaction_thread is not None: + raise RuntimeError("disk row transactions cannot be nested") + stream = torch.cuda.current_stream(self.device) + if self._cache_used: + stream.wait_event(self._cache_done) + self._transaction_thread = threading.get_ident() + self._transaction_stream = stream + try: + yield self + finally: + self._cache_done.record(stream) + self._cache_used = True + self._transaction_stream = None + self._transaction_thread = None + + def read_rows(self, ids: torch.Tensor, count: int) -> None: + self._stage_ids(ids, count) + self._read_staged(count) + + def _stage_ids(self, ids: torch.Tensor, count: int) -> None: + if self._transaction_thread != threading.get_ident(): + raise RuntimeError("read_rows requires an active disk row transaction") + count = operator.index(count) + if not 0 <= count <= self.max_lookups: + raise ValueError("disk lookup count exceeds batch capacity") + if ( + ids.device != self.device + or ids.dtype != torch.int64 + or not ids.is_contiguous() + ): + raise ValueError( + "disk row IDs must be contiguous CUDA int64 on the cache device" + ) + if ids.numel() < count: + raise ValueError("disk row IDs do not cover the requested count") + self.ids_host[:count].copy_(ids.view(-1)[:count], non_blocking=True) + self._ids_ready.record(self._transaction_stream) + + def _read_staged(self, count: int) -> None: + # Completes ID production and all prior cache readers before host writes. + with torch.cuda.device(self.device): + self._ids_ready.synchronize() + if self._gds is not None: + self._gds.read(self._ids_buffer, count, self._transaction_stream) + return + self._native.ple_reader_run( + self._reader, + self._ids_buffer, + self._weight_buffer, + self._scale_buffer, + count, + ) + + def stats(self) -> dict[str, int | float]: + self._require_open() + with self._lock: + result = dict( + self._gds.native.stats(self._gds.reader) + if self._gds is not None + else self._native.ple_reader_stats(self._reader) + ) + result["ids_host_bytes"] = ( + self.ids_host.numel() * self.ids_host.element_size() + ) + result["weight_cache_bytes"] = ( + self.weight.numel() * self.weight.element_size() + ) + result["scale_cache_bytes"] = ( + self.scale.numel() * self.scale.element_size() + if self.scale is not None + else 0 + ) + result["cache_bytes"] = ( + result["weight_cache_bytes"] + result["scale_cache_bytes"] + ) + result["owned_staging_bytes"] = ( + result["staging_bytes"] + + result["ids_host_bytes"] + + result["cache_bytes"] + ) + descriptors = result.get("descriptor_bytes", 0) + result["gds_enabled"] = int(self._gds is not None) + result["device_staging_bytes"] = ( + (result["staging_bytes"] + result["cache_bytes"] + descriptors) + if self._gds + else 0 + ) + result["owned_host_bytes"] = ( + result["ids_host_bytes"] + descriptors + result["metadata_bytes"] + ) + if self._gds is None: + result["owned_host_bytes"] += ( + result["staging_bytes"] + result["cache_bytes"] + ) + result["owned_staging_bytes"] += 2 * descriptors + return result diff --git a/tests/fixtures/spark-redhat-config/LICENSE b/tests/fixtures/spark-redhat-config/LICENSE new file mode 100644 index 0000000..ba557f2 --- /dev/null +++ b/tests/fixtures/spark-redhat-config/LICENSE @@ -0,0 +1,229 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------------------------------------------------------------------------------------- +Some of the code in this project are adapted from other open-source projects with different +licenses. This product also bundles some third-party components under other open source licenses. +This section summarizes those components and their licenses. +See licenses/ for text of these licenses. + +BSD 3-Clause License +-------------------- + +include/flashinfer/attention/hopper/epilogue.cuh +include/flashinfer/attention/hopper/mainloop.cuh +include/flashinfer/attention/hopper/kernel_traits.cuh +include/flashinfer/attention/hopper/named_barrier.cuh +include/flashinfer/attention/hopper/tile_scheduler.cuh +include/flashinfer/attention/hopper/utils.cuh + +BSD 3-Clause "New" License +-------------------------- + +3rdparty/cutlass +include/flashinfer/attention/hopper/block_sparse_gather.cuh + +MIT License +----------- + +3rdparty/spdlog +3rdparty/spdlog/include/spdlog/fmt/bundled (fmt library) diff --git a/tests/fixtures/spark-redhat-config/NOTICE b/tests/fixtures/spark-redhat-config/NOTICE new file mode 100644 index 0000000..cebc9b6 --- /dev/null +++ b/tests/fixtures/spark-redhat-config/NOTICE @@ -0,0 +1,10 @@ +The frozen config.py fixture is vLLM's models/qwen4_exp/config.py, distributed +under Apache-2.0 (see LICENSE and its original copyright header). + +It was extracted without modification from the declared ARM64 dependency: +eugr/spark-vllm-b12x@sha256:03a956544e9f9b4b7ba91690469fbe7848bdbef29a7d2e0f5788368467bb3595 +Runtime revision: b12x-8244d1c44-20261015 +Source SHA-256: 71230ef5e53e016416f54ae96b9f8b4e2c10a6311d8e6035ac063ac48fea2471 + +Tests apply the repository's content-addressed compatibility patch only to a +temporary copy. Keep this upstream fixture immutable. diff --git a/tests/fixtures/spark-redhat-config/config.py b/tests/fixtures/spark-redhat-config/config.py new file mode 100644 index 0000000..341d391 --- /dev/null +++ b/tests/fixtures/spark-redhat-config/config.py @@ -0,0 +1,312 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Qwen4Exp model configuration.""" + +from pathlib import Path +from typing import Any, ClassVar, cast + +from transformers import PretrainedConfig +from transformers.models.qwen3_vl.configuration_qwen3_vl import ( + Qwen3VLVisionConfig, +) + +from vllm.logger import init_logger +from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig + +logger = init_logger(__name__) + +_QSA_CONFIG_FIELDS = ( + "indexer_n_heads", + "indexer_kv_heads", + "indexer_head_dim", + "indexer_budget", + "indexer_compress_ratio", +) + + +def resolve_ple_embedding_dtype( + config: PretrainedConfig, + config_dict: dict[str, Any], + model: str | Path, + revision: str | None, +) -> None: + """Infer omitted PLE storage dtype from checkpoint headers before allocation.""" + text_config = config.get_text_config() + text_config_dict = config_dict.get("text_config", config_dict) + if not text_config.ple_layer_ids or text_config_dict.get("ple_embedding_dtype"): + return + + from vllm.transformers_utils.config import get_safetensors_params_metadata + + source = str(model) + quant_config = ( + config_dict.get("quantization_config") + or text_config_dict.get("quantization_config") + or {} + ) + if quant_config.get("quant_method") == "mxfp4_csf" and ( + root := quant_config.get("checkpoint_root") + ): + # An FP4-CSF serving directory holds metadata only; the checkpoint's + # tensors, PLE shards included, live under its root. + source = str(Path(root) / "tensors") + metadata = get_safetensors_params_metadata(source, revision=revision) + dtypes = { + info["dtype"] + for name, info in metadata.items() + if ".ple.ple_embedding.ngram_embedding.shard_" in name + and name.endswith(".weight") + } + if not dtypes: + return + storage_dtypes = { + "BF16": "bfloat16", + "F8_E4M3": "float8_e4m3fn", + "U8": "nvfp4", + } + if len(dtypes) != 1 or not dtypes.issubset(storage_dtypes): + raise ValueError(f"Unsupported PLE checkpoint storage dtypes: {sorted(dtypes)}") + text_config.ple_embedding_dtype = storage_dtypes[dtypes.pop()] + logger.info( + "Resolved PLE embedding storage dtype from checkpoint: %s", + text_config.ple_embedding_dtype, + ) + + +class Qwen4ExpVisionConfig(Qwen3VLVisionConfig): + model_type = "qwen4_exp" + base_config_key = "vision_config" + + +class Qwen4ExpTextConfig(Qwen3NextConfig): + model_type = "qwen4_exp_text" + base_config_key = "text_config" + keys_to_ignore_at_inference = ["past_key_values"] + # B12X QSA gathers KV across DCP ranks, so DCP may equal TP even with fewer + # KV heads than ranks. Other QSA backends refuse DCP > 1 on their own. + supports_full_tp_dcp_with_kv_gather = True + + def __init__( + self, + ple_embedding_dtype: str | None = None, + index_share_for_mtp_iteration: bool = True, + hc_count: int = 4, + hc_lowrank: int = 320, + ple_layer_ids: list[int] | None = None, + ple_embed_dim: int | None = None, + ple_conv_kernel_size: int = 4, + ngram_size: int = 3, + heads_per_ngram: int = 8, + ngram_vocab_size_base: int = 20_000_000, + make_ngram_vocab_size_divisible_by: int = 128, + output_gate_type: str = "sigmoid", + rope_parameters: dict[str, Any] | None = None, + layer_types: list[str] | None = None, + **kwargs: Any, + ) -> None: + if hc_count <= 1: + raise ValueError(f"Qwen4Exp requires hc_count > 1, got {hc_count}.") + + if rope_parameters is not None: + if kwargs.get("rope_scaling") is None: + kwargs["rope_scaling"] = rope_parameters + if kwargs.get("rope_theta") is None and "rope_theta" in rope_parameters: + kwargs["rope_theta"] = rope_parameters["rope_theta"] + if ( + kwargs.get("partial_rotary_factor") is None + and "partial_rotary_factor" in rope_parameters + ): + kwargs["partial_rotary_factor"] = rope_parameters[ + "partial_rotary_factor" + ] + + rope_scaling = kwargs.get("rope_scaling") + rope_theta = kwargs.get("rope_theta", 10_000.0) + super().__init__(layer_types=layer_types, **kwargs) + + normalized_rope_parameters = self.rope_parameters + self.rope_scaling = ( + rope_scaling or rope_parameters or normalized_rope_parameters + ) + self.rope_parameters = rope_parameters or normalized_rope_parameters + self.rope_theta = rope_theta + + self.ple_embedding_dtype = ple_embedding_dtype or "bfloat16" + self.index_share_for_mtp_iteration = index_share_for_mtp_iteration + self.hc_count = hc_count + self.hc_lowrank = hc_lowrank + self.ple_layer_ids = ple_layer_ids or [] + self.ple_embed_dim = ( + self.hidden_size if ple_embed_dim is None else ple_embed_dim + ) + self.ple_conv_kernel_size = ple_conv_kernel_size + self.ngram_size = ngram_size + self.heads_per_ngram = heads_per_ngram + self.ngram_vocab_size_base = ngram_vocab_size_base + self.make_ngram_vocab_size_divisible_by = make_ngram_vocab_size_divisible_by + self.output_gate_type = output_gate_type + + self._validate_ple_config() + self._validate_ple_layer_ids() + self._validate_qsa_config() + + def _validate_ple_config(self) -> None: + if self.hc_lowrank <= 0: + raise ValueError(f"hc_lowrank must be positive, got {self.hc_lowrank}") + if self.ngram_size < 2: + raise ValueError(f"ngram_size must be >= 2, got {self.ngram_size}") + if self.heads_per_ngram <= 0: + raise ValueError( + f"heads_per_ngram must be positive, got {self.heads_per_ngram}" + ) + if self.ple_embed_dim <= 0: + raise ValueError( + f"ple_embed_dim must be positive, got {self.ple_embed_dim}" + ) + ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram + if self.ple_embed_dim % ngram_heads: + raise ValueError( + "ple_embed_dim must be divisible by total ngram heads: " + f"{self.ple_embed_dim} % {ngram_heads} != 0" + ) + if self.ple_conv_kernel_size <= 0: + raise ValueError( + "ple_conv_kernel_size must be positive, got " + f"{self.ple_conv_kernel_size}" + ) + if self.ngram_vocab_size_base <= 0: + raise ValueError("ngram_vocab_size_base must be positive") + if self.make_ngram_vocab_size_divisible_by <= 0: + raise ValueError("make_ngram_vocab_size_divisible_by must be positive") + + def _validate_ple_layer_ids(self) -> None: + invalid = [ + layer_id + for layer_id in self.ple_layer_ids + if not 1 <= int(layer_id) <= self.num_hidden_layers + ] + if invalid: + raise ValueError( + "ple_layer_ids are 1-based and must refer to an existing layer; " + f"got {invalid} for {self.num_hidden_layers} layers" + ) + + def _validate_qsa_config(self) -> None: + configured = {name: getattr(self, name, None) for name in _QSA_CONFIG_FIELDS} + if all(value is None for value in configured.values()): + return + + missing = [name for name, value in configured.items() if value is None] + if missing: + raise ValueError(f"QSA config is missing required fields: {missing}") + + values = {name: int(cast(int, value)) for name, value in configured.items()} + if any(value <= 0 for value in values.values()): + raise ValueError(f"QSA config values must be positive: {values}") + if values["indexer_kv_heads"] != 1: + raise ValueError("the QSA MQA operators require indexer_kv_heads=1") + if values["indexer_budget"] % values["indexer_compress_ratio"] != 0: + raise ValueError( + "indexer_budget must be divisible by indexer_compress_ratio" + ) + block_topk = values["indexer_budget"] // values["indexer_compress_ratio"] + if block_topk not in (512, 2048): + raise ValueError( + "QSA requires indexer_budget / indexer_compress_ratio " + f"to be 512 or 2048, got {block_topk}" + ) + rotary_dim = int(self.head_dim * self.partial_rotary_factor) + if rotary_dim > values["indexer_head_dim"]: + raise ValueError( + "QSA indexer_head_dim must cover the attention rotary " + f"dimension, got {values['indexer_head_dim']} < {rotary_dim}" + ) + + @property + def layers_block_type(self) -> list[str]: + return [ + "attention" if layer_type == "full_attention" else layer_type + for layer_type in self.layer_types + ] + + @property + def short_conv_layer_ids(self) -> list[int]: + if not self.ple_layer_ids: + return [] + return sorted({int(layer_id) - 1 for layer_id in self.ple_layer_ids}) + + @property + def short_conv_state_shape(self) -> tuple[int, int] | None: + if not self.short_conv_layer_ids: + return None + ple_state_len = (self.ple_conv_kernel_size - 1) * self.ngram_size + ple_channels = self.hidden_size * self.hc_count + return ple_channels, ple_state_len + + @property + def ngram_context_len(self) -> int: + if not self.ple_layer_ids: + return 0 + return max(int(self.ngram_size) - 1, 0) + + +class Qwen4ExpConfig(PretrainedConfig): + model_type = "qwen4_exp" + sub_configs: ClassVar[dict[str, type[PretrainedConfig]]] = { + "vision_config": Qwen4ExpVisionConfig, + "text_config": Qwen4ExpTextConfig, + } + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + text_config: Qwen4ExpTextConfig | dict[str, Any] | None = None, + vision_config: Qwen4ExpVisionConfig | dict[str, Any] | None = None, + image_token_id: int = 248056, + video_token_id: int = 248057, + vision_start_token_id: int = 248053, + vision_end_token_id: int = 248054, + tie_word_embeddings: bool = False, + rope_parameters: dict[str, Any] | None = None, + **kwargs: Any, + ) -> None: + if text_config is not None: + kwargs.pop("split_ngram_parts", None) + + text_kwargs = ( + dict(kwargs) + if text_config is None + and "hidden_size" in kwargs + and "num_hidden_layers" in kwargs + else {} + ) + + if isinstance(vision_config, dict): + self.vision_config = self.sub_configs["vision_config"](**vision_config) + elif vision_config is None: + self.vision_config = self.sub_configs["vision_config"]() + else: + self.vision_config = vision_config + + if isinstance(text_config, dict): + self.text_config = self.sub_configs["text_config"](**text_config) + elif text_config is None: + self.text_config = self.sub_configs["text_config"](**text_kwargs) + else: + self.text_config = text_config + + self.image_token_id = image_token_id + self.video_token_id = video_token_id + self.vision_start_token_id = vision_start_token_id + self.vision_end_token_id = vision_end_token_id + self.rope_parameters = rope_parameters or getattr( + self.text_config, "rope_parameters", {} + ) + super().__init__(**kwargs, tie_word_embeddings=tie_word_embeddings) + + +__all__ = [ + "Qwen4ExpConfig", + "Qwen4ExpTextConfig", + "Qwen4ExpVisionConfig", +] diff --git a/tests/spark_anthropic_e2e.rs b/tests/spark_anthropic_e2e.rs index e69fa1e..8bb6750 100644 --- a/tests/spark_anthropic_e2e.rs +++ b/tests/spark_anthropic_e2e.rs @@ -5,6 +5,7 @@ use sparkplane::spark::image_history; #[path = "../src/spark/gateway.rs"] #[cfg_attr(test, allow(dead_code))] mod gateway; +use sparkplane::spark::wire; #[path = "../src/spark/upstream.rs"] #[cfg_attr(test, allow(dead_code))] mod upstream; diff --git a/tests/spark_fp4_local_recipe.rs b/tests/spark_fp4_local_recipe.rs new file mode 100644 index 0000000..92f5075 --- /dev/null +++ b/tests/spark_fp4_local_recipe.rs @@ -0,0 +1,171 @@ +#![cfg(feature = "appliance")] + +use sparkplane::spark::{engine::EngineCatalog, model_catalog::ModelCatalog}; +use std::{fs, process::Command}; + +#[test] +fn fp4_checkpoint_reader_preserves_bytes_strides_and_failure_ownership() { + let directory = tempfile::tempdir().unwrap(); + let fixture = "tests/fixtures/spark-fp4-loader"; + for name in ["_bounce.c", "_batch.c", "_direct.c"] { + fs::copy(format!("{fixture}/{name}"), directory.path().join(name)).unwrap(); + } + let dockerfile = + fs::read_to_string("configs/sparkplane/engines/vllm-qwen38-fp4-local.Dockerfile").unwrap(); + let patch = dockerfile + .lines() + .find_map(|line| line.strip_prefix("COPY patches/")) + .unwrap() + .split_whitespace() + .next() + .unwrap(); + let patch = format!("configs/sparkplane/engines/patches/{patch}"); + assert!( + Command::new("python3") + .arg(&patch) + .arg(directory.path()) + .status() + .unwrap() + .success() + ); + assert!( + Command::new("python3") + .arg("configs/sparkplane/engines/checks/qwen38_fp4_checkpoint.py") + .arg(directory.path()) + .status() + .unwrap() + .success() + ); + // The source guard refuses repeated edits or a different upstream reader. + assert!( + !Command::new("python3") + .arg(&patch) + .arg(directory.path()) + .output() + .unwrap() + .status + .success() + ); +} + +#[test] +fn fp4_local_abliterated_selects_its_runtime_without_changing_stock_flash_next() { + let models = ModelCatalog::load("configs/sparkplane/models.toml".as_ref()).unwrap(); + let files = fs::read_dir("configs/sparkplane/engines") + .unwrap() + .filter_map(Result::ok) + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "toml")) + .map(|entry| { + ( + entry.file_name().to_string_lossy().into_owned(), + fs::read_to_string(entry.path()).unwrap(), + ) + }) + .collect::>(); + let engines = EngineCatalog::parse_files(files).unwrap(); + let stock = models.resolve("qwen3.8:flash-next-radixark").unwrap(); + let abliterated = models.resolve("qwen3.8:flash-next-abliterated").unwrap(); + let stock_engine = engines.select(stock.artifacts()).unwrap(); + let engine = engines.select(abliterated.artifacts()).unwrap(); + assert_ne!(stock_engine.config().id, engine.config().id); + assert_eq!(stock_engine.config().id, "vllm-qwen38-mmap-arm64"); + assert_eq!(abliterated.artifacts().auxiliary.len(), 21); + assert!(abliterated.artifacts().auxiliary.iter().all(|file| { + file.path.starts_with("tensors/") && file.sha256.is_some() && file.bytes > 0 + })); + + let config = engine.config(); + assert_eq!(config.seccomp, "default"); + assert_eq!(config.network, "sparkplane-internal"); + assert_eq!(config.run_as_uid, 65534); + assert!( + config + .environment + .iter() + .any(|entry| entry == "VLLM_PLE_TABLE_MEMORY=disk") + ); + assert!( + config + .environment + .iter() + .any(|entry| entry == "B12X_CHECKPOINT_PREAD=1") + ); + assert!( + config + .environment + .iter() + .any(|entry| entry == "B12X_DISK_BACKEND=pread") + ); + let profile = engine + .profile_for(Some("qwen4_exp"), abliterated.artifacts()) + .unwrap(); + assert_eq!(profile.context_window, 262144); + assert!(profile.startup_protocol_probe); + for pair in [ + ["--load-format", "b12x"], + ["--quantization", "modelopt_mixed"], + ["--kv-cache-dtype", "fp8"], + ["--tool-call-parser", "qwen3_xml"], + ] { + assert!(profile.arguments.windows(2).any(|args| args == pair)); + } + assert!(profile.vision.is_some()); + assert_eq!(profile.sampling.temperature, Some(1.0)); +} + +#[test] +fn fp4_disk_ple_preserves_rows_and_drains_failed_reads() { + let directory = tempfile::tempdir().unwrap(); + let loader = directory.path().join("loader"); + fs::create_dir(&loader).unwrap(); + let sequence = directory.path().join("sequence/_shared"); + fs::create_dir_all(&sequence).unwrap(); + fs::copy( + "tests/fixtures/spark-fp4-loader/disk_table.py", + sequence.join("disk_table.py"), + ) + .unwrap(); + for name in ["_ple_reader.c", "_row_plan.h"] { + fs::copy( + format!("tests/fixtures/spark-fp4-loader/{name}"), + loader.join(name), + ) + .unwrap(); + } + let dockerfile = + fs::read_to_string("configs/sparkplane/engines/vllm-qwen38-fp4-local.Dockerfile").unwrap(); + let patch = dockerfile + .lines() + .filter_map(|line| line.strip_prefix("COPY patches/")) + .nth(1) + .unwrap() + .split_whitespace() + .next() + .unwrap(); + let patch = format!("configs/sparkplane/engines/patches/{patch}"); + assert!( + Command::new("python3") + .arg(&patch) + .arg(&loader) + .status() + .unwrap() + .success() + ); + assert!( + Command::new("python3") + .arg("configs/sparkplane/engines/checks/qwen38_fp4_ple.py") + .arg(&loader) + .status() + .unwrap() + .success() + ); + assert!( + !Command::new("python3") + .arg(&patch) + .arg(&loader) + .output() + .unwrap() + .status + .success() + ); +} diff --git a/tests/spark_modalities_e2e.rs b/tests/spark_modalities_e2e.rs index 518f5b3..7b50d73 100644 --- a/tests/spark_modalities_e2e.rs +++ b/tests/spark_modalities_e2e.rs @@ -1,6 +1,7 @@ #![cfg(feature = "appliance")] use sparkplane::spark::image_history; +use sparkplane::spark::wire; #[path = "../src/spark/gateway.rs"] #[cfg_attr(test, allow(dead_code))] @@ -38,6 +39,7 @@ fn vision_profile() -> gateway::GatewayProfile { }), embeddings: None, startup_protocol_probe: true, + reasoning: None, parallel_tool_calls: false, verbosity: false, native_responses: false, native_response_timeout_seconds: 0, stream_idle_timeout_seconds: 0, diff --git a/tests/spark_openai_e2e.rs b/tests/spark_openai_e2e.rs index 02892c4..a66d84b 100644 --- a/tests/spark_openai_e2e.rs +++ b/tests/spark_openai_e2e.rs @@ -5,6 +5,7 @@ use sparkplane::spark::image_history; #[path = "../src/spark/gateway.rs"] #[cfg_attr(test, allow(dead_code))] mod gateway; +use sparkplane::spark::wire; #[path = "../src/spark/upstream.rs"] #[cfg_attr(test, allow(dead_code))] mod upstream; diff --git a/tests/spark_qwen_vllm_optimization.rs b/tests/spark_qwen_vllm_optimization.rs index 3b148c5..08a5db4 100644 --- a/tests/spark_qwen_vllm_optimization.rs +++ b/tests/spark_qwen_vllm_optimization.rs @@ -1,9 +1,26 @@ #[cfg(feature = "appliance")] #[test] fn qwen38_checkpoint_and_selected_engine_accept_inline_images() { + assert_qwen_vision_policy( + "qwen3.8:flash-next", + "bfbc24af59a3e73a9cd0653b8d4ae758dfaec4e3e6c15dfb1c6c8ee8d5683c85", + ); +} + +#[cfg(feature = "appliance")] +#[test] +fn retained_radixark_checkpoint_keeps_its_vision_policy() { + assert_qwen_vision_policy( + "qwen3.8:flash-next-radixark", + "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516", + ); +} + +#[cfg(feature = "appliance")] +fn assert_qwen_vision_policy(alias: &str, processor_sha256: &str) { use sparkplane::spark::{engine::EngineCatalog, gateway, model_catalog::ModelCatalog}; let models = ModelCatalog::parse(include_str!("../configs/sparkplane/models.toml")).unwrap(); - let artifacts = models.resolve("qwen3.8:flash-next").unwrap().artifacts(); + let artifacts = models.resolve(alias).unwrap().artifacts(); let directory = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("configs/sparkplane/engines"); let files = std::fs::read_dir(directory) @@ -28,10 +45,7 @@ fn qwen38_checkpoint_and_selected_engine_accept_inline_images() { .vision .as_ref() .expect("Qwen image input must have a verified vision policy"); - assert_eq!( - vision.processor_sha256, - "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516" - ); + assert_eq!(vision.processor_sha256, processor_sha256); assert_eq!(vision.max_count, 16); assert_eq!(vision.max_total_bytes, 524288); let image = serde_json::json!({"type":"input_image", "image_url":format!("data:image/png;base64,{}", vision.health_image_base64)}); diff --git a/tests/spark_redhat_recipe.rs b/tests/spark_redhat_recipe.rs new file mode 100644 index 0000000..1cb71ef --- /dev/null +++ b/tests/spark_redhat_recipe.rs @@ -0,0 +1,162 @@ +#![cfg(feature = "appliance")] + +use sparkplane::spark::{engine::EngineCatalog, model_catalog::ModelCatalog}; +use std::fs; + +#[test] +fn flash_next_defaults_to_redhat_and_preserves_explicit_alternatives() { + let models = ModelCatalog::load("configs/sparkplane/models.toml".as_ref()).unwrap(); + for alias in [ + "qwen3.8:flash-next", + "qwen3.8:flash-next-nvfp4", + "qwen3.8:flash-next-redhat", + ] { + let model = models.resolve(alias).unwrap(); + assert_eq!(model.repository(), "RedHatAI/Qwen3.8-Flash-Next-NVFP4"); + assert_eq!(model.revision(), "c8f2fb1b9869f686b214782036123b10ff96d14a"); + } + let radix = models.resolve("qwen3.8:flash-next-radixark").unwrap(); + assert_eq!(radix.repository(), "RadixArk/Qwen3.8-Flash-Next-NVFP4"); + assert_eq!( + radix.artifacts().engine_profile.as_deref(), + Some("qwen3.8-flash-next-nvfp4") + ); + let abliterated = models.resolve("qwen3.8:flash-next-abliterated").unwrap(); + assert_eq!( + abliterated.repository(), + "DevelopingDad/Qwen3.8-Flash-Next-FP4-Local-Abliterated" + ); + + let files = fs::read_dir("configs/sparkplane/engines") + .unwrap() + .filter_map(Result::ok) + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "toml")) + .map(|entry| { + ( + entry.file_name().to_string_lossy().into_owned(), + fs::read_to_string(entry.path()).unwrap(), + ) + }) + .collect::>(); + let engines = EngineCatalog::parse_files(files).unwrap(); + let redhat = models.resolve("qwen3.8:flash-next").unwrap(); + let engine = engines.select(redhat.artifacts()).unwrap(); + assert_eq!(engine.config().id, "vllm-qwen38-redhat-arm64"); + assert_eq!( + engines.select(radix.artifacts()).unwrap().config().id, + "vllm-qwen38-mmap-arm64" + ); + assert_eq!( + engines.select(abliterated.artifacts()).unwrap().config().id, + "vllm-qwen38-fp4-local-arm64" + ); + assert_eq!( + engine.config().image_repository, + "sparkplane/vllm-qwen38-redhat" + ); + assert_eq!( + engine.config().image_digest, + "sha256:5cfc1fe12892c42654abd337fae28be0dea8e67f575e669ba9c547c2b9fb0a62" + ); + assert_eq!(engine.config().seccomp, "default"); + assert_eq!(engine.config().network, "sparkplane-internal"); + assert_eq!(engine.config().run_as_uid, 65534); + let profile = engine + .profile_for(Some("qwen4_exp"), redhat.artifacts()) + .unwrap(); + assert_eq!(profile.context_window, 262144); + assert!(profile.startup_protocol_probe); + assert!(profile.vision.is_some()); +} + +#[test] +fn redhat_config_patch_accepts_only_the_pinned_upstream_source() { + use sha2::{Digest, Sha256}; + use std::process::Command; + let patch = "configs/sparkplane/engines/patches/sha256-abfa66016b36ea4035777f9b02a5c9a6c3badc49d48e12c3320f016ba005efe4.py"; + assert_eq!( + format!("{:x}", Sha256::digest(fs::read(patch).unwrap())), + "abfa66016b36ea4035777f9b02a5c9a6c3badc49d48e12c3320f016ba005efe4" + ); + let original = fs::read("tests/fixtures/spark-redhat-config/config.py").unwrap(); + assert_eq!( + format!("{:x}", Sha256::digest(&original)), + "71230ef5e53e016416f54ae96b9f8b4e2c10a6311d8e6035ac063ac48fea2471" + ); + let temp = tempfile::tempdir().unwrap(); + let config = temp.path().join("config.py"); + fs::write(&config, &original).unwrap(); + assert!( + Command::new("python3") + .args([patch]) + .arg(&config) + .output() + .unwrap() + .status + .success() + ); + assert!( + Command::new("python3") + .args(["-m", "py_compile"]) + .arg(&config) + .output() + .unwrap() + .status + .success() + ); + let patched = fs::read(&config).unwrap(); + assert!( + !Command::new("python3") + .args([patch]) + .arg(&config) + .output() + .unwrap() + .status + .success() + ); + assert_eq!(fs::read(&config).unwrap(), patched); + let mut changed = original; + changed.extend_from_slice(b"\n# Different upstream revision.\n"); + fs::write(&config, &changed).unwrap(); + assert!( + !Command::new("python3") + .args([patch]) + .arg(&config) + .output() + .unwrap() + .status + .success() + ); + assert_eq!(fs::read(&config).unwrap(), changed); +} + +#[test] +fn reasoning_is_an_explicit_validated_recipe_contract() { + let source = fs::read_to_string("configs/sparkplane/engines/vllm-qwen38-redhat.toml").unwrap(); + let parse = |text: &str| sparkplane::spark::engine::EnginePolicy::parse(text); + let policy = parse(&source).unwrap(); + let models = ModelCatalog::load("configs/sparkplane/models.toml".as_ref()).unwrap(); + let model = models.resolve("qwen3.8:flash-next").unwrap(); + let profile = policy + .gateway_profile_for(None, model.artifacts(), None) + .unwrap(); + let capabilities = profile.inference_capabilities(); + let reasoning = capabilities.reasoning.unwrap(); + assert_eq!(reasoning.default_effort, "xhigh"); + assert!(reasoning.summaries && reasoning.supported_efforts.contains(&"xhigh".into())); + assert!( + capabilities.tool_calling && !capabilities.verbosity && !capabilities.parallel_tool_calls + ); + for invalid in [ + source.replace("default_effort = \"xhigh\"", "default_effort = \"ultra\""), + source.replace("\"high\", \"xhigh\"]", "\"high\", \"high\"]"), + source.replace( + "startup_protocol_probe = true", + "startup_protocol_probe = false", + ), + source.replace("verbosity = false", "verbosity = true"), + ] { + assert_ne!(invalid, source); + assert!(parse(&invalid).is_err()); + } +} diff --git a/tests/spark_vllm_e2e.rs b/tests/spark_vllm_e2e.rs index a6683d2..209c56d 100644 --- a/tests/spark_vllm_e2e.rs +++ b/tests/spark_vllm_e2e.rs @@ -5,6 +5,7 @@ use sparkplane::spark::image_history; #[path = "../src/spark/gateway.rs"] #[cfg_attr(test, allow(dead_code))] mod gateway; +use sparkplane::spark::wire; #[path = "../src/spark/upstream.rs"] #[cfg_attr(test, allow(dead_code))] mod upstream;