From 6f95a4941f237bfca6ada2a9b2eb95f440149734 Mon Sep 17 00:00:00 2001 From: RickZ <121943721+rickisba@users.noreply.github.com> Date: Mon, 27 Jul 2026 19:55:56 +0800 Subject: [PATCH 1/4] env: add CUDA 13 serving constraints --- env/docker/cu130/constraints.txt | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 env/docker/cu130/constraints.txt diff --git a/env/docker/cu130/constraints.txt b/env/docker/cu130/constraints.txt new file mode 100644 index 0000000..2ba2d36 --- /dev/null +++ b/env/docker/cu130/constraints.txt @@ -0,0 +1,23 @@ +# CacheRoute CUDA 13 serving profile. +# +# Keep the binary-sensitive serving stack exact. Application dependencies are +# installed separately from requirements-dev.txt under these constraints. + +torch==2.11.0 +torchvision==0.26.0 +torchaudio==2.11.0 +vllm==0.25.1 +lmcache==0.5.2 + +# Shared compatibility ranges required by the target serving stack. +transformers>=5.5.3,<6.0 +sentence-transformers>=5.6,<6.0 +huggingface-hub>=1.5,<2.0 +fastapi>=0.133,<0.137 +starlette>=1.0.1,<2.0 +aiohttp>=3.13.3,<4.0 +pydantic>=2.12,<3.0 +numpy>=1.26,<2.3 +setuptools>=77,<81 +requests>=2.33,<3.0 +redis>=5.1,<9 From dc9ee1687bb0e61bf4ec5f0a15daf4ce57fad568 Mon Sep 17 00:00:00 2001 From: RickZ <121943721+rickisba@users.noreply.github.com> Date: Mon, 27 Jul 2026 19:56:10 +0800 Subject: [PATCH 2/4] env: add CUDA 13 application dependency profile --- env/docker/cu130/requirements-dev.txt | 44 +++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 env/docker/cu130/requirements-dev.txt diff --git a/env/docker/cu130/requirements-dev.txt b/env/docker/cu130/requirements-dev.txt new file mode 100644 index 0000000..d52b755 --- /dev/null +++ b/env/docker/cu130/requirements-dev.txt @@ -0,0 +1,44 @@ +# CacheRoute application and development dependencies for the CUDA 13 profile. +# +# PyTorch, vLLM, and LMCache are intentionally installed by the Dockerfile and +# must not be added here. This file is additive and does not replace the legacy +# root requirements.txt profile. + +sentence-transformers>=5.6,<6.0 +faiss-cpu==1.13.1 + +fastapi>=0.133,<0.137 +uvicorn>=0.38,<0.52 +starlette>=1.0.1,<2.0 +pydantic>=2.12,<3.0 +httpx>=0.28,<0.29 +aiohttp>=3.13.3,<4.0 +requests>=2.33,<3.0 +redis>=5.1,<9 + +transformers>=5.5.3,<6.0 +huggingface-hub>=1.5,<2.0 +safetensors>=0.7,<0.9 +datasets>=4.4,<5.0 + +numpy>=1.26,<2.3 +scipy>=1.16,<1.17 +pandas>=2.3,<2.4 +scikit-learn>=1.7,<1.8 +matplotlib>=3.10,<3.11 + +jupyter_client>=8.6,<9.0 +pyzmq>=27.1,<28.0 + +pyyaml>=6.0,<7.0 +tqdm>=4.67,<5.0 +warcio>=1.7,<2.0 +beautifulsoup4>=4.14,<5.0 + +# Booktype 1.5 is intentionally excluded. It is a Python-2-era package that can +# overwrite the modern redis package with Python 2 source files. + +pytest>=9.0,<10.0 +black>=24,<27 +isort>=5.13,<9 +mypy>=1.11,<3 From 843e89effd66e695c8b0bdf8493392797145f459 Mon Sep 17 00:00:00 2001 From: RickZ <121943721+rickisba@users.noreply.github.com> Date: Mon, 27 Jul 2026 19:56:39 +0800 Subject: [PATCH 3/4] env: add isolated CUDA 13 development image --- env/docker/cu130/Dockerfile | 119 ++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 env/docker/cu130/Dockerfile diff --git a/env/docker/cu130/Dockerfile b/env/docker/cu130/Dockerfile new file mode 100644 index 0000000..3e6a337 --- /dev/null +++ b/env/docker/cu130/Dockerfile @@ -0,0 +1,119 @@ +ARG CUDA_VERSION=13.0.0 +ARG PYTHON_VERSION=3.12 +ARG RUST_TOOLCHAIN=stable + +FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04 + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ARG PYTHON_VERSION +ARG RUST_TOOLCHAIN + +ENV DEBIAN_FRONTEND=noninteractive +ENV RUSTUP_HOME=/opt/rustup +ENV CARGO_HOME=/opt/cargo +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +ENV PYTHONNOUSERSITE=1 +ENV VIRTUAL_ENV=/opt/venv +ENV PATH="/opt/venv/bin:/opt/cargo/bin:${PATH}" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + software-properties-common \ + ca-certificates \ + curl \ + git \ + git-lfs \ + wget \ + vim \ + nano \ + less \ + tmux \ + unzip \ + build-essential \ + pkg-config \ + cmake \ + ninja-build \ + libssl-dev \ + libnuma-dev \ + libibverbs-dev \ + libcurl4-openssl-dev \ + libhiredis-dev \ + libgl1 \ + ffmpeg \ + net-tools \ + iproute2 \ + iputils-ping \ + iperf \ + htop \ + jq \ + lsof \ + openssh-client && \ + rm -rf /var/lib/apt/lists/* + +# Ubuntu 22.04 does not provide Python 3.12 as its default interpreter. +RUN add-apt-repository ppa:deadsnakes/ppa && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + "python${PYTHON_VERSION}" \ + "python${PYTHON_VERSION}-dev" \ + "python${PYTHON_VERSION}-venv" \ + "python${PYTHON_VERSION}-tk" && \ + rm -rf /var/lib/apt/lists/* + +RUN update-alternatives --install \ + /usr/bin/python3 \ + python3 \ + "/usr/bin/python${PYTHON_VERSION}" \ + 1 && \ + python3 -m venv "${VIRTUAL_ENV}" + +# Isolate the serving stack from Ubuntu's system Python packages. +RUN python3 -m pip install --no-cache-dir --upgrade \ + "pip<26" \ + "setuptools>=77,<81" \ + wheel \ + uv + +# Rust/Cargo are required by instance/resource_agent. +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + sh -s -- -y \ + --profile minimal \ + --default-toolchain "${RUST_TOOLCHAIN}" && \ + chmod -R a+rwX "${RUSTUP_HOME}" "${CARGO_HOME}" && \ + rustc --version && \ + cargo --version + +RUN python3 -c "import sys, tkinter; print('python:', sys.executable); print('tkinter:', tkinter.TkVersion)" && \ + ffmpeg -version >/dev/null + +COPY constraints.txt /opt/cacheroute/constraints.txt +COPY requirements-dev.txt /opt/cacheroute/requirements-dev.txt + +# Install the CUDA 13 PyTorch wheel set before vLLM and LMCache. +RUN python3 -m pip install --no-cache-dir \ + --index-url https://download.pytorch.org/whl/cu130 \ + -c /opt/cacheroute/constraints.txt \ + torch==2.11.0 \ + torchvision==0.26.0 \ + torchaudio==2.11.0 + +RUN python3 -m pip install --no-cache-dir \ + -c /opt/cacheroute/constraints.txt \ + vllm==0.25.1 \ + lmcache==0.5.2 + +RUN python3 -m pip install --no-cache-dir \ + -c /opt/cacheroute/constraints.txt \ + -r /opt/cacheroute/requirements-dev.txt + +# Validate metadata and real imports. Build steps normally do not receive GPUs, +# so torch.cuda.is_available() is intentionally checked only at container run time. +RUN python3 -m pip check && \ + python3 -c 'from importlib.metadata import version; import sys; import redis; import torch; import torchcodec; import vllm; import lmcache; print("Python serving environment:"); print("python:", sys.version); print("torch:", torch.__version__); print("torch CUDA:", torch.version.cuda); print("vllm:", version("vllm")); print("lmcache:", version("lmcache")); print("transformers:", version("transformers")); print("redis:", version("redis")); print("torchcodec:", version("torchcodec"))' + +WORKDIR /workspace/llm-stack/CacheRoute + +CMD ["sleep", "infinity"] From 66a9a26e272854d84843f8a2751418764e220ae1 Mon Sep 17 00:00:00 2001 From: RickZ <121943721+rickisba@users.noreply.github.com> Date: Mon, 27 Jul 2026 19:57:14 +0800 Subject: [PATCH 4/4] docs: document additive CUDA 13 environment profile --- env/docker/cu130/README.md | 154 +++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 env/docker/cu130/README.md diff --git a/env/docker/cu130/README.md b/env/docker/cu130/README.md new file mode 100644 index 0000000..222f9cf --- /dev/null +++ b/env/docker/cu130/README.md @@ -0,0 +1,154 @@ +# CUDA 13 / vLLM 0.25.1 / LMCache 0.5.2 profile + +This directory provides an additive development-image profile for the target +serving baseline tracked by Issue #148. It does not replace or modify the +existing CUDA 12.8 / vLLM 0.13.x / LMCache 0.3.11 environment. + +## Target stack + +| Component | Version | +|---|---| +| Base image | `nvidia/cuda:13.0.0-devel-ubuntu22.04` | +| Python | `3.12.x` | +| PyTorch | `2.11.0+cu130` | +| torchvision | `0.26.0+cu130` | +| torchaudio | `2.11.0+cu130` | +| vLLM | `0.25.1` | +| LMCache | `0.5.2` | + +The image uses `/opt/venv` to isolate serving dependencies from Ubuntu system +Python packages. FFmpeg is installed because current TorchCodec wheels require +its shared libraries. Rust/Cargo and Tkinter remain available for the existing +CacheRoute resource-agent and desktop-dashboard workflows. + +## Files + +- `Dockerfile`: complete CUDA 13 development image. +- `constraints.txt`: exact serving-stack versions and shared compatibility + ranges. +- `requirements-dev.txt`: CacheRoute application/development dependencies that + are compatible with the target stack. + +`requirements-dev.txt` intentionally excludes `Booktype==1.5`. That package is +from the Python 2 era and can overwrite the modern `redis` module with invalid +Python 2 source files. + +## Build + +Use this directory as the Docker build context. This keeps the build context +small and avoids sending models, KDN databases, logs, or other repository data +to the Docker daemon. + +```bash +cd /llm-stack/CacheRoute + +sudo docker build \ + -f env/docker/cu130/Dockerfile \ + -t cacheroute:dev-vllm0.25.1-lmcache0.5.2-cu130 \ + env/docker/cu130 +``` + +The Dockerfile is compatible with both the legacy Docker builder and BuildKit. +With Buildx, the equivalent command is: + +```bash +sudo docker buildx build \ + --load \ + --progress=plain \ + -f env/docker/cu130/Dockerfile \ + -t cacheroute:dev-vllm0.25.1-lmcache0.5.2-cu130 \ + env/docker/cu130 +``` + +## Create the development container + +```bash +sudo docker run -d \ + --name cacheroute-dev-cu130 \ + --gpus all \ + --network host \ + --ipc=host \ + --shm-size=64g \ + --ulimit memlock=-1 \ + --ulimit stack=67108864 \ + -e HF_HOME=/workspace/llm-stack/cache/hf \ + -e TORCH_HOME=/workspace/llm-stack/cache/torch \ + -e LMCACHE_HOME=/workspace/llm-stack/cache/lmcache \ + -e PYTHONPATH=/workspace/llm-stack/CacheRoute \ + -e PYTHONHASHSEED=0 \ + -v /llm-stack:/workspace/llm-stack \ + -w /workspace/llm-stack/CacheRoute \ + cacheroute:dev-vllm0.25.1-lmcache0.5.2-cu130 +``` + +Enter the container: + +```bash +sudo docker exec -it cacheroute-dev-cu130 bash +``` + +Install the mounted CacheRoute source without re-resolving the serving stack: + +```bash +cd /workspace/llm-stack/CacheRoute +python3 -m pip install -e . --no-deps +python3 -m pip check +``` + +## Runtime sanity check + +```bash +python3 - <<'PY' +import sys +import torch + +print("python:", sys.executable) +print("torch:", torch.__version__) +print("torch CUDA runtime:", torch.version.cuda) +print("CUDA available:", torch.cuda.is_available()) +print("GPU count:", torch.cuda.device_count()) +PY +``` + +The active interpreter should be `/opt/venv/bin/python3`, and CUDA should be +available when the container is started with `--gpus all`. + +## LMCache MP runtime note + +This profile targets LMCache MP mode and vLLM `LMCacheMPConnector`. Do not use +the legacy `LMCACHE_CONFIG_FILE`, `remote_url`, or +`--kv-offloading-backend lmcache` startup path as the primary interface. + +When using LMCache MP, either unset PyTorch expandable segments: + +```bash +unset PYTORCH_CUDA_ALLOC_CONF +``` + +or explicitly enable vLLM's CuMem allocator. The conservative validation path +uses `unset PYTORCH_CUDA_ALLOC_CONF`. + +A minimal LMCache server uses a dedicated MP endpoint and optional RESP L2 +adapter, while vLLM connects through `--kv-transfer-config` with +`LMCacheMPConnector`. + +## Validation status and scope + +The image build has been validated with: + +- Python `3.12.13`; +- PyTorch `2.11.0+cu130`; +- vLLM `0.25.1`; +- LMCache `0.5.2`; +- Transformers `5.12.1`; +- Redis Python client `8.0.1`; +- successful `pip check`; +- successful imports of Torch, TorchCodec, vLLM, LMCache, and Redis; +- successful startup of the existing CacheRoute Scheduler, KDN, Proxy, and + Instance workflow after installing FFmpeg. + +This environment profile does not claim that Issue #148 is complete. In +particular, migration of the KDN raw Redis dump/inject contract to LMCache MP's +RESP L2 behavior remains a runtime-integration task. This PR therefore adds the +new installation/build interface without changing the legacy environment or +CacheRoute routing, queueing, and storage behavior.