Introduce v1 runtime compatibility, modern serving profile, and MP startup path

core/runtime_compat.py

@@ -0,0 +1,116 @@
+"""Runtime compatibility helpers for vLLM/LMCache generations.
+
+The compatibility layer keeps legacy CacheRoute deployments working while
+allowing newer vLLM/LMCache stacks to use their current Redis key layout.
+"""
+from __future__ import annotations
+
+import os
+import re
+from typing import Iterable, Optional
+
+RUNTIME_PROFILE_AUTO = "auto"
+RUNTIME_PROFILE_LEGACY = "legacy"
+RUNTIME_PROFILE_V1 = "v1"
+SUPPORTED_RUNTIME_PROFILES = {
+    RUNTIME_PROFILE_AUTO,
+    RUNTIME_PROFILE_LEGACY,
+    RUNTIME_PROFILE_V1,
+}
+
+_HEX_RE = re.compile(r"^[0-9a-fA-F]{32,}$")
+
+
+def normalize_runtime_profile(value: Optional[str] = None) -> str:
+    """Normalize a runtime profile, defaulting to the environment or ``auto``."""
+    raw = value
+    if raw is None:
+        raw = os.getenv("CACHEROUTE_RUNTIME_PROFILE", RUNTIME_PROFILE_AUTO)
+    profile = str(raw or RUNTIME_PROFILE_AUTO).strip().lower()
+    aliases = {
+        "old": RUNTIME_PROFILE_LEGACY,
+        "v0": RUNTIME_PROFILE_LEGACY,
+        "modern": RUNTIME_PROFILE_V1,
+        "new": RUNTIME_PROFILE_V1,
+        "current": RUNTIME_PROFILE_V1,
+    }
+    profile = aliases.get(profile, profile)
+    if profile not in SUPPORTED_RUNTIME_PROFILES:
+        raise ValueError(
+            f"unsupported CACHEROUTE_RUNTIME_PROFILE={profile!r}; "
+            f"expected one of {sorted(SUPPORTED_RUNTIME_PROFILES)}"
+        )
+    return profile
+
+
+def resolve_scan_match(profile: str, requested_match: Optional[str]) -> str:
+    """Resolve the Redis SCAN pattern for a compatibility profile.
+
+    ``vllm@*`` was the historic CacheRoute default. Unless strict ``legacy``
+    mode is selected, it is treated as a compatibility sentinel rather than a
+    forced pattern because older API/CLI callers may still send it implicitly.
+    """
+    profile = normalize_runtime_profile(profile)
+    requested = str(requested_match or "").strip()
+
+    if requested and requested.lower() != "auto":
+        historic_default = requested == "vllm@*"
+        if not (historic_default and profile != RUNTIME_PROFILE_LEGACY):
+            return requested
+
+    if profile == RUNTIME_PROFILE_LEGACY:
+        return "vllm@*"
+    return "*"
+
+
+def classify_lmcache_redis_key(key: bytes) -> Optional[str]:
+    """Classify supported LMCache Redis key layouts.
+
+    Legacy CacheRoute used keys beginning with ``vllm@``. Newer LMCache
+    deployments include the model identifier/path and one or more metadata
+    fields before a long hexadecimal chunk hash, for example
+    ``/models/llama@...@@<hash>``.
+    """
+    try:
+        text = key.decode("utf-8")
+    except UnicodeDecodeError:
+        return None
+
+    if text.startswith("vllm@"):
+        return "legacy"
+
+    parts = text.split("@")
+    if len(parts) < 4 or not parts[0]:
+        return None
+
+    # LMCache versions/connectors differ in the exact field count, but all
+    # current layouts carry a long hexadecimal chunk hash after the model id.
+    if any(_HEX_RE.fullmatch(part or "") for part in parts[1:]):
+        return "v1"
+    return None
+
+
+def filter_supported_keys(
+    keys: Iterable[bytes],
+    profile: str,
+    requested_match: Optional[str],
+) -> set[bytes]:
+    """Filter Redis keys when automatic discovery is active."""
+    profile = normalize_runtime_profile(profile)
+    scan_match = resolve_scan_match(profile, requested_match)
+    key_set = set(keys)
+
+    # An explicit non-auto pattern is authoritative; SCAN already filtered it.
+    if scan_match != "*":
+        return key_set
+
+    allowed = {"legacy", "v1"}
+    if profile == RUNTIME_PROFILE_LEGACY:
+        allowed = {"legacy"}
+    elif profile == RUNTIME_PROFILE_V1:
+        allowed = {"v1"}
+
+    return {
+        key for key in key_set
+        if classify_lmcache_redis_key(key) in allowed
+    }

docs/runtime_compatibility_v1.md

@@ -0,0 +1,159 @@
+# CacheRoute v1 runtime compatibility
+
+This branch introduces a compatibility layer for two serving generations:
+
+| Profile | Intended serving stack | Redis KV key handling |
+|---|---|---|
+| `legacy` | vLLM 0.13.x + LMCache 0.3.11 | historical `vllm@*` namespace |
+| `v1` | vLLM 0.25.1 + LMCache 0.5.2 | model-scoped LMCache keys |
+| `auto` | default | discovers and validates either layout |
+
+Set the profile with:
+
+```bash
+export CACHEROUTE_RUNTIME_PROFILE=auto   # default
+# or: legacy / v1
+```
+
+The additive CUDA 13 image under [`env/docker/cu130`](../env/docker/cu130)
+sets `CACHEROUTE_RUNTIME_PROFILE=v1` by default. The legacy image and root
+requirements remain unchanged.
+
+## Startup interfaces are versioned too
+
+The serving commands are part of the runtime compatibility contract. The two
+stacks do not share the same primary LMCache/vLLM startup interface.
+
+| Profile | LMCache process | vLLM connector path |
+|---|---|---|
+| `legacy` | YAML selected by `LMCACHE_CONFIG_FILE` | historical LMCache offloading arguments |
+| `v1` | standalone `lmcache server` with RESP L2 | `vllm serve` with `LMCacheMPConnector` in `--kv-transfer-config` |
+
+Do not mix the interfaces. In particular, the v1 path must unset the legacy
+`LMCACHE_CONFIG_FILE` and must not depend on `remote_url` or
+`--kv-offloading-backend lmcache`.
+
+The complete validated commands and environment-variable overrides are in
+[`env/docker/cu130/README.md`](../env/docker/cu130/README.md). Existing
+containers can use the repository scripts directly, without rebuilding:
+
+```bash
+export CACHEROUTE_RUNTIME_PROFILE=v1
+bash env/docker/cu130/scripts/start_lmcache_mp.sh
+# In another terminal after port 5555 is listening:
+bash env/docker/cu130/scripts/start_vllm_mp.sh
+```
+
+The v1 startup order is:
+
+```text
+Redis RESP L2 -> LMCache MP :5555 -> vLLM :8000 -> CacheRoute components
+```
+
+## KDN KV construction
+
+Manual `--match` is no longer required in the normal CLI workflow. `auto` and
+`v1` modes treat the historical implicit `vllm@*` argument as a compatibility
+sentinel and scan for supported LMCache key layouts.
+
+An explicit `--match` remains authoritative for debugging or custom backends.
+For strict legacy behavior, set `CACHEROUTE_RUNTIME_PROFILE=legacy`.
+
+LMCache remote writes are asynchronous. KDN therefore uses:
+
+- a polling interval (default `0.2s`),
+- a maximum first-key timeout (default `30s`), and
+- a quiet period after the last key-set change (default `1.5s`).
+
+The 30-second value is an upper bound, not a fixed delay. A normal build exits
+as soon as keys appear and remain unchanged for the quiet period.
+
+A build that captures zero keys now fails and removes its partial output
+directory. It is never marked `kv_ready`.
+
+## Deployment profiles
+
+The serving stack remains owned by its Docker image. CacheRoute application
+dependencies must not upgrade CUDA-sensitive packages.
+
+### Legacy profile
+
+The existing environment remains the stable path for:
+
+```text
+CUDA 12.8 / PyTorch 2.9.x / vLLM 0.13.x / LMCache 0.3.11
+```
+
+Use:
+
+```bash
+export CACHEROUTE_RUNTIME_PROFILE=legacy
+```
+
+when strict historical Redis-key behavior is required.
+
+### v1 profile
+
+The isolated modern image is defined in [`env/docker/cu130`](../env/docker/cu130):
+
+```text
+CUDA 13.0 / Python 3.12 / PyTorch 2.11.0+cu130
+vLLM 0.25.1 / LMCache 0.5.2
+```
+
+It uses `/opt/venv`, keeps the modern serving stack separate from Ubuntu system
+packages, installs FFmpeg for TorchCodec, and preserves Rust/Cargo and Tkinter
+for existing CacheRoute auxiliary components.
+
+The v1 dependency files are additive. They do not modify:
+
+- `env/docker/Dockerfile`,
+- root `requirements.txt`,
+- `pyproject.toml`, or
+- legacy runtime behavior.
+
+For reliable differential capture, use a dedicated Redis database or Redis
+instance for KDN construction. `--flushdb` must not target an online shared DB.
+
+## Cache identity compatibility
+
+Runtime-profile selection does not make incompatible KV blocks reusable. KDN
+construction and the target Instance must still agree on the cache identity,
+including:
+
+- model identity/path and model configuration;
+- tensor-parallel topology and worker layout;
+- KV dtype;
+- LMCache chunk size;
+- LMCache hash algorithm;
+- request tags or other key-affecting configuration;
+- RESP L2 endpoint/database semantics.
+
+The validated v1 baseline currently uses chunk size `256` and hash algorithm
+`sha256_cbor`.
+
+## Validation boundary
+
+The modern image has been validated for installation, imports, `pip check`, GPU
+runtime availability, and startup of the CacheRoute Scheduler/KDN/Proxy/Instance
+workflow.
+
+The KDN raw Redis dump/restore path has also been verified byte-for-byte for the
+observed LMCache 0.5.2 key/value layout. This proves storage integrity, but does
+not yet prove that a request consumes the injected blocks through LMCache MP.
+End-to-end validation must compare LMCache hit-token and remote-read metrics.
+
+## Follow-up migration scope
+
+Subsequent v1 changes should remain behind the compatibility layer where the
+vLLM/LMCache interfaces differ, including:
+
+- Instance-side observability and metric collection,
+- request metadata and prompt-prefix compatibility,
+- injected-cache hit validation,
+- dual-profile integration tests,
+- startup/configuration validation for each runtime profile.
+
+Avoid scattering version checks through Scheduler, Proxy, Instance, and KDN
+code. Add profile-specific behavior to the compatibility layer or focused
+adapters.

env/docker/cu130/Dockerfile

@@ -0,0 +1,120 @@
+ARG CUDA_VERSION=13.0.0
+ARG PYTHON_VERSION=3.12
+ARG RUST_TOOLCHAIN=stable
+
+FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+ARG PYTHON_VERSION
+ARG RUST_TOOLCHAIN
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV RUSTUP_HOME=/opt/rustup
+ENV CARGO_HOME=/opt/cargo
+ENV PIP_DISABLE_PIP_VERSION_CHECK=1
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONNOUSERSITE=1
+ENV VIRTUAL_ENV=/opt/venv
+ENV PATH="/opt/venv/bin:/opt/cargo/bin:${PATH}"
+ENV CACHEROUTE_RUNTIME_PROFILE=v1
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        software-properties-common \
+        ca-certificates \
+        curl \
+        git \
+        git-lfs \
+        wget \
+        vim \
+        nano \
+        less \
+        tmux \
+        unzip \
+        build-essential \
+        pkg-config \
+        cmake \
+        ninja-build \
+        libssl-dev \
+        libnuma-dev \
+        libibverbs-dev \
+        libcurl4-openssl-dev \
+        libhiredis-dev \
+        libgl1 \
+        ffmpeg \
+        net-tools \
+        iproute2 \
+        iputils-ping \
+        iperf \
+        htop \
+        jq \
+        lsof \
+        openssh-client && \
+    rm -rf /var/lib/apt/lists/*
+
+# Ubuntu 22.04 does not provide Python 3.12 as its default interpreter.
+RUN add-apt-repository ppa:deadsnakes/ppa && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends \
+        "python${PYTHON_VERSION}" \
+        "python${PYTHON_VERSION}-dev" \
+        "python${PYTHON_VERSION}-venv" \
+        "python${PYTHON_VERSION}-tk" && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN update-alternatives --install \
+        /usr/bin/python3 \
+        python3 \
+        "/usr/bin/python${PYTHON_VERSION}" \
+        1 && \
+    python3 -m venv "${VIRTUAL_ENV}"
+
+# Isolate the serving stack from Ubuntu's system Python packages.
+RUN python3 -m pip install --no-cache-dir --upgrade \
+        "pip<26" \
+        "setuptools>=77,<81" \
+        wheel \
+        uv
+
+# Rust/Cargo are required by instance/resource_agent.
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
+        sh -s -- -y \
+        --profile minimal \
+        --default-toolchain "${RUST_TOOLCHAIN}" && \
+    chmod -R a+rwX "${RUSTUP_HOME}" "${CARGO_HOME}" && \
+    rustc --version && \
+    cargo --version
+
+RUN python3 -c "import sys, tkinter; print('python:', sys.executable); print('tkinter:', tkinter.TkVersion)" && \
+    ffmpeg -version >/dev/null
+
+COPY constraints.txt /opt/cacheroute/constraints.txt
+COPY requirements-dev.txt /opt/cacheroute/requirements-dev.txt
+
+# Install the CUDA 13 PyTorch wheel set before vLLM and LMCache.
+RUN python3 -m pip install --no-cache-dir \
+        --index-url https://download.pytorch.org/whl/cu130 \
+        -c /opt/cacheroute/constraints.txt \
+        torch==2.11.0 \
+        torchvision==0.26.0 \
+        torchaudio==2.11.0
+
+RUN python3 -m pip install --no-cache-dir \
+        -c /opt/cacheroute/constraints.txt \
+        vllm==0.25.1 \
+        lmcache==0.5.2
+
+RUN python3 -m pip install --no-cache-dir \
+        -c /opt/cacheroute/constraints.txt \
+        -r /opt/cacheroute/requirements-dev.txt
+
+# Validate metadata and real imports. Build steps normally do not receive GPUs,
+# so torch.cuda.is_available() is intentionally checked only at container run time.
+RUN python3 -m pip check && \
+    python3 -c 'from importlib.metadata import version; import sys; import redis; import torch; import torchcodec; import vllm; import lmcache; print("Python serving environment:"); print("python:", sys.version); print("torch:", torch.__version__); print("torch CUDA:", torch.version.cuda); print("vllm:", version("vllm")); print("lmcache:", version("lmcache")); print("transformers:", version("transformers")); print("redis:", version("redis")); print("torchcodec:", version("torchcodec"))'
+
+WORKDIR /workspace/llm-stack/CacheRoute
+
+CMD ["sleep", "infinity"]