diff --git a/core/runtime_compat.py b/core/runtime_compat.py new file mode 100644 index 0000000..f45bcdb --- /dev/null +++ b/core/runtime_compat.py @@ -0,0 +1,116 @@ +"""Runtime compatibility helpers for vLLM/LMCache generations. + +The compatibility layer keeps legacy CacheRoute deployments working while +allowing newer vLLM/LMCache stacks to use their current Redis key layout. +""" +from __future__ import annotations + +import os +import re +from typing import Iterable, Optional + +RUNTIME_PROFILE_AUTO = "auto" +RUNTIME_PROFILE_LEGACY = "legacy" +RUNTIME_PROFILE_V1 = "v1" +SUPPORTED_RUNTIME_PROFILES = { + RUNTIME_PROFILE_AUTO, + RUNTIME_PROFILE_LEGACY, + RUNTIME_PROFILE_V1, +} + +_HEX_RE = re.compile(r"^[0-9a-fA-F]{32,}$") + + +def normalize_runtime_profile(value: Optional[str] = None) -> str: + """Normalize a runtime profile, defaulting to the environment or ``auto``.""" + raw = value + if raw is None: + raw = os.getenv("CACHEROUTE_RUNTIME_PROFILE", RUNTIME_PROFILE_AUTO) + profile = str(raw or RUNTIME_PROFILE_AUTO).strip().lower() + aliases = { + "old": RUNTIME_PROFILE_LEGACY, + "v0": RUNTIME_PROFILE_LEGACY, + "modern": RUNTIME_PROFILE_V1, + "new": RUNTIME_PROFILE_V1, + "current": RUNTIME_PROFILE_V1, + } + profile = aliases.get(profile, profile) + if profile not in SUPPORTED_RUNTIME_PROFILES: + raise ValueError( + f"unsupported CACHEROUTE_RUNTIME_PROFILE={profile!r}; " + f"expected one of {sorted(SUPPORTED_RUNTIME_PROFILES)}" + ) + return profile + + +def resolve_scan_match(profile: str, requested_match: Optional[str]) -> str: + """Resolve the Redis SCAN pattern for a compatibility profile. + + ``vllm@*`` was the historic CacheRoute default. Unless strict ``legacy`` + mode is selected, it is treated as a compatibility sentinel rather than a + forced pattern because older API/CLI callers may still send it implicitly. + """ + profile = normalize_runtime_profile(profile) + requested = str(requested_match or "").strip() + + if requested and requested.lower() != "auto": + historic_default = requested == "vllm@*" + if not (historic_default and profile != RUNTIME_PROFILE_LEGACY): + return requested + + if profile == RUNTIME_PROFILE_LEGACY: + return "vllm@*" + return "*" + + +def classify_lmcache_redis_key(key: bytes) -> Optional[str]: + """Classify supported LMCache Redis key layouts. + + Legacy CacheRoute used keys beginning with ``vllm@``. Newer LMCache + deployments include the model identifier/path and one or more metadata + fields before a long hexadecimal chunk hash, for example + ``/models/llama@...@@``. + """ + try: + text = key.decode("utf-8") + except UnicodeDecodeError: + return None + + if text.startswith("vllm@"): + return "legacy" + + parts = text.split("@") + if len(parts) < 4 or not parts[0]: + return None + + # LMCache versions/connectors differ in the exact field count, but all + # current layouts carry a long hexadecimal chunk hash after the model id. + if any(_HEX_RE.fullmatch(part or "") for part in parts[1:]): + return "v1" + return None + + +def filter_supported_keys( + keys: Iterable[bytes], + profile: str, + requested_match: Optional[str], +) -> set[bytes]: + """Filter Redis keys when automatic discovery is active.""" + profile = normalize_runtime_profile(profile) + scan_match = resolve_scan_match(profile, requested_match) + key_set = set(keys) + + # An explicit non-auto pattern is authoritative; SCAN already filtered it. + if scan_match != "*": + return key_set + + allowed = {"legacy", "v1"} + if profile == RUNTIME_PROFILE_LEGACY: + allowed = {"legacy"} + elif profile == RUNTIME_PROFILE_V1: + allowed = {"v1"} + + return { + key for key in key_set + if classify_lmcache_redis_key(key) in allowed + } diff --git a/docs/runtime_compatibility_v1.md b/docs/runtime_compatibility_v1.md new file mode 100644 index 0000000..1a15cb9 --- /dev/null +++ b/docs/runtime_compatibility_v1.md @@ -0,0 +1,159 @@ +# CacheRoute v1 runtime compatibility + +This branch introduces a compatibility layer for two serving generations: + +| Profile | Intended serving stack | Redis KV key handling | +|---|---|---| +| `legacy` | vLLM 0.13.x + LMCache 0.3.11 | historical `vllm@*` namespace | +| `v1` | vLLM 0.25.1 + LMCache 0.5.2 | model-scoped LMCache keys | +| `auto` | default | discovers and validates either layout | + +Set the profile with: + +```bash +export CACHEROUTE_RUNTIME_PROFILE=auto # default +# or: legacy / v1 +``` + +The additive CUDA 13 image under [`env/docker/cu130`](../env/docker/cu130) +sets `CACHEROUTE_RUNTIME_PROFILE=v1` by default. The legacy image and root +requirements remain unchanged. + +## Startup interfaces are versioned too + +The serving commands are part of the runtime compatibility contract. The two +stacks do not share the same primary LMCache/vLLM startup interface. + +| Profile | LMCache process | vLLM connector path | +|---|---|---| +| `legacy` | YAML selected by `LMCACHE_CONFIG_FILE` | historical LMCache offloading arguments | +| `v1` | standalone `lmcache server` with RESP L2 | `vllm serve` with `LMCacheMPConnector` in `--kv-transfer-config` | + +Do not mix the interfaces. In particular, the v1 path must unset the legacy +`LMCACHE_CONFIG_FILE` and must not depend on `remote_url` or +`--kv-offloading-backend lmcache`. + +The complete validated commands and environment-variable overrides are in +[`env/docker/cu130/README.md`](../env/docker/cu130/README.md). Existing +containers can use the repository scripts directly, without rebuilding: + +```bash +export CACHEROUTE_RUNTIME_PROFILE=v1 +bash env/docker/cu130/scripts/start_lmcache_mp.sh +# In another terminal after port 5555 is listening: +bash env/docker/cu130/scripts/start_vllm_mp.sh +``` + +The v1 startup order is: + +```text +Redis RESP L2 -> LMCache MP :5555 -> vLLM :8000 -> CacheRoute components +``` + +## KDN KV construction + +Manual `--match` is no longer required in the normal CLI workflow. `auto` and +`v1` modes treat the historical implicit `vllm@*` argument as a compatibility +sentinel and scan for supported LMCache key layouts. + +An explicit `--match` remains authoritative for debugging or custom backends. +For strict legacy behavior, set `CACHEROUTE_RUNTIME_PROFILE=legacy`. + +LMCache remote writes are asynchronous. KDN therefore uses: + +- a polling interval (default `0.2s`), +- a maximum first-key timeout (default `30s`), and +- a quiet period after the last key-set change (default `1.5s`). + +The 30-second value is an upper bound, not a fixed delay. A normal build exits +as soon as keys appear and remain unchanged for the quiet period. + +A build that captures zero keys now fails and removes its partial output +directory. It is never marked `kv_ready`. + +## Deployment profiles + +The serving stack remains owned by its Docker image. CacheRoute application +dependencies must not upgrade CUDA-sensitive packages. + +### Legacy profile + +The existing environment remains the stable path for: + +```text +CUDA 12.8 / PyTorch 2.9.x / vLLM 0.13.x / LMCache 0.3.11 +``` + +Use: + +```bash +export CACHEROUTE_RUNTIME_PROFILE=legacy +``` + +when strict historical Redis-key behavior is required. + +### v1 profile + +The isolated modern image is defined in [`env/docker/cu130`](../env/docker/cu130): + +```text +CUDA 13.0 / Python 3.12 / PyTorch 2.11.0+cu130 +vLLM 0.25.1 / LMCache 0.5.2 +``` + +It uses `/opt/venv`, keeps the modern serving stack separate from Ubuntu system +packages, installs FFmpeg for TorchCodec, and preserves Rust/Cargo and Tkinter +for existing CacheRoute auxiliary components. + +The v1 dependency files are additive. They do not modify: + +- `env/docker/Dockerfile`, +- root `requirements.txt`, +- `pyproject.toml`, or +- legacy runtime behavior. + +For reliable differential capture, use a dedicated Redis database or Redis +instance for KDN construction. `--flushdb` must not target an online shared DB. + +## Cache identity compatibility + +Runtime-profile selection does not make incompatible KV blocks reusable. KDN +construction and the target Instance must still agree on the cache identity, +including: + +- model identity/path and model configuration; +- tensor-parallel topology and worker layout; +- KV dtype; +- LMCache chunk size; +- LMCache hash algorithm; +- request tags or other key-affecting configuration; +- RESP L2 endpoint/database semantics. + +The validated v1 baseline currently uses chunk size `256` and hash algorithm +`sha256_cbor`. + +## Validation boundary + +The modern image has been validated for installation, imports, `pip check`, GPU +runtime availability, and startup of the CacheRoute Scheduler/KDN/Proxy/Instance +workflow. + +The KDN raw Redis dump/restore path has also been verified byte-for-byte for the +observed LMCache 0.5.2 key/value layout. This proves storage integrity, but does +not yet prove that a request consumes the injected blocks through LMCache MP. +End-to-end validation must compare LMCache hit-token and remote-read metrics. + +## Follow-up migration scope + +Subsequent v1 changes should remain behind the compatibility layer where the +vLLM/LMCache interfaces differ, including: + +- Instance-side observability and metric collection, +- request metadata and prompt-prefix compatibility, +- injected-cache hit validation, +- dual-profile integration tests, +- startup/configuration validation for each runtime profile. + +Avoid scattering version checks through Scheduler, Proxy, Instance, and KDN +code. Add profile-specific behavior to the compatibility layer or focused +adapters. diff --git a/env/docker/cu130/Dockerfile b/env/docker/cu130/Dockerfile new file mode 100644 index 0000000..966abf1 --- /dev/null +++ b/env/docker/cu130/Dockerfile @@ -0,0 +1,120 @@ +ARG CUDA_VERSION=13.0.0 +ARG PYTHON_VERSION=3.12 +ARG RUST_TOOLCHAIN=stable + +FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04 + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ARG PYTHON_VERSION +ARG RUST_TOOLCHAIN + +ENV DEBIAN_FRONTEND=noninteractive +ENV RUSTUP_HOME=/opt/rustup +ENV CARGO_HOME=/opt/cargo +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +ENV PYTHONNOUSERSITE=1 +ENV VIRTUAL_ENV=/opt/venv +ENV PATH="/opt/venv/bin:/opt/cargo/bin:${PATH}" +ENV CACHEROUTE_RUNTIME_PROFILE=v1 + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + software-properties-common \ + ca-certificates \ + curl \ + git \ + git-lfs \ + wget \ + vim \ + nano \ + less \ + tmux \ + unzip \ + build-essential \ + pkg-config \ + cmake \ + ninja-build \ + libssl-dev \ + libnuma-dev \ + libibverbs-dev \ + libcurl4-openssl-dev \ + libhiredis-dev \ + libgl1 \ + ffmpeg \ + net-tools \ + iproute2 \ + iputils-ping \ + iperf \ + htop \ + jq \ + lsof \ + openssh-client && \ + rm -rf /var/lib/apt/lists/* + +# Ubuntu 22.04 does not provide Python 3.12 as its default interpreter. +RUN add-apt-repository ppa:deadsnakes/ppa && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + "python${PYTHON_VERSION}" \ + "python${PYTHON_VERSION}-dev" \ + "python${PYTHON_VERSION}-venv" \ + "python${PYTHON_VERSION}-tk" && \ + rm -rf /var/lib/apt/lists/* + +RUN update-alternatives --install \ + /usr/bin/python3 \ + python3 \ + "/usr/bin/python${PYTHON_VERSION}" \ + 1 && \ + python3 -m venv "${VIRTUAL_ENV}" + +# Isolate the serving stack from Ubuntu's system Python packages. +RUN python3 -m pip install --no-cache-dir --upgrade \ + "pip<26" \ + "setuptools>=77,<81" \ + wheel \ + uv + +# Rust/Cargo are required by instance/resource_agent. +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + sh -s -- -y \ + --profile minimal \ + --default-toolchain "${RUST_TOOLCHAIN}" && \ + chmod -R a+rwX "${RUSTUP_HOME}" "${CARGO_HOME}" && \ + rustc --version && \ + cargo --version + +RUN python3 -c "import sys, tkinter; print('python:', sys.executable); print('tkinter:', tkinter.TkVersion)" && \ + ffmpeg -version >/dev/null + +COPY constraints.txt /opt/cacheroute/constraints.txt +COPY requirements-dev.txt /opt/cacheroute/requirements-dev.txt + +# Install the CUDA 13 PyTorch wheel set before vLLM and LMCache. +RUN python3 -m pip install --no-cache-dir \ + --index-url https://download.pytorch.org/whl/cu130 \ + -c /opt/cacheroute/constraints.txt \ + torch==2.11.0 \ + torchvision==0.26.0 \ + torchaudio==2.11.0 + +RUN python3 -m pip install --no-cache-dir \ + -c /opt/cacheroute/constraints.txt \ + vllm==0.25.1 \ + lmcache==0.5.2 + +RUN python3 -m pip install --no-cache-dir \ + -c /opt/cacheroute/constraints.txt \ + -r /opt/cacheroute/requirements-dev.txt + +# Validate metadata and real imports. Build steps normally do not receive GPUs, +# so torch.cuda.is_available() is intentionally checked only at container run time. +RUN python3 -m pip check && \ + python3 -c 'from importlib.metadata import version; import sys; import redis; import torch; import torchcodec; import vllm; import lmcache; print("Python serving environment:"); print("python:", sys.version); print("torch:", torch.__version__); print("torch CUDA:", torch.version.cuda); print("vllm:", version("vllm")); print("lmcache:", version("lmcache")); print("transformers:", version("transformers")); print("redis:", version("redis")); print("torchcodec:", version("torchcodec"))' + +WORKDIR /workspace/llm-stack/CacheRoute + +CMD ["sleep", "infinity"] diff --git a/env/docker/cu130/README.md b/env/docker/cu130/README.md new file mode 100644 index 0000000..16f04f0 --- /dev/null +++ b/env/docker/cu130/README.md @@ -0,0 +1,353 @@ +# CUDA 13 / vLLM 0.25.1 / LMCache 0.5.2 profile + +This directory provides the additive v1 development-image profile for the +modern CacheRoute runtime. It does not replace or modify the existing CUDA 12.8 +/ vLLM 0.13.x / LMCache 0.3.11 environment. + +The runtime compatibility architecture is described in +[`docs/runtime_compatibility_v1.md`](../../../docs/runtime_compatibility_v1.md). + +## Choose this profile only for the modern stack + +| Profile | Serving stack | LMCache startup interface | +|---|---|---| +| Legacy | vLLM 0.13.x + LMCache 0.3.11 | YAML through `LMCACHE_CONFIG_FILE` and the historical vLLM offloading flags | +| v1 | vLLM 0.25.1 + LMCache 0.5.2 | Standalone `lmcache server` plus vLLM `LMCacheMPConnector` | + +Do not combine the two interfaces. In v1 mode, unset `LMCACHE_CONFIG_FILE` and +do not use `remote_url`, `--kv-offloading-backend lmcache`, or the old +`python3 -m vllm.entrypoints.openai.api_server` example as the primary startup +path. + +## Target stack + +| Component | Version | +|---|---| +| Base image | `nvidia/cuda:13.0.0-devel-ubuntu22.04` | +| Python | `3.12.x` | +| PyTorch | `2.11.0+cu130` | +| torchvision | `0.26.0+cu130` | +| torchaudio | `2.11.0+cu130` | +| vLLM | `0.25.1` | +| LMCache | `0.5.2` | + +The image uses `/opt/venv` to isolate serving dependencies from Ubuntu system +Python packages. FFmpeg is installed because current TorchCodec wheels require +its shared libraries. Rust/Cargo and Tkinter remain available for the existing +CacheRoute resource-agent and desktop-dashboard workflows. + +The image sets: + +```bash +CACHEROUTE_RUNTIME_PROFILE=v1 +``` + +This selects the modern compatibility path by default. Override it at container +startup only when explicitly testing `auto` or `legacy` behavior. + +## Files + +- `Dockerfile`: complete CUDA 13 development image. +- `constraints.txt`: exact serving-stack versions and shared compatibility + ranges. +- `requirements-dev.txt`: CacheRoute application/development dependencies that + are compatible with the target stack. +- `scripts/start_lmcache_mp.sh`: reusable LMCache 0.5.2 MP + RESP L2 startup. +- `scripts/start_vllm_mp.sh`: reusable vLLM 0.25.1 + `LMCacheMPConnector` + startup. + +`requirements-dev.txt` intentionally excludes `Booktype==1.5`. That package is +from the Python 2 era and can overwrite the modern `redis` module with invalid +Python 2 source files. + +## Use an existing container without rebuilding + +The scripts live in the mounted CacheRoute repository, so an already-created +container does not need to run through the Dockerfile again. + +Update the branch and select the v1 profile: + +```bash +cd /workspace/llm-stack/CacheRoute +git fetch origin +git switch v1/runtime-compat +git pull --ff-only origin v1/runtime-compat + +export CACHEROUTE_RUNTIME_PROFILE=v1 +``` + +Persist the profile for interactive root shells when useful: + +```bash +grep -q 'CACHEROUTE_RUNTIME_PROFILE' /root/.bashrc || \ + echo 'export CACHEROUTE_RUNTIME_PROFILE=v1' >> /root/.bashrc +``` + +Long-running services only see environment variables present when they start. +Restart LMCache, vLLM, KDN, Proxy, and Instance after switching profiles. + +## Runtime startup order + +Use separate terminals in this order: + +1. Redis RESP L2 backend; +2. LMCache MP server; +3. vLLM with `LMCacheMPConnector`; +4. CacheRoute Scheduler, KDN, Proxy, Instance, and Client. + +### 1. Verify Redis + +```bash +redis-cli -h 127.0.0.1 -p 6379 ping +# Expected: PONG +``` + +### 2. Start LMCache MP + +Recommended repository script: + +```bash +cd /workspace/llm-stack/CacheRoute +export CACHEROUTE_RUNTIME_PROFILE=v1 +bash env/docker/cu130/scripts/start_lmcache_mp.sh +``` + +Equivalent validated command: + +```bash +unset LMCACHE_CONFIG_FILE +export CACHEROUTE_RUNTIME_PROFILE=v1 +export LMCACHE_LOG_LEVEL=INFO +export PYTHONHASHSEED=0 + +lmcache server \ + --instance-id cacheroute-lmcache-1 \ + --host 127.0.0.1 \ + --port 5555 \ + --http-host 127.0.0.1 \ + --http-port 8080 \ + --l1-size-gb 80 \ + --chunk-size 256 \ + --hash-algorithm sha256_cbor \ + --eviction-policy LRU \ + --max-workers 8 \ + --l2-store-policy default \ + --l2-adapter '{ + "type": "resp", + "host": "127.0.0.1", + "port": 6379, + "num_workers": 8 + }' +``` + +Confirm that the MP and HTTP ports are listening: + +```bash +ss -lntp | grep -E ':(5555|8080)\b' +``` + +### 3. Start vLLM + +Start this only after LMCache is listening on port `5555`. + +Recommended repository script: + +```bash +cd /workspace/llm-stack/CacheRoute +export CACHEROUTE_RUNTIME_PROFILE=v1 +bash env/docker/cu130/scripts/start_vllm_mp.sh +``` + +Equivalent validated command: + +```bash +export CACHEROUTE_RUNTIME_PROFILE=v1 +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +export MODEL_DIR=/workspace/llm-stack/models/LLM-Research/Meta-Llama-3-70B-Instruct +export PYTHONHASHSEED=0 +export OMP_NUM_THREADS=8 + +unset LMCACHE_CONFIG_FILE +unset PYTORCH_CUDA_ALLOC_CONF + +vllm serve "$MODEL_DIR" \ + --served-model-name llama3-70b \ + --host 0.0.0.0 \ + --port 8000 \ + --tensor-parallel-size 8 \ + --gpu-memory-utilization 0.75 \ + --dtype auto \ + --max-model-len 4096 \ + --max-num-seqs 8 \ + --max-num-batched-tokens 16384 \ + --disable-hybrid-kv-cache-manager \ + --no-enable-prefix-caching \ + --kv-transfer-config '{ + "kv_connector": "LMCacheMPConnector", + "kv_connector_module_path": "lmcache.integration.vllm.lmcache_mp_connector", + "kv_role": "kv_both", + "kv_connector_extra_config": { + "lmcache.mp.host": "tcp://127.0.0.1", + "lmcache.mp.port": 5555 + } + }' \ + --kv-cache-metrics +``` + +Wait for the OpenAI-compatible endpoint: + +```bash +curl -sS http://127.0.0.1:8000/v1/models | python3 -m json.tool +``` + +The scripts expose the main values as environment variables. For example: + +```bash +MODEL_DIR=/other/model \ +TENSOR_PARALLEL_SIZE=4 \ +CUDA_VISIBLE_DEVICES=0,1,2,3 \ +bash env/docker/cu130/scripts/start_vllm_mp.sh +``` + +```bash +LMCACHE_L1_SIZE_GB=40 \ +REDIS_HOST=172.18.0.121 \ +bash env/docker/cu130/scripts/start_lmcache_mp.sh +``` + +Additional command-line arguments are appended unchanged to each underlying +command. + +## Cache identity requirements + +KDN construction and the target Instance must agree on the cache identity. At a +minimum, keep the following aligned when building and consuming reusable KV: + +- model identity/path and served model; +- tensor-parallel topology and worker layout; +- KV dtype and model configuration; +- LMCache chunk size (`256` in this profile); +- LMCache hash algorithm (`sha256_cbor` in this profile); +- Redis RESP L2 endpoint and database semantics. + +A successful Redis `SET` or byte-for-byte KDN round trip does not by itself +prove a cache hit. Validate consumption with LMCache hit-token and remote-read +metrics. + +## Build the image for a new container + +Use this directory as the Docker build context. This keeps the context small and +avoids sending models, KDN databases, logs, or other repository data to the +Docker daemon. + +```bash +cd /llm-stack/CacheRoute + +sudo docker build \ + -f env/docker/cu130/Dockerfile \ + -t cacheroute:dev-vllm0.25.1-lmcache0.5.2-cu130 \ + env/docker/cu130 +``` + +With Buildx: + +```bash +sudo docker buildx build \ + --load \ + --progress=plain \ + -f env/docker/cu130/Dockerfile \ + -t cacheroute:dev-vllm0.25.1-lmcache0.5.2-cu130 \ + env/docker/cu130 +``` + +## Create a new development container + +```bash +sudo docker run -d \ + --name cacheroute-dev-cu130 \ + --gpus all \ + --network host \ + --ipc=host \ + --shm-size=64g \ + --ulimit memlock=-1 \ + --ulimit stack=67108864 \ + -e CACHEROUTE_RUNTIME_PROFILE=v1 \ + -e HF_HOME=/workspace/llm-stack/cache/hf \ + -e TORCH_HOME=/workspace/llm-stack/cache/torch \ + -e LMCACHE_HOME=/workspace/llm-stack/cache/lmcache \ + -e PYTHONPATH=/workspace/llm-stack/CacheRoute \ + -e PYTHONHASHSEED=0 \ + -v /llm-stack:/workspace/llm-stack \ + -w /workspace/llm-stack/CacheRoute \ + cacheroute:dev-vllm0.25.1-lmcache0.5.2-cu130 +``` + +Enter the container: + +```bash +sudo docker exec -it cacheroute-dev-cu130 bash +``` + +Install the mounted CacheRoute source without re-resolving the serving stack: + +```bash +cd /workspace/llm-stack/CacheRoute +python3 -m pip install -e . --no-deps +python3 -m pip check +``` + +## Runtime sanity check + +```bash +python3 - <<'PY' +import os +import sys +import torch + +print("python:", sys.executable) +print("torch:", torch.__version__) +print("torch CUDA runtime:", torch.version.cuda) +print("CUDA available:", torch.cuda.is_available()) +print("GPU count:", torch.cuda.device_count()) +print("CacheRoute profile:", os.getenv("CACHEROUTE_RUNTIME_PROFILE")) +PY +``` + +The active interpreter should be `/opt/venv/bin/python3`, CUDA should be +available when the container is started with `--gpus all`, and the CacheRoute +profile should be `v1`. + +## KDN migration status + +The v1 branch contains the first runtime compatibility slice for KDN KV +construction: + +- modern model-scoped LMCache Redis keys are discovered automatically; +- legacy `vllm@*` keys remain supported; +- asynchronous remote writes use a first-key timeout and quiet-period finish; +- zero-key builds fail and are never marked `kv_ready`; +- raw key/value dump and restore were verified byte-for-byte in the migration + environment. + +This does not yet prove an end-to-end LMCache MP cache hit after injection. +Validation must compare LMCache hit-token and remote-read metrics before the PR +is considered complete. + +## Validation status and scope + +The image build from the environment work was validated with: + +- Python `3.12.13`; +- PyTorch `2.11.0+cu130`; +- vLLM `0.25.1`; +- LMCache `0.5.2`; +- Transformers `5.12.1`; +- Redis Python client `8.0.1`; +- successful `pip check`; +- successful imports of Torch, TorchCodec, vLLM, LMCache, and Redis; +- successful startup of the existing CacheRoute Scheduler, KDN, Proxy, and + Instance workflow after installing FFmpeg. + +The modern environment profile and the runtime compatibility work are additive. +The legacy image, root `requirements.txt`, Scheduler routing, and stable legacy +runtime remain unchanged. diff --git a/env/docker/cu130/constraints.txt b/env/docker/cu130/constraints.txt new file mode 100644 index 0000000..2ba2d36 --- /dev/null +++ b/env/docker/cu130/constraints.txt @@ -0,0 +1,23 @@ +# CacheRoute CUDA 13 serving profile. +# +# Keep the binary-sensitive serving stack exact. Application dependencies are +# installed separately from requirements-dev.txt under these constraints. + +torch==2.11.0 +torchvision==0.26.0 +torchaudio==2.11.0 +vllm==0.25.1 +lmcache==0.5.2 + +# Shared compatibility ranges required by the target serving stack. +transformers>=5.5.3,<6.0 +sentence-transformers>=5.6,<6.0 +huggingface-hub>=1.5,<2.0 +fastapi>=0.133,<0.137 +starlette>=1.0.1,<2.0 +aiohttp>=3.13.3,<4.0 +pydantic>=2.12,<3.0 +numpy>=1.26,<2.3 +setuptools>=77,<81 +requests>=2.33,<3.0 +redis>=5.1,<9 diff --git a/env/docker/cu130/requirements-dev.txt b/env/docker/cu130/requirements-dev.txt new file mode 100644 index 0000000..d52b755 --- /dev/null +++ b/env/docker/cu130/requirements-dev.txt @@ -0,0 +1,44 @@ +# CacheRoute application and development dependencies for the CUDA 13 profile. +# +# PyTorch, vLLM, and LMCache are intentionally installed by the Dockerfile and +# must not be added here. This file is additive and does not replace the legacy +# root requirements.txt profile. + +sentence-transformers>=5.6,<6.0 +faiss-cpu==1.13.1 + +fastapi>=0.133,<0.137 +uvicorn>=0.38,<0.52 +starlette>=1.0.1,<2.0 +pydantic>=2.12,<3.0 +httpx>=0.28,<0.29 +aiohttp>=3.13.3,<4.0 +requests>=2.33,<3.0 +redis>=5.1,<9 + +transformers>=5.5.3,<6.0 +huggingface-hub>=1.5,<2.0 +safetensors>=0.7,<0.9 +datasets>=4.4,<5.0 + +numpy>=1.26,<2.3 +scipy>=1.16,<1.17 +pandas>=2.3,<2.4 +scikit-learn>=1.7,<1.8 +matplotlib>=3.10,<3.11 + +jupyter_client>=8.6,<9.0 +pyzmq>=27.1,<28.0 + +pyyaml>=6.0,<7.0 +tqdm>=4.67,<5.0 +warcio>=1.7,<2.0 +beautifulsoup4>=4.14,<5.0 + +# Booktype 1.5 is intentionally excluded. It is a Python-2-era package that can +# overwrite the modern redis package with Python 2 source files. + +pytest>=9.0,<10.0 +black>=24,<27 +isort>=5.13,<9 +mypy>=1.11,<3 diff --git a/env/docker/cu130/scripts/start_lmcache_mp.sh b/env/docker/cu130/scripts/start_lmcache_mp.sh new file mode 100644 index 0000000..4afa183 --- /dev/null +++ b/env/docker/cu130/scripts/start_lmcache_mp.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -euo pipefail + +# CacheRoute v1 / LMCache 0.5.2 MP server startup. +# Run with: bash env/docker/cu130/scripts/start_lmcache_mp.sh + +export CACHEROUTE_RUNTIME_PROFILE="${CACHEROUTE_RUNTIME_PROFILE:-v1}" +export LMCACHE_LOG_LEVEL="${LMCACHE_LOG_LEVEL:-INFO}" +export PYTHONHASHSEED="${PYTHONHASHSEED:-0}" + +# The legacy YAML-based LMCache configuration must not leak into MP mode. +unset LMCACHE_CONFIG_FILE + +LMCACHE_INSTANCE_ID="${LMCACHE_INSTANCE_ID:-cacheroute-lmcache-1}" +LMCACHE_MP_HOST="${LMCACHE_MP_HOST:-127.0.0.1}" +LMCACHE_MP_PORT="${LMCACHE_MP_PORT:-5555}" +LMCACHE_HTTP_HOST="${LMCACHE_HTTP_HOST:-127.0.0.1}" +LMCACHE_HTTP_PORT="${LMCACHE_HTTP_PORT:-8080}" +LMCACHE_L1_SIZE_GB="${LMCACHE_L1_SIZE_GB:-80}" +LMCACHE_CHUNK_SIZE="${LMCACHE_CHUNK_SIZE:-256}" +LMCACHE_HASH_ALGORITHM="${LMCACHE_HASH_ALGORITHM:-sha256_cbor}" +LMCACHE_EVICTION_POLICY="${LMCACHE_EVICTION_POLICY:-LRU}" +LMCACHE_MAX_WORKERS="${LMCACHE_MAX_WORKERS:-8}" +LMCACHE_L2_STORE_POLICY="${LMCACHE_L2_STORE_POLICY:-default}" + +REDIS_HOST="${REDIS_HOST:-127.0.0.1}" +REDIS_PORT="${REDIS_PORT:-6379}" +LMCACHE_L2_WORKERS="${LMCACHE_L2_WORKERS:-8}" + +if [[ -n "${LMCACHE_L2_ADAPTER_JSON:-}" ]]; then + L2_ADAPTER_JSON="${LMCACHE_L2_ADAPTER_JSON}" +else + L2_ADAPTER_JSON="$(printf '{"type":"resp","host":"%s","port":%s,"num_workers":%s}' \ + "${REDIS_HOST}" "${REDIS_PORT}" "${LMCACHE_L2_WORKERS}")" +fi + +printf '[CacheRoute v1] starting LMCache MP server\n' +printf ' profile: %s\n' "${CACHEROUTE_RUNTIME_PROFILE}" +printf ' MP endpoint: %s:%s\n' "${LMCACHE_MP_HOST}" "${LMCACHE_MP_PORT}" +printf ' HTTP endpoint: %s:%s\n' "${LMCACHE_HTTP_HOST}" "${LMCACHE_HTTP_PORT}" +printf ' RESP L2: %s:%s\n' "${REDIS_HOST}" "${REDIS_PORT}" +printf ' chunk/hash: %s / %s\n' "${LMCACHE_CHUNK_SIZE}" "${LMCACHE_HASH_ALGORITHM}" + +exec lmcache server \ + --instance-id "${LMCACHE_INSTANCE_ID}" \ + --host "${LMCACHE_MP_HOST}" \ + --port "${LMCACHE_MP_PORT}" \ + --http-host "${LMCACHE_HTTP_HOST}" \ + --http-port "${LMCACHE_HTTP_PORT}" \ + --l1-size-gb "${LMCACHE_L1_SIZE_GB}" \ + --chunk-size "${LMCACHE_CHUNK_SIZE}" \ + --hash-algorithm "${LMCACHE_HASH_ALGORITHM}" \ + --eviction-policy "${LMCACHE_EVICTION_POLICY}" \ + --max-workers "${LMCACHE_MAX_WORKERS}" \ + --l2-store-policy "${LMCACHE_L2_STORE_POLICY}" \ + --l2-adapter "${L2_ADAPTER_JSON}" \ + "$@" diff --git a/env/docker/cu130/scripts/start_vllm_mp.sh b/env/docker/cu130/scripts/start_vllm_mp.sh new file mode 100644 index 0000000..b65433d --- /dev/null +++ b/env/docker/cu130/scripts/start_vllm_mp.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +set -euo pipefail + +# CacheRoute v1 / vLLM 0.25.1 startup with LMCacheMPConnector. +# Run LMCache MP first, then: +# bash env/docker/cu130/scripts/start_vllm_mp.sh + +export CACHEROUTE_RUNTIME_PROFILE="${CACHEROUTE_RUNTIME_PROFILE:-v1}" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +export MODEL_DIR="${MODEL_DIR:-/workspace/llm-stack/models/LLM-Research/Meta-Llama-3-70B-Instruct}" +export PYTHONHASHSEED="${PYTHONHASHSEED:-0}" +export OMP_NUM_THREADS="${OMP_NUM_THREADS:-8}" + +# The validated v1 path does not use the legacy YAML/offloading interface. +unset LMCACHE_CONFIG_FILE +unset PYTORCH_CUDA_ALLOC_CONF + +SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-llama3-70b}" +VLLM_HOST="${VLLM_HOST:-0.0.0.0}" +VLLM_PORT="${VLLM_PORT:-8000}" +TENSOR_PARALLEL_SIZE="${TENSOR_PARALLEL_SIZE:-8}" +GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.75}" +MODEL_DTYPE="${MODEL_DTYPE:-auto}" +MAX_MODEL_LEN="${MAX_MODEL_LEN:-4096}" +MAX_NUM_SEQS="${MAX_NUM_SEQS:-8}" +MAX_NUM_BATCHED_TOKENS="${MAX_NUM_BATCHED_TOKENS:-16384}" + +LMCACHE_MP_CONNECT_HOST="${LMCACHE_MP_CONNECT_HOST:-tcp://127.0.0.1}" +LMCACHE_MP_PORT="${LMCACHE_MP_PORT:-5555}" + +if [[ -n "${VLLM_KV_TRANSFER_CONFIG_JSON:-}" ]]; then + KV_TRANSFER_CONFIG_JSON="${VLLM_KV_TRANSFER_CONFIG_JSON}" +else + KV_TRANSFER_CONFIG_JSON="$(printf '{"kv_connector":"LMCacheMPConnector","kv_connector_module_path":"lmcache.integration.vllm.lmcache_mp_connector","kv_role":"kv_both","kv_connector_extra_config":{"lmcache.mp.host":"%s","lmcache.mp.port":%s}}' \ + "${LMCACHE_MP_CONNECT_HOST}" "${LMCACHE_MP_PORT}")" +fi + +printf '[CacheRoute v1] starting vLLM with LMCacheMPConnector\n' +printf ' profile: %s\n' "${CACHEROUTE_RUNTIME_PROFILE}" +printf ' model: %s (%s)\n' "${MODEL_DIR}" "${SERVED_MODEL_NAME}" +printf ' GPUs / TP: %s / %s\n' "${CUDA_VISIBLE_DEVICES}" "${TENSOR_PARALLEL_SIZE}" +printf ' LMCache MP: %s:%s\n' "${LMCACHE_MP_CONNECT_HOST}" "${LMCACHE_MP_PORT}" + +exec vllm serve "${MODEL_DIR}" \ + --served-model-name "${SERVED_MODEL_NAME}" \ + --host "${VLLM_HOST}" \ + --port "${VLLM_PORT}" \ + --tensor-parallel-size "${TENSOR_PARALLEL_SIZE}" \ + --gpu-memory-utilization "${GPU_MEMORY_UTILIZATION}" \ + --dtype "${MODEL_DTYPE}" \ + --max-model-len "${MAX_MODEL_LEN}" \ + --max-num-seqs "${MAX_NUM_SEQS}" \ + --max-num-batched-tokens "${MAX_NUM_BATCHED_TOKENS}" \ + --disable-hybrid-kv-cache-manager \ + --no-enable-prefix-caching \ + --kv-transfer-config "${KV_TRANSFER_CONFIG_JSON}" \ + --kv-cache-metrics \ + "$@" diff --git a/kdn_server/kv_builder.py b/kdn_server/kv_builder.py index 7dc911a..f989499 100644 --- a/kdn_server/kv_builder.py +++ b/kdn_server/kv_builder.py @@ -4,16 +4,21 @@ import base64 import json -import os import shutil import time from dataclasses import dataclass from pathlib import Path -from typing import Dict, Iterable, List, Optional, Tuple +from typing import Dict, Iterable, List, Optional import requests import redis +from core.runtime_compat import ( + classify_lmcache_redis_key, + filter_supported_keys, + normalize_runtime_profile, + resolve_scan_match, +) from .text_db import compute_kid, _normalize_text @@ -35,37 +40,40 @@ class KVBuildConfig: redis_db: int = 0 redis_password: Optional[str] = None - match: str = "vllm@*" # Previous dump key pattern. + # ``auto`` supports both the historical vllm@* layout and newer LMCache + # model-scoped key layouts. Explicit patterns remain supported. + match: str = "auto" + runtime_profile: Optional[str] = None scan_count: int = 1000 - settle_wait_s: float = 0.2 # Wait after request for keys to stabilize. - settle_rounds: int = 3 # Treat keys as stable after N consecutive rounds with unchanged key count. + settle_wait_s: float = 0.2 # Redis polling interval. + settle_rounds: int = 3 # Deprecated compatibility field. + first_key_timeout_s: float = 30.0 + quiet_period_s: float = 1.5 flushdb: bool = False # Warning: clears the entire Redis DB; dangerous. class KVCacheBuilder: - """ - Unified implementation of triggering inference, scanning Redis KV keys, and dumping them to KV_database//. - """ + """Trigger inference, capture LMCache Redis blocks, and persist them.""" def __init__(self, cfg: KVBuildConfig, text_db=None): self.cfg = cfg + self.runtime_profile = normalize_runtime_profile(cfg.runtime_profile) + self.scan_match = resolve_scan_match(self.runtime_profile, cfg.match) self.rds = redis.Redis( host=cfg.redis_host, port=cfg.redis_port, db=cfg.redis_db, password=cfg.redis_password, - decode_responses=False, # Must be False so key/value are handled as bytes. + decode_responses=False, ) self.text_db = text_db - def build_from_text_file(self, txt_path: str) -> Dict: p = Path(txt_path).resolve() content = p.read_text(encoding="utf-8") return self.build_from_text(content) - def build_from_text(self, text: str) -> Dict: norm = _normalize_text(text) if not norm: @@ -74,60 +82,88 @@ def build_from_text(self, text: str) -> Dict: kid = compute_kid(norm) out_dir = Path(self.cfg.kv_root).resolve() / kid - # Overwrite refresh: delete the old directory. if out_dir.exists(): shutil.rmtree(out_dir) (out_dir / "blocks").mkdir(parents=True, exist_ok=True) - if self.cfg.flushdb: - # High risk: affects other tasks in the same DB. - self.rds.flushdb() - - # (A) Record before state. - keys_before = self._scan_keys_set() - - # (B) Trigger one inference request. - self._trigger_infer(norm) - - # (C) Wait for stability and capture after state. - keys_after = self._wait_keys_settle_set() - - # (D) Diff: dump only newly added keys. - keys_new = list(keys_after - keys_before) - - # 3) dump keys -> files - manifest_path = out_dir / "manifest.jsonl" - dumped = self._dump_keys(keys_new, out_dir / "blocks", manifest_path) - - # 4) Write run_meta. - meta = { - "kid": kid, - "time": int(time.time()), - "api_url": self.cfg.api_url, - "model": self.cfg.model, - "max_tokens": self.cfg.max_tokens, - "temperature": self.cfg.temperature, - "redis": { - "host": self.cfg.redis_host, - "port": self.cfg.redis_port, - "db": self.cfg.redis_db, - "match": self.cfg.match, - }, - "dumped_keys": dumped, - "keys_before": len(keys_before), - "keys_after": len(keys_after), - "keys_new": len(keys_new), - } - (out_dir / "run_meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2), encoding="utf-8") - - if self.text_db is not None: - self.text_db.mark_kv_ready(kid=kid, kv_rel_dir=kid, dumped_keys=dumped, updated_at=meta["time"]) - - return {"kid": kid, "kv_dir": str(out_dir), "dumped_keys": dumped} - + wait_started = time.monotonic() + try: + if self.cfg.flushdb: + self.rds.flushdb() + + keys_before = self._scan_keys_set() + self._trigger_infer(norm) + keys_after = self._wait_keys_settle_set(keys_before) + keys_new = sorted(keys_after - keys_before) + + manifest_path = out_dir / "manifest.jsonl" + dumped = self._dump_keys(keys_new, out_dir / "blocks", manifest_path) + if dumped <= 0: + raise RuntimeError( + "KV build failed: no LMCache Redis keys were dumped; " + f"profile={self.runtime_profile!r}, requested_match={self.cfg.match!r}, " + f"scan_match={self.scan_match!r}, keys_before={len(keys_before)}, " + f"keys_after={len(keys_after)}, keys_new={len(keys_new)}" + ) + + key_formats = sorted({ + classify_lmcache_redis_key(key) or "explicit" + for key in keys_new + }) + meta = { + "kid": kid, + "time": int(time.time()), + "api_url": self.cfg.api_url, + "model": self.cfg.model, + "max_tokens": self.cfg.max_tokens, + "temperature": self.cfg.temperature, + "runtime_profile": self.runtime_profile, + "key_formats": key_formats, + "redis": { + "host": self.cfg.redis_host, + "port": self.cfg.redis_port, + "db": self.cfg.redis_db, + "requested_match": self.cfg.match, + "scan_match": self.scan_match, + }, + "capture": { + "poll_interval_s": self.cfg.settle_wait_s, + "first_key_timeout_s": self.cfg.first_key_timeout_s, + "quiet_period_s": self.cfg.quiet_period_s, + "elapsed_s": round(time.monotonic() - wait_started, 3), + }, + "dumped_keys": dumped, + "keys_before": len(keys_before), + "keys_after": len(keys_after), + "keys_new": len(keys_new), + } + (out_dir / "run_meta.json").write_text( + json.dumps(meta, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + if self.text_db is not None: + self.text_db.mark_kv_ready( + kid=kid, + kv_rel_dir=kid, + dumped_keys=dumped, + updated_at=meta["time"], + ) + + return { + "kid": kid, + "kv_dir": str(out_dir), + "dumped_keys": dumped, + "runtime_profile": self.runtime_profile, + "key_formats": key_formats, + } + except Exception: + # Never leave an empty/partial KV directory that could later be + # mistaken for a ready knowledge block. + shutil.rmtree(out_dir, ignore_errors=True) + raise def _trigger_infer(self, prompt: str) -> None: - # Preserve previous use of /v1/chat/completions here. payload = { "model": self.cfg.model, "messages": [{"role": "system", "content": prompt}], @@ -135,80 +171,106 @@ def _trigger_infer(self, prompt: str) -> None: "temperature": self.cfg.temperature, "stream": False, } - r = requests.post(self.cfg.api_url, json=payload, timeout=300) - r.raise_for_status() + response = requests.post(self.cfg.api_url, json=payload, timeout=300) + response.raise_for_status() def _scan_keys_set(self) -> set[bytes]: - return set(self._scan_keys()) - - def _wait_keys_settle_set(self) -> set[bytes]: - last_n = -1 - stable = 0 - keys_set: set[bytes] = set() - - for _ in range(max(1, self.cfg.settle_rounds * 3)): - time.sleep(self.cfg.settle_wait_s) - keys_set = self._scan_keys_set() - n = len(keys_set) - if n == last_n: - stable += 1 - if stable >= self.cfg.settle_rounds: - break - else: - stable = 0 - last_n = n - return keys_set + raw_keys = set(self._scan_keys()) + return filter_supported_keys( + raw_keys, + profile=self.runtime_profile, + requested_match=self.cfg.match, + ) + + def _wait_keys_settle_set(self, keys_before: set[bytes]) -> set[bytes]: + """Wait for asynchronous LMCache remote writes without fixed delay. + + Empty scans never count as stable. Once new keys appear, the function + returns after the set remains unchanged for ``quiet_period_s``. The + timeout is an upper bound, not a per-registration sleep. + """ + timeout_s = max(0.1, float(self.cfg.first_key_timeout_s)) + quiet_period_s = max(0.0, float(self.cfg.quiet_period_s)) + poll_s = max(0.01, float(self.cfg.settle_wait_s)) + deadline = time.monotonic() + timeout_s + + latest_keys = set(keys_before) + last_new_keys: Optional[set[bytes]] = None + last_change_ts: Optional[float] = None + + while time.monotonic() < deadline: + time.sleep(poll_s) + latest_keys = self._scan_keys_set() + new_keys = latest_keys - keys_before + now = time.monotonic() + + if not new_keys: + continue + + if last_new_keys is None or new_keys != last_new_keys: + last_new_keys = set(new_keys) + last_change_ts = now + continue + + if last_change_ts is not None and now - last_change_ts >= quiet_period_s: + return latest_keys + + dbsize = None + try: + dbsize = int(self.rds.dbsize()) + except Exception: + pass + + if not last_new_keys: + raise TimeoutError( + "LMCache produced no matching Redis keys before timeout; " + f"timeout_s={timeout_s}, profile={self.runtime_profile!r}, " + f"requested_match={self.cfg.match!r}, scan_match={self.scan_match!r}, " + f"redis={self.cfg.redis_host}:{self.cfg.redis_port}/{self.cfg.redis_db}, " + f"dbsize={dbsize}" + ) + + raise TimeoutError( + "LMCache Redis keys appeared but did not become quiet before timeout; " + f"timeout_s={timeout_s}, captured_new_keys={len(last_new_keys)}" + ) def _scan_keys(self) -> List[bytes]: - # SCAN match pattern cursor = 0 out: List[bytes] = [] while True: - cursor, batch = self.rds.scan(cursor=cursor, match=self.cfg.match, count=self.cfg.scan_count) + cursor, batch = self.rds.scan( + cursor=cursor, + match=self.scan_match, + count=self.cfg.scan_count, + ) out.extend(batch) if cursor == 0: break return out - def _wait_keys_settle(self) -> List[bytes]: - last_n = -1 - stable = 0 - keys: List[bytes] = [] - - for _ in range(max(1, self.cfg.settle_rounds * 3)): - time.sleep(self.cfg.settle_wait_s) - keys = self._scan_keys() - n = len(keys) - if n == last_n: - stable += 1 - if stable >= self.cfg.settle_rounds: - break - else: - stable = 0 - last_n = n - return keys - + """Deprecated compatibility wrapper for older direct callers.""" + before = self._scan_keys_set() + return list(self._wait_keys_settle_set(before)) def _dump_keys(self, keys: Iterable[bytes], blocks_dir: Path, manifest_path: Path) -> int: dumped = 0 with manifest_path.open("w", encoding="utf-8") as mf: - for k in keys: - v = self.rds.get(k) - if v is None: + for key in keys: + value = self.rds.get(key) + if value is None: continue - fname = _b64url(k) + ".dump" + fname = _b64url(key) + ".dump" fpath = blocks_dir / fname - # Persist bytes as-is. - fpath.write_bytes(v) + fpath.write_bytes(value) rec = { - "key_b64url": _b64url(k), + "key_b64url": _b64url(key), "file": f"blocks/{fname}", - "bytes": len(v), + "bytes": len(value), } mf.write(json.dumps(rec, ensure_ascii=False) + "\n") dumped += 1 return dumped - diff --git a/tests/test_runtime_compat.py b/tests/test_runtime_compat.py new file mode 100644 index 0000000..cd2cea7 --- /dev/null +++ b/tests/test_runtime_compat.py @@ -0,0 +1,39 @@ +from core.runtime_compat import ( + classify_lmcache_redis_key, + filter_supported_keys, + normalize_runtime_profile, + resolve_scan_match, +) + + +def test_profile_aliases(): + assert normalize_runtime_profile("old") == "legacy" + assert normalize_runtime_profile("modern") == "v1" + + +def test_historic_default_match_is_profile_aware(): + assert resolve_scan_match("auto", "vllm@*") == "*" + assert resolve_scan_match("v1", "vllm@*") == "*" + assert resolve_scan_match("legacy", "vllm@*") == "vllm@*" + + +def test_classifies_legacy_and_v1_keys(): + legacy = b"vllm@0123456789abcdef" + modern = ( + b"/workspace/llm-stack/models/Meta-Llama-3-70B-Instruct" + b"@08000800@@d51d1eac7122c3b998e188f97d30caedd7a4dcfad7d5c91187097efee4eaf9d0" + ) + assert classify_lmcache_redis_key(legacy) == "legacy" + assert classify_lmcache_redis_key(modern) == "v1" + assert classify_lmcache_redis_key(b"unrelated") is None + + +def test_auto_filter_keeps_both_generations(): + legacy = b"vllm@0123456789abcdef" + modern = b"/model@1@0@" + b"a" * 64 + b"@bfloat16" + unrelated = b"healthcheck" + assert filter_supported_keys( + {legacy, modern, unrelated}, + profile="auto", + requested_match="auto", + ) == {legacy, modern}