From 1cfef32c7202ff5f5c82e45ff00046c2c6739104 Mon Sep 17 00:00:00 2001
From: RickZ <121943721+rickisba@users.noreply.github.com>
Date: Tue, 28 Jul 2026 10:23:27 +0800
Subject: [PATCH 1/7] env: add reusable v1 shell activation

---
 env/docker/cu130/scripts/activate_v1.sh | 35 +++++++++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 env/docker/cu130/scripts/activate_v1.sh

diff --git a/env/docker/cu130/scripts/activate_v1.sh b/env/docker/cu130/scripts/activate_v1.sh
new file mode 100644
index 0000000..a5f99d6
--- /dev/null
+++ b/env/docker/cu130/scripts/activate_v1.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+# Source this file in every terminal that starts a modern CacheRoute component:
+#   source env/docker/cu130/scripts/activate_v1.sh
+
+if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
+  printf 'This script must be sourced, not executed.\n' >&2
+  printf 'Run: source env/docker/cu130/scripts/activate_v1.sh\n' >&2
+  exit 2
+fi
+
+_SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+_REPO_ROOT="$(cd -- "${_SCRIPT_DIR}/../../../.." && pwd)"
+
+export CACHEROUTE_RUNTIME_PROFILE=v1
+export PYTHONHASHSEED="${PYTHONHASHSEED:-0}"
+export OMP_NUM_THREADS="${OMP_NUM_THREADS:-8}"
+
+case ":${PYTHONPATH:-}:" in
+  *":${_REPO_ROOT}:"*) ;;
+  *) export PYTHONPATH="${_REPO_ROOT}${PYTHONPATH:+:${PYTHONPATH}}" ;;
+esac
+
+# The modern MP path must not inherit legacy LMCache/vLLM settings.
+unset LMCACHE_CONFIG_FILE
+unset PYTORCH_CUDA_ALLOC_CONF
+
+printf '[CacheRoute v1] environment activated\n'
+printf '  repository: %s\n' "${_REPO_ROOT}"
+printf '  profile: %s\n' "${CACHEROUTE_RUNTIME_PROFILE}"
+printf '  PYTHONPATH: %s\n' "${PYTHONPATH}"
+printf '  PYTHONHASHSEED / OMP_NUM_THREADS: %s / %s\n' \
+  "${PYTHONHASHSEED}" "${OMP_NUM_THREADS}"
+
+unset _SCRIPT_DIR _REPO_ROOT

From 25022362a79b6d177a8f3e74652e67c45d99cce7 Mon Sep 17 00:00:00 2001
From: RickZ <121943721+rickisba@users.noreply.github.com>
Date: Tue, 28 Jul 2026 10:24:17 +0800
Subject: [PATCH 2/7] env: add non-destructive v1 environment check

---
 .../cu130/scripts/check_v1_environment.py     | 259 ++++++++++++++++++
 1 file changed, 259 insertions(+)
 create mode 100644 env/docker/cu130/scripts/check_v1_environment.py

diff --git a/env/docker/cu130/scripts/check_v1_environment.py b/env/docker/cu130/scripts/check_v1_environment.py
new file mode 100644
index 0000000..8b4cfd5
--- /dev/null
+++ b/env/docker/cu130/scripts/check_v1_environment.py
@@ -0,0 +1,259 @@
+#!/usr/bin/env python3
+"""Non-destructive validation for the CacheRoute v1 serving environment."""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from importlib import metadata
+from pathlib import Path
+from typing import Any
+from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
+
+REPO_ROOT = Path(__file__).resolve().parents[4]
+TARGET_VERSIONS = {
+    "torch": "2.11.0",
+    "vllm": "0.25.1",
+    "lmcache": "0.5.2",
+}
+
+
+class Report:
+    def __init__(self) -> None:
+        self.errors: list[str] = []
+        self.warnings: list[str] = []
+        self.details: dict[str, Any] = {}
+
+    def ok(self, message: str) -> None:
+        print(f"[OK] {message}")
+
+    def warn(self, message: str) -> None:
+        self.warnings.append(message)
+        print(f"[WARN] {message}")
+
+    def error(self, message: str) -> None:
+        self.errors.append(message)
+        print(f"[ERROR] {message}")
+
+
+def normalized_version(value: str) -> str:
+    return value.split("+", 1)[0]
+
+
+def http_get(url: str, timeout: float = 5.0) -> tuple[int, str]:
+    request = Request(url, headers={"User-Agent": "CacheRoute-v1-check"})
+    try:
+        with urlopen(request, timeout=timeout) as response:
+            return int(response.status), response.read().decode("utf-8", errors="replace")
+    except HTTPError as exc:
+        return int(exc.code), exc.read().decode("utf-8", errors="replace")
+
+
+def check_http(
+    report: Report,
+    name: str,
+    url: str,
+    marker: str | None,
+    require_running: bool,
+) -> None:
+    try:
+        status, body = http_get(url)
+    except (URLError, TimeoutError, OSError) as exc:
+        message = f"{name} unavailable at {url}: {exc}"
+        if require_running:
+            report.error(message)
+        else:
+            report.warn(message)
+        return
+
+    if status != 200:
+        message = f"{name} returned HTTP {status} at {url}"
+        if require_running:
+            report.error(message)
+        else:
+            report.warn(message)
+        return
+
+    if marker and marker not in body:
+        message = f"{name} is reachable but marker {marker!r} is absent"
+        if require_running:
+            report.error(message)
+        else:
+            report.warn(message)
+        return
+
+    report.ok(f"{name}: HTTP 200 ({len(body)} bytes)")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--model-dir",
+        default=os.getenv(
+            "MODEL_DIR",
+            "/workspace/llm-stack/models/LLM-Research/Meta-Llama-3-70B-Instruct",
+        ),
+    )
+    parser.add_argument("--redis-host", default=os.getenv("REDIS_HOST", "127.0.0.1"))
+    parser.add_argument("--redis-port", type=int, default=int(os.getenv("REDIS_PORT", "6379")))
+    parser.add_argument("--redis-db", type=int, default=int(os.getenv("REDIS_DB", "0")))
+    parser.add_argument("--vllm-url", default="http://127.0.0.1:8000")
+    parser.add_argument("--lmcache-http-url", default="http://127.0.0.1:8080")
+    parser.add_argument("--expected-gpus", type=int, default=None)
+    parser.add_argument("--allow-no-gpu", action="store_true")
+    parser.add_argument("--skip-redis", action="store_true")
+    parser.add_argument(
+        "--require-running",
+        action="store_true",
+        help="Treat unavailable Redis/LMCache/vLLM services as errors.",
+    )
+    args = parser.parse_args()
+
+    report = Report()
+    print("===== CacheRoute v1 environment check =====")
+
+    report.details["repo_root"] = str(REPO_ROOT)
+    if (REPO_ROOT / "core" / "runtime_compat.py").is_file():
+        report.ok(f"repository root: {REPO_ROOT}")
+    else:
+        report.error(f"repository root is invalid: {REPO_ROOT}")
+
+    profile = os.getenv("CACHEROUTE_RUNTIME_PROFILE", "")
+    report.details["runtime_profile"] = profile
+    if profile == "v1":
+        report.ok("CACHEROUTE_RUNTIME_PROFILE=v1")
+    else:
+        report.error(
+            "CACHEROUTE_RUNTIME_PROFILE must be v1; source "
+            "env/docker/cu130/scripts/activate_v1.sh first"
+        )
+
+    python_version = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
+    report.details["python"] = python_version
+    if sys.version_info >= (3, 12):
+        report.ok(f"Python {python_version}: {sys.executable}")
+    else:
+        report.error(f"Python >=3.12 required, found {python_version}")
+
+    for package, expected in TARGET_VERSIONS.items():
+        try:
+            actual = metadata.version(package)
+        except metadata.PackageNotFoundError:
+            report.error(f"missing package: {package}=={expected}")
+            continue
+        report.details[package] = actual
+        if normalized_version(actual) == expected:
+            report.ok(f"{package} {actual}")
+        else:
+            report.error(f"{package} expected {expected}, found {actual}")
+
+    try:
+        import torch
+    except Exception as exc:
+        report.error(f"torch import failed: {exc}")
+    else:
+        cuda_runtime = str(torch.version.cuda)
+        cuda_available = bool(torch.cuda.is_available())
+        gpu_count = int(torch.cuda.device_count())
+        report.details.update(
+            {
+                "torch_cuda_runtime": cuda_runtime,
+                "cuda_available": cuda_available,
+                "gpu_count": gpu_count,
+            }
+        )
+        if cuda_runtime.startswith("13.0"):
+            report.ok(f"PyTorch CUDA runtime {cuda_runtime}")
+        else:
+            report.error(f"expected PyTorch CUDA runtime 13.0, found {cuda_runtime}")
+
+        if cuda_available:
+            report.ok(f"CUDA available; visible GPU count={gpu_count}")
+        elif args.allow_no_gpu:
+            report.warn("CUDA is unavailable, accepted by --allow-no-gpu")
+        else:
+            report.error("CUDA is unavailable; run the container with --gpus all")
+
+        expected_gpus = args.expected_gpus
+        if expected_gpus is None:
+            visible = os.getenv("CUDA_VISIBLE_DEVICES", "").strip()
+            expected_gpus = len([item for item in visible.split(",") if item.strip()]) if visible else 8
+        if cuda_available and gpu_count < expected_gpus:
+            report.error(f"expected at least {expected_gpus} visible GPUs, found {gpu_count}")
+
+    model_dir = Path(args.model_dir)
+    report.details["model_dir"] = str(model_dir)
+    if model_dir.is_dir():
+        report.ok(f"model directory exists: {model_dir}")
+    else:
+        report.error(f"model directory not found: {model_dir}")
+
+    required_scripts = [
+        REPO_ROOT / "env/docker/cu130/scripts/start_lmcache_mp.sh",
+        REPO_ROOT / "env/docker/cu130/scripts/start_vllm_mp.sh",
+        REPO_ROOT / "scripts/validate_v1_kdn_roundtrip.py",
+    ]
+    for script in required_scripts:
+        if script.is_file():
+            report.ok(f"script exists: {script.relative_to(REPO_ROOT)}")
+        else:
+            report.error(f"script missing: {script.relative_to(REPO_ROOT)}")
+
+    if not args.skip_redis:
+        try:
+            import redis
+
+            client = redis.Redis(
+                host=args.redis_host,
+                port=args.redis_port,
+                db=args.redis_db,
+                decode_responses=False,
+                socket_connect_timeout=3,
+                socket_timeout=5,
+            )
+            pong = bool(client.ping())
+            size = int(client.dbsize())
+            report.details["redis_dbsize"] = size
+            if pong:
+                report.ok(
+                    f"Redis {args.redis_host}:{args.redis_port}/{args.redis_db}; DBSIZE={size}"
+                )
+        except Exception as exc:
+            message = f"Redis unavailable: {exc}"
+            if args.require_running:
+                report.error(message)
+            else:
+                report.warn(message)
+
+    check_http(
+        report,
+        "vLLM model endpoint",
+        f"{args.vllm_url.rstrip('/')}/v1/models",
+        None,
+        args.require_running,
+    )
+    check_http(
+        report,
+        "vLLM metrics",
+        f"{args.vllm_url.rstrip('/')}/metrics",
+        "vllm:external_prefix_cache_queries_total",
+        args.require_running,
+    )
+    check_http(
+        report,
+        "LMCache MP metrics",
+        f"{args.lmcache_http_url.rstrip('/')}/metrics",
+        "lmcache_mp_l2_adapters",
+        args.require_running,
+    )
+
+    print("\n===== Summary =====")
+    print(json.dumps(report.details, ensure_ascii=False, indent=2, sort_keys=True))
+    print(f"errors={len(report.errors)} warnings={len(report.warnings)}")
+    return 1 if report.errors else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

From 99bef7889218eaf9febff5da1103d482e9800c1a Mon Sep 17 00:00:00 2001
From: RickZ <121943721+rickisba@users.noreply.github.com>
Date: Tue, 28 Jul 2026 10:25:57 +0800
Subject: [PATCH 3/7] test: add staged v1 KDN round-trip validator

---
 scripts/validate_v1_kdn_roundtrip.py | 505 +++++++++++++++++++++++++++
 1 file changed, 505 insertions(+)
 create mode 100644 scripts/validate_v1_kdn_roundtrip.py

diff --git a/scripts/validate_v1_kdn_roundtrip.py b/scripts/validate_v1_kdn_roundtrip.py
new file mode 100644
index 0000000..40688af
--- /dev/null
+++ b/scripts/validate_v1_kdn_roundtrip.py
@@ -0,0 +1,505 @@
+#!/usr/bin/env python3
+"""Build, inspect, inject, and consume CacheRoute v1 KDN artifacts.
+
+The subcommands are deliberately staged because a reliable L2-consumption test
+requires LMCache and vLLM to be restarted after Redis restore, clearing L1/GPU
+state before ``consume``.
+"""
+from __future__ import annotations
+
+import argparse
+import base64
+import hashlib
+import json
+import sys
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterable
+from urllib.error import HTTPError
+from urllib.request import Request, urlopen
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+import redis  # noqa: E402
+
+from kdn_server.kv_injector import KVCacheInjector  # noqa: E402
+from kdn_server.text_db import _normalize_text, compute_kid  # noqa: E402
+
+VLLM_METRICS = (
+    "vllm:external_prefix_cache_queries_total",
+    "vllm:external_prefix_cache_hits_total",
+    "vllm:prompt_tokens_cached_total",
+    "vllm:prompt_tokens_total",
+)
+LMCACHE_METRICS = (
+    "lmcache_mp_lookup_requested_tokens_total",
+    "lmcache_mp_lookup_hit_tokens_total",
+    "lmcache_mp_l2_prefetch_lookup_requests_total",
+    "lmcache_mp_l2_prefetch_lookup_objects_chunks_total",
+    "lmcache_mp_l2_prefetch_hit_chunks_total",
+    "lmcache_mp_l2_prefetch_load_submitted_requests_total",
+    "lmcache_mp_l2_prefetch_load_submitted_objects_chunks_total",
+    "lmcache_mp_l2_prefetch_load_completed_chunks_total",
+    "lmcache_mp_l2_load_completed_requests_total",
+    "lmcache_mp_l2_prefetch_failure_chunks_total",
+    "lmcache_mp_l1_read_chunks_total",
+    "lmcache_mp_l1_write_chunks_total",
+)
+
+
+@dataclass(frozen=True)
+class ManifestRecord:
+    key: bytes
+    file: Path
+    size: int
+
+
+def http_json(url: str, payload: dict[str, Any], timeout: float) -> dict[str, Any]:
+    data = json.dumps(payload).encode("utf-8")
+    request = Request(
+        url,
+        data=data,
+        headers={"Content-Type": "application/json", "User-Agent": "CacheRoute-v1-validator"},
+        method="POST",
+    )
+    try:
+        with urlopen(request, timeout=timeout) as response:
+            body = response.read().decode("utf-8", errors="replace")
+            if response.status < 200 or response.status >= 300:
+                raise RuntimeError(f"HTTP {response.status}: {body[:1000]}")
+    except HTTPError as exc:
+        body = exc.read().decode("utf-8", errors="replace")
+        raise RuntimeError(f"HTTP {exc.code} from {url}: {body[:1000]}") from exc
+    value = json.loads(body)
+    if not isinstance(value, dict):
+        raise RuntimeError(f"expected JSON object from {url}, got {type(value).__name__}")
+    return value
+
+
+def http_text(url: str, timeout: float = 20.0) -> str:
+    request = Request(url, headers={"User-Agent": "CacheRoute-v1-validator"})
+    with urlopen(request, timeout=timeout) as response:
+        return response.read().decode("utf-8", errors="replace")
+
+
+def document_identity(path: Path) -> tuple[str, str]:
+    raw = path.read_text(encoding="utf-8")
+    normalized = _normalize_text(raw)
+    if not normalized:
+        raise ValueError(f"document is empty after normalization: {path}")
+    return normalized, compute_kid(normalized)
+
+
+def decode_key(value: str) -> bytes:
+    return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4))
+
+
+def load_manifest(kv_dir: Path) -> list[ManifestRecord]:
+    manifest = kv_dir / "manifest.jsonl"
+    if not manifest.is_file():
+        raise FileNotFoundError(f"manifest not found: {manifest}")
+    records: list[ManifestRecord] = []
+    with manifest.open("r", encoding="utf-8") as handle:
+        for line_no, line in enumerate(handle, start=1):
+            if not line.strip():
+                continue
+            value = json.loads(line)
+            key_b64 = value.get("key_b64url")
+            rel_file = value.get("file")
+            size = value.get("bytes")
+            if not isinstance(key_b64, str) or not isinstance(rel_file, str):
+                raise ValueError(f"invalid manifest record at line {line_no}: {value}")
+            file_path = kv_dir / rel_file
+            if not file_path.is_file():
+                raise FileNotFoundError(f"missing dump at line {line_no}: {file_path}")
+            actual_size = file_path.stat().st_size
+            expected_size = int(size) if size is not None else actual_size
+            if actual_size != expected_size:
+                raise ValueError(
+                    f"dump size mismatch at line {line_no}: expected={expected_size} actual={actual_size}"
+                )
+            records.append(ManifestRecord(decode_key(key_b64), file_path, expected_size))
+    if not records:
+        raise ValueError(f"manifest has no records: {manifest}")
+    return records
+
+
+def resolve_artifact(document: Path, kv_root: Path) -> tuple[str, str, Path, list[ManifestRecord]]:
+    normalized, kid = document_identity(document)
+    kv_dir = kv_root / kid
+    records = load_manifest(kv_dir)
+    return normalized, kid, kv_dir, records
+
+
+def print_json(title: str, value: dict[str, Any]) -> None:
+    print(f"\n===== {title} =====")
+    print(json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True))
+
+
+def inspect_artifact(document: Path, kv_root: Path) -> dict[str, Any]:
+    normalized, kid, kv_dir, records = resolve_artifact(document, kv_root)
+    run_meta_path = kv_dir / "run_meta.json"
+    run_meta = json.loads(run_meta_path.read_text(encoding="utf-8")) if run_meta_path.is_file() else {}
+    dump_files = list((kv_dir / "blocks").glob("*.dump"))
+    payload_bytes = sum(record.size for record in records)
+    errors: list[str] = []
+
+    if len(dump_files) != len(records):
+        errors.append(f"manifest records={len(records)} but dump files={len(dump_files)}")
+    if run_meta:
+        if int(run_meta.get("dumped_keys", -1)) != len(records):
+            errors.append(
+                f"run_meta dumped_keys={run_meta.get('dumped_keys')} but manifest={len(records)}"
+            )
+        if run_meta.get("runtime_profile") != "v1":
+            errors.append(f"run_meta runtime_profile={run_meta.get('runtime_profile')!r}, expected 'v1'")
+        if "v1" not in (run_meta.get("key_formats") or []):
+            errors.append(f"run_meta key_formats={run_meta.get('key_formats')!r} lacks 'v1'")
+
+    result = {
+        "ok": not errors,
+        "document": str(document),
+        "document_bytes": document.stat().st_size,
+        "normalized_characters": len(normalized),
+        "kid": kid,
+        "kv_dir": str(kv_dir),
+        "manifest_records": len(records),
+        "dump_files": len(dump_files),
+        "payload_bytes": payload_bytes,
+        "run_meta": run_meta,
+        "errors": errors,
+    }
+    print_json("ARTIFACT INSPECTION", result)
+    if errors:
+        raise RuntimeError("artifact inspection failed")
+    return result
+
+
+def redis_client(args: argparse.Namespace) -> redis.Redis:
+    return redis.Redis(
+        host=args.redis_host,
+        port=args.redis_port,
+        db=args.redis_db,
+        password=args.redis_password,
+        decode_responses=False,
+        socket_connect_timeout=5,
+        socket_timeout=120,
+    )
+
+
+def sample_indices(length: int, count: int) -> list[int]:
+    if count <= 0 or length <= 0:
+        return []
+    if count >= length:
+        return list(range(length))
+    if count == 1:
+        return [0]
+    return sorted({round(index * (length - 1) / (count - 1)) for index in range(count)})
+
+
+def inject_artifact(args: argparse.Namespace) -> dict[str, Any]:
+    _, kid, kv_dir, records = resolve_artifact(args.document, args.kv_root)
+    client = redis_client(args)
+    if not client.ping():
+        raise RuntimeError("Redis ping failed")
+    before_size = int(client.dbsize())
+
+    if args.flush_redis:
+        if not args.yes:
+            raise RuntimeError("--flush-redis requires --yes because it deletes the entire selected DB")
+        client.flushdb()
+    elif before_size and not args.allow_extra_keys:
+        raise RuntimeError(
+            f"Redis DB contains {before_size} keys; use an empty DB, --flush-redis --yes, "
+            "or explicitly allow extras with --allow-extra-keys"
+        )
+
+    injector = KVCacheInjector(
+        redis_host=args.redis_host,
+        redis_port=args.redis_port,
+        redis_db=args.redis_db,
+        redis_password=args.redis_password,
+        socket_timeout_s=120,
+    )
+    started = time.perf_counter()
+    injected = injector.inject_kv_dir(str(kv_dir), return_keys=False)
+    elapsed = time.perf_counter() - started
+
+    expected_keys = {record.key for record in records}
+    actual_keys = set(client.scan_iter(match=b"*", count=1000))
+    missing_keys = expected_keys - actual_keys
+    extra_keys = actual_keys - expected_keys
+    size_mismatches: list[dict[str, Any]] = []
+    actual_payload_bytes = 0
+    for record in records:
+        if record.key not in actual_keys:
+            continue
+        actual_size = int(client.strlen(record.key))
+        actual_payload_bytes += actual_size
+        if actual_size != record.size:
+            size_mismatches.append(
+                {"key": repr(record.key), "expected": record.size, "actual": actual_size}
+            )
+
+    hash_checks: list[dict[str, Any]] = []
+    for index in sample_indices(len(records), args.sample_hashes):
+        record = records[index]
+        expected = record.file.read_bytes()
+        actual = client.get(record.key)
+        hash_checks.append(
+            {
+                "index": index,
+                "bytes": record.size,
+                "match": actual is not None
+                and hashlib.sha256(expected).digest() == hashlib.sha256(actual).digest(),
+            }
+        )
+
+    expected_payload_bytes = sum(record.size for record in records)
+    ok = (
+        injected.injected == len(records)
+        and injected.missing_files == 0
+        and not missing_keys
+        and (args.allow_extra_keys or not extra_keys)
+        and not size_mismatches
+        and actual_payload_bytes == expected_payload_bytes
+        and all(item["match"] for item in hash_checks)
+    )
+    result = {
+        "ok": ok,
+        "kid": kid,
+        "redis": f"{args.redis_host}:{args.redis_port}/{args.redis_db}",
+        "dbsize_before": before_size,
+        "dbsize_after": int(client.dbsize()),
+        "injected": injected.injected,
+        "missing_files": injected.missing_files,
+        "manifest_records": len(records),
+        "missing_keys": len(missing_keys),
+        "extra_keys": len(extra_keys),
+        "size_mismatches": size_mismatches,
+        "expected_payload_bytes": expected_payload_bytes,
+        "actual_payload_bytes": actual_payload_bytes,
+        "sample_sha256": hash_checks,
+        "elapsed_seconds": round(elapsed, 3),
+    }
+    print_json("REDIS INJECTION", result)
+    if not ok:
+        raise RuntimeError("Redis injection verification failed")
+    return result
+
+
+def parse_prometheus(text: str, names: Iterable[str]) -> dict[str, float]:
+    result = {name: 0.0 for name in names}
+    for raw_line in text.splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith("#"):
+            continue
+        for name in result:
+            if line.startswith(name + "{") or line.startswith(name + " "):
+                try:
+                    result[name] += float(line.rsplit(None, 1)[1])
+                except (IndexError, ValueError):
+                    pass
+                break
+    return result
+
+
+def metric_delta(before: dict[str, float], after: dict[str, float]) -> dict[str, float]:
+    return {name: after[name] - before[name] for name in before}
+
+
+def consume_artifact(args: argparse.Namespace) -> dict[str, Any]:
+    normalized, kid, _, records = resolve_artifact(args.document, args.kv_root)
+    client = redis_client(args)
+    dbsize = int(client.dbsize())
+    if dbsize <= 0:
+        raise RuntimeError("Redis DB is empty; run the inject stage first")
+
+    vllm_metrics_url = f"{args.vllm_url.rstrip('/')}/metrics"
+    lmcache_metrics_url = f"{args.lmcache_http_url.rstrip('/')}/metrics"
+    vllm_before = parse_prometheus(http_text(vllm_metrics_url), VLLM_METRICS)
+    lmcache_before = parse_prometheus(http_text(lmcache_metrics_url), LMCACHE_METRICS)
+
+    payload = {
+        "model": args.model,
+        "messages": [{"role": "system", "content": normalized}],
+        "max_tokens": 1,
+        "temperature": 0.0,
+        "stream": False,
+    }
+    started = time.perf_counter()
+    response = http_json(
+        f"{args.vllm_url.rstrip('/')}/v1/chat/completions",
+        payload,
+        timeout=args.request_timeout,
+    )
+    elapsed = time.perf_counter() - started
+    time.sleep(args.metrics_wait)
+
+    vllm_after = parse_prometheus(http_text(vllm_metrics_url), VLLM_METRICS)
+    lmcache_after = parse_prometheus(http_text(lmcache_metrics_url), LMCACHE_METRICS)
+    vllm_delta = metric_delta(vllm_before, vllm_after)
+    lmcache_delta = metric_delta(lmcache_before, lmcache_after)
+
+    requested = lmcache_delta["lmcache_mp_lookup_requested_tokens_total"]
+    hit = lmcache_delta["lmcache_mp_lookup_hit_tokens_total"]
+    hit_rate = hit / requested if requested > 0 else 0.0
+    failures = lmcache_delta["lmcache_mp_l2_prefetch_failure_chunks_total"]
+    l2_hits = lmcache_delta["lmcache_mp_l2_prefetch_hit_chunks_total"]
+    l2_loaded = lmcache_delta["lmcache_mp_l2_prefetch_load_completed_chunks_total"]
+
+    checks = {
+        "vllm_external_queries_positive": vllm_delta[
+            "vllm:external_prefix_cache_queries_total"
+        ]
+        > 0,
+        "vllm_external_hits_positive": vllm_delta["vllm:external_prefix_cache_hits_total"] > 0,
+        "lookup_requested_positive": requested > 0,
+        "lookup_full_hit": requested > 0 and hit == requested,
+        "l2_prefetch_failure_zero": failures == 0,
+        "redis_keys_preserved": int(client.dbsize()) >= len(records),
+    }
+    if not args.allow_l1_hit:
+        checks["l2_hit_positive"] = l2_hits > 0
+        checks["l2_load_completed_positive"] = l2_loaded > 0
+
+    result = {
+        "ok": all(checks.values()),
+        "kid": kid,
+        "request_elapsed_seconds": round(elapsed, 3),
+        "usage": response.get("usage"),
+        "vllm_delta": vllm_delta,
+        "lmcache_delta": lmcache_delta,
+        "lookup_requested_tokens": int(requested),
+        "lookup_hit_tokens": int(hit),
+        "lookup_hit_rate": hit_rate,
+        "redis_dbsize_after": int(client.dbsize()),
+        "checks": checks,
+    }
+    print_json("CACHE CONSUMPTION", result)
+    if not result["ok"]:
+        raise RuntimeError(
+            "cache-consumption verification failed; restart LMCache/vLLM after injection "
+            "to clear L1/GPU state before retrying"
+        )
+    return result
+
+
+def build_artifact(args: argparse.Namespace) -> dict[str, Any]:
+    normalized, kid = document_identity(args.document)
+    base_url = args.kdn_url.rstrip("/")
+    register = http_json(
+        f"{base_url}/knowledge/register_text",
+        {"content": normalized},
+        timeout=args.request_timeout,
+    )
+    returned_kid = str(register.get("kid", "")).strip().lower()
+    if returned_kid != kid:
+        raise RuntimeError(f"KDN returned kid={returned_kid!r}, expected {kid!r}")
+
+    build_payload = {
+        "kid": kid,
+        "api_url": f"{args.vllm_url.rstrip('/')}/v1/chat/completions",
+        "model": args.model,
+        "max_tokens": 1,
+        "temperature": 0.0,
+        "redis_host": args.redis_host,
+        "redis_port": args.redis_port,
+        "redis_db": args.redis_db,
+        "redis_password": args.redis_password,
+        "scan_count": 1000,
+        "flushdb": False,
+    }
+    build = http_json(
+        f"{base_url}/knowledge/build_kv",
+        build_payload,
+        timeout=max(args.request_timeout, 600),
+    )
+    if int(build.get("dumped_keys", 0)) <= 0:
+        raise RuntimeError(f"KDN build produced no keys: {build}")
+    result = {"kid": kid, "register": register, "build": build}
+    print_json("KDN BUILD", result)
+    inspect_artifact(args.document, args.kv_root)
+    return result
+
+
+def add_common(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--document", type=Path, required=True)
+    parser.add_argument(
+        "--kv-root",
+        type=Path,
+        default=REPO_ROOT / "kdn_server" / "KV_database",
+    )
+    parser.add_argument("--redis-host", default="127.0.0.1")
+    parser.add_argument("--redis-port", type=int, default=6379)
+    parser.add_argument("--redis-db", type=int, default=0)
+    parser.add_argument("--redis-password", default=None)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    build = subparsers.add_parser("build", help="Register a document and build its KV artifact.")
+    add_common(build)
+    build.add_argument("--kdn-url", default="http://127.0.0.1:9101")
+    build.add_argument("--vllm-url", default="http://127.0.0.1:8000")
+    build.add_argument("--model", default="llama3-70b")
+    build.add_argument("--request-timeout", type=float, default=300.0)
+
+    inspect = subparsers.add_parser("inspect", help="Validate local manifest/dump metadata.")
+    add_common(inspect)
+
+    inject = subparsers.add_parser("inject", help="Restore a local KV artifact into Redis.")
+    add_common(inject)
+    inject.add_argument("--flush-redis", action="store_true")
+    inject.add_argument("--yes", action="store_true", help="Confirm destructive Redis FLUSHDB.")
+    inject.add_argument("--allow-extra-keys", action="store_true")
+    inject.add_argument("--sample-hashes", type=int, default=3)
+
+    consume = subparsers.add_parser(
+        "consume", help="Send the exact builder-style prefix and verify LMCache/vLLM metrics."
+    )
+    add_common(consume)
+    consume.add_argument("--vllm-url", default="http://127.0.0.1:8000")
+    consume.add_argument("--lmcache-http-url", default="http://127.0.0.1:8080")
+    consume.add_argument("--model", default="llama3-70b")
+    consume.add_argument("--request-timeout", type=float, default=300.0)
+    consume.add_argument("--metrics-wait", type=float, default=2.0)
+    consume.add_argument(
+        "--allow-l1-hit",
+        action="store_true",
+        help="Do not require a new Redis L2 prefetch (useful for repeated runs).",
+    )
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    args.document = args.document.resolve()
+    args.kv_root = args.kv_root.resolve()
+    if not args.document.is_file():
+        raise FileNotFoundError(f"document not found: {args.document}")
+
+    if args.command == "build":
+        build_artifact(args)
+    elif args.command == "inspect":
+        inspect_artifact(args.document, args.kv_root)
+    elif args.command == "inject":
+        inject_artifact(args)
+    elif args.command == "consume":
+        consume_artifact(args)
+    else:
+        raise AssertionError(args.command)
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except Exception as exc:
+        print(f"[FAIL] {type(exc).__name__}: {exc}", file=sys.stderr)
+        raise SystemExit(1)

From bee6f7e299feb13f6aae1e2007e33795bcf1996d Mon Sep 17 00:00:00 2001
From: RickZ <121943721+rickisba@users.noreply.github.com>
Date: Tue, 28 Jul 2026 10:27:02 +0800
Subject: [PATCH 4/7] docs: record completed v1 migration validation

---
 docs/v1_migration_closeout.md | 260 ++++++++++++++++++++++++++++++++++
 1 file changed, 260 insertions(+)
 create mode 100644 docs/v1_migration_closeout.md

diff --git a/docs/v1_migration_closeout.md b/docs/v1_migration_closeout.md
new file mode 100644
index 0000000..a83a3ff
--- /dev/null
+++ b/docs/v1_migration_closeout.md
@@ -0,0 +1,260 @@
+# CacheRoute v1 migration closeout
+
+This document records the validated baseline for the modern CacheRoute serving
+stack and defines the boundary between environment migration work and subsequent
+feature development.
+
+## Validated serving baseline
+
+| Component | Validated value |
+|---|---|
+| Base image | CUDA 13.0 / Ubuntu 22.04 |
+| Python | 3.12.x |
+| PyTorch | 2.11.0+cu130 |
+| vLLM | 0.25.1 |
+| LMCache | 0.5.2 |
+| LMCache mode | standalone MP server |
+| vLLM connector | `LMCacheMPConnector` |
+| L2 backend | Redis RESP, DB 0 |
+| Tensor parallel size | 8 |
+| LMCache chunk size | 256 tokens |
+| Hash algorithm | `sha256_cbor` |
+| Model | `Meta-Llama-3-70B-Instruct` served as `llama3-70b` |
+
+The modern startup contract is:
+
+```text
+Redis RESP L2
+  -> LMCache MP :5555 / HTTP :8080
+  -> vLLM + LMCacheMPConnector :8000
+  -> Scheduler -> KDN -> Proxy -> Instance -> Client
+```
+
+The legacy and modern startup interfaces must not be mixed:
+
+- legacy uses `LMCACHE_CONFIG_FILE`, YAML `remote_url`, and historical vLLM
+  offloading flags;
+- v1 uses `lmcache server`, RESP L2, and `vllm serve --kv-transfer-config`.
+
+## Runtime compatibility result
+
+`CACHEROUTE_RUNTIME_PROFILE` provides three modes:
+
+| Profile | Behavior |
+|---|---|
+| `legacy` | strict historical `vllm@*` Redis namespace |
+| `v1` | modern model-scoped LMCache Redis keys |
+| `auto` | accepts either supported key generation |
+
+Normal v1 registration no longer requires a manual `--match`. The historical
+CLI default `vllm@*` is treated as a compatibility sentinel in `v1` and `auto`
+profiles, so the builder scans and validates the modern key layout automatically.
+An explicit `--match` remains available for debugging and custom backends.
+
+The 30-second build setting is a maximum first-key timeout, not a fixed sleep.
+After keys appear, capture completes when the key set remains unchanged for the
+configured quiet period.
+
+## End-to-end KDN validation evidence
+
+The closeout validation used:
+
+```text
+data/CacheRoute_dataset/knowledge_document/
+668f6bd1ad2419d9dfbcda0b689311b8b6696c7ed772001bea7bd12573137b4a.txt
+```
+
+The document-derived KID matched the filename:
+
+```text
+668f6bd1ad2419d9dfbcda0b689311b8b6696c7ed772001bea7bd12573137b4a
+```
+
+### Build and local persistence
+
+The v1 KDN builder produced:
+
+```text
+runtime_profile:  v1
+key_formats:      [v1]
+scan_match:       *
+keys_before:      0
+keys_after:       96
+keys_new:         96
+dumped_keys:      96
+manifest records: 96
+dump files:       96
+local size:       approximately 961 MiB
+capture elapsed:  8.931 seconds
+```
+
+The registration command did not include `--match` or `--flushdb`.
+
+### Redis restore integrity
+
+After Redis DB 0 was explicitly cleared, `kdn_server/kv_injector.py` restored
+all saved blocks:
+
+```text
+injected:                 96
+missing_files:            0
+Redis keys:               96
+missing_keys:             0
+extra_keys:               0
+size_mismatches:          0
+expected_payload_bytes:   1,006,632,960
+actual_payload_bytes:     1,006,632,960
+sample SHA256 mismatches: 0
+```
+
+This proves byte-for-byte preservation of the original Redis key/value pairs.
+
+### Actual LMCache consumption
+
+LMCache and vLLM were restarted after restore so that LMCache L1 and the vLLM
+GPU cache started empty. An exact builder-style request was sent with the
+normalized document as the only `system` message.
+
+vLLM reported:
+
+```text
+prompt tokens:                         3293
+external prefix-cache queries:         3293
+external prefix-cache hits:            3072
+prompt tokens cached:                  3072
+```
+
+LMCache MP reported:
+
+```text
+lookup requested tokens:               3072
+lookup hit tokens:                     3072
+lookup hit rate:                       100%
+L2 prefetch lookup requests:           1
+L2 prefetch lookup objects:            96
+L2 prefetch hit chunks:                96
+L2 prefetch load completed chunks:     96
+L2 prefetch failure chunks:            0
+L1 write chunks:                       96
+L1 read chunks:                        96
+```
+
+The physical-key count is consistent with the runtime topology:
+
+```text
+96 physical Redis keys / 8 TP ranks = 12 logical chunks
+12 logical chunks * 256 tokens      = 3072 cached tokens
+```
+
+The remaining 221 prompt tokens were not a full 256-token chunk and were
+recomputed by vLLM:
+
+```text
+3293 total prompt tokens - 3072 cached tokens = 221 recomputed tokens
+```
+
+This closes the previous validation gap: the KDN artifact was not only saved and
+restored correctly; it was discovered in Redis, prefetched into LMCache L1, and
+consumed by vLLM as external prefix cache.
+
+## Metrics endpoints
+
+The two metrics endpoints are distinct:
+
+| Endpoint | Owner | Important metrics |
+|---|---|---|
+| `http://127.0.0.1:8000/metrics` | vLLM | `vllm:external_prefix_cache_queries_total`, `vllm:external_prefix_cache_hits_total`, `vllm:prompt_tokens_cached_total` |
+| `http://127.0.0.1:8080/metrics` | LMCache MP | `lmcache_mp_lookup_requested_tokens_total`, `lmcache_mp_lookup_hit_tokens_total`, `lmcache_mp_l2_prefetch_hit_chunks_total`, `lmcache_mp_l2_prefetch_load_completed_chunks_total`, `lmcache_mp_l2_prefetch_failure_chunks_total` |
+
+LMCache metrics may not exist until the first store or lookup event. A direct KDN
+Redis injection also bypasses LMCache's own store accounting, so
+`lmcache_mp_l2_usage_bytes` must not be used as proof that Redis is empty.
+
+## Reproducible validation tools
+
+Activate v1 in every terminal:
+
+```bash
+source env/docker/cu130/scripts/activate_v1.sh
+```
+
+Run the non-destructive environment check:
+
+```bash
+python3 env/docker/cu130/scripts/check_v1_environment.py --require-running
+```
+
+Use the staged KDN validator:
+
+```bash
+DOC=/workspace/llm-stack/CacheRoute/data/CacheRoute_dataset/knowledge_document/668f6bd1ad2419d9dfbcda0b689311b8b6696c7ed772001bea7bd12573137b4a.txt
+
+python3 scripts/validate_v1_kdn_roundtrip.py build --document "$DOC"
+python3 scripts/validate_v1_kdn_roundtrip.py inspect --document "$DOC"
+```
+
+The restore stage is destructive only when explicitly confirmed:
+
+```bash
+python3 scripts/validate_v1_kdn_roundtrip.py inject \
+  --document "$DOC" \
+  --flush-redis \
+  --yes
+```
+
+After injection, restart LMCache and vLLM to clear L1/GPU state, then run:
+
+```bash
+python3 scripts/validate_v1_kdn_roundtrip.py consume --document "$DOC"
+```
+
+The validator deliberately does not combine injection and consumption into one
+unattended command because the service restart between the two stages is part of
+the correctness contract.
+
+## Cache-identity compatibility requirements
+
+A reusable artifact is valid only when producer and consumer agree on at least:
+
+- model identity and actual model path used by LMCache keys;
+- served model and tokenizer;
+- chat template and exact token prefix;
+- tensor-parallel size and worker/rank layout;
+- model/KV dtype;
+- LMCache chunk size;
+- LMCache hash algorithm;
+- connector generation and key serialization;
+- Redis database semantics.
+
+A byte-perfect Redis restore proves transport integrity but does not override an
+identity mismatch. Cache consumption metrics remain the final acceptance test.
+
+## Safety rules
+
+- Do not use `--flushdb` during normal KDN registration.
+- Do not run `FLUSHDB` against a shared or production Redis database.
+- The round-trip validator requires both `--flush-redis` and `--yes` before it
+  performs destructive cleanup.
+- `redis-cli` is optional; the documented tools use the Python Redis client.
+- The correct standalone injector is `kdn_server/kv_injector.py`.
+
+## Migration definition of done
+
+The following migration goals are complete:
+
+- legacy environment and startup documentation remain available;
+- isolated CUDA 13 / vLLM 0.25.1 / LMCache 0.5.2 image profile exists;
+- LMCache MP and vLLM startup scripts exist;
+- v1 profile activation is explicit and repeatable;
+- KDN recognizes modern LMCache keys without manual `--match`;
+- asynchronous writes are captured without a fixed 30-second delay;
+- zero-key builds fail instead of becoming `kv_ready`;
+- local dump/manifest integrity is verified;
+- Redis restore is byte-for-byte verified;
+- LMCache RESP L2 lookup/load is verified;
+- vLLM external-prefix-cache consumption is verified;
+- environment and round-trip checks are reproducible from repository scripts.
+
+Future Scheduler, Proxy, Instance, predictor, and UI work can now use this v1
+runtime as the modern development baseline without reopening the environment
+migration unless the serving-stack versions or cache identity change.

From 2e1217ffa0941808fc94383c4280310887d88b46 Mon Sep 17 00:00:00 2001
From: RickZ <121943721+rickisba@users.noreply.github.com>
Date: Tue, 28 Jul 2026 10:29:17 +0800
Subject: [PATCH 5/7] docs: align v1 quickstart with validated pipeline

---
 docs/quickstart_v1.md | 230 +++++++++++++++++++++++++++++-------------
 1 file changed, 158 insertions(+), 72 deletions(-)

diff --git a/docs/quickstart_v1.md b/docs/quickstart_v1.md
index b770ba2..22bfe6f 100644
--- a/docs/quickstart_v1.md
+++ b/docs/quickstart_v1.md
@@ -1,10 +1,11 @@
 # CacheRoute v1 end-to-end quick start
 
-This guide covers the complete single-machine path for the modern CacheRoute runtime:
+This guide covers the complete single-machine path for the modern CacheRoute
+runtime:
 
 ```text
 Redis RESP L2
-  -> LMCache 0.5.2 MP server :5555
+  -> LMCache 0.5.2 MP server :5555 / HTTP :8080
   -> vLLM 0.25.1 + LMCacheMPConnector :8000
   -> Scheduler :7001/:7002
   -> KDN Server :9101
@@ -13,53 +14,58 @@ Redis RESP L2
   -> Client/UI :7071
 ```
 
-Use this guide for the CUDA 13 / PyTorch 2.11 / vLLM 0.25.1 / LMCache 0.5.2 profile. Do not mix it with the legacy YAML-based LMCache startup path.
+Use this path for CUDA 13 / PyTorch 2.11 / vLLM 0.25.1 / LMCache
+0.5.2. The existing root README deployment remains the legacy stable path.
+Do not mix its YAML/offloading commands with the v1 MP startup interface.
+
+For the completed migration evidence and compatibility boundary, see
+[`v1_migration_closeout.md`](v1_migration_closeout.md).
 
 ## 0. Assumptions
 
 The commands assume:
 
 - the repository is mounted at `/workspace/llm-stack/CacheRoute`;
-- the model is located at `/workspace/llm-stack/models/LLM-Research/Meta-Llama-3-70B-Instruct`;
+- the model is at
+  `/workspace/llm-stack/models/LLM-Research/Meta-Llama-3-70B-Instruct`;
 - the served model name is `llama3-70b`;
-- Redis, LMCache, vLLM, and CacheRoute use host networking and loopback addresses;
+- Redis, LMCache, vLLM, and CacheRoute use host networking and loopback
+  addresses;
 - each long-running process is started in a separate terminal;
-- `core/config.py` has `USE_MOCK = False` and model paths matching this deployment.
+- `core/config.py` has `USE_MOCK = False` and matching model paths.
 
-For a new image/container, first follow [`env/docker/cu130/README.md`](../env/docker/cu130/README.md). An already-built compatible container does not need to rebuild the Docker image.
+A newly created image/container should first follow
+[`env/docker/cu130/README.md`](../env/docker/cu130/README.md). An existing
+compatible container does not need to rebuild the Docker image.
 
-## 1. Select the v1 runtime profile
+## 1. Update the repository and activate v1
 
 ```bash
 cd /workspace/llm-stack/CacheRoute
 git switch main
 git pull --ff-only origin main
 
-export CACHEROUTE_RUNTIME_PROFILE=v1
-export PYTHONPATH=/workspace/llm-stack/CacheRoute
+source env/docker/cu130/scripts/activate_v1.sh
 ```
 
-Verify the compatibility layer:
+Source the activation script in every terminal that starts LMCache, vLLM, KDN,
+Proxy, Instance, or validation tools. Environment variables are inherited only
+by processes started after activation.
+
+Run the non-destructive environment check before starting services:
 
 ```bash
-python3 - <<'PY'
-from core.runtime_compat import normalize_runtime_profile
-print(normalize_runtime_profile())
-PY
-# Expected: v1
+python3 env/docker/cu130/scripts/check_v1_environment.py
 ```
 
-Every new terminal that starts a CacheRoute component should export `CACHEROUTE_RUNTIME_PROFILE=v1`. Environment variables are inherited only by processes started after the export.
-
-## 2. Start or verify Redis
-
-When Redis is already running:
+When Redis, LMCache, and vLLM are expected to be running, use the strict form:
 
 ```bash
-redis-cli -h 127.0.0.1 -p 6379 ping
-# Expected: PONG
+python3 env/docker/cu130/scripts/check_v1_environment.py --require-running
 ```
 
+## 2. Start or verify Redis
+
 To create a local Redis container for the first time:
 
 ```bash
@@ -82,13 +88,25 @@ For later runs:
 sudo docker start lmcache-redis
 ```
 
-Do not run KDN `--flushdb` against an online shared Redis database unless its contents are disposable.
+`redis-cli` is optional. The Python client check works in the v1 container:
+
+```bash
+python3 - <<'PY'
+import redis
+r = redis.Redis(host="127.0.0.1", port=6379, db=0)
+print("PING:", r.ping())
+print("DBSIZE:", r.dbsize())
+PY
+```
+
+Do not use KDN `--flushdb` during normal registration. Do not clear a shared or
+production Redis database.
 
 ## 3. Terminal 1: start LMCache MP
 
 ```bash
 cd /workspace/llm-stack/CacheRoute
-export CACHEROUTE_RUNTIME_PROFILE=v1
+source env/docker/cu130/scripts/activate_v1.sh
 bash env/docker/cu130/scripts/start_lmcache_mp.sh
 ```
 
@@ -106,7 +124,8 @@ Confirm the listening ports:
 ss -lntp | grep -E ':(5555|8080)\b'
 ```
 
-The v1 path deliberately unsets `LMCACHE_CONFIG_FILE`. It does not use the legacy `remote_url` YAML interface.
+The v1 startup deliberately unsets `LMCACHE_CONFIG_FILE` and does not use the
+legacy `remote_url` YAML interface.
 
 ## 4. Terminal 2: start vLLM with LMCacheMPConnector
 
@@ -114,7 +133,7 @@ Start vLLM only after LMCache MP is listening on port `5555`:
 
 ```bash
 cd /workspace/llm-stack/CacheRoute
-export CACHEROUTE_RUNTIME_PROFILE=v1
+source env/docker/cu130/scripts/activate_v1.sh
 export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
 export MODEL_DIR=/workspace/llm-stack/models/LLM-Research/Meta-Llama-3-70B-Instruct
 
@@ -127,17 +146,35 @@ Wait for the model endpoint:
 curl -sS http://127.0.0.1:8000/v1/models | python3 -m json.tool
 ```
 
-Check that LMCache metrics are exposed:
+The metrics endpoints are separate:
+
+```text
+vLLM:       http://127.0.0.1:8000/metrics
+LMCache MP: http://127.0.0.1:8080/metrics
+```
+
+Check vLLM external-cache metrics:
+
+```bash
+curl -sS http://127.0.0.1:8000/metrics \
+  | grep -E '^vllm:(external_prefix_cache_(queries|hits)|prompt_tokens_cached)'
+```
+
+Check LMCache MP state:
 
 ```bash
-curl -sS http://127.0.0.1:8000/metrics | grep -i lmcache | head -n 50
+curl -sS http://127.0.0.1:8080/metrics \
+  | grep -E '^lmcache_mp_(lookup|l1_|l2_)' \
+  | head -n 100
 ```
 
+Lookup counters may not exist until the first request.
+
 ## 5. Terminal 3: start the Scheduler
 
 ```bash
 cd /workspace/llm-stack/CacheRoute/test
-export CACHEROUTE_RUNTIME_PROFILE=v1
+source ../env/docker/cu130/scripts/activate_v1.sh
 
 python3 demo_scheduler.py \
   --cacheroute \
@@ -147,57 +184,56 @@ python3 demo_scheduler.py \
   --cacheroute-log-decision 1
 ```
 
-The Scheduler service plane listens on `127.0.0.1:7001` and its control plane listens on `127.0.0.1:7002`.
+The Scheduler service plane listens on `127.0.0.1:7001` and its control plane
+listens on `127.0.0.1:7002`.
 
 ## 6. Terminal 4: start the KDN Server
 
 ```bash
 cd /workspace/llm-stack/CacheRoute/test
-export CACHEROUTE_RUNTIME_PROFILE=v1
+source ../env/docker/cu130/scripts/activate_v1.sh
 python3 demo_kdn.py
 ```
 
-The KDN Server listens on `127.0.0.1:9101`. Normal v1 knowledge builds do not require a manual `--match`.
+The KDN Server listens on `127.0.0.1:9101`. Normal v1 builds do not require a
+manual `--match`.
 
 ## 7. Terminal 5: register knowledge and build KVCache
 
-Create a sample knowledge file:
+The standard non-interactive command is:
 
 ```bash
 cd /workspace/llm-stack/CacheRoute
-mkdir -p data/quickstart
-
-cat > data/quickstart/cacheroute_v1.txt <<'EOF'
-CacheRoute is a knowledge-oriented LLM scheduling framework. It reduces repeated prefill computation by storing and reusing KVCache blocks for frequently requested external knowledge. The Scheduler selects a KDN server and a Proxy, while the Proxy chooses between text recomputation and KVCache reuse.
-EOF
-```
+source env/docker/cu130/scripts/activate_v1.sh
 
-Start the KDN CLI:
+DOC=/workspace/llm-stack/CacheRoute/data/CacheRoute_dataset/knowledge_document/668f6bd1ad2419d9dfbcda0b689311b8b6696c7ed772001bea7bd12573137b4a.txt
 
-```bash
-python3 kdn_server/kdn_register_cli.py
+python3 kdn_server/kdn_register_cli.py \
+  --build-kv-file "$DOC" \
+  --api-url http://127.0.0.1:8000/v1/chat/completions \
+  --model llama3-70b \
+  --redis-host 127.0.0.1 \
+  --redis-port 6379 \
+  --redis-db 0
 ```
 
-At the CLI prompt:
-
-```text
-:buildkv_file /workspace/llm-stack/CacheRoute/data/quickstart/cacheroute_v1.txt --api-url http://127.0.0.1:8000/v1/chat/completions --model llama3-70b
-:pool
-```
+Do not add `--match` for the normal v1 path. Do not add `--flushdb` unless the
+selected Redis DB is an isolated disposable test database.
 
-A successful build must report `dumped_keys > 0`. Inspect the generated metadata:
+A successful build reports `dumped_keys > 0`. Inspect it with:
 
 ```bash
-cat kdn_server/KV_database/<kid>/run_meta.json
+python3 scripts/validate_v1_kdn_roundtrip.py inspect --document "$DOC"
 ```
 
-Expected fields include `"runtime_profile": "v1"`, `"key_formats": ["v1"]`, and a non-zero `dumped_keys` value.
+The result verifies the document-derived KID, `run_meta.json`, manifest count,
+dump count, and payload size.
 
 ## 8. Terminal 6: start the Proxy
 
 ```bash
 cd /workspace/llm-stack/CacheRoute/test
-export CACHEROUTE_RUNTIME_PROFILE=v1
+source ../env/docker/cu130/scripts/activate_v1.sh
 
 python3 demo_proxy.py \
   --strategy round_robin \
@@ -205,13 +241,15 @@ python3 demo_proxy.py \
   --ready-release-policy text_bypass
 ```
 
-The Proxy listens on `127.0.0.1:8001`, its control plane on `127.0.0.1:8002`, and its UI is normally available at `http://127.0.0.1:8202`.
+The Proxy listens on `127.0.0.1:8001`, its control plane on
+`127.0.0.1:8002`, and the browser UI is normally available at
+`http://127.0.0.1:8202`.
 
 ## 9. Terminal 7: start the Instance
 
 ```bash
 cd /workspace/llm-stack/CacheRoute/test
-export CACHEROUTE_RUNTIME_PROFILE=v1
+source ../env/docker/cu130/scripts/activate_v1.sh
 
 python3 demo_instance.py \
   --host 127.0.0.1 \
@@ -219,28 +257,31 @@ python3 demo_instance.py \
   --proxy-cp-url http://127.0.0.1:8002
 ```
 
-For a minimal functional test without the resource agent, add `--no-resource-monitor`.
+For a minimal functional test without the resource agent, add
+`--no-resource-monitor`.
 
 Confirm control-plane registration:
 
 ```bash
 curl -sS http://127.0.0.1:7001/debug/status | python3 -m json.tool
-curl -sS 'http://127.0.0.1:8002/v1/instance/list?include_dead=true' | python3 -m json.tool
+curl -sS 'http://127.0.0.1:8002/v1/instance/list?include_dead=true' \
+  | python3 -m json.tool
 ```
 
 ## 10. Terminal 8: start the Client
 
 ```bash
 cd /workspace/llm-stack/CacheRoute/test
-export CACHEROUTE_RUNTIME_PROFILE=v1
+source ../env/docker/cu130/scripts/activate_v1.sh
 python3 demo_client.py --with-ui
 ```
 
-Open `http://127.0.0.1:7071/ui/client`. For terminal-only operation, run `python3 demo_client.py`.
+Open `http://127.0.0.1:7071/ui/client`. For terminal-only operation, run
+`python3 demo_client.py`.
 
-## 11. Send an end-to-end request
+## 11. Send an end-to-end CacheRoute request
 
-Send the request through the Scheduler rather than directly to vLLM:
+Send requests through the Scheduler rather than directly to vLLM:
 
 ```bash
 curl http://127.0.0.1:7001/v1/chat/completions \
@@ -260,34 +301,79 @@ curl http://127.0.0.1:7001/v1/chat/completions \
   }'
 ```
 
-Inspect routing decisions:
+Inspect the most recent routing decision:
 
 ```bash
 curl -sS http://127.0.0.1:7001/debug/strategy | python3 -m json.tool
 ```
 
-Expected path:
+Expected service path:
 
 ```text
 Client -> Scheduler :7001 -> Proxy :8001 -> Instance :9001
        -> vLLM :8000 -> LMCache MP :5555 -> Redis RESP L2 :6379
 ```
 
-## 12. Validate actual cache consumption
+## 12. Reproduce the KDN build/restore/consume validation
+
+The validator is staged because LMCache and vLLM must be restarted after Redis
+restore to clear L1/GPU state before proving an L2 load.
 
-A successful KDN Redis injection proves transport integrity, not an LMCache hit. Capture LMCache hit-token and remote-read metrics before and after a request:
+### Stage A: build or inspect
 
 ```bash
-curl -sS http://127.0.0.1:8000/metrics \
-  | grep -E 'lmcache.*(hit|requested|remote_read|retrieve)'
+python3 scripts/validate_v1_kdn_roundtrip.py build --document "$DOC"
+python3 scripts/validate_v1_kdn_roundtrip.py inspect --document "$DOC"
+```
+
+### Stage B: restore into an isolated Redis DB
+
+Stop LMCache and vLLM before the destructive restore. The command refuses to
+run `FLUSHDB` unless both confirmation flags are present:
+
+```bash
+python3 scripts/validate_v1_kdn_roundtrip.py inject \
+  --document "$DOC" \
+  --flush-redis \
+  --yes
+```
+
+### Stage C: prove cache consumption
+
+Restart LMCache MP and vLLM, preserving the restored Redis keys, then run:
+
+```bash
+python3 scripts/validate_v1_kdn_roundtrip.py consume --document "$DOC"
+```
+
+Acceptance conditions include:
+
+```text
+missing files/keys/size mismatches = 0
+LMCache lookup requested tokens > 0
+LMCache lookup hit tokens == requested tokens
+L2 prefetch hit/load > 0
+L2 prefetch failures == 0
+vLLM external prefix-cache hits > 0
 ```
 
-Do not rely only on TTFT differences.
+The completed migration run produced 96 physical Redis keys, restored
+1,006,632,960 bytes, loaded all 96 keys from RESP L2, and hit 3072/3072
+chunk-aligned tokens.
 
 ## Troubleshooting boundaries
 
-- No keys produced during KDN build: verify the v1 profile, Redis endpoint, model request success, and LMCache MP logs.
-- Redis keys exist but the request misses: compare model path/identity, tensor-parallel layout, KV dtype, chunk size, hash algorithm, and exact token prefix.
+- No keys during build: verify `CACHEROUTE_RUNTIME_PROFILE=v1`, Redis, the vLLM
+  request, and LMCache logs. Reusing the same prompt while LMCache L1 still holds
+  it may prevent a new L2 store; restart LMCache or use a new document.
+- Redis keys exist but the request misses: compare model path/identity, tokenizer,
+  chat template, tensor parallel layout, KV dtype, chunk size, hash algorithm,
+  and exact token prefix.
 - `vllm serve` cannot connect: start LMCache MP first and verify port `5555`.
-- Old YAML settings are still active: unset `LMCACHE_CONFIG_FILE` and restart LMCache/vLLM.
-- Profile changes do not appear: restart all affected processes after exporting `CACHEROUTE_RUNTIME_PROFILE=v1`.
+- Old YAML settings remain active: source `activate_v1.sh`, then restart LMCache
+  and vLLM.
+- LMCache metrics appear empty: query port `8080`, not port `8000`, and drive at
+  least one lookup.
+- `lmcache_mp_l2_usage_bytes` is zero after direct injection: the injector
+  bypasses LMCache store accounting; use lookup/hit/load counters instead.
+- Profile changes do not appear: restart every affected process after activation.

From 54e8b9ef53c1457b2257e151736758be55fe3ff0 Mon Sep 17 00:00:00 2001
From: RickZ <121943721+rickisba@users.noreply.github.com>
Date: Tue, 28 Jul 2026 10:31:38 +0800
Subject: [PATCH 6/7] docs: finish the CUDA 13 environment runbook

---
 env/docker/cu130/README.md | 268 ++++++++++++++++++++++---------------
 1 file changed, 161 insertions(+), 107 deletions(-)

diff --git a/env/docker/cu130/README.md b/env/docker/cu130/README.md
index 16f04f0..45c3e79 100644
--- a/env/docker/cu130/README.md
+++ b/env/docker/cu130/README.md
@@ -4,18 +4,21 @@ This directory provides the additive v1 development-image profile for the
 modern CacheRoute runtime. It does not replace or modify the existing CUDA 12.8
 / vLLM 0.13.x / LMCache 0.3.11 environment.
 
-The runtime compatibility architecture is described in
-[`docs/runtime_compatibility_v1.md`](../../../docs/runtime_compatibility_v1.md).
+Start with the repository-level guides:
+
+- complete runtime path: [`docs/quickstart_v1.md`](../../../docs/quickstart_v1.md);
+- compatibility design: [`docs/runtime_compatibility_v1.md`](../../../docs/runtime_compatibility_v1.md);
+- completed migration evidence: [`docs/v1_migration_closeout.md`](../../../docs/v1_migration_closeout.md).
 
 ## Choose this profile only for the modern stack
 
 | Profile | Serving stack | LMCache startup interface |
 |---|---|---|
-| Legacy | vLLM 0.13.x + LMCache 0.3.11 | YAML through `LMCACHE_CONFIG_FILE` and the historical vLLM offloading flags |
-| v1 | vLLM 0.25.1 + LMCache 0.5.2 | Standalone `lmcache server` plus vLLM `LMCacheMPConnector` |
+| Legacy | vLLM 0.13.x + LMCache 0.3.11 | YAML through `LMCACHE_CONFIG_FILE` and historical vLLM offloading flags |
+| v1 | vLLM 0.25.1 + LMCache 0.5.2 | standalone `lmcache server` plus vLLM `LMCacheMPConnector` |
 
 Do not combine the two interfaces. In v1 mode, unset `LMCACHE_CONFIG_FILE` and
-do not use `remote_url`, `--kv-offloading-backend lmcache`, or the old
+do not use `remote_url`, `--kv-offloading-backend lmcache`, or the historical
 `python3 -m vllm.entrypoints.openai.api_server` example as the primary startup
 path.
 
@@ -32,59 +35,95 @@ path.
 | LMCache | `0.5.2` |
 
 The image uses `/opt/venv` to isolate serving dependencies from Ubuntu system
-Python packages. FFmpeg is installed because current TorchCodec wheels require
-its shared libraries. Rust/Cargo and Tkinter remain available for the existing
-CacheRoute resource-agent and desktop-dashboard workflows.
+Python packages. FFmpeg is installed for TorchCodec. Rust/Cargo and Tkinter
+remain available for the resource-agent and dashboard workflows.
+
+The image sets `CACHEROUTE_RUNTIME_PROFILE=v1` by default.
+
+## Files
+
+- `Dockerfile`: complete CUDA 13 development image;
+- `constraints.txt`: exact serving-stack versions and shared compatibility
+  ranges;
+- `requirements-dev.txt`: CacheRoute dependencies compatible with the target
+  serving stack;
+- `scripts/activate_v1.sh`: shared shell activation for every service terminal;
+- `scripts/check_v1_environment.py`: non-destructive version, CUDA, model,
+  Redis, and endpoint validation;
+- `scripts/start_lmcache_mp.sh`: LMCache 0.5.2 MP + RESP L2 startup;
+- `scripts/start_vllm_mp.sh`: vLLM 0.25.1 + `LMCacheMPConnector` startup.
 
-The image sets:
+`requirements-dev.txt` intentionally excludes `Booktype==1.5`. That Python-2-era
+package can overwrite the modern `redis` module with invalid source files.
+
+## Existing compatible container: no rebuild required
+
+The runtime scripts live in the mounted repository, so an already-created
+container can use the merged `main` branch directly:
 
 ```bash
-CACHEROUTE_RUNTIME_PROFILE=v1
+cd /workspace/llm-stack/CacheRoute
+git switch main
+git pull --ff-only origin main
+
+source env/docker/cu130/scripts/activate_v1.sh
 ```
 
-This selects the modern compatibility path by default. Override it at container
-startup only when explicitly testing `auto` or `legacy` behavior.
+Do not switch back to the historical `v1/runtime-compat` development branch;
+that work has already been merged into `main`.
 
-## Files
+The activation script:
 
-- `Dockerfile`: complete CUDA 13 development image.
-- `constraints.txt`: exact serving-stack versions and shared compatibility
-  ranges.
-- `requirements-dev.txt`: CacheRoute application/development dependencies that
-  are compatible with the target stack.
-- `scripts/start_lmcache_mp.sh`: reusable LMCache 0.5.2 MP + RESP L2 startup.
-- `scripts/start_vllm_mp.sh`: reusable vLLM 0.25.1 + `LMCacheMPConnector`
-  startup.
+- sets `CACHEROUTE_RUNTIME_PROFILE=v1`;
+- adds the repository root to `PYTHONPATH` without duplicating it;
+- sets stable `PYTHONHASHSEED` and `OMP_NUM_THREADS` defaults;
+- clears legacy `LMCACHE_CONFIG_FILE`;
+- clears `PYTORCH_CUDA_ALLOC_CONF` for the validated MP path.
 
-`requirements-dev.txt` intentionally excludes `Booktype==1.5`. That package is
-from the Python 2 era and can overwrite the modern `redis` module with invalid
-Python 2 source files.
+It must be sourced in every new terminal:
 
-## Use an existing container without rebuilding
+```bash
+source env/docker/cu130/scripts/activate_v1.sh
+```
+
+Long-running services only see variables present when they start. Restart
+LMCache, vLLM, KDN, Proxy, and Instance after changing profiles.
 
-The scripts live in the mounted CacheRoute repository, so an already-created
-container does not need to run through the Dockerfile again.
+## Non-destructive environment validation
 
-Update the branch and select the v1 profile:
+Before starting services:
 
 ```bash
-cd /workspace/llm-stack/CacheRoute
-git fetch origin
-git switch v1/runtime-compat
-git pull --ff-only origin v1/runtime-compat
+python3 env/docker/cu130/scripts/check_v1_environment.py
+```
 
-export CACHEROUTE_RUNTIME_PROFILE=v1
+This checks:
+
+- repository location and v1 profile;
+- Python, PyTorch, vLLM, and LMCache versions;
+- PyTorch CUDA runtime, CUDA availability, and GPU count;
+- model directory;
+- required repository scripts;
+- Redis connectivity when available;
+- vLLM and LMCache endpoints as warnings when they are not running.
+
+After Redis, LMCache, and vLLM are started, require all endpoints:
+
+```bash
+python3 env/docker/cu130/scripts/check_v1_environment.py --require-running
 ```
 
-Persist the profile for interactive root shells when useful:
+Useful overrides include:
 
 ```bash
-grep -q 'CACHEROUTE_RUNTIME_PROFILE' /root/.bashrc || \
-  echo 'export CACHEROUTE_RUNTIME_PROFILE=v1' >> /root/.bashrc
+python3 env/docker/cu130/scripts/check_v1_environment.py \
+  --model-dir /other/model \
+  --expected-gpus 4 \
+  --redis-host 172.18.0.121
 ```
 
-Long-running services only see environment variables present when they start.
-Restart LMCache, vLLM, KDN, Proxy, and Instance after switching profiles.
+Use `--allow-no-gpu` only for image/CI inspection where GPU access is not
+expected.
 
 ## Runtime startup order
 
@@ -97,18 +136,22 @@ Use separate terminals in this order:
 
 ### 1. Verify Redis
 
+`redis-cli` is optional. The Python client is sufficient:
+
 ```bash
-redis-cli -h 127.0.0.1 -p 6379 ping
-# Expected: PONG
+python3 - <<'PY'
+import redis
+r = redis.Redis(host="127.0.0.1", port=6379, db=0)
+print("PING:", r.ping())
+print("DBSIZE:", r.dbsize())
+PY
 ```
 
 ### 2. Start LMCache MP
 
-Recommended repository script:
-
 ```bash
 cd /workspace/llm-stack/CacheRoute
-export CACHEROUTE_RUNTIME_PROFILE=v1
+source env/docker/cu130/scripts/activate_v1.sh
 bash env/docker/cu130/scripts/start_lmcache_mp.sh
 ```
 
@@ -140,7 +183,7 @@ lmcache server \
   }'
 ```
 
-Confirm that the MP and HTTP ports are listening:
+Confirm the MP and HTTP ports:
 
 ```bash
 ss -lntp | grep -E ':(5555|8080)\b'
@@ -148,13 +191,13 @@ ss -lntp | grep -E ':(5555|8080)\b'
 
 ### 3. Start vLLM
 
-Start this only after LMCache is listening on port `5555`.
-
-Recommended repository script:
+Start vLLM only after LMCache is listening on port `5555`:
 
 ```bash
 cd /workspace/llm-stack/CacheRoute
-export CACHEROUTE_RUNTIME_PROFILE=v1
+source env/docker/cu130/scripts/activate_v1.sh
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+export MODEL_DIR=/workspace/llm-stack/models/LLM-Research/Meta-Llama-3-70B-Instruct
 bash env/docker/cu130/scripts/start_vllm_mp.sh
 ```
 
@@ -200,7 +243,9 @@ Wait for the OpenAI-compatible endpoint:
 curl -sS http://127.0.0.1:8000/v1/models | python3 -m json.tool
 ```
 
-The scripts expose the main values as environment variables. For example:
+### Parameter overrides
+
+The repository scripts expose their main settings as environment variables:
 
 ```bash
 MODEL_DIR=/other/model \
@@ -215,30 +260,42 @@ REDIS_HOST=172.18.0.121 \
 bash env/docker/cu130/scripts/start_lmcache_mp.sh
 ```
 
-Additional command-line arguments are appended unchanged to each underlying
-command.
+Additional command-line arguments are appended unchanged to the underlying
+commands.
+
+## Metrics endpoints
+
+The endpoints and namespaces are different:
+
+| Endpoint | Metrics |
+|---|---|
+| `http://127.0.0.1:8000/metrics` | vLLM, including `vllm:external_prefix_cache_queries_total`, `vllm:external_prefix_cache_hits_total`, and `vllm:prompt_tokens_cached_total` |
+| `http://127.0.0.1:8080/metrics` | LMCache MP, including `lmcache_mp_lookup_*`, `lmcache_mp_l1_*`, and `lmcache_mp_l2_*` |
+
+LMCache lookup metrics may not exist before the first request. A direct KDN
+Redis restore bypasses LMCache store accounting, so `lmcache_mp_l2_usage_bytes`
+is not a reliable test for injected data. Use lookup/hit/load metrics.
 
 ## Cache identity requirements
 
-KDN construction and the target Instance must agree on the cache identity. At a
-minimum, keep the following aligned when building and consuming reusable KV:
+KDN construction and the consuming Instance must agree on:
 
 - model identity/path and served model;
+- tokenizer, chat template, and exact token prefix;
 - tensor-parallel topology and worker layout;
 - KV dtype and model configuration;
-- LMCache chunk size (`256` in this profile);
-- LMCache hash algorithm (`sha256_cbor` in this profile);
-- Redis RESP L2 endpoint and database semantics.
+- LMCache chunk size (`256` here);
+- LMCache hash algorithm (`sha256_cbor` here);
+- connector/key generation;
+- Redis database semantics.
 
-A successful Redis `SET` or byte-for-byte KDN round trip does not by itself
-prove a cache hit. Validate consumption with LMCache hit-token and remote-read
-metrics.
+A successful Redis `SET` or byte-for-byte KDN round trip does not alone prove a
+cache hit. The final acceptance test is LMCache/vLLM consumption metrics.
 
 ## Build the image for a new container
 
-Use this directory as the Docker build context. This keeps the context small and
-avoids sending models, KDN databases, logs, or other repository data to the
-Docker daemon.
+Use this directory as the Docker build context. It avoids sending models, KDN
+databases, logs, or other repository data to the Docker daemon.
 
 ```bash
 cd /llm-stack/CacheRoute
@@ -294,60 +351,57 @@ Install the mounted CacheRoute source without re-resolving the serving stack:
 cd /workspace/llm-stack/CacheRoute
 python3 -m pip install -e . --no-deps
 python3 -m pip check
+source env/docker/cu130/scripts/activate_v1.sh
+python3 env/docker/cu130/scripts/check_v1_environment.py
 ```
 
-## Runtime sanity check
-
-```bash
-python3 - <<'PY'
-import os
-import sys
-import torch
-
-print("python:", sys.executable)
-print("torch:", torch.__version__)
-print("torch CUDA runtime:", torch.version.cuda)
-print("CUDA available:", torch.cuda.is_available())
-print("GPU count:", torch.cuda.device_count())
-print("CacheRoute profile:", os.getenv("CACHEROUTE_RUNTIME_PROFILE"))
-PY
+## Completed KDN migration validation
+
+The modern runtime was validated end to end with the dataset document whose KID
+is `668f6bd1ad2419d9dfbcda0b689311b8b6696c7ed772001bea7bd12573137b4a`:
+
+```text
+KDN keys persisted locally:          96
+Redis keys restored:                 96
+Payload bytes restored:              1,006,632,960
+Missing/extra/mismatched entries:    0
+LMCache L2 lookup hits:              96/96 chunks
+LMCache lookup tokens:               3072/3072
+LMCache token hit rate:              100%
+vLLM external cached tokens:         3072
+LMCache L2 prefetch failures:        0
 ```
 
-The active interpreter should be `/opt/venv/bin/python3`, CUDA should be
-available when the container is started with `--gpus all`, and the CacheRoute
-profile should be `v1`.
+Reproduce the staged validation with:
+
+```bash
+DOC=/workspace/llm-stack/CacheRoute/data/CacheRoute_dataset/knowledge_document/668f6bd1ad2419d9dfbcda0b689311b8b6696c7ed772001bea7bd12573137b4a.txt
 
-## KDN migration status
+python3 scripts/validate_v1_kdn_roundtrip.py build --document "$DOC"
+python3 scripts/validate_v1_kdn_roundtrip.py inspect --document "$DOC"
+```
 
-The v1 branch contains the first runtime compatibility slice for KDN KV
-construction:
+For a destructive restore against an isolated Redis DB:
 
-- modern model-scoped LMCache Redis keys are discovered automatically;
-- legacy `vllm@*` keys remain supported;
-- asynchronous remote writes use a first-key timeout and quiet-period finish;
-- zero-key builds fail and are never marked `kv_ready`;
-- raw key/value dump and restore were verified byte-for-byte in the migration
-  environment.
+```bash
+python3 scripts/validate_v1_kdn_roundtrip.py inject \
+  --document "$DOC" \
+  --flush-redis \
+  --yes
+```
 
-This does not yet prove an end-to-end LMCache MP cache hit after injection.
-Validation must compare LMCache hit-token and remote-read metrics before the PR
-is considered complete.
+Restart LMCache and vLLM after restore, then prove consumption:
 
-## Validation status and scope
+```bash
+python3 scripts/validate_v1_kdn_roundtrip.py consume --document "$DOC"
+```
 
-The image build from the environment work was validated with:
+The injector implementation is `kdn_server/kv_injector.py`. Normal v1
+registration does not require `--match` and should not use `--flushdb`.
 
-- Python `3.12.13`;
-- PyTorch `2.11.0+cu130`;
-- vLLM `0.25.1`;
-- LMCache `0.5.2`;
-- Transformers `5.12.1`;
-- Redis Python client `8.0.1`;
-- successful `pip check`;
-- successful imports of Torch, TorchCodec, vLLM, LMCache, and Redis;
-- successful startup of the existing CacheRoute Scheduler, KDN, Proxy, and
-  Instance workflow after installing FFmpeg.
+## Compatibility and scope
 
-The modern environment profile and the runtime compatibility work are additive.
-The legacy image, root `requirements.txt`, Scheduler routing, and stable legacy
-runtime remain unchanged.
+The modern environment profile is additive. The legacy Dockerfile, root
+`requirements.txt`, Scheduler routing, and stable legacy runtime remain
+unchanged. Subsequent Scheduler, Proxy, Instance, predictor, and UI development
+can use this v1 stack as the modern baseline.

From 3fe246f83fc13368554e629b6465501bc22a3e41 Mon Sep 17 00:00:00 2001
From: RickZ <121943721+rickisba@users.noreply.github.com>
Date: Tue, 28 Jul 2026 10:34:16 +0800
Subject: [PATCH 7/7] docs: close the v1 runtime validation gap

---
 docs/runtime_compatibility_v1.md | 192 +++++++++++++++++--------------
 1 file changed, 105 insertions(+), 87 deletions(-)

diff --git a/docs/runtime_compatibility_v1.md b/docs/runtime_compatibility_v1.md
index 1a15cb9..ee3c5fd 100644
--- a/docs/runtime_compatibility_v1.md
+++ b/docs/runtime_compatibility_v1.md
@@ -1,159 +1,177 @@
 # CacheRoute v1 runtime compatibility
 
-This branch introduces a compatibility layer for two serving generations:
+CacheRoute supports two serving generations through one compatibility layer:
 
 | Profile | Intended serving stack | Redis KV key handling |
 |---|---|---|
 | `legacy` | vLLM 0.13.x + LMCache 0.3.11 | historical `vllm@*` namespace |
 | `v1` | vLLM 0.25.1 + LMCache 0.5.2 | model-scoped LMCache keys |
-| `auto` | default | discovers and validates either layout |
+| `auto` | default outside the dedicated v1 image | discovers and validates either layout |
 
 Set the profile with:
 
 ```bash
-export CACHEROUTE_RUNTIME_PROFILE=auto   # default
+export CACHEROUTE_RUNTIME_PROFILE=auto
 # or: legacy / v1
 ```
 
-The additive CUDA 13 image under [`env/docker/cu130`](../env/docker/cu130)
-sets `CACHEROUTE_RUNTIME_PROFILE=v1` by default. The legacy image and root
+For the modern stack, source the repository activation script instead of
+repeating environment variables in every terminal:
+
+```bash
+source env/docker/cu130/scripts/activate_v1.sh
+```
+
+The additive CUDA 13 image under [`env/docker/cu130`](../env/docker/cu130) sets
+`CACHEROUTE_RUNTIME_PROFILE=v1` by default. The legacy image and root
 requirements remain unchanged.
 
-## Startup interfaces are versioned too
+## Startup interfaces are versioned
 
-The serving commands are part of the runtime compatibility contract. The two
-stacks do not share the same primary LMCache/vLLM startup interface.
+The serving commands are part of the runtime compatibility contract:
 
 | Profile | LMCache process | vLLM connector path |
 |---|---|---|
 | `legacy` | YAML selected by `LMCACHE_CONFIG_FILE` | historical LMCache offloading arguments |
 | `v1` | standalone `lmcache server` with RESP L2 | `vllm serve` with `LMCacheMPConnector` in `--kv-transfer-config` |
 
-Do not mix the interfaces. In particular, the v1 path must unset the legacy
-`LMCACHE_CONFIG_FILE` and must not depend on `remote_url` or
-`--kv-offloading-backend lmcache`.
-
-The complete validated commands and environment-variable overrides are in
-[`env/docker/cu130/README.md`](../env/docker/cu130/README.md). Existing
-containers can use the repository scripts directly, without rebuilding:
+Do not mix the interfaces. The v1 path must not depend on `remote_url`,
+`LMCACHE_CONFIG_FILE`, or `--kv-offloading-backend lmcache`.
 
-```bash
-export CACHEROUTE_RUNTIME_PROFILE=v1
-bash env/docker/cu130/scripts/start_lmcache_mp.sh
-# In another terminal after port 5555 is listening:
-bash env/docker/cu130/scripts/start_vllm_mp.sh
-```
-
-The v1 startup order is:
+The validated v1 startup order is:
 
 ```text
 Redis RESP L2 -> LMCache MP :5555 -> vLLM :8000 -> CacheRoute components
 ```
 
+Complete commands and overrides are documented in
+[`env/docker/cu130/README.md`](../env/docker/cu130/README.md). Existing
+compatible containers can use the merged `main` branch without rebuilding.
+
 ## KDN KV construction
 
 Manual `--match` is no longer required in the normal CLI workflow. `auto` and
-`v1` modes treat the historical implicit `vllm@*` argument as a compatibility
-sentinel and scan for supported LMCache key layouts.
-
-An explicit `--match` remains authoritative for debugging or custom backends.
-For strict legacy behavior, set `CACHEROUTE_RUNTIME_PROFILE=legacy`.
-
-LMCache remote writes are asynchronous. KDN therefore uses:
+`v1` treat the historical implicit `vllm@*` value as a compatibility sentinel
+and scan for supported LMCache key layouts. An explicit `--match` remains
+authoritative for debugging or custom backends.
 
-- a polling interval (default `0.2s`),
-- a maximum first-key timeout (default `30s`), and
-- a quiet period after the last key-set change (default `1.5s`).
+LMCache remote writes are asynchronous. KDN uses:
 
-The 30-second value is an upper bound, not a fixed delay. A normal build exits
-as soon as keys appear and remain unchanged for the quiet period.
+- polling interval: `0.2s` by default;
+- maximum first-key timeout: `30s` by default;
+- quiet period after the last key-set change: `1.5s` by default.
 
-A build that captures zero keys now fails and removes its partial output
-directory. It is never marked `kv_ready`.
+The 30-second value is an upper bound, not a fixed delay. A zero-key build fails,
+removes its partial output, and is never marked `kv_ready`.
 
 ## Deployment profiles
 
-The serving stack remains owned by its Docker image. CacheRoute application
-dependencies must not upgrade CUDA-sensitive packages.
-
 ### Legacy profile
 
-The existing environment remains the stable path for:
+The stable legacy path remains:
 
 ```text
 CUDA 12.8 / PyTorch 2.9.x / vLLM 0.13.x / LMCache 0.3.11
 ```
 
-Use:
-
-```bash
-export CACHEROUTE_RUNTIME_PROFILE=legacy
-```
-
-when strict historical Redis-key behavior is required.
+Use `CACHEROUTE_RUNTIME_PROFILE=legacy` for strict historical Redis-key
+behavior.
 
 ### v1 profile
 
-The isolated modern image is defined in [`env/docker/cu130`](../env/docker/cu130):
+The modern image is defined in [`env/docker/cu130`](../env/docker/cu130):
 
 ```text
 CUDA 13.0 / Python 3.12 / PyTorch 2.11.0+cu130
 vLLM 0.25.1 / LMCache 0.5.2
 ```
 
-It uses `/opt/venv`, keeps the modern serving stack separate from Ubuntu system
-packages, installs FFmpeg for TorchCodec, and preserves Rust/Cargo and Tkinter
-for existing CacheRoute auxiliary components.
-
-The v1 dependency files are additive. They do not modify:
+It uses `/opt/venv`, isolates the serving stack from Ubuntu system packages,
+installs FFmpeg for TorchCodec, and preserves Rust/Cargo and Tkinter for
+CacheRoute auxiliary components.
 
-- `env/docker/Dockerfile`,
-- root `requirements.txt`,
-- `pyproject.toml`, or
-- legacy runtime behavior.
-
-For reliable differential capture, use a dedicated Redis database or Redis
-instance for KDN construction. `--flushdb` must not target an online shared DB.
+The v1 dependency files are additive. They do not modify the legacy Dockerfile,
+root `requirements.txt`, `pyproject.toml`, or legacy runtime behavior.
 
 ## Cache identity compatibility
 
 Runtime-profile selection does not make incompatible KV blocks reusable. KDN
-construction and the target Instance must still agree on the cache identity,
-including:
+construction and the consuming Instance must agree on:
 
-- model identity/path and model configuration;
+- model identity/path and served model;
+- tokenizer, chat template, and exact token prefix;
 - tensor-parallel topology and worker layout;
-- KV dtype;
+- model/KV dtype;
 - LMCache chunk size;
 - LMCache hash algorithm;
-- request tags or other key-affecting configuration;
-- RESP L2 endpoint/database semantics.
+- connector generation and key serialization;
+- Redis endpoint/database semantics.
 
-The validated v1 baseline currently uses chunk size `256` and hash algorithm
+The validated v1 baseline uses TP=8, chunk size `256`, and hash algorithm
 `sha256_cbor`.
 
-## Validation boundary
+## Completed validation
+
+The environment migration has been validated beyond byte-level storage:
+
+- modern LMCache keys were captured without manual `--match`;
+- 96 physical key/value blocks were persisted locally;
+- Redis restore reproduced 96 keys and 1,006,632,960 payload bytes with no
+  missing, extra, size-mismatched, or sampled SHA256-mismatched entries;
+- after LMCache/vLLM restart, LMCache found and loaded all 96 blocks from RESP
+  L2;
+- LMCache reported 3072 requested and 3072 hit tokens;
+- vLLM reported 3072 externally cached prompt tokens;
+- LMCache reported zero L2 prefetch failures.
+
+The relationship between physical keys and cached tokens is:
+
+```text
+96 Redis keys / 8 TP ranks = 12 logical chunks
+12 chunks * 256 tokens     = 3072 cached tokens
+```
+
+Detailed evidence and the reproducible staged workflow are in
+[`v1_migration_closeout.md`](v1_migration_closeout.md).
 
-The modern image has been validated for installation, imports, `pip check`, GPU
-runtime availability, and startup of the CacheRoute Scheduler/KDN/Proxy/Instance
-workflow.
+## Metrics contract
 
-The KDN raw Redis dump/restore path has also been verified byte-for-byte for the
-observed LMCache 0.5.2 key/value layout. This proves storage integrity, but does
-not yet prove that a request consumes the injected blocks through LMCache MP.
-End-to-end validation must compare LMCache hit-token and remote-read metrics.
+Use the correct endpoint for each subsystem:
+
+```text
+vLLM metrics:       http://127.0.0.1:8000/metrics
+LMCache MP metrics: http://127.0.0.1:8080/metrics
+```
+
+Important vLLM counters include:
+
+```text
+vllm:external_prefix_cache_queries_total
+vllm:external_prefix_cache_hits_total
+vllm:prompt_tokens_cached_total
+```
+
+Important LMCache MP counters include:
+
+```text
+lmcache_mp_lookup_requested_tokens_total
+lmcache_mp_lookup_hit_tokens_total
+lmcache_mp_l2_prefetch_hit_chunks_total
+lmcache_mp_l2_prefetch_load_completed_chunks_total
+lmcache_mp_l2_prefetch_failure_chunks_total
+```
 
-## Follow-up migration scope
+LMCache lookup metrics may not exist before the first request. Direct KDN Redis
+injection bypasses LMCache store accounting, so `lmcache_mp_l2_usage_bytes` is
+not a reliable indicator for injected data.
 
-Subsequent v1 changes should remain behind the compatibility layer where the
-vLLM/LMCache interfaces differ, including:
+## Future compatibility work
 
-- Instance-side observability and metric collection,
-- request metadata and prompt-prefix compatibility,
-- injected-cache hit validation,
-- dual-profile integration tests,
-- startup/configuration validation for each runtime profile.
+Subsequent changes should continue to isolate version-specific behavior behind
+`core/runtime_compat.py` or focused adapters rather than scattering version
+checks through Scheduler, Proxy, Instance, and KDN.
 
-Avoid scattering version checks through Scheduler, Proxy, Instance, and KDN
-code. Add profile-specific behavior to the compatibility layer or focused
-adapters.
+Future feature development may extend Instance observability, request metadata,
+cache placement, predictors, and UI behavior while using the validated v1
+runtime as the modern baseline. Reopen environment migration only when the
+serving-stack version or cache-identity contract changes.