diff --git a/Justfile b/Justfile index f2dc1b23..9826d02b 100644 --- a/Justfile +++ b/Justfile @@ -336,6 +336,23 @@ test-installer-artifact: DEADLINE_SECS="${SHOW_ME_THE_FUTURE_DEADLINE:-${SHOW_ME_THE_FUTURE_TIMEOUT:-600}}" START_TIME=$(date +%s) + + # Upstream synthetic demo fixtures (console pkg/api/handlers/demo_data.go + # GetDemoClusters). The console only sets .source == "demo" for requests + # carrying X-Demo-Mode: true, which this plain curl never sends, so + # matching these fixture names is what actually rejects demo data here. + DEMO_CLUSTER_NAMES='["kind-local","minikube","k3s-edge","eks-prod-us-east-1","gke-staging","aks-dev-westeu","openshift-prod","oci-oke-phoenix","alibaba-ack-shanghai","do-nyc1-prod","rancher-mgmt","vllm-gpu-cluster"]' + # Require live discovery: a k8s/mcp source that reports at least one + # cluster, none of which is a demo fixture. An empty .clusters array is + # rejected because the k8s fallback returns {"clusters": [], "source": "k8s"} + # before any cluster is reachable. + CLUSTERS_FILTER='.source != "demo" and (.source == "k8s" or .source == "mcp") and ((.clusters // []) | length) > 0 and (any(.clusters[]?.name; . as $name | $demo | index($name) != null) | not)' + + # Tracks the furthest readiness stage reached so a timeout reports which + # probe actually blocked the gate instead of a generic readiness failure. + LAST_STAGE="healthz" + LAST_CLUSTERS_JSON="" + echo "==> Polling KubeStellar Console readiness at http://127.0.0.1:8080 (deadline: ${DEADLINE_SECS}s)..." while true; do @@ -350,17 +367,35 @@ test-installer-artifact: HEALTHZ_JSON=$(curl --silent --fail --max-time 2 http://127.0.0.1:8080/healthz 2>/dev/null || true) if [ -n "$HEALTHZ_JSON" ] && echo "$HEALTHZ_JSON" | jq -e '.status == "ok"' >/dev/null 2>&1; then + LAST_STAGE="root" ROOT_CODE=$(curl --silent --fail --max-time 2 --output /dev/null --write-out "%{http_code}" http://127.0.0.1:8080/ 2>/dev/null || true) if [ "$ROOT_CODE" = "200" ]; then - echo "==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200" - break + LAST_STAGE="clusters" + CLUSTERS_JSON=$(curl --silent --fail --max-time 2 http://127.0.0.1:8080/api/mcp/clusters 2>/dev/null || true) + LAST_CLUSTERS_JSON="$CLUSTERS_JSON" + # Ensure actual cluster discovery is active and not silently passing on synthetic demo fixtures + if [ -n "$CLUSTERS_JSON" ] && echo "$CLUSTERS_JSON" | jq -e --argjson demo "$DEMO_CLUSTER_NAMES" "$CLUSTERS_FILTER" >/dev/null 2>&1; then + echo "==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200, and live cluster telemetry verified" + break + fi fi fi NOW=$(date +%s) ELAPSED=$((NOW - START_TIME)) if [ "$ELAPSED" -ge "$DEADLINE_SECS" ]; then - echo "ERROR: Timed out after ${DEADLINE_SECS}s waiting for KubeStellar Console readiness!" >&2 + case "$LAST_STAGE" in + healthz) + echo "ERROR: Timed out after ${DEADLINE_SECS}s waiting for KubeStellar Console readiness: /healthz never reported status ok!" >&2 + ;; + root) + echo "ERROR: Timed out after ${DEADLINE_SECS}s waiting for KubeStellar Console readiness: /healthz was ok but / never returned HTTP 200!" >&2 + ;; + clusters) + echo "ERROR: Timed out after ${DEADLINE_SECS}s waiting for KubeStellar Console readiness: /healthz and / passed but /api/mcp/clusters never reported live (non-demo, non-empty) cluster telemetry!" >&2 + echo "==> Last /api/mcp/clusters payload: ${LAST_CLUSTERS_JSON:-}" >&2 + ;; + esac if [ -f "$SERIAL_LOG" ]; then echo "==> Serial log tail (last 100 lines):" >&2 tail -n 100 "$SERIAL_LOG" >&2 diff --git a/tests/e2e/test_kubestellar_browser_login.py b/tests/e2e/test_kubestellar_browser_login.py index bf975b32..3c2bc0b0 100755 --- a/tests/e2e/test_kubestellar_browser_login.py +++ b/tests/e2e/test_kubestellar_browser_login.py @@ -4,7 +4,7 @@ Tests: 1. Console availability at http://127.0.0.1:8080/ 2. Automated login flow (Cluster Access / GitHub OAuth / Dev Mode / Token Login) -3. Dashboard navigation and DOM rendering +3. Dashboard navigation and DOM rendering of live cluster resources (rejecting demo mode) 4. Kiosk gate overlay and kc-agent interaction on port 8585 """ @@ -17,6 +17,7 @@ from typing import Any from selenium import webdriver +from selenium.common.exceptions import WebDriverException from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC @@ -41,6 +42,16 @@ def parse_args() -> argparse.Namespace: default=60, help="Max timeout in seconds to wait for console readiness (default: 60)", ) + parser.add_argument( + "--cluster-telemetry-timeout", + type=int, + default=CLUSTER_TELEMETRY_TIMEOUT, + help=( + "Max timeout in seconds to wait for live cluster resources to render " + f"(default: {CLUSTER_TELEMETRY_TIMEOUT}); upstream populates cluster health " + "asynchronously after boot" + ), + ) return parser.parse_args() def wait_for_http_ready(url: str, timeout: int, check_healthz: bool = True) -> bool: @@ -72,7 +83,128 @@ def wait_for_http_ready(url: str, timeout: int, check_healthz: bool = True) -> b return False -def run_browser_verification(console_url: str, agent_url: str) -> None: +# Upstream synthetic demo fixtures injected by the console demo data +# (pkg/api/handlers/demo_data.go GetDemoClusters). +DEMO_CLUSTER_NAMES = ( + "kind-local", + "minikube", + "k3s-edge", + "eks-prod-us-east-1", + "gke-staging", + "aks-dev-westeu", + "openshift-prod", + "oci-oke-phoenix", + "alibaba-ack-shanghai", + "do-nyc1-prod", + "rancher-mgmt", + "vllm-gpu-cluster", +) + +# Console routes that render one DOM element per discovered cluster, paired +# with the selector that only exists once a cluster was actually returned by +# the API. Static card containers and their headings are deliberately excluded: +# they render with empty data and would let the gate pass without any +# discovered cluster. +# /clusters -> ClusterGrid renders SortableClusterItem with +# data-testid="cluster-row-${cluster.name}" +# (web/src/components/clusters/components/ClusterDragReorder.tsx) +# /workloads -> Clusters Overview grid renders one +# data-testid="cluster-card" per cluster +# (web/src/components/workloads/Workloads.parts.tsx) +# The dashboard landing page has no such per-cluster testid, so these pages +# must be visited explicitly for the DOM verification to mean anything. +CLUSTER_RESOURCE_PAGES = ( + ("/clusters", "[data-testid^='cluster-row-']"), + ("/workloads", "[data-testid='cluster-card']"), +) + +# Live control-plane / in-cluster identifiers that never appear in demo data. +LIVE_CLUSTER_TEXT_MARKERS = ("its1", "wds1", "in-cluster") + +# Default budget for live cluster resources to render. Upstream ListClusters +# returns clusters with HealthUnknown and fills health in asynchronously after +# boot, so a short budget flakes on an otherwise healthy install. Override with +# --cluster-telemetry-timeout. +CLUSTER_TELEMETRY_TIMEOUT = 120 + + +def assert_no_demo_fixtures(driver: webdriver.Chrome) -> None: + """Fail if the rendered page contains upstream synthetic demo cluster fixtures.""" + page_text = driver.find_element(By.TAG_NAME, "body").text + for demo_name in DEMO_CLUSTER_NAMES: + if demo_name in page_text: + raise AssertionError( + f"Detected synthetic demo cluster '{demo_name}' in console DOM; demo mode was not rejected!" + ) + + +def verify_live_cluster_resources( + driver: webdriver.Chrome, + console_url: str, + timeout: int = CLUSTER_TELEMETRY_TIMEOUT, +) -> None: + """Verify that actual Kubernetes cluster resources are discovered and rendered in the DOM, + ensuring unconfigured demo placeholders or synthetic dev mode do not pass silently. + """ + print("==> Verifying live cluster resources are rendered in the DOM (rejecting demo mode)...") + + # 1. Reject synthetic demo mode flag in localStorage if explicitly active + try: + demo_mode_flag = driver.execute_script("return window.localStorage.getItem('kc-demo-mode')") + except WebDriverException as exc: + print(f"==> Warning: could not read kc-demo-mode from localStorage: {exc}") + else: + if demo_mode_flag == "true": + raise AssertionError("Console is running in synthetic demo mode (kc-demo-mode=true in localStorage)") + + # 2. Visit each cluster-listing route and wait for a real per-cluster + # element (or live cluster identifiers), re-checking for demo fixtures on + # every pass so late-rendered async demo content cannot slip through a + # single early scan. + base_url = console_url.rstrip("/") + start = time.time() + found_resource = False + while not found_resource and time.time() - start < timeout: + for route, selector in CLUSTER_RESOURCE_PAGES: + driver.get(f"{base_url}{route}") + WebDriverWait(driver, 15).until(EC.presence_of_element_located((By.TAG_NAME, "body"))) + + assert_no_demo_fixtures(driver) + + elements = driver.find_elements(By.CSS_SELECTOR, selector) + if elements and any(el.is_displayed() for el in elements): + found_resource = True + print(f"==> Found live cluster resource element on {route}: {selector}") + break + + body_text = driver.find_element(By.TAG_NAME, "body").text + if any(term in body_text for term in LIVE_CLUSTER_TEXT_MARKERS): + if "No clusters connected" not in body_text: + found_resource = True + print(f"==> Discovered live cluster workload indicators in {route} DOM text.") + break + + if not found_resource: + time.sleep(2) + + if not found_resource: + raise AssertionError( + f"Timed out after {timeout}s waiting for live Kubernetes cluster resources " + f"(per-cluster rows or cards on {', '.join(route for route, _ in CLUSTER_RESOURCE_PAGES)}, " + "or initialized ControlPlanes) to be rendered in the DOM." + ) + + # 3. Final demo scan after the async content settled. + assert_no_demo_fixtures(driver) + + print("==> Live cluster workload DOM rendering verified successfully!") + + +def run_browser_verification( + console_url: str, + agent_url: str, + cluster_telemetry_timeout: int = CLUSTER_TELEMETRY_TIMEOUT, +) -> None: chrome_opts = Options() chrome_opts.add_argument("--headless=new") chrome_opts.add_argument("--no-sandbox") @@ -129,7 +261,13 @@ def run_browser_verification(console_url: str, agent_url: str) -> None: ) print(f"==> Successfully verified console UI! Page title: {driver.title}") - # 3. Check kiosk gate behavior + # 3. Verify actual Kubernetes cluster resources are rendered (rejecting demo mode) + verify_live_cluster_resources(driver, console_url, timeout=cluster_telemetry_timeout) + + # 4. Check kiosk gate behavior on the landing page again, since the + # cluster verification navigated away to the cluster listing routes. + driver.get(console_url) + WebDriverWait(driver, 15).until(EC.presence_of_element_located((By.TAG_NAME, "body"))) gate_elements = driver.find_elements(By.ID, "kubestellar-kiosk-gate") agent_healthy = False try: @@ -158,7 +296,11 @@ def main() -> None: print(f"ERROR: Timed out waiting for console at {args.console_url}", file=sys.stderr) sys.exit(1) - run_browser_verification(args.console_url, args.agent_url) + run_browser_verification( + args.console_url, + args.agent_url, + cluster_telemetry_timeout=args.cluster_telemetry_timeout, + ) if __name__ == "__main__": diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index d423c685..bc6df57e 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +import subprocess from pathlib import Path import yaml @@ -135,3 +136,91 @@ def test_installer_and_ddi_strip_vmlinux_and_static_archives() -> None: assert "find /layer -type f -name '*.a' -delete" in installer_element assert 'rm -f "/layer/usr/lib/modules/${KVER}/vmlinux"' in ddi_element assert "find /layer -type f -name '*.a' -delete" in ddi_element + + +def _smoke_recipe() -> str: + justfile = JUSTFILE.read_text(encoding="utf-8") + start = justfile.index("test-installer-artifact:") + end = justfile.index("install-vm:", start) + return justfile[start:end] + + +def _smoke_gate_jq_filter() -> str: + match = re.search(r"CLUSTERS_FILTER='([^']+)'", _smoke_recipe()) + assert match, "smoke recipe must define CLUSTERS_FILTER" + return match.group(1) + + +def _smoke_gate_demo_names() -> str: + match = re.search(r"DEMO_CLUSTER_NAMES='([^']+)'", _smoke_recipe()) + assert match, "smoke recipe must define DEMO_CLUSTER_NAMES" + return match.group(1) + + +def test_installer_smoke_probes_live_cluster_telemetry_and_rejects_demo_mode() -> None: + recipe = _smoke_recipe() + + assert "/api/mcp/clusters" in recipe + assert '.source != "demo"' in recipe + assert "/healthz" in recipe + assert "200" in recipe + # Demo fixtures are what the unauthenticated probe can actually observe, + # because the console only reports source "demo" for X-Demo-Mode requests. + assert "kind-local" in _smoke_gate_demo_names() + + +def test_smoke_gate_timeout_reports_which_stage_blocked() -> None: + recipe = _smoke_recipe() + + # A timeout must name the stage that never passed so CI triage does not + # have to guess between healthz, root and cluster telemetry. + assert "LAST_STAGE" in recipe + assert "/healthz never reported status ok" in recipe + assert "never returned HTTP 200" in recipe + assert "never reported live (non-demo, non-empty) cluster telemetry" in recipe + + +def test_smoke_gate_jq_filter_accepts_live_telemetry_and_rejects_demo_mode() -> None: + filter_expr = _smoke_gate_jq_filter() + demo_names = _smoke_gate_demo_names() + + def eval_filter(payload: str) -> int: + proc = subprocess.run( + ["jq", "-e", "--argjson", "demo", demo_names, filter_expr], + input=payload, + capture_output=True, + text=True, + ) + return proc.returncode + + # Demo mode with synthetic clusters must be rejected + demo_payload = '{"clusters": [{"name": "kind-local"}], "source": "demo"}' + assert eval_filter(demo_payload) != 0 + + # Synthetic demo fixtures served without the demo source must be rejected + demo_fixture_payload = '{"clusters": [{"name": "minikube"}], "source": "k8s"}' + assert eval_filter(demo_fixture_payload) != 0 + + # A single demo fixture alongside a live cluster must still be rejected + mixed_payload = '{"clusters": [{"name": "its1"}, {"name": "k3s-edge"}], "source": "mcp"}' + assert eval_filter(mixed_payload) != 0 + + # Service unavailable / No cluster access must be rejected + error_payload = '{"error": "No cluster access"}' + assert eval_filter(error_payload) != 0 + + # Zero discovered clusters must be rejected: the console k8s fallback + # returns this shape before any cluster becomes reachable. + empty_k8s_payload = '{"clusters": [], "source": "k8s"}' + assert eval_filter(empty_k8s_payload) != 0 + + # A missing clusters key must be rejected too + assert eval_filter('{"source": "k8s"}') != 0 + + # Live k8s cluster telemetry must be accepted + k8s_payload = '{"clusters": [{"name": "in-cluster", "healthy": true}], "source": "k8s"}' + assert eval_filter(k8s_payload) == 0 + + # Live mcp cluster telemetry must be accepted + mcp_payload = '{"clusters": [{"name": "its1"}], "source": "mcp"}' + assert eval_filter(mcp_payload) == 0 diff --git a/tests/unit/test_lima_e2e_contract.py b/tests/unit/test_lima_e2e_contract.py index a4799a8b..5ec1ef91 100644 --- a/tests/unit/test_lima_e2e_contract.py +++ b/tests/unit/test_lima_e2e_contract.py @@ -37,3 +37,25 @@ def test_justfile_defines_setup_kubestellar_and_e2e_targets() -> None: justfile = JUSTFILE.read_text(encoding="utf-8") assert "test-e2e-browser" in justfile assert "test-e2e-lima:" in justfile + + +def test_e2e_browser_test_script_verifies_live_cluster_telemetry_and_rejects_demo_mode() -> None: + content = E2E_SCRIPT.read_text(encoding="utf-8") + assert "verify_live_cluster_resources" in content + assert "kc-demo-mode" in content + assert "kind-local" in content + assert "minikube" in content + assert "No clusters connected" in content + # Demo fixtures must be re-scanned while async content loads, not once up front. + assert "assert_no_demo_fixtures" in content + # Per-cluster DOM elements only exist on the cluster listing routes, which + # the script must navigate to explicitly. + assert "/clusters" in content + assert "cluster-row-" in content + assert "cluster-card" in content + # Cluster health is populated asynchronously after boot, so the render + # budget must be configurable rather than a short hardcoded wait. + assert "cluster-telemetry-timeout" in content + assert "CLUSTER_TELEMETRY_TIMEOUT" in content + # WebDriver failures must be caught by type, not by matching message text. + assert "except WebDriverException" in content