From cc40da81017ecc03bd3674bd66814b6f09a475d4 Mon Sep 17 00:00:00 2001 From: mrbobbytables Date: Sat, 19 Sep 2026 22:46:36 +0000 Subject: [PATCH 1/3] test(installer): probe cluster telemetry and reject demo mode in smoke gates Enhance test-installer-artifact readiness check to verify /api/mcp/clusters rejecting demo mode (.source != "demo") alongside /healthz and root HTTP 200. Update E2E browser verification to assert live Kubernetes cluster resources in the DOM and reject synthetic demo fixtures and demo mode localStorage flags. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: mrbobbytables --- Justfile | 8 ++- tests/e2e/test_kubestellar_browser_login.py | 76 ++++++++++++++++++++- tests/unit/test_installer_contract.py | 44 ++++++++++++ tests/unit/test_lima_e2e_contract.py | 10 +++ 4 files changed, 134 insertions(+), 4 deletions(-) diff --git a/Justfile b/Justfile index f2dc1b23..e4c1b33f 100644 --- a/Justfile +++ b/Justfile @@ -352,8 +352,12 @@ test-installer-artifact: if [ -n "$HEALTHZ_JSON" ] && echo "$HEALTHZ_JSON" | jq -e '.status == "ok"' >/dev/null 2>&1; then ROOT_CODE=$(curl --silent --fail --max-time 2 --output /dev/null --write-out "%{http_code}" http://127.0.0.1:8080/ 2>/dev/null || true) if [ "$ROOT_CODE" = "200" ]; then - echo "==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200" - break + CLUSTERS_JSON=$(curl --silent --fail --max-time 2 http://127.0.0.1:8080/api/mcp/clusters 2>/dev/null || true) + # Ensure actual cluster discovery is active and not silently passing on synthetic demo mode + if [ -n "$CLUSTERS_JSON" ] && echo "$CLUSTERS_JSON" | jq -e '.source != "demo" and ((.clusters | length) > 0 or .source == "k8s" or .source == "mcp")' >/dev/null 2>&1; then + echo "==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200, and live cluster telemetry verified" + break + fi fi fi diff --git a/tests/e2e/test_kubestellar_browser_login.py b/tests/e2e/test_kubestellar_browser_login.py index bf975b32..bf638810 100755 --- a/tests/e2e/test_kubestellar_browser_login.py +++ b/tests/e2e/test_kubestellar_browser_login.py @@ -4,7 +4,7 @@ Tests: 1. Console availability at http://127.0.0.1:8080/ 2. Automated login flow (Cluster Access / GitHub OAuth / Dev Mode / Token Login) -3. Dashboard navigation and DOM rendering +3. Dashboard navigation and DOM rendering of live cluster resources (rejecting demo mode) 4. Kiosk gate overlay and kc-agent interaction on port 8585 """ @@ -72,6 +72,75 @@ def wait_for_http_ready(url: str, timeout: int, check_healthz: bool = True) -> b return False +def verify_live_cluster_resources(driver: webdriver.Chrome, timeout: int = 15) -> None: + """Verify that actual Kubernetes cluster resources are discovered and rendered in the DOM, + ensuring unconfigured demo placeholders or synthetic dev mode do not pass silently. + """ + print("==> Verifying live cluster resources are rendered in the DOM (rejecting demo mode)...") + + # 1. Reject synthetic demo mode flag in localStorage if explicitly active + try: + demo_mode_flag = driver.execute_script("return window.localStorage.getItem('kc-demo-mode')") + if demo_mode_flag == "true": + raise AssertionError("Console is running in synthetic demo mode (kc-demo-mode=true in localStorage)") + except Exception as e: + if "synthetic demo mode" in str(e): + raise + + # 2. Reject synthetic demo cluster mock names in the page + # Upstream demo data injects synthetic clusters: "kind-local", "minikube", "k3s-edge", "eks-prod-us-east-1" + demo_clusters = ["kind-local", "minikube", "k3s-edge", "eks-prod-us-east-1"] + page_text = driver.find_element(By.TAG_NAME, "body").text + for demo_name in demo_clusters: + if demo_name in page_text: + raise AssertionError(f"Detected synthetic demo cluster '{demo_name}' in console DOM; demo mode was not rejected!") + + # 3. Wait for real cluster resource elements, cards, or metrics in the DOM + cluster_resource_selectors = [ + "[data-testid='cluster-card']", + "[data-testid='clusters-page']", + "[data-testid='card-cluster-health']", + "[data-testid='card-node-status']", + "[data-testid='card-resource-usage']", + "[data-testid='card-top-pods']", + "[data-testid='stat-block-healthy-count']", + "[data-testid='stat-block-total-nodes']", + "[data-testid='stat-block-total-pods']", + "[data-testid='node-row']", + "[data-testid='pod-row']", + ] + + start = time.time() + found_resource = False + while time.time() - start < timeout: + for selector in cluster_resource_selectors: + elements = driver.find_elements(By.CSS_SELECTOR, selector) + if elements and any(el.is_displayed() for el in elements): + found_resource = True + print(f"==> Found live cluster resource element: {selector}") + break + if found_resource: + break + + body_text = driver.find_element(By.TAG_NAME, "body").text + # Look for live cluster/controlplane identifiers or node/pod telemetry + if any(term in body_text for term in ["its1", "wds1", "in-cluster", "Node Status", "Cluster Health", "ControlPlane", "Top Pods"]): + if "No clusters connected" not in body_text: + found_resource = True + print("==> Discovered live cluster workload indicators in DOM text.") + break + + time.sleep(1) + + if not found_resource: + raise AssertionError( + "Timed out waiting for live Kubernetes cluster resources (nodes, pods, or initialized ControlPlanes) " + "to be rendered in the DOM." + ) + + print("==> Live cluster workload DOM rendering verified successfully!") + + def run_browser_verification(console_url: str, agent_url: str) -> None: chrome_opts = Options() chrome_opts.add_argument("--headless=new") @@ -129,7 +198,10 @@ def run_browser_verification(console_url: str, agent_url: str) -> None: ) print(f"==> Successfully verified console UI! Page title: {driver.title}") - # 3. Check kiosk gate behavior + # 3. Verify actual Kubernetes cluster resources are rendered (rejecting demo mode) + verify_live_cluster_resources(driver, timeout=15) + + # 4. Check kiosk gate behavior gate_elements = driver.find_elements(By.ID, "kubestellar-kiosk-gate") agent_healthy = False try: diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index d423c685..3465d154 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +import subprocess from pathlib import Path import yaml @@ -135,3 +136,46 @@ def test_installer_and_ddi_strip_vmlinux_and_static_archives() -> None: assert "find /layer -type f -name '*.a' -delete" in installer_element assert 'rm -f "/layer/usr/lib/modules/${KVER}/vmlinux"' in ddi_element assert "find /layer -type f -name '*.a' -delete" in ddi_element + + +def test_installer_smoke_probes_live_cluster_telemetry_and_rejects_demo_mode() -> None: + justfile = JUSTFILE.read_text(encoding="utf-8") + start = justfile.index("test-installer-artifact:") + end = justfile.index("install-vm:", start) + recipe = justfile[start:end] + + assert "/api/mcp/clusters" in recipe + assert '.source != "demo"' in recipe + assert "/healthz" in recipe + assert "200" in recipe + + +def test_smoke_gate_jq_filter_accepts_live_telemetry_and_rejects_demo_mode() -> None: + filter_expr = '.source != "demo" and ((.clusters | length) > 0 or .source == "k8s" or .source == "mcp")' + + def eval_filter(payload: str) -> int: + proc = subprocess.run( + ["jq", "-e", filter_expr], + input=payload, + capture_output=True, + text=True, + ) + return proc.returncode + + # Demo mode with synthetic clusters must be rejected + demo_payload = '{"clusters": [{"name": "kind-local"}], "source": "demo"}' + assert eval_filter(demo_payload) != 0 + + # Service unavailable / No cluster access must be rejected + error_payload = '{"error": "No cluster access"}' + assert eval_filter(error_payload) != 0 + + # Live k8s cluster telemetry must be accepted + k8s_payload = '{"clusters": [{"name": "in-cluster", "healthy": true}], "source": "k8s"}' + assert eval_filter(k8s_payload) == 0 + + # Live mcp cluster telemetry must be accepted + mcp_payload = '{"clusters": [{"name": "its1"}], "source": "mcp"}' + assert eval_filter(mcp_payload) == 0 + + diff --git a/tests/unit/test_lima_e2e_contract.py b/tests/unit/test_lima_e2e_contract.py index a4799a8b..0fcb30f4 100644 --- a/tests/unit/test_lima_e2e_contract.py +++ b/tests/unit/test_lima_e2e_contract.py @@ -37,3 +37,13 @@ def test_justfile_defines_setup_kubestellar_and_e2e_targets() -> None: justfile = JUSTFILE.read_text(encoding="utf-8") assert "test-e2e-browser" in justfile assert "test-e2e-lima:" in justfile + + +def test_e2e_browser_test_script_verifies_live_cluster_telemetry_and_rejects_demo_mode() -> None: + content = E2E_SCRIPT.read_text(encoding="utf-8") + assert "verify_live_cluster_resources" in content + assert "kc-demo-mode" in content + assert "kind-local" in content + assert "minikube" in content + assert "No clusters connected" in content + From 23f1228c79f22a28572e04864f55dfbed02f65bf Mon Sep 17 00:00:00 2001 From: "sec-check[bot]" Date: Mon, 21 Sep 2026 21:49:15 -0400 Subject: [PATCH 2/3] test(installer): require non-empty live cluster discovery in smoke gates The smoke gate accepted {"clusters": [], "source": "k8s"}, which the console k8s fallback returns before any cluster is reachable, so it did not enforce active cluster discovery. It also checked .source != "demo", which is unreachable for a plain curl because the console only serves demo data for requests carrying X-Demo-Mode: true. Require a k8s/mcp source with at least one discovered cluster and reject responses containing upstream synthetic demo fixture names, which is what the unauthenticated probe can actually observe. Extract the filter and the demo fixture list into shell variables and read them from the Justfile in the unit test so the test cannot drift from the gate. Tighten the browser check as well: drop static card headings and container selectors that render with empty data, re-scan for demo fixtures on every poll so late-rendered demo content cannot slip through, and catch WebDriverException by type instead of matching exception message text. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: sec-check[bot] --- Justfile | 16 +++- tests/e2e/test_kubestellar_browser_login.py | 84 ++++++++++++++------- tests/unit/test_installer_contract.py | 46 +++++++++-- tests/unit/test_lima_e2e_contract.py | 8 +- 4 files changed, 116 insertions(+), 38 deletions(-) diff --git a/Justfile b/Justfile index e4c1b33f..91feb977 100644 --- a/Justfile +++ b/Justfile @@ -336,6 +336,18 @@ test-installer-artifact: DEADLINE_SECS="${SHOW_ME_THE_FUTURE_DEADLINE:-${SHOW_ME_THE_FUTURE_TIMEOUT:-600}}" START_TIME=$(date +%s) + + # Upstream synthetic demo fixtures (console pkg/api/handlers/demo_data.go + # GetDemoClusters). The console only sets .source == "demo" for requests + # carrying X-Demo-Mode: true, which this plain curl never sends, so + # matching these fixture names is what actually rejects demo data here. + DEMO_CLUSTER_NAMES='["kind-local","minikube","k3s-edge","eks-prod-us-east-1","gke-staging","aks-dev-westeu","openshift-prod","oci-oke-phoenix","alibaba-ack-shanghai","do-nyc1-prod","rancher-mgmt","vllm-gpu-cluster"]' + # Require live discovery: a k8s/mcp source that reports at least one + # cluster, none of which is a demo fixture. An empty .clusters array is + # rejected because the k8s fallback returns {"clusters": [], "source": "k8s"} + # before any cluster is reachable. + CLUSTERS_FILTER='.source != "demo" and (.source == "k8s" or .source == "mcp") and ((.clusters // []) | length) > 0 and (any(.clusters[]?.name; . as $name | $demo | index($name) != null) | not)' + echo "==> Polling KubeStellar Console readiness at http://127.0.0.1:8080 (deadline: ${DEADLINE_SECS}s)..." while true; do @@ -353,8 +365,8 @@ test-installer-artifact: ROOT_CODE=$(curl --silent --fail --max-time 2 --output /dev/null --write-out "%{http_code}" http://127.0.0.1:8080/ 2>/dev/null || true) if [ "$ROOT_CODE" = "200" ]; then CLUSTERS_JSON=$(curl --silent --fail --max-time 2 http://127.0.0.1:8080/api/mcp/clusters 2>/dev/null || true) - # Ensure actual cluster discovery is active and not silently passing on synthetic demo mode - if [ -n "$CLUSTERS_JSON" ] && echo "$CLUSTERS_JSON" | jq -e '.source != "demo" and ((.clusters | length) > 0 or .source == "k8s" or .source == "mcp")' >/dev/null 2>&1; then + # Ensure actual cluster discovery is active and not silently passing on synthetic demo fixtures + if [ -n "$CLUSTERS_JSON" ] && echo "$CLUSTERS_JSON" | jq -e --argjson demo "$DEMO_CLUSTER_NAMES" "$CLUSTERS_FILTER" >/dev/null 2>&1; then echo "==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200, and live cluster telemetry verified" break fi diff --git a/tests/e2e/test_kubestellar_browser_login.py b/tests/e2e/test_kubestellar_browser_login.py index bf638810..a3889f67 100755 --- a/tests/e2e/test_kubestellar_browser_login.py +++ b/tests/e2e/test_kubestellar_browser_login.py @@ -17,6 +17,7 @@ from typing import Any from selenium import webdriver +from selenium.common.exceptions import WebDriverException from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC @@ -72,6 +73,47 @@ def wait_for_http_ready(url: str, timeout: int, check_healthz: bool = True) -> b return False +# Upstream synthetic demo fixtures injected by the console demo data +# (pkg/api/handlers/demo_data.go GetDemoClusters). +DEMO_CLUSTER_NAMES = ( + "kind-local", + "minikube", + "k3s-edge", + "eks-prod-us-east-1", + "gke-staging", + "aks-dev-westeu", + "openshift-prod", + "oci-oke-phoenix", + "alibaba-ack-shanghai", + "do-nyc1-prod", + "rancher-mgmt", + "vllm-gpu-cluster", +) + +# Selectors that only render once real cluster resources are returned. Static +# dashboard card containers and their headings are deliberately excluded: they +# render with empty data and would let the gate pass without any discovered +# cluster. +CLUSTER_RESOURCE_SELECTORS = ( + "[data-testid='cluster-card']", + "[data-testid='node-row']", + "[data-testid='pod-row']", +) + +# Live control-plane / in-cluster identifiers that never appear in demo data. +LIVE_CLUSTER_TEXT_MARKERS = ("its1", "wds1", "in-cluster") + + +def assert_no_demo_fixtures(driver: webdriver.Chrome) -> None: + """Fail if the rendered page contains upstream synthetic demo cluster fixtures.""" + page_text = driver.find_element(By.TAG_NAME, "body").text + for demo_name in DEMO_CLUSTER_NAMES: + if demo_name in page_text: + raise AssertionError( + f"Detected synthetic demo cluster '{demo_name}' in console DOM; demo mode was not rejected!" + ) + + def verify_live_cluster_resources(driver: webdriver.Chrome, timeout: int = 15) -> None: """Verify that actual Kubernetes cluster resources are discovered and rendered in the DOM, ensuring unconfigured demo placeholders or synthetic dev mode do not pass silently. @@ -81,39 +123,21 @@ def verify_live_cluster_resources(driver: webdriver.Chrome, timeout: int = 15) - # 1. Reject synthetic demo mode flag in localStorage if explicitly active try: demo_mode_flag = driver.execute_script("return window.localStorage.getItem('kc-demo-mode')") + except WebDriverException as exc: + print(f"==> Warning: could not read kc-demo-mode from localStorage: {exc}") + else: if demo_mode_flag == "true": raise AssertionError("Console is running in synthetic demo mode (kc-demo-mode=true in localStorage)") - except Exception as e: - if "synthetic demo mode" in str(e): - raise - - # 2. Reject synthetic demo cluster mock names in the page - # Upstream demo data injects synthetic clusters: "kind-local", "minikube", "k3s-edge", "eks-prod-us-east-1" - demo_clusters = ["kind-local", "minikube", "k3s-edge", "eks-prod-us-east-1"] - page_text = driver.find_element(By.TAG_NAME, "body").text - for demo_name in demo_clusters: - if demo_name in page_text: - raise AssertionError(f"Detected synthetic demo cluster '{demo_name}' in console DOM; demo mode was not rejected!") - - # 3. Wait for real cluster resource elements, cards, or metrics in the DOM - cluster_resource_selectors = [ - "[data-testid='cluster-card']", - "[data-testid='clusters-page']", - "[data-testid='card-cluster-health']", - "[data-testid='card-node-status']", - "[data-testid='card-resource-usage']", - "[data-testid='card-top-pods']", - "[data-testid='stat-block-healthy-count']", - "[data-testid='stat-block-total-nodes']", - "[data-testid='stat-block-total-pods']", - "[data-testid='node-row']", - "[data-testid='pod-row']", - ] + # 2. Wait for real cluster resource elements or live cluster identifiers, + # re-checking for demo fixtures on every pass so late-rendered async + # demo content cannot slip through a single early scan. start = time.time() found_resource = False while time.time() - start < timeout: - for selector in cluster_resource_selectors: + assert_no_demo_fixtures(driver) + + for selector in CLUSTER_RESOURCE_SELECTORS: elements = driver.find_elements(By.CSS_SELECTOR, selector) if elements and any(el.is_displayed() for el in elements): found_resource = True @@ -123,8 +147,7 @@ def verify_live_cluster_resources(driver: webdriver.Chrome, timeout: int = 15) - break body_text = driver.find_element(By.TAG_NAME, "body").text - # Look for live cluster/controlplane identifiers or node/pod telemetry - if any(term in body_text for term in ["its1", "wds1", "in-cluster", "Node Status", "Cluster Health", "ControlPlane", "Top Pods"]): + if any(term in body_text for term in LIVE_CLUSTER_TEXT_MARKERS): if "No clusters connected" not in body_text: found_resource = True print("==> Discovered live cluster workload indicators in DOM text.") @@ -138,6 +161,9 @@ def verify_live_cluster_resources(driver: webdriver.Chrome, timeout: int = 15) - "to be rendered in the DOM." ) + # 3. Final demo scan after the async content settled. + assert_no_demo_fixtures(driver) + print("==> Live cluster workload DOM rendering verified successfully!") diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index 3465d154..4e74399c 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -138,24 +138,44 @@ def test_installer_and_ddi_strip_vmlinux_and_static_archives() -> None: assert "find /layer -type f -name '*.a' -delete" in ddi_element -def test_installer_smoke_probes_live_cluster_telemetry_and_rejects_demo_mode() -> None: +def _smoke_recipe() -> str: justfile = JUSTFILE.read_text(encoding="utf-8") start = justfile.index("test-installer-artifact:") end = justfile.index("install-vm:", start) - recipe = justfile[start:end] + return justfile[start:end] + + +def _smoke_gate_jq_filter() -> str: + match = re.search(r"CLUSTERS_FILTER='([^']+)'", _smoke_recipe()) + assert match, "smoke recipe must define CLUSTERS_FILTER" + return match.group(1) + + +def _smoke_gate_demo_names() -> str: + match = re.search(r"DEMO_CLUSTER_NAMES='([^']+)'", _smoke_recipe()) + assert match, "smoke recipe must define DEMO_CLUSTER_NAMES" + return match.group(1) + + +def test_installer_smoke_probes_live_cluster_telemetry_and_rejects_demo_mode() -> None: + recipe = _smoke_recipe() assert "/api/mcp/clusters" in recipe assert '.source != "demo"' in recipe assert "/healthz" in recipe assert "200" in recipe + # Demo fixtures are what the unauthenticated probe can actually observe, + # because the console only reports source "demo" for X-Demo-Mode requests. + assert "kind-local" in _smoke_gate_demo_names() def test_smoke_gate_jq_filter_accepts_live_telemetry_and_rejects_demo_mode() -> None: - filter_expr = '.source != "demo" and ((.clusters | length) > 0 or .source == "k8s" or .source == "mcp")' + filter_expr = _smoke_gate_jq_filter() + demo_names = _smoke_gate_demo_names() def eval_filter(payload: str) -> int: proc = subprocess.run( - ["jq", "-e", filter_expr], + ["jq", "-e", "--argjson", "demo", demo_names, filter_expr], input=payload, capture_output=True, text=True, @@ -166,10 +186,26 @@ def eval_filter(payload: str) -> int: demo_payload = '{"clusters": [{"name": "kind-local"}], "source": "demo"}' assert eval_filter(demo_payload) != 0 + # Synthetic demo fixtures served without the demo source must be rejected + demo_fixture_payload = '{"clusters": [{"name": "minikube"}], "source": "k8s"}' + assert eval_filter(demo_fixture_payload) != 0 + + # A single demo fixture alongside a live cluster must still be rejected + mixed_payload = '{"clusters": [{"name": "its1"}, {"name": "k3s-edge"}], "source": "mcp"}' + assert eval_filter(mixed_payload) != 0 + # Service unavailable / No cluster access must be rejected error_payload = '{"error": "No cluster access"}' assert eval_filter(error_payload) != 0 + # Zero discovered clusters must be rejected: the console k8s fallback + # returns this shape before any cluster becomes reachable. + empty_k8s_payload = '{"clusters": [], "source": "k8s"}' + assert eval_filter(empty_k8s_payload) != 0 + + # A missing clusters key must be rejected too + assert eval_filter('{"source": "k8s"}') != 0 + # Live k8s cluster telemetry must be accepted k8s_payload = '{"clusters": [{"name": "in-cluster", "healthy": true}], "source": "k8s"}' assert eval_filter(k8s_payload) == 0 @@ -177,5 +213,3 @@ def eval_filter(payload: str) -> int: # Live mcp cluster telemetry must be accepted mcp_payload = '{"clusters": [{"name": "its1"}], "source": "mcp"}' assert eval_filter(mcp_payload) == 0 - - diff --git a/tests/unit/test_lima_e2e_contract.py b/tests/unit/test_lima_e2e_contract.py index 0fcb30f4..ac009d51 100644 --- a/tests/unit/test_lima_e2e_contract.py +++ b/tests/unit/test_lima_e2e_contract.py @@ -46,4 +46,10 @@ def test_e2e_browser_test_script_verifies_live_cluster_telemetry_and_rejects_dem assert "kind-local" in content assert "minikube" in content assert "No clusters connected" in content - + # Demo fixtures must be re-scanned while async content loads, not once up front. + assert "assert_no_demo_fixtures" in content + # Static card headings render with empty data, so they must not gate the check. + assert "Node Status" not in content + assert "Top Pods" not in content + # WebDriver failures must be caught by type, not by matching message text. + assert "except WebDriverException" in content From 6c6b1592259fc5361a0ba3c704f9ab831f9ef312 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:59:30 -0400 Subject: [PATCH 3/3] test(installer): probe real cluster routes and report blocking smoke stage - Replace non-existent node-row/pod-row selectors with the per-cluster testids the console actually renders (cluster-row- on /clusters, cluster-card on /workloads) and navigate to those routes, so the DOM verification is actually exercised instead of falling back to text markers. - Make the live-cluster render budget configurable (default 120s, new --cluster-telemetry-timeout flag); upstream populates cluster health asynchronously after boot, so 15s could flake on a healthy install. - Report which readiness stage blocked the smoke gate on timeout (healthz, root, or cluster telemetry) and echo the last /api/mcp/clusters payload. - Drop the forbidden-string asserts in the e2e contract test in favour of positive asserts on the new routes, selectors and timeout flag. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Justfile | 21 +++- tests/e2e/test_kubestellar_browser_login.py | 106 ++++++++++++++------ tests/unit/test_installer_contract.py | 11 ++ tests/unit/test_lima_e2e_contract.py | 12 ++- 4 files changed, 115 insertions(+), 35 deletions(-) diff --git a/Justfile b/Justfile index 91feb977..9826d02b 100644 --- a/Justfile +++ b/Justfile @@ -348,6 +348,11 @@ test-installer-artifact: # before any cluster is reachable. CLUSTERS_FILTER='.source != "demo" and (.source == "k8s" or .source == "mcp") and ((.clusters // []) | length) > 0 and (any(.clusters[]?.name; . as $name | $demo | index($name) != null) | not)' + # Tracks the furthest readiness stage reached so a timeout reports which + # probe actually blocked the gate instead of a generic readiness failure. + LAST_STAGE="healthz" + LAST_CLUSTERS_JSON="" + echo "==> Polling KubeStellar Console readiness at http://127.0.0.1:8080 (deadline: ${DEADLINE_SECS}s)..." while true; do @@ -362,9 +367,12 @@ test-installer-artifact: HEALTHZ_JSON=$(curl --silent --fail --max-time 2 http://127.0.0.1:8080/healthz 2>/dev/null || true) if [ -n "$HEALTHZ_JSON" ] && echo "$HEALTHZ_JSON" | jq -e '.status == "ok"' >/dev/null 2>&1; then + LAST_STAGE="root" ROOT_CODE=$(curl --silent --fail --max-time 2 --output /dev/null --write-out "%{http_code}" http://127.0.0.1:8080/ 2>/dev/null || true) if [ "$ROOT_CODE" = "200" ]; then + LAST_STAGE="clusters" CLUSTERS_JSON=$(curl --silent --fail --max-time 2 http://127.0.0.1:8080/api/mcp/clusters 2>/dev/null || true) + LAST_CLUSTERS_JSON="$CLUSTERS_JSON" # Ensure actual cluster discovery is active and not silently passing on synthetic demo fixtures if [ -n "$CLUSTERS_JSON" ] && echo "$CLUSTERS_JSON" | jq -e --argjson demo "$DEMO_CLUSTER_NAMES" "$CLUSTERS_FILTER" >/dev/null 2>&1; then echo "==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200, and live cluster telemetry verified" @@ -376,7 +384,18 @@ test-installer-artifact: NOW=$(date +%s) ELAPSED=$((NOW - START_TIME)) if [ "$ELAPSED" -ge "$DEADLINE_SECS" ]; then - echo "ERROR: Timed out after ${DEADLINE_SECS}s waiting for KubeStellar Console readiness!" >&2 + case "$LAST_STAGE" in + healthz) + echo "ERROR: Timed out after ${DEADLINE_SECS}s waiting for KubeStellar Console readiness: /healthz never reported status ok!" >&2 + ;; + root) + echo "ERROR: Timed out after ${DEADLINE_SECS}s waiting for KubeStellar Console readiness: /healthz was ok but / never returned HTTP 200!" >&2 + ;; + clusters) + echo "ERROR: Timed out after ${DEADLINE_SECS}s waiting for KubeStellar Console readiness: /healthz and / passed but /api/mcp/clusters never reported live (non-demo, non-empty) cluster telemetry!" >&2 + echo "==> Last /api/mcp/clusters payload: ${LAST_CLUSTERS_JSON:-}" >&2 + ;; + esac if [ -f "$SERIAL_LOG" ]; then echo "==> Serial log tail (last 100 lines):" >&2 tail -n 100 "$SERIAL_LOG" >&2 diff --git a/tests/e2e/test_kubestellar_browser_login.py b/tests/e2e/test_kubestellar_browser_login.py index a3889f67..3c2bc0b0 100755 --- a/tests/e2e/test_kubestellar_browser_login.py +++ b/tests/e2e/test_kubestellar_browser_login.py @@ -42,6 +42,16 @@ def parse_args() -> argparse.Namespace: default=60, help="Max timeout in seconds to wait for console readiness (default: 60)", ) + parser.add_argument( + "--cluster-telemetry-timeout", + type=int, + default=CLUSTER_TELEMETRY_TIMEOUT, + help=( + "Max timeout in seconds to wait for live cluster resources to render " + f"(default: {CLUSTER_TELEMETRY_TIMEOUT}); upstream populates cluster health " + "asynchronously after boot" + ), + ) return parser.parse_args() def wait_for_http_ready(url: str, timeout: int, check_healthz: bool = True) -> bool: @@ -90,19 +100,33 @@ def wait_for_http_ready(url: str, timeout: int, check_healthz: bool = True) -> b "vllm-gpu-cluster", ) -# Selectors that only render once real cluster resources are returned. Static -# dashboard card containers and their headings are deliberately excluded: they -# render with empty data and would let the gate pass without any discovered -# cluster. -CLUSTER_RESOURCE_SELECTORS = ( - "[data-testid='cluster-card']", - "[data-testid='node-row']", - "[data-testid='pod-row']", +# Console routes that render one DOM element per discovered cluster, paired +# with the selector that only exists once a cluster was actually returned by +# the API. Static card containers and their headings are deliberately excluded: +# they render with empty data and would let the gate pass without any +# discovered cluster. +# /clusters -> ClusterGrid renders SortableClusterItem with +# data-testid="cluster-row-${cluster.name}" +# (web/src/components/clusters/components/ClusterDragReorder.tsx) +# /workloads -> Clusters Overview grid renders one +# data-testid="cluster-card" per cluster +# (web/src/components/workloads/Workloads.parts.tsx) +# The dashboard landing page has no such per-cluster testid, so these pages +# must be visited explicitly for the DOM verification to mean anything. +CLUSTER_RESOURCE_PAGES = ( + ("/clusters", "[data-testid^='cluster-row-']"), + ("/workloads", "[data-testid='cluster-card']"), ) # Live control-plane / in-cluster identifiers that never appear in demo data. LIVE_CLUSTER_TEXT_MARKERS = ("its1", "wds1", "in-cluster") +# Default budget for live cluster resources to render. Upstream ListClusters +# returns clusters with HealthUnknown and fills health in asynchronously after +# boot, so a short budget flakes on an otherwise healthy install. Override with +# --cluster-telemetry-timeout. +CLUSTER_TELEMETRY_TIMEOUT = 120 + def assert_no_demo_fixtures(driver: webdriver.Chrome) -> None: """Fail if the rendered page contains upstream synthetic demo cluster fixtures.""" @@ -114,7 +138,11 @@ def assert_no_demo_fixtures(driver: webdriver.Chrome) -> None: ) -def verify_live_cluster_resources(driver: webdriver.Chrome, timeout: int = 15) -> None: +def verify_live_cluster_resources( + driver: webdriver.Chrome, + console_url: str, + timeout: int = CLUSTER_TELEMETRY_TIMEOUT, +) -> None: """Verify that actual Kubernetes cluster resources are discovered and rendered in the DOM, ensuring unconfigured demo placeholders or synthetic dev mode do not pass silently. """ @@ -129,36 +157,41 @@ def verify_live_cluster_resources(driver: webdriver.Chrome, timeout: int = 15) - if demo_mode_flag == "true": raise AssertionError("Console is running in synthetic demo mode (kc-demo-mode=true in localStorage)") - # 2. Wait for real cluster resource elements or live cluster identifiers, - # re-checking for demo fixtures on every pass so late-rendered async - # demo content cannot slip through a single early scan. + # 2. Visit each cluster-listing route and wait for a real per-cluster + # element (or live cluster identifiers), re-checking for demo fixtures on + # every pass so late-rendered async demo content cannot slip through a + # single early scan. + base_url = console_url.rstrip("/") start = time.time() found_resource = False - while time.time() - start < timeout: - assert_no_demo_fixtures(driver) + while not found_resource and time.time() - start < timeout: + for route, selector in CLUSTER_RESOURCE_PAGES: + driver.get(f"{base_url}{route}") + WebDriverWait(driver, 15).until(EC.presence_of_element_located((By.TAG_NAME, "body"))) + + assert_no_demo_fixtures(driver) - for selector in CLUSTER_RESOURCE_SELECTORS: elements = driver.find_elements(By.CSS_SELECTOR, selector) if elements and any(el.is_displayed() for el in elements): found_resource = True - print(f"==> Found live cluster resource element: {selector}") + print(f"==> Found live cluster resource element on {route}: {selector}") break - if found_resource: - break - body_text = driver.find_element(By.TAG_NAME, "body").text - if any(term in body_text for term in LIVE_CLUSTER_TEXT_MARKERS): - if "No clusters connected" not in body_text: - found_resource = True - print("==> Discovered live cluster workload indicators in DOM text.") - break + body_text = driver.find_element(By.TAG_NAME, "body").text + if any(term in body_text for term in LIVE_CLUSTER_TEXT_MARKERS): + if "No clusters connected" not in body_text: + found_resource = True + print(f"==> Discovered live cluster workload indicators in {route} DOM text.") + break - time.sleep(1) + if not found_resource: + time.sleep(2) if not found_resource: raise AssertionError( - "Timed out waiting for live Kubernetes cluster resources (nodes, pods, or initialized ControlPlanes) " - "to be rendered in the DOM." + f"Timed out after {timeout}s waiting for live Kubernetes cluster resources " + f"(per-cluster rows or cards on {', '.join(route for route, _ in CLUSTER_RESOURCE_PAGES)}, " + "or initialized ControlPlanes) to be rendered in the DOM." ) # 3. Final demo scan after the async content settled. @@ -167,7 +200,11 @@ def verify_live_cluster_resources(driver: webdriver.Chrome, timeout: int = 15) - print("==> Live cluster workload DOM rendering verified successfully!") -def run_browser_verification(console_url: str, agent_url: str) -> None: +def run_browser_verification( + console_url: str, + agent_url: str, + cluster_telemetry_timeout: int = CLUSTER_TELEMETRY_TIMEOUT, +) -> None: chrome_opts = Options() chrome_opts.add_argument("--headless=new") chrome_opts.add_argument("--no-sandbox") @@ -225,9 +262,12 @@ def run_browser_verification(console_url: str, agent_url: str) -> None: print(f"==> Successfully verified console UI! Page title: {driver.title}") # 3. Verify actual Kubernetes cluster resources are rendered (rejecting demo mode) - verify_live_cluster_resources(driver, timeout=15) + verify_live_cluster_resources(driver, console_url, timeout=cluster_telemetry_timeout) - # 4. Check kiosk gate behavior + # 4. Check kiosk gate behavior on the landing page again, since the + # cluster verification navigated away to the cluster listing routes. + driver.get(console_url) + WebDriverWait(driver, 15).until(EC.presence_of_element_located((By.TAG_NAME, "body"))) gate_elements = driver.find_elements(By.ID, "kubestellar-kiosk-gate") agent_healthy = False try: @@ -256,7 +296,11 @@ def main() -> None: print(f"ERROR: Timed out waiting for console at {args.console_url}", file=sys.stderr) sys.exit(1) - run_browser_verification(args.console_url, args.agent_url) + run_browser_verification( + args.console_url, + args.agent_url, + cluster_telemetry_timeout=args.cluster_telemetry_timeout, + ) if __name__ == "__main__": diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index 4e74399c..bc6df57e 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -169,6 +169,17 @@ def test_installer_smoke_probes_live_cluster_telemetry_and_rejects_demo_mode() - assert "kind-local" in _smoke_gate_demo_names() +def test_smoke_gate_timeout_reports_which_stage_blocked() -> None: + recipe = _smoke_recipe() + + # A timeout must name the stage that never passed so CI triage does not + # have to guess between healthz, root and cluster telemetry. + assert "LAST_STAGE" in recipe + assert "/healthz never reported status ok" in recipe + assert "never returned HTTP 200" in recipe + assert "never reported live (non-demo, non-empty) cluster telemetry" in recipe + + def test_smoke_gate_jq_filter_accepts_live_telemetry_and_rejects_demo_mode() -> None: filter_expr = _smoke_gate_jq_filter() demo_names = _smoke_gate_demo_names() diff --git a/tests/unit/test_lima_e2e_contract.py b/tests/unit/test_lima_e2e_contract.py index ac009d51..5ec1ef91 100644 --- a/tests/unit/test_lima_e2e_contract.py +++ b/tests/unit/test_lima_e2e_contract.py @@ -48,8 +48,14 @@ def test_e2e_browser_test_script_verifies_live_cluster_telemetry_and_rejects_dem assert "No clusters connected" in content # Demo fixtures must be re-scanned while async content loads, not once up front. assert "assert_no_demo_fixtures" in content - # Static card headings render with empty data, so they must not gate the check. - assert "Node Status" not in content - assert "Top Pods" not in content + # Per-cluster DOM elements only exist on the cluster listing routes, which + # the script must navigate to explicitly. + assert "/clusters" in content + assert "cluster-row-" in content + assert "cluster-card" in content + # Cluster health is populated asynchronously after boot, so the render + # budget must be configurable rather than a short hardcoded wait. + assert "cluster-telemetry-timeout" in content + assert "CLUSTER_TELEMETRY_TIMEOUT" in content # WebDriver failures must be caught by type, not by matching message text. assert "except WebDriverException" in content