Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 38 additions & 3 deletions Justfile
Original file line number Diff line number Diff line change
Expand Up @@ -336,6 +336,23 @@ test-installer-artifact:

DEADLINE_SECS="${SHOW_ME_THE_FUTURE_DEADLINE:-${SHOW_ME_THE_FUTURE_TIMEOUT:-600}}"
START_TIME=$(date +%s)

# Upstream synthetic demo fixtures (console pkg/api/handlers/demo_data.go
# GetDemoClusters). The console only sets .source == "demo" for requests
# carrying X-Demo-Mode: true, which this plain curl never sends, so
# matching these fixture names is what actually rejects demo data here.
DEMO_CLUSTER_NAMES='["kind-local","minikube","k3s-edge","eks-prod-us-east-1","gke-staging","aks-dev-westeu","openshift-prod","oci-oke-phoenix","alibaba-ack-shanghai","do-nyc1-prod","rancher-mgmt","vllm-gpu-cluster"]'
# Require live discovery: a k8s/mcp source that reports at least one
# cluster, none of which is a demo fixture. An empty .clusters array is
# rejected because the k8s fallback returns {"clusters": [], "source": "k8s"}
# before any cluster is reachable.
CLUSTERS_FILTER='.source != "demo" and (.source == "k8s" or .source == "mcp") and ((.clusters // []) | length) > 0 and (any(.clusters[]?.name; . as $name | $demo | index($name) != null) | not)'

# Tracks the furthest readiness stage reached so a timeout reports which
# probe actually blocked the gate instead of a generic readiness failure.
LAST_STAGE="healthz"
LAST_CLUSTERS_JSON=""

echo "==> Polling KubeStellar Console readiness at http://127.0.0.1:8080 (deadline: ${DEADLINE_SECS}s)..."

while true; do
Expand All @@ -350,17 +367,35 @@ test-installer-artifact:

HEALTHZ_JSON=$(curl --silent --fail --max-time 2 http://127.0.0.1:8080/healthz 2>/dev/null || true)
if [ -n "$HEALTHZ_JSON" ] && echo "$HEALTHZ_JSON" | jq -e '.status == "ok"' >/dev/null 2>&1; then
LAST_STAGE="root"
ROOT_CODE=$(curl --silent --fail --max-time 2 --output /dev/null --write-out "%{http_code}" http://127.0.0.1:8080/ 2>/dev/null || true)
if [ "$ROOT_CODE" = "200" ]; then
echo "==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200"
break
LAST_STAGE="clusters"
CLUSTERS_JSON=$(curl --silent --fail --max-time 2 http://127.0.0.1:8080/api/mcp/clusters 2>/dev/null || true)
LAST_CLUSTERS_JSON="$CLUSTERS_JSON"
# Ensure actual cluster discovery is active and not silently passing on synthetic demo fixtures
if [ -n "$CLUSTERS_JSON" ] && echo "$CLUSTERS_JSON" | jq -e --argjson demo "$DEMO_CLUSTER_NAMES" "$CLUSTERS_FILTER" >/dev/null 2>&1; then
echo "==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200, and live cluster telemetry verified"
break
fi
fi
fi

NOW=$(date +%s)
ELAPSED=$((NOW - START_TIME))
if [ "$ELAPSED" -ge "$DEADLINE_SECS" ]; then
echo "ERROR: Timed out after ${DEADLINE_SECS}s waiting for KubeStellar Console readiness!" >&2
case "$LAST_STAGE" in
healthz)
echo "ERROR: Timed out after ${DEADLINE_SECS}s waiting for KubeStellar Console readiness: /healthz never reported status ok!" >&2
;;
root)
echo "ERROR: Timed out after ${DEADLINE_SECS}s waiting for KubeStellar Console readiness: /healthz was ok but / never returned HTTP 200!" >&2
;;
clusters)
echo "ERROR: Timed out after ${DEADLINE_SECS}s waiting for KubeStellar Console readiness: /healthz and / passed but /api/mcp/clusters never reported live (non-demo, non-empty) cluster telemetry!" >&2
echo "==> Last /api/mcp/clusters payload: ${LAST_CLUSTERS_JSON:-<empty>}" >&2
;;
esac
if [ -f "$SERIAL_LOG" ]; then
echo "==> Serial log tail (last 100 lines):" >&2
tail -n 100 "$SERIAL_LOG" >&2
Expand Down
150 changes: 146 additions & 4 deletions tests/e2e/test_kubestellar_browser_login.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
Tests:
1. Console availability at http://127.0.0.1:8080/
2. Automated login flow (Cluster Access / GitHub OAuth / Dev Mode / Token Login)
3. Dashboard navigation and DOM rendering
3. Dashboard navigation and DOM rendering of live cluster resources (rejecting demo mode)
4. Kiosk gate overlay and kc-agent interaction on port 8585
"""

Expand All @@ -17,6 +17,7 @@
from typing import Any

from selenium import webdriver
from selenium.common.exceptions import WebDriverException
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Expand All @@ -41,6 +42,16 @@ def parse_args() -> argparse.Namespace:
default=60,
help="Max timeout in seconds to wait for console readiness (default: 60)",
)
parser.add_argument(
"--cluster-telemetry-timeout",
type=int,
default=CLUSTER_TELEMETRY_TIMEOUT,
help=(
"Max timeout in seconds to wait for live cluster resources to render "
f"(default: {CLUSTER_TELEMETRY_TIMEOUT}); upstream populates cluster health "
"asynchronously after boot"
),
)
return parser.parse_args()

def wait_for_http_ready(url: str, timeout: int, check_healthz: bool = True) -> bool:
Expand Down Expand Up @@ -72,7 +83,128 @@ def wait_for_http_ready(url: str, timeout: int, check_healthz: bool = True) -> b
return False


def run_browser_verification(console_url: str, agent_url: str) -> None:
# Upstream synthetic demo fixtures injected by the console demo data
# (pkg/api/handlers/demo_data.go GetDemoClusters).
DEMO_CLUSTER_NAMES = (
"kind-local",
"minikube",
"k3s-edge",
"eks-prod-us-east-1",
"gke-staging",
"aks-dev-westeu",
"openshift-prod",
"oci-oke-phoenix",
"alibaba-ack-shanghai",
"do-nyc1-prod",
"rancher-mgmt",
"vllm-gpu-cluster",
)

# Console routes that render one DOM element per discovered cluster, paired
# with the selector that only exists once a cluster was actually returned by
# the API. Static card containers and their headings are deliberately excluded:
# they render with empty data and would let the gate pass without any
# discovered cluster.
# /clusters -> ClusterGrid renders SortableClusterItem with
# data-testid="cluster-row-${cluster.name}"
# (web/src/components/clusters/components/ClusterDragReorder.tsx)
# /workloads -> Clusters Overview grid renders one
# data-testid="cluster-card" per cluster
# (web/src/components/workloads/Workloads.parts.tsx)
# The dashboard landing page has no such per-cluster testid, so these pages
# must be visited explicitly for the DOM verification to mean anything.
CLUSTER_RESOURCE_PAGES = (
("/clusters", "[data-testid^='cluster-row-']"),
("/workloads", "[data-testid='cluster-card']"),
)

# Live control-plane / in-cluster identifiers that never appear in demo data.
LIVE_CLUSTER_TEXT_MARKERS = ("its1", "wds1", "in-cluster")

# Default budget for live cluster resources to render. Upstream ListClusters
# returns clusters with HealthUnknown and fills health in asynchronously after
# boot, so a short budget flakes on an otherwise healthy install. Override with
# --cluster-telemetry-timeout.
CLUSTER_TELEMETRY_TIMEOUT = 120


def assert_no_demo_fixtures(driver: webdriver.Chrome) -> None:
"""Fail if the rendered page contains upstream synthetic demo cluster fixtures."""
page_text = driver.find_element(By.TAG_NAME, "body").text
for demo_name in DEMO_CLUSTER_NAMES:
if demo_name in page_text:
raise AssertionError(
f"Detected synthetic demo cluster '{demo_name}' in console DOM; demo mode was not rejected!"
)


def verify_live_cluster_resources(
driver: webdriver.Chrome,
console_url: str,
timeout: int = CLUSTER_TELEMETRY_TIMEOUT,
) -> None:
"""Verify that actual Kubernetes cluster resources are discovered and rendered in the DOM,
ensuring unconfigured demo placeholders or synthetic dev mode do not pass silently.
"""
print("==> Verifying live cluster resources are rendered in the DOM (rejecting demo mode)...")

# 1. Reject synthetic demo mode flag in localStorage if explicitly active
try:
demo_mode_flag = driver.execute_script("return window.localStorage.getItem('kc-demo-mode')")
except WebDriverException as exc:
print(f"==> Warning: could not read kc-demo-mode from localStorage: {exc}")
else:
if demo_mode_flag == "true":
raise AssertionError("Console is running in synthetic demo mode (kc-demo-mode=true in localStorage)")

# 2. Visit each cluster-listing route and wait for a real per-cluster
# element (or live cluster identifiers), re-checking for demo fixtures on
# every pass so late-rendered async demo content cannot slip through a
# single early scan.
base_url = console_url.rstrip("/")
start = time.time()
found_resource = False
while not found_resource and time.time() - start < timeout:
for route, selector in CLUSTER_RESOURCE_PAGES:
driver.get(f"{base_url}{route}")
WebDriverWait(driver, 15).until(EC.presence_of_element_located((By.TAG_NAME, "body")))

assert_no_demo_fixtures(driver)

elements = driver.find_elements(By.CSS_SELECTOR, selector)
if elements and any(el.is_displayed() for el in elements):
found_resource = True
print(f"==> Found live cluster resource element on {route}: {selector}")
break

body_text = driver.find_element(By.TAG_NAME, "body").text
if any(term in body_text for term in LIVE_CLUSTER_TEXT_MARKERS):
if "No clusters connected" not in body_text:
found_resource = True
print(f"==> Discovered live cluster workload indicators in {route} DOM text.")
break

if not found_resource:
time.sleep(2)

if not found_resource:
raise AssertionError(
f"Timed out after {timeout}s waiting for live Kubernetes cluster resources "
f"(per-cluster rows or cards on {', '.join(route for route, _ in CLUSTER_RESOURCE_PAGES)}, "
"or initialized ControlPlanes) to be rendered in the DOM."
)

# 3. Final demo scan after the async content settled.
assert_no_demo_fixtures(driver)

print("==> Live cluster workload DOM rendering verified successfully!")


def run_browser_verification(
console_url: str,
agent_url: str,
cluster_telemetry_timeout: int = CLUSTER_TELEMETRY_TIMEOUT,
) -> None:
chrome_opts = Options()
chrome_opts.add_argument("--headless=new")
chrome_opts.add_argument("--no-sandbox")
Expand Down Expand Up @@ -129,7 +261,13 @@ def run_browser_verification(console_url: str, agent_url: str) -> None:
)
print(f"==> Successfully verified console UI! Page title: {driver.title}")

# 3. Check kiosk gate behavior
# 3. Verify actual Kubernetes cluster resources are rendered (rejecting demo mode)
verify_live_cluster_resources(driver, console_url, timeout=cluster_telemetry_timeout)

# 4. Check kiosk gate behavior on the landing page again, since the
# cluster verification navigated away to the cluster listing routes.
driver.get(console_url)
WebDriverWait(driver, 15).until(EC.presence_of_element_located((By.TAG_NAME, "body")))
gate_elements = driver.find_elements(By.ID, "kubestellar-kiosk-gate")
agent_healthy = False
try:
Expand Down Expand Up @@ -158,7 +296,11 @@ def main() -> None:
print(f"ERROR: Timed out waiting for console at {args.console_url}", file=sys.stderr)
sys.exit(1)

run_browser_verification(args.console_url, args.agent_url)
run_browser_verification(
args.console_url,
args.agent_url,
cluster_telemetry_timeout=args.cluster_telemetry_timeout,
)


if __name__ == "__main__":
Expand Down
89 changes: 89 additions & 0 deletions tests/unit/test_installer_contract.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from __future__ import annotations

import re
import subprocess
from pathlib import Path

import yaml
Expand Down Expand Up @@ -135,3 +136,91 @@ def test_installer_and_ddi_strip_vmlinux_and_static_archives() -> None:
assert "find /layer -type f -name '*.a' -delete" in installer_element
assert 'rm -f "/layer/usr/lib/modules/${KVER}/vmlinux"' in ddi_element
assert "find /layer -type f -name '*.a' -delete" in ddi_element


def _smoke_recipe() -> str:
justfile = JUSTFILE.read_text(encoding="utf-8")
start = justfile.index("test-installer-artifact:")
end = justfile.index("install-vm:", start)
return justfile[start:end]


def _smoke_gate_jq_filter() -> str:
match = re.search(r"CLUSTERS_FILTER='([^']+)'", _smoke_recipe())
assert match, "smoke recipe must define CLUSTERS_FILTER"
return match.group(1)


def _smoke_gate_demo_names() -> str:
match = re.search(r"DEMO_CLUSTER_NAMES='([^']+)'", _smoke_recipe())
assert match, "smoke recipe must define DEMO_CLUSTER_NAMES"
return match.group(1)


def test_installer_smoke_probes_live_cluster_telemetry_and_rejects_demo_mode() -> None:
recipe = _smoke_recipe()

assert "/api/mcp/clusters" in recipe
assert '.source != "demo"' in recipe
assert "/healthz" in recipe
assert "200" in recipe
# Demo fixtures are what the unauthenticated probe can actually observe,
# because the console only reports source "demo" for X-Demo-Mode requests.
assert "kind-local" in _smoke_gate_demo_names()


def test_smoke_gate_timeout_reports_which_stage_blocked() -> None:
recipe = _smoke_recipe()

# A timeout must name the stage that never passed so CI triage does not
# have to guess between healthz, root and cluster telemetry.
assert "LAST_STAGE" in recipe
assert "/healthz never reported status ok" in recipe
assert "never returned HTTP 200" in recipe
assert "never reported live (non-demo, non-empty) cluster telemetry" in recipe


def test_smoke_gate_jq_filter_accepts_live_telemetry_and_rejects_demo_mode() -> None:
filter_expr = _smoke_gate_jq_filter()
demo_names = _smoke_gate_demo_names()

def eval_filter(payload: str) -> int:
proc = subprocess.run(
["jq", "-e", "--argjson", "demo", demo_names, filter_expr],
input=payload,
capture_output=True,
text=True,
)
return proc.returncode

# Demo mode with synthetic clusters must be rejected
demo_payload = '{"clusters": [{"name": "kind-local"}], "source": "demo"}'
assert eval_filter(demo_payload) != 0

# Synthetic demo fixtures served without the demo source must be rejected
demo_fixture_payload = '{"clusters": [{"name": "minikube"}], "source": "k8s"}'
assert eval_filter(demo_fixture_payload) != 0

# A single demo fixture alongside a live cluster must still be rejected
mixed_payload = '{"clusters": [{"name": "its1"}, {"name": "k3s-edge"}], "source": "mcp"}'
assert eval_filter(mixed_payload) != 0

# Service unavailable / No cluster access must be rejected
error_payload = '{"error": "No cluster access"}'
assert eval_filter(error_payload) != 0

# Zero discovered clusters must be rejected: the console k8s fallback
# returns this shape before any cluster becomes reachable.
empty_k8s_payload = '{"clusters": [], "source": "k8s"}'
assert eval_filter(empty_k8s_payload) != 0

# A missing clusters key must be rejected too
assert eval_filter('{"source": "k8s"}') != 0

# Live k8s cluster telemetry must be accepted
k8s_payload = '{"clusters": [{"name": "in-cluster", "healthy": true}], "source": "k8s"}'
assert eval_filter(k8s_payload) == 0

# Live mcp cluster telemetry must be accepted
mcp_payload = '{"clusters": [{"name": "its1"}], "source": "mcp"}'
assert eval_filter(mcp_payload) == 0
22 changes: 22 additions & 0 deletions tests/unit/test_lima_e2e_contract.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,25 @@ def test_justfile_defines_setup_kubestellar_and_e2e_targets() -> None:
justfile = JUSTFILE.read_text(encoding="utf-8")
assert "test-e2e-browser" in justfile
assert "test-e2e-lima:" in justfile


def test_e2e_browser_test_script_verifies_live_cluster_telemetry_and_rejects_demo_mode() -> None:
content = E2E_SCRIPT.read_text(encoding="utf-8")
assert "verify_live_cluster_resources" in content
assert "kc-demo-mode" in content
assert "kind-local" in content
assert "minikube" in content
assert "No clusters connected" in content
# Demo fixtures must be re-scanned while async content loads, not once up front.
assert "assert_no_demo_fixtures" in content
# Per-cluster DOM elements only exist on the cluster listing routes, which
# the script must navigate to explicitly.
assert "/clusters" in content
assert "cluster-row-" in content
assert "cluster-card" in content
# Cluster health is populated asynchronously after boot, so the render
# budget must be configurable rather than a short hardcoded wait.
assert "cluster-telemetry-timeout" in content
assert "CLUSTER_TELEMETRY_TIMEOUT" in content
# WebDriver failures must be caught by type, not by matching message text.
assert "except WebDriverException" in content