OTA-1927: Eval cluster update prompts

Makefile

@@ -1,7 +1,7 @@
 # Put targets here if there is a risk that a target name might conflict with a filename.
 # this list is probably overkill right now.
 # See: https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html
-.PHONY: test test-unit test-e2e test-eval test-lseval-periodic test-lseval-troubleshooting images run format verify get-embeddings get-embeddings-byok get-embeddings-okp tls-scan
+.PHONY: test test-unit test-e2e test-eval test-lseval-periodic test-lseval-troubleshooting test-cluster-updates images run format verify get-embeddings get-embeddings-byok get-embeddings-okp tls-scan
 
 export PATH := $(HOME)/.local/bin:$(PATH)
 
@@ -92,7 +92,7 @@ test-eval: ## Run evaluation tests - requires running OLS server
 	@echo "Running evaluation tests..."
 	@echo "Reports will be written to ${ARTIFACT_DIR}"
 	uv run --extra evaluation pytest tests/e2e/evaluation -vv -s --durations=0 -o junit_suite_name="${SUITE_ID}" --junit-prefix="${SUITE_ID}" --junit-xml="${ARTIFACT_DIR}/junit_e2e_${SUITE_ID}.xml" \
-	--eval_out_dir ${ARTIFACT_DIR} -m "not lseval"
+	--eval_out_dir ${ARTIFACT_DIR} -m "not lseval and not cluster_updates"
 
 test-lseval-periodic: ## Run LSEval periodic evaluation (full 797-question dataset) - requires running OLS server with OpenAI keys
 	@echo "Running LSEval periodic evaluation..."
@@ -110,6 +110,12 @@ test-lseval-periodic: ## Run LSEval periodic evaluation (full 797-question datas
 test-lseval-troubleshooting: ## DISABLED: LSEval troubleshooting (uncomment Makefile to re-enable)
 	@echo "test-lseval-troubleshooting is disabled for now (troubleshooting LSEval commented out)." >&2
 
+test-cluster-updates: ## Run cluster-updates evaluation (18 conversations, 35 evaluations) - requires running OLS server with OpenAI keys
+	@echo "Running cluster-updates evaluation..."
+	@echo "Reports will be written to ${ARTIFACT_DIR}"
+	uv run --extra lseval --extra evaluation pytest tests/e2e/evaluation -vv -s --durations=0 -o junit_suite_name="${SUITE_ID}" --junit-prefix="${SUITE_ID}" --junit-xml="${ARTIFACT_DIR}/junit_e2e_${SUITE_ID}.xml" \
+	--eval_out_dir ${ARTIFACT_DIR} -m cluster_updates
+
 coverage-report:	unit-tests-coverage-report integration-tests-coverage-report ## Export coverage reports into interactive HTML
 
 unit-tests-coverage-report:	test-unit ## Export unit test coverage report into interactive HTML

eval/README.md

@@ -32,6 +32,27 @@ source venv/bin/activate
 pip install git+https://github.com/lightspeed-core/lightspeed-evaluation.git
 ```
 
+### RBAC Setup for Cluster-Updates Evaluation
+
+**IMPORTANT:** Cluster-updates evaluation requires proper RBAC permissions for OLS to access cluster resources via MCP servers.
+
+```bash
+# Apply the RBAC manifest (creates ServiceAccount with cluster-reader + monitoring-edit permissions)
+oc apply -f eval/rbac-cluster-updates.yaml
+
+# Generate an API token for the service account (valid for 24 hours)
+export API_KEY=$(oc create token cluster-update-user -n openshift-lightspeed --duration=24h)
+```
+
+**Required Permissions:**
+
+- **cluster-reader**: Access to ClusterVersion, ClusterOperator, Node resources
+- **monitoring-edit**: Access to Prometheus metrics and Alertmanager alerts
+
+**Why These Permissions Matter:**
+
+Without proper RBAC, OLS will respond with "unable to retrieve... due to access restrictions" instead of providing accurate cluster status analysis. The evaluation will still run but will fail quality checks with ~79% pass rate instead of the expected 85%+.
+
 ### Run Evaluation
 
 ```bash
@@ -50,17 +71,65 @@ lightspeed-eval --system-config system.yaml --eval-data eval_data_short.yaml --o
 
 # Full evaluation (797 questions)
 lightspeed-eval --system-config system.yaml --eval-data eval_data.yaml --output-dir ./results
+
+# Cluster-updates evaluation (19 conversations, 20 test turns) - uses optimized config
+lightspeed-eval --system-config system_cluster_updates.yaml \
+                --eval-data eval_data_cluster_updates.yaml \
+                --output-dir ./results
+
+# Run specific cluster-updates test category (e.g., critical tests)
+lightspeed-eval --system-config system_cluster_updates.yaml \
+                --eval-data eval_data_cluster_updates.yaml \
+                --tags cluster-updates-critical \
+                --output-dir ./results
 ```
 
 ## What's Included
 
 ### Datasets
-- **`eval_data_short.yaml`**: 10 conversations
-- **`eval_data.yaml`**: 797 conversations
-
-### Configuration
-- **`system.yaml`**: Pre-configured for OLS at `localhost:8080`
-- **Default metrics**: answer correctness
+- **`eval_data_short.yaml`**: 10 conversations (quick smoke test)
+- **`eval_data.yaml`**: 797 general OpenShift knowledge questions (conv_001-797)
+- **`eval_data_cluster_updates.yaml`**: 19 cluster-updates test conversations (conv_001-019, 20 test turns)
+
+### Test Categories (by tag)
+- **cluster-updates-scenarios**: Comprehensive health assessment with extensive constraints (conv_001-005, 5 conversations)
+- **cluster-updates-critical**: Condition status interpretation - MUST pass 100% (conv_006)
+- **cluster-updates-format**: Output format compliance (Summary + TL;DR) (conv_007)
+- **cluster-updates-blockers**: Admin-ack gates and upgrade blockers (conv_008)
+- **cluster-updates-risks**: Conditional update risk analysis (conv_009)
+- **cluster-updates-path**: Upgrade path validation (conv_010)
+- **cluster-updates-troubleshoot**: Upgrade failure diagnosis and remediation (conv_011)
+- **cluster-updates-conversation**: Multi-turn conversation handling (conv_012, 2 turns)
+- **cluster-updates-no-updates**: Cluster at latest version scenarios (conv_013)
+- **cluster-updates-channels**: Update channel understanding (conv_014)
+- **cluster-updates-mcp**: MachineConfigPool upgrade behavior (conv_015)
+- **cluster-updates-pdb**: PodDisruptionBudget impact on upgrades (conv_016)
+- **cluster-updates-eus**: Extended Update Support (EUS) upgrade paths (conv_017)
+- **cluster-updates-conditions**: Condition status interpretation (conv_018, conv_019)
+
+### Configuration Files
+
+Two configuration files are available depending on your use case:
+
+#### `system.yaml` - Default Configuration
+- **Use for:** General OpenShift knowledge evaluation (conv_001-797)
+- **API Base:** `http://localhost:8080` (local development)
+- **Max Tokens:** 512 (standard responses)
+- **API Provider:** `openai`
+- **Metrics:** All standard metrics available (Ragas, DeepEval, custom)
+
+#### `system_cluster_updates.yaml` - Cluster-Updates Optimized
+- **Use for:** Cluster-updates evaluation (conv_001-019)
+- **API Base:** `http://localhost:8080` (same as default)
+- **Max Tokens:** 2048 (detailed cluster analysis - 4x larger for complex responses)
+- **API Provider:** `openai` (cluster-specific configuration)
+- **Output Directory:** `./results` (organized test output)
+- **Available Metrics:**
+  - `custom:answer_correctness` - Basic correctness evaluation
+  - `geval:condition_status_accuracy` - Kubernetes condition interpretation (threshold: 0.99 - CRITICAL!)
+  - `geval:output_format_compliance` - Response format validation (threshold: 0.80)
+  - `geval:technical_accuracy` - OpenShift/Kubernetes domain knowledge (threshold: 0.80)
+  - `geval:actionable_guidance` - Specific remediation steps (threshold: 0.7)
 
 
 ## Results
@@ -72,4 +141,4 @@ Results are saved in output directories:
 
 
 ## Data & Eval system setup
-Refer [Lightspeed Evaluation tool](https://github.com/lightspeed-core/lightspeed-evaluation#readme)
+Refer [Lightspeed Evaluation tool](https://github.com/lightspeed-core/lightspeed-evaluation#readme)