diff --git a/Makefile b/Makefile
index bb77bea4e..734505c2e 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
# Put targets here if there is a risk that a target name might conflict with a filename.
# this list is probably overkill right now.
# See: https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html
-.PHONY: test test-unit test-e2e test-eval test-lseval-periodic test-lseval-troubleshooting images run format verify get-embeddings get-embeddings-byok get-embeddings-okp tls-scan
+.PHONY: test test-unit test-e2e test-eval test-lseval-periodic test-lseval-troubleshooting test-cluster-updates images run format verify get-embeddings get-embeddings-byok get-embeddings-okp tls-scan
export PATH := $(HOME)/.local/bin:$(PATH)
@@ -92,7 +92,7 @@ test-eval: ## Run evaluation tests - requires running OLS server
@echo "Running evaluation tests..."
@echo "Reports will be written to ${ARTIFACT_DIR}"
uv run --extra evaluation pytest tests/e2e/evaluation -vv -s --durations=0 -o junit_suite_name="${SUITE_ID}" --junit-prefix="${SUITE_ID}" --junit-xml="${ARTIFACT_DIR}/junit_e2e_${SUITE_ID}.xml" \
- --eval_out_dir ${ARTIFACT_DIR} -m "not lseval"
+ --eval_out_dir ${ARTIFACT_DIR} -m "not lseval and not cluster_updates"
test-lseval-periodic: ## Run LSEval periodic evaluation (full 797-question dataset) - requires running OLS server with OpenAI keys
@echo "Running LSEval periodic evaluation..."
@@ -110,6 +110,12 @@ test-lseval-periodic: ## Run LSEval periodic evaluation (full 797-question datas
test-lseval-troubleshooting: ## DISABLED: LSEval troubleshooting (uncomment Makefile to re-enable)
@echo "test-lseval-troubleshooting is disabled for now (troubleshooting LSEval commented out)." >&2
+test-cluster-updates: ## Run cluster-updates evaluation (18 conversations, 35 evaluations) - requires running OLS server with OpenAI keys
+ @echo "Running cluster-updates evaluation..."
+ @echo "Reports will be written to ${ARTIFACT_DIR}"
+ uv run --extra lseval --extra evaluation pytest tests/e2e/evaluation -vv -s --durations=0 -o junit_suite_name="${SUITE_ID}" --junit-prefix="${SUITE_ID}" --junit-xml="${ARTIFACT_DIR}/junit_e2e_${SUITE_ID}.xml" \
+ --eval_out_dir ${ARTIFACT_DIR} -m cluster_updates
+
coverage-report: unit-tests-coverage-report integration-tests-coverage-report ## Export coverage reports into interactive HTML
unit-tests-coverage-report: test-unit ## Export unit test coverage report into interactive HTML
diff --git a/eval/README.md b/eval/README.md
index 69bbc1877..5ff51241f 100644
--- a/eval/README.md
+++ b/eval/README.md
@@ -32,6 +32,27 @@ source venv/bin/activate
pip install git+https://github.com/lightspeed-core/lightspeed-evaluation.git
```
+### RBAC Setup for Cluster-Updates Evaluation
+
+**IMPORTANT:** Cluster-updates evaluation requires proper RBAC permissions for OLS to access cluster resources via MCP servers.
+
+```bash
+# Apply the RBAC manifest (creates ServiceAccount with cluster-reader + monitoring-edit permissions)
+oc apply -f eval/rbac-cluster-updates.yaml
+
+# Generate an API token for the service account (valid for 24 hours)
+export API_KEY=$(oc create token cluster-update-user -n openshift-lightspeed --duration=24h)
+```
+
+**Required Permissions:**
+
+- **cluster-reader**: Access to ClusterVersion, ClusterOperator, Node resources
+- **monitoring-edit**: Access to Prometheus metrics and Alertmanager alerts
+
+**Why These Permissions Matter:**
+
+Without proper RBAC, OLS will respond with "unable to retrieve... due to access restrictions" instead of providing accurate cluster status analysis. The evaluation will still run but will fail quality checks with ~79% pass rate instead of the expected 85%+.
+
### Run Evaluation
```bash
@@ -50,17 +71,65 @@ lightspeed-eval --system-config system.yaml --eval-data eval_data_short.yaml --o
# Full evaluation (797 questions)
lightspeed-eval --system-config system.yaml --eval-data eval_data.yaml --output-dir ./results
+
+# Cluster-updates evaluation (19 conversations, 20 test turns) - uses optimized config
+lightspeed-eval --system-config system_cluster_updates.yaml \
+ --eval-data eval_data_cluster_updates.yaml \
+ --output-dir ./results
+
+# Run specific cluster-updates test category (e.g., critical tests)
+lightspeed-eval --system-config system_cluster_updates.yaml \
+ --eval-data eval_data_cluster_updates.yaml \
+ --tags cluster-updates-critical \
+ --output-dir ./results
```
## What's Included
### Datasets
-- **`eval_data_short.yaml`**: 10 conversations
-- **`eval_data.yaml`**: 797 conversations
-
-### Configuration
-- **`system.yaml`**: Pre-configured for OLS at `localhost:8080`
-- **Default metrics**: answer correctness
+- **`eval_data_short.yaml`**: 10 conversations (quick smoke test)
+- **`eval_data.yaml`**: 797 general OpenShift knowledge questions (conv_001-797)
+- **`eval_data_cluster_updates.yaml`**: 19 cluster-updates test conversations (conv_001-019, 20 test turns)
+
+### Test Categories (by tag)
+- **cluster-updates-scenarios**: Comprehensive health assessment with extensive constraints (conv_001-005, 5 conversations)
+- **cluster-updates-critical**: Condition status interpretation - MUST pass 100% (conv_006)
+- **cluster-updates-format**: Output format compliance (Summary + TL;DR) (conv_007)
+- **cluster-updates-blockers**: Admin-ack gates and upgrade blockers (conv_008)
+- **cluster-updates-risks**: Conditional update risk analysis (conv_009)
+- **cluster-updates-path**: Upgrade path validation (conv_010)
+- **cluster-updates-troubleshoot**: Upgrade failure diagnosis and remediation (conv_011)
+- **cluster-updates-conversation**: Multi-turn conversation handling (conv_012, 2 turns)
+- **cluster-updates-no-updates**: Cluster at latest version scenarios (conv_013)
+- **cluster-updates-channels**: Update channel understanding (conv_014)
+- **cluster-updates-mcp**: MachineConfigPool upgrade behavior (conv_015)
+- **cluster-updates-pdb**: PodDisruptionBudget impact on upgrades (conv_016)
+- **cluster-updates-eus**: Extended Update Support (EUS) upgrade paths (conv_017)
+- **cluster-updates-conditions**: Condition status interpretation (conv_018, conv_019)
+
+### Configuration Files
+
+Two configuration files are available depending on your use case:
+
+#### `system.yaml` - Default Configuration
+- **Use for:** General OpenShift knowledge evaluation (conv_001-797)
+- **API Base:** `http://localhost:8080` (local development)
+- **Max Tokens:** 512 (standard responses)
+- **API Provider:** `openai`
+- **Metrics:** All standard metrics available (Ragas, DeepEval, custom)
+
+#### `system_cluster_updates.yaml` - Cluster-Updates Optimized
+- **Use for:** Cluster-updates evaluation (conv_001-019)
+- **API Base:** `http://localhost:8080` (same as default)
+- **Max Tokens:** 2048 (detailed cluster analysis - 4x larger for complex responses)
+- **API Provider:** `openai` (cluster-specific configuration)
+- **Output Directory:** `./results` (organized test output)
+- **Available Metrics:**
+ - `custom:answer_correctness` - Basic correctness evaluation
+ - `geval:condition_status_accuracy` - Kubernetes condition interpretation (threshold: 0.99 - CRITICAL!)
+ - `geval:output_format_compliance` - Response format validation (threshold: 0.80)
+ - `geval:technical_accuracy` - OpenShift/Kubernetes domain knowledge (threshold: 0.80)
+ - `geval:actionable_guidance` - Specific remediation steps (threshold: 0.7)
## Results
@@ -72,4 +141,4 @@ Results are saved in output directories:
## Data & Eval system setup
-Refer [Lightspeed Evaluation tool](https://github.com/lightspeed-core/lightspeed-evaluation#readme)
\ No newline at end of file
+Refer [Lightspeed Evaluation tool](https://github.com/lightspeed-core/lightspeed-evaluation#readme)
diff --git a/eval/eval_data_cluster_updates.yaml b/eval/eval_data_cluster_updates.yaml
new file mode 100644
index 000000000..98bc81f55
--- /dev/null
+++ b/eval/eval_data_cluster_updates.yaml
@@ -0,0 +1,696 @@
+- conversation_group_id: conv_001
+ tag: cluster-updates-critical
+ turns:
+ - expected_response: 'Response should provide a comprehensive pre-upgrade analysis with the following structure:
+
+
+ ## Summary
+
+ - Update Service Health: Cincinnati service status, last update check timestamp, update channel (stable-4.21), cluster ID
+
+ - Cluster History Context: Initial version, upgrade path, last completed upgrade, cluster age
+
+ - Available Updates: Count of recommended updates with version numbers, conditional updates analysis
+
+ - Upgrade Readiness Assessment: ClusterVersion conditions (Failing=False, Upgradeable=True/missing, Available=True, Progressing status, RetrievedUpdates=True)
+
+ - Admin-Ack Gates: List of defined gates and acknowledged gates status
+
+ - ClusterOperator Health: Total operator count, list of any operators with issues (Available=False, Degraded=True, Upgradeable=False)
+
+ - Infrastructure Health: MachineConfigPools status, Node status, Resource pressure, PDBs, CSRs
+
+ - Recent Events and Active Alerts if available
+
+
+ ## TL;DR
+
+ - Current Version: 4.21.14
+
+ - Available Updates: Specific count
+
+ - Upgrade Blocked: Yes/No with specific reasons
+
+ - Cluster health summary with operator counts
+
+ - Clear recommendation
+
+
+ The response must reference actual cluster data with real version numbers, timestamps, and condition statuses.'
+ query: "# OpenShift Cluster Upgrade Pre-Check Analysis\n\n\n- LANGUAGE REQUIREMENT: Respond in English. All analysis, explanations, recommendations, and text must be in English.\n\n- YOU MUST ALWAYS CALL THE TOOLS TO GET THE INFORMATION. YOU SHOULD NEVER TREAT DATA FROM EXAMPLES AS REAL DATA.\n- YOU MUST ALWAYS REFERENCE REAL DATA FROM TOOL CALLS. IF REAL DATA IS NOT AVAILABLE, NOTIFY THE USER AND REFUSE TO ANSWER USING INCORRECT DATA BUT DO NOT USE PLACEHOLDER OR DUMMY DATA.\n- NEVER use placeholder or dummy data - only reference real data from tool calls.\n- ONLY report issues that are actually present in the data.\n- ONLY OUTPUT the Summary and TL;DR sections.\n- Be specific about the source of any issues identified.\n- CRITICAL: When counting available updates, count ALL array elements in status.availableUpdates AND status.conditionalUpdates separately.\n\n\n**IN SCOPE - Issues that affect OCP cluster updates:**\n- ClusterVersion conditions that block\
+ \ or signal upgrade problems\n- ClusterOperator health that blocks operator-phase progression during upgrade\n- MachineConfigPool state that blocks node-phase rollout\n- Node conditions that prevent draining, rebooting, or rejoining during upgrade\n- PodDisruptionBudgets that prevent node draining during rolling MCP updates\n- Conditional update risks that apply to this specific cluster (Recommended=False)\n- Admin-ack gates required before minor-version upgrades\n- Deprecated API usage that will break after upgrade\n- Pending CSRs that will prevent nodes from rejoining post-reboot\n- OLM Subscription health for layered operators that must be compatible with target release\n- Update path validity (channel, skip-level, EUS constraints)\n- Resource pressure that prevents upgrade surge capacity\n- Active alerts directly relevant to upgrade readiness\n- Configuration overrides that mask CVO reconciliation\n\n**OUT OF SCOPE - Do NOT flag these unless they directly affect upgrades:**\n-\
+ \ General application performance issues\n- User workload errors unrelated to PDBs or node drain\n- Non-upgrade-related warnings or events\n- Cosmetic issues\n- Issues that are recovering on their own and are not blocking\n- Anything that does not have a documented effect on oc adm upgrade or the upgrade process\n\nIf you cannot tie an issue to a specific upgrade-blocking or upgrade-disrupting mechanism, do not report it.\n\n**CRITICAL: Timeout and Error Handling**\n**Timeout Awareness:**\n- You have a 60-second timeout - manage your time wisely\n- Prioritize essential data (ClusterVersion, ClusterOperators, admin-acks/admin-gates) first\n- Track execution time and stop making new tool calls after 50 seconds\n- Provide analysis with available data rather than timing out trying to fetch everything\n**Error Handling Rules:**\n1. **Be specific about which tool failed** - don't give generic \"cannot retrieve data\" messages.\n2. **Explain what data you're missing** -\
+ \ e.g., \"Unable to fetch ClusterVersion resource\" vs \"Unable to retrieve data\".\n3. **Try alternative approaches**:\n - If resources_list fails for all ClusterOperators, note this specifically.\n - If nodes_top fails, continue with other analysis - it's optional.\n - If get_alerts fails, skip it - alerts are optional.\n - If events_list fails, continue without event data.\n - If APIRequestCount fails or is unavailable on the cluster version, note it and skip.\n4. **Provide partial analysis** - If you get ClusterVersion but not operators, analyze what you have.\n5. **Give actionable troubleshooting** when tools fail:\n - Check if OpenShift MCP server is running: 'oc get pods -n openshift-lightspeed'\n - Verify cluster connectivity.\n - Suggest checking MCP server logs for specific errors.\n6. **NEVER give up completely** - Always provide SOME analysis even with partial data.\n7. **RBAC and Permission Errors - Critical for cluster-wide resources**:\n - If tool calls fail with \"Forbidden\" (403) or \"access denied\" errors, this means insufficient RBAC permissions.\n - Be SPECIFIC about which resources are inaccessible (e.g., \"Cannot access ClusterVersion resource\", \"Cannot access Prometheus metrics\").\n - Provide analysis based ONLY on accessible resources - DO NOT invent data.\n - For comprehensive cluster analysis, state required permissions: \"For full upgrade analysis, the service account requires cluster-reader ClusterRoleBinding (ClusterVersion, ClusterOperator, Node access) and monitoring-edit ClusterRoleBinding (Prometheus and Alertmanager access). Apply: oc apply -f eval/rbac-cluster-updates.yaml\"\n**Example of good error handling:**\n- BAD: \"I cannot retrieve necessary data from the\
+ \ cluster\"\n- GOOD: \"Successfully retrieved ClusterVersion (current: 4.21.14, 7 updates available, 2 conditional updates with risks). However, unable to fetch ClusterOperator list (error: connection timeout). Based on ClusterVersion alone, the cluster reports Upgradeable=True and Failing=False. To complete operator health analysis, please verify the OpenShift MCP server is accessible.\"\n- GOOD (RBAC error): \"Successfully retrieved MachineConfigPools and namespace-scoped resources. However, cannot access ClusterVersion and ClusterOperator resources due to insufficient RBAC permissions (HTTP 403 Forbidden). For comprehensive upgrade readiness analysis, ensure the service account has cluster-reader and monitoring-edit ClusterRoleBindings. Apply: oc apply -f eval/rbac-cluster-updates.yaml\"\n**Example of good timeout handling:**\n- GOOD: \"Retrieved ClusterVersion, all 28 ClusterOperators, admin-acks, and admin-gates (execution time: 38 seconds). Skipping APIRequestCount and alerts to avoid timeout. All admin-ack gates are satisfied; cluster is on track for upgrade.\"\n**Tool Call Priority to Avoid Timeouts:**\n**PHASE 1 - ESSENTIAL (Always fetch, target under 25 seconds):**\n1. resources_get: ClusterVersion (apiVersion: \"config.openshift.io/v1\", kind: \"ClusterVersion\", name: \"version\")\n - Capture full status including: conditions, availableUpdates, conditionalUpdates, history,\
+ \ capabilities, desired\n2. resources_list: ClusterOperator (apiVersion: \"config.openshift.io/v1\", kind: \"ClusterOperator\")\n3. resources_get: ConfigMap \"admin-gates\" in namespace \"openshift-config-managed\"\n (lists upgrade-blocking gate keys defined by the cluster's components)\n4. resources_get: ConfigMap \"admin-acks\" in namespace \"openshift-config\"\n (lists administrator acknowledgements)\n**PHASE 2 - IMPORTANT (Fetch if time permits, under 45 seconds total):**\n5. resources_list: Node (apiVersion: \"v1\", kind: \"Node\") - Quick check for NotReady nodes and pressure conditions\n6. resources_list: MachineConfigPool (apiVersion: \"machineconfiguration.openshift.io/v1\", kind: \"MachineConfigPool\")\n7. events_list: Get recent warning/error events from last 30 minutes in upgrade-relevant namespaces (openshift-cluster-version, openshift-machine-config-operator, openshift-etcd, openshift-kube-apiserver, openshift-apiserver, openshift-authentication, openshift-network-operator)\n\
+ **PHASE 3 - OPTIONAL (Only if under 50 seconds total):**\n8. nodes_top: Check node CPU/memory usage\n9. resources_list: PodDisruptionBudget (apiVersion: \"policy/v1\", kind: \"PodDisruptionBudget\") - Filter out openshift-*, kube-*\n10. resources_list: APIRequestCount (apiVersion: \"apiserver.openshift.io/v1\", kind: \"APIRequestCount\") - Identify deprecated APIs in use\n11. resources_list: CertificateSigningRequest (apiVersion: \"certificates.k8s.io/v1\", kind: \"CertificateSigningRequest\") - Filter for Pending state\n12. resources_list: MachineHealthCheck (apiVersion: \"machine.openshift.io/v1beta1\", kind: \"MachineHealthCheck\") - Check for unpaused MHCs\n13. resources_list: Subscription (apiVersion: \"operators.coreos.com/v1alpha1\", kind: \"Subscription\") - Layered operator health\n14. get_alerts: Check for critical/warning alerts\n**CRITICAL EFFICIENCY RULES:**\n- If approaching 50 seconds of execution time, STOP making new tool calls and provide analysis with data collected\n\
+ - NEVER let total execution exceed 55 seconds to avoid timeout\n- Prioritize breadth over depth: Get ClusterVersion + ClusterOperators + admin-acks fully before diving into logs/events\n- Skip optional data if essential data took longer than expected\n\nBEFORE providing your response, verify:\n1. Every word in your response is in the target language (except system identifiers like file paths, URLs, command names).\n2. Technical terms are translated or explained in the target language.\n3. No English phrases or mixed language content exists in your explanations.\n4. All section headers and content follow the target language requirements.\n\n\n\n\nThis is a pre-upgrade analysis for OpenShift cluster version 4.21.14. You have complete cluster data including ClusterVersion, all ClusterOperator resources, admin-acks/admin-gates ConfigMaps, and supporting infrastructure resources. Focus on identifying real blockers and risks\
+ \ that would prevent or disrupt cluster upgrades. Stay strictly within the upgrade-impact scope defined above.\n\n\n\nCRITICAL: Understanding Kubernetes/OpenShift Conditions\n\nConditions have TWO important fields you MUST check:\n- **type**: The name of the condition (e.g., \"Failing\", \"Available\", \"Progressing\", \"Upgradeable\", \"Recommended\")\n- **status**: The state of the condition (ONLY these values: \"True\", \"False\", or \"Unknown\")\n**MANDATORY CHECKING PROCESS:**\nFor EVERY condition you analyze, you MUST:\n1. First, locate the condition by its type field.\n2. Second, read the EXACT value of the status field.\n3. Third, interpret based ONLY on the status field value:\n - If status=\"True\" → The condition IS active/present.\n - If status=\"False\" → The condition is NOT active/NOT present.\n - If status=\"Unknown\" → The condition state is uncertain.\n**DO NOT report a problem unless status=\"True\" for negative conditions\
+ \ OR status=\"False\" for positive conditions!**\n**Critical Examples - MEMORIZE THESE:**\nClusterVersion / ClusterOperator / general:\n- {type: \"Failing\", status: \"False\"} → Cluster is NOT failing → NO PROBLEM\n- {type: \"Failing\", status: \"True\"} → Cluster IS failing → PROBLEM\n- {type: \"Available\", status: \"True\"} → Component IS available → NO PROBLEM\n- {type: \"Available\", status: \"False\"} → Component is NOT available → PROBLEM\n- {type: \"Degraded\", status: \"False\"} → NOT degraded → NO PROBLEM\n- {type: \"Degraded\", status: \"True\"} → IS degraded → PROBLEM\n- {type: \"Upgradeable\", status: \"True\"} or absent → Upgrades allowed → NO PROBLEM\n- {type: \"Upgradeable\", status: \"False\"} → Upgrades BLOCKED → PROBLEM (read message/reason)\n- {type: \"Progressing\", status: \"True\"} → Currently changing state. Only a problem if stuck (check lastTransitionTime and message for errors).\n- {type: \"RetrievedUpdates\", status: \"True\"} → Update service healthy →\
+ \ NO PROBLEM\n- {type: \"RetrievedUpdates\", status: \"False\"} → Cannot reach update service → PROBLEM\n- {type: \"ReleaseAccepted\", status: \"True\"} → Release image accepted → NO PROBLEM\n- {type: \"ReleaseAccepted\", status: \"False\"} → Release image rejected (signature/manifest issue) → PROBLEM\n- {type: \"ImplicitlyEnabledCapabilities\", status: \"False\"} → No capability surprise → NO PROBLEM\n- {type: \"ImplicitlyEnabledCapabilities\", status: \"True\"} → Disabled capability was implicitly enabled → INFORMATIONAL/WARNING\nConditional update entries (status.conditionalUpdates[].conditions[]):\n- {type: \"Recommended\", status: \"True\"} → Update IS recommended for this cluster → SAFE\n- {type: \"Recommended\", status: \"False\"} → Update has KNOWN RISK matching this cluster → REPORT RISK (name, message, url)\n- {type: \"Recommended\", status: \"Unknown\"} → CVO still evaluating → INFORMATIONAL\nNode conditions:\n- {type: \"Ready\", status: \"True\"} → Node is ready → NO PROBLEM\n\
+ - {type: \"Ready\", status: \"False\"} or \"Unknown\" → Node NotReady → PROBLEM (will block drain/upgrade on that node)\n- {type: \"MemoryPressure\", status: \"True\"} → Memory pressure → PROBLEM\n- {type: \"DiskPressure\", status: \"True\"} → Disk pressure → PROBLEM (often blocks image pulls during upgrade)\n- {type: \"PIDPressure\", status: \"True\"} → PID pressure → PROBLEM\n- {type: \"NetworkUnavailable\", status: \"True\"} → Network unavailable → PROBLEM\nMachineConfigPool conditions:\n- {type: \"Updated\", status: \"True\"} → Pool is at desired config → NO PROBLEM\n- {type: \"Updated\", status: \"False\"} → Pool not yet updated. Only a problem if stuck or paused inappropriately.\n- {type: \"Updating\", status: \"True\"} → Pool currently rolling. Acceptable mid-upgrade; problem if stuck for hours.\n- {type: \"Degraded\", status: \"True\"} → Pool degraded → PROBLEM (blocks further node updates)\n- {type: \"NodeDegraded\", status: \"True\"} → A node in pool failed config apply →\
+ \ PROBLEM\n- {type: \"RenderDegraded\", status: \"True\"} → Could not render config → PROBLEM (blocks any update for the pool)\n**VERIFICATION REQUIREMENT:**\nBefore making ANY conclusion about a condition, you MUST internally state:\n\"Condition type='X' has status='Y'\" and then interpret it correctly.\n**NEVER assume a condition is true just because the type exists - ALWAYS check the status field!**\n**The presence of a condition type does NOT mean it is active - check the status field!**\n\n\n\n### 1. Available Updates and Conditional Updates Analysis\n**Available updates (status.availableUpdates):**\n- Count EXACTLY how many items are in status.availableUpdates array.\n- For each entry, extract: version, image, channels[], url (errata).\n- Identify the latest recommended z-stream and the latest recommended y-stream (if any).\n**Conditional updates (status.conditionalUpdates) — REQUIRED:**\n- Count EXACTLY how many items\
+ \ are in status.conditionalUpdates array.\n- For each conditional update, locate the conditions[] entry with type=\"Recommended\":\n - If status=\"False\": the cluster matches a known risk for this target. Extract:\n - Target release.version and release.image\n - The reason and message from the Recommended condition\n - All risks[] entries: name, message, url (Red Hat KCS or bug link)\n - If status=\"Unknown\": CVO has not finished evaluating risks yet — note as informational.\n - If status=\"True\": cluster does NOT match any risk for this target — treat as effectively recommended.\n- Conditional update presence with Recommended=False is NOT itself an upgrade blocker, but it IS a risk the administrator must explicitly accept; surface it prominently.\n\n### 2. ClusterVersion Conditions - VERIFICATION REQUIRED\nFor each of the following, locate the condition in status.conditions[], read the status field, and interpret per . Quote the actual reason\
+ \ and message if reporting a problem.\na) **Failing**: status=\"True\" → reconciliation failure, report reason and message.\nb) **Upgradeable**: status=\"False\" → upgrades are explicitly blocked; report reason and message verbatim. Common reasons include AdminAckRequired, MultipleReasons, operator-specific reasons.\nc) **Available**: status=\"False\" → cluster operationally impaired.\nd) **Progressing**: status=\"True\" AND not currently in an admin-initiated upgrade → may indicate a stuck reconciliation.\ne) **RetrievedUpdates**: status=\"False\" → Cincinnati/update service unreachable; cluster cannot discover updates.\nf) **ReleaseAccepted**: status=\"False\" → desired release image was rejected (signature verification, manifest validation, or image pull failure).\ng) **ImplicitlyEnabledCapabilities**: status=\"True\" → a capability disabled in spec was implicitly enabled; surface as informational warning.\n\n### 3. Admin-Ack Gate Analysis (CRITICAL FOR MINOR UPGRADES)\nOpenShift\
+ \ requires administrators to acknowledge specific upgrade gates before minor-version upgrades. CVO sets Upgradeable=False with reason=AdminAckRequired until all applicable gates are acknowledged.\nProcedure:\n- Read keys from ConfigMap admin-gates in namespace openshift-config-managed. These are the gate keys the cluster's components have declared (example key shape: ack-4.13-kube-1.27-api-removals-in-4.14).\n- Read keys from ConfigMap admin-acks in namespace openshift-config. An acknowledgement is valid only if the value is the literal string \"true\".\n- For each key in admin-gates:\n - If the same key exists in admin-acks with value \"true\" → ACKNOWLEDGED.\n - If missing OR value is anything other than \"true\" → NOT ACKNOWLEDGED, and minor upgrade is blocked until administrator runs:\n 'oc -n openshift-config patch cm admin-acks --patch '{\"data\":{\"\":\"true\"}}' --type=merge'\n- Report each unacknowledged gate by its exact key name. Do NOT invent gate keys; only\
+ \ report what is actually present in admin-gates.\n- If admin-gates is empty or absent → no current admin-ack gates apply.\n- If either ConfigMap is unreadable, note it explicitly and indicate that gate state cannot be confirmed.\n\n### 4. ClusterOperator Health Check (per-operator condition matrix)\nFor each ClusterOperator, check status.conditions[]. Report as upgrade-impacting only when status fields match these patterns:\n- Available=False → BLOCKER. Operator is down, will block its phase of the upgrade.\n- Degraded=True → WARNING/POTENTIAL BLOCKER. Operator is reconciling with errors. If Available=True, upgrade may still proceed but with risk; if Available=False as well, treat as blocker.\n- Upgradeable=False → BLOCKER for minor (and sometimes z-stream) upgrades. Report exact reason and message.\n- Progressing=True for an extended period (no admin-initiated upgrade in flight) with error messages → POTENTIAL BLOCKER (stuck reconciliation).\n**Pay special attention to these critical\
+ \ operators** (failures here are upgrade-blocking by nature):\n- cluster-version (the CVO itself)\n- etcd — quorum and member health gate the entire control plane upgrade\n- kube-apiserver, kube-controller-manager, kube-scheduler — revision rollouts must converge before next phase\n- openshift-apiserver\n- machine-config — drives node-side updates\n- machine-api — provisions/replaces nodes\n- authentication\n- network — SDN/OVN health is required for any rolling reboot\n- dns\n- ingress\n- monitoring\n- image-registry\n- storage and any CSI driver operators\nFor each problematic operator, report: name, the failing condition (type and status), the reason, and the message.\n\n### 5. MachineConfigPool Status (Node Rollout Readiness)\nFor each MachineConfigPool (focus on master and worker, plus any custom pools):\n- spec.paused == true → Pool is paused. Paused master pool is almost always wrong. Paused worker pool is acceptable only as part of a documented EUS upgrade workflow; flag it\
+ \ for administrator awareness because paused pools block node-level updates and inhibit certificate rotation.\n- status.conditions[]:\n - Degraded=True → BLOCKER for that pool's node updates. Report message.\n - NodeDegraded=True → BLOCKER. A node in the pool failed to apply config; identify the node from the message.\n - RenderDegraded=True → BLOCKER. The MCO cannot render a valid config for the pool.\n - Updated=False AND Updating=False → Pool has pending changes but is not progressing — investigate.\n- Configuration drift: If status.observedGeneration != metadata.generation, the pool is behind; mention if the gap is significant.\n- status.machineCount, status.readyMachineCount, status.updatedMachineCount, status.degradedMachineCount — report if degradedMachineCount > 0 or if readyMachineCount < machineCount outside an active upgrade.\n\n### 6. Node Health and Resource Pressure\na) **Node Readiness and Pressure** (per Node, from status.conditions):\n - Ready=False or Ready=Unknown\
+ \ → BLOCKER. The node cannot drain, reboot, and rejoin during a rolling update. Report node name and the condition's reason and message.\n - MemoryPressure=True → BLOCKER/WARNING. Pods will be evicted and reschedule may not converge during upgrade surge; flag node name.\n - DiskPressure=True → BLOCKER. Image pulls for new release content will fail. Flag node name and check /var/lib/containers if message indicates.\n - PIDPressure=True → BLOCKER.\n - NetworkUnavailable=True → BLOCKER.\n - spec.unschedulable=true (cordoned) outside an active drain → flag for administrator awareness.\nb) **Resource Utilization** (using nodes_top, optional):\n - Flag any node with CPU usage > 90% or memory usage > 90%.\n - Explain impact: Rolling upgrades require surge capacity (control-plane revisions roll one node at a time; worker pools drain one node at a time). Saturated nodes can prevent successful drain and pod rescheduling.\n - For control-plane nodes, memory pressure is especially\
+ \ impactful because etcd is sensitive to I/O contention.\n\n### 7. PodDisruptionBudget Analysis (User Workload Drain Blockers)\nPDBs become upgrade-relevant because the MachineConfigOperator drains worker nodes one at a time during rolling updates. A PDB that does not allow eviction will block the drain indefinitely.\nProcedure:\n- Query PDBs in ALL namespaces EXCEPT OpenShift system namespaces:\n - All namespaces with prefix openshift-\n - All namespaces with prefix kube-\n - Namespaces default and openshift\n- For each remaining (user workload) PDB, evaluate as a drain blocker if ANY of the following are true:\n - status.disruptionsAllowed == 0 AND status.currentHealthy <= status.desiredHealthy (eviction blocked right now)\n - spec.minAvailable equals 100% (or equals status.expectedPods) — no pod can be evicted\n - spec.maxUnavailable == 0 — explicitly forbids any disruption\n - The PDB selector matches zero pods (status.expectedPods == 0) AND minAvailable >= 1 — misconfigured,\
+ \ will block drain\n- For each problematic PDB, report: namespace, name, the offending field, and status.disruptionsAllowed.\n- Ignore all PDBs in OpenShift system namespaces — these are managed by Red Hat.\n- If no problematic user workload PDBs exist, state \"No problematic user workload PDBs found\".\n\n### 8. Update Path Validation\na) **Channel correctness**:\n - Read spec.channel (e.g., stable-4.21, fast-4.21, eus-4.18).\n - Check status.desired.channels[] for channels available for the current version.\n - If spec.channel is not present in status.desired.channels AND RetrievedUpdates=False, the channel may be invalid for this version — flag it.\nb) **Skip-level detection**:\n - Examine status.history[0].version (current) vs any administrator-mentioned target.\n - OpenShift does NOT support skipping minor versions (e.g., 4.18 → 4.20 directly). Upgrades must go through each intermediate minor, except via the EUS-to-EUS path where worker pools are paused.\n - If the\
+ \ latest available update or conditional update is more than one minor ahead of the current version, surface this as an informational note about path constraints.\nc) **EUS path indicators**:\n - If spec.channel starts with eus-, note that the cluster is on the EUS path and that worker MCP pause/unpause is part of the workflow.\n\n### 9. Deprecated API Usage (Affects Minor Upgrades)\nIf the APIRequestCount resource is available on this cluster (apiserver.openshift.io/v1):\n- List APIRequestCount objects.\n- For each, read status.removedInRelease. If a removal release is set AND the upcoming minor target matches or exceeds it:\n - Read status.currentHour.byUser[] and status.last24h[].byUser[] to identify which clients are still calling the API.\n - Report: API name (e.g., flowschemas.v1beta2.flowcontrol.apiserver.k8s.io), removedInRelease, and a deduplicated list of top callers (username and userAgent).\n- If no deprecated APIs are in use OR none are removed by the target release\
+ \ → state so explicitly.\n- If APIRequestCount is unavailable, skip with a note.\n\n### 10. Pending CertificateSigningRequests\nDuring upgrades, nodes that reboot must have their kubelet client and serving certificates approved. A backlog of pending CSRs can prevent nodes from rejoining the cluster.\n- List CertificateSigningRequest objects.\n- Filter to those with no Approved condition AND no Denied condition (i.e., still pending) OR with status.certificate empty.\n- Group by spec.signerName (e.g., kubernetes.io/kube-apiserver-client-kubelet, kubernetes.io/kubelet-serving).\n- If 5 or more node-related CSRs are pending, flag as a node rejoin risk and report counts and signer names.\n- Pending CSRs unrelated to nodes (custom signers) can be ignored unless explicitly tied to upgrade workflows.\n\n### 11. MachineHealthCheck Status\nActive MachineHealthChecks can interfere with upgrades by remediating nodes that are intentionally drained or rebooted as part of the upgrade. Red Hat documentation\
+ \ recommends pausing MHCs during upgrades.\n- List MachineHealthChecks.\n- For each, check metadata.annotations[\"cluster.x-k8s.io/paused\"] or metadata.annotations[\"machine.openshift.io/cluster-api-cluster\"] paused-style annotations. Different OCP versions use different paused annotations; if any pause annotation is present, treat as paused.\n- Report MHCs that are NOT paused and target node sets that will roll during the upgrade — surface as a recommendation, not a blocker.\n\n### 12. OLM Subscription Health (Layered Operators)\nLayered operators installed via OLM must be on a channel/version compatible with the target OpenShift release before upgrade.\n- List Subscriptions.\n- For each Subscription, examine status.conditions[]:\n - CatalogSourcesUnhealthy=True → operator catalog cannot be reached (will block any operator updates)\n - InstallPlanFailed=True or ResolutionFailed=True → operator cannot install/update; flag the operator\n - InstallPlanPending=True AND not progressing\
+ \ → manual approval may be required before upgrade\n- Report by namespace and Subscription name. Do not flag healthy Subscriptions.\n\n### 13. Cluster Capabilities Assessment\n- Extract enabled capabilities from status.capabilities.enabledCapabilities.\n- Extract known capabilities from status.capabilities.knownCapabilities.\n- Disabled capabilities = known minus enabled. Note these.\n- If ImplicitlyEnabledCapabilities=True, surface that the upgrade target implicitly enables a capability that was disabled in spec.capabilities.\n- Capabilities themselves are rarely upgrade blockers, but capability transitions can change which operators are reconciled.\n\n### 14. Cincinnati Update Service Health\n- spec.upstream: if set, the cluster uses a custom update service; if unset, default Red Hat update service is used.\n- Verify RetrievedUpdates condition: status, lastTransitionTime, message.\n- If status.availableUpdates is empty AND RetrievedUpdates=True → cluster is on the latest known version\
+ \ in its channel.\n- If status.availableUpdates is empty AND RetrievedUpdates=False → update discovery is broken.\n- spec.clusterID: report for telemetry context.\n- spec.signatureStores: if present, custom signature stores are configured (relevant for disconnected clusters and ReleaseAccepted failures).\n\n### 15. Cluster Version History Context\n- Initial version: status.history[] last entry (oldest).\n- Most recent completed upgrade: most recent status.history[] entry with state=\"Completed\".\n- Any state=\"Partial\" entries indicate failed or interrupted upgrades — surface them.\n- Cluster age: derive from oldest history entry's startedTime or completionTime.\n\n### 16. Configuration Overrides\n- Review spec.overrides[]. Each entry with unmanaged=true means the CVO will not reconcile that resource.\n- Overrides are not upgrade blockers per se, but they can mask drift and cause post-upgrade inconsistencies. Surface any overrides as informational.\n\n### 17. Recent Events Analysis\
+ \ (Upgrade-Relevant Only)\n- Query events from last 30 minutes, type Warning or higher.\n- Restrict to upgrade-relevant namespaces: openshift-cluster-version, openshift-machine-config-operator, openshift-etcd, openshift-kube-apiserver, openshift-apiserver, openshift-authentication, openshift-network-operator, openshift-monitoring.\n- Group by reason and involvedObject to avoid noise.\n- Translate technical reasons into plain language:\n - ImagePullBackOff → \"Operator pod cannot pull its container image — check registry connectivity or pull secrets\"\n - FailedScheduling → \"Operator pod cannot be scheduled — check node taints, resources, or selectors\"\n - Unhealthy (for etcd) → \"etcd member health check failing — investigate before upgrading\"\n- Skip events unrelated to upgrade readiness.\n\n### 18. Active Alerts (Optional)\nIf get_alerts is available:\n- Focus on severity=critical and severity=warning.\n- Prioritize alerts whose names indicate upgrade impact, including but\
+ \ not limited to: ClusterNotUpgradeable, ClusterOperatorDown, ClusterOperatorDegraded, KubeAPIDown, etcdMembersDown, etcdInsufficientMembers, KubePersistentVolumeFillingUp, NodeFilesystemSpaceFillingUp, KubeNodeNotReady, MachineConfigDaemonReboot-style alerts.\n- Translate each fired alert into an actionable recommendation.\n- If get_alerts is unavailable, skip this section.\n\n\n\n## Summary\n**Update Service Health**\n- **Cincinnati Service**: [spec.upstream URL if configured, otherwise \"Default Red Hat update service\"]\n- **Service Status**: [RetrievedUpdates condition status and message]\n- **Last Update Check**: [From RetrievedUpdates condition lastTransitionTime]\n- **Update Channel**: [Current spec.channel]\n- **Channel Validity**: [Confirmed valid for current version, or flagged as not in status.desired.channels]\n- **Cluster ID**: [spec.clusterID]\n**Cluster History Context**\n- **Initial Version**: [First entry from status.history\
+ \ with date]\n- **Upgrade Path**: [Recent version progression from history]\n- **Last Completed Upgrade**: [Most recent Completed entry with timeframe]\n- **Partial/Failed Upgrade History**: [Any Partial entries, otherwise \"None\"]\n- **Cluster Age**: [Time since initial installation]\n**Available Updates**\n- **Recommended Updates**: [Count from status.availableUpdates with versions]\n- **Conditional Updates**: [Count from status.conditionalUpdates]\n- **Conditional Update Risk Analysis**: For each conditional update with Recommended=False, list:\n - Target version, risk name, risk message, reference URL\n - Otherwise: \"No conditional update risks apply to this cluster\"\n**Upgrade Readiness Assessment**\n\nYOU MUST explicitly state the status field value for each condition you check.\n**ClusterVersion Conditions:**\n- **Failing**: [type=\"Failing\" found with status=\"X\"] → [interpretation]\n- **Upgradeable**: [type=\"Upgradeable\" found with status=\"X\" OR not found] → [interpretation,\
+ \ including reason and message if status=\"False\"]\n- **Available**: [type=\"Available\" found with status=\"X\"] → [interpretation]\n- **Progressing**: [type=\"Progressing\" found with status=\"X\"] → [interpretation; only flag if stuck and not in admin-initiated upgrade]\n- **RetrievedUpdates**: [status=\"X\"] → [interpretation]\n- **ReleaseAccepted**: [status=\"X\"] → [interpretation]\n- **ImplicitlyEnabledCapabilities**: [status=\"X\"] → [interpretation]\n**Admin-Ack Gates (Minor Upgrade Prerequisite):**\n- **Defined Gates** (from openshift-config-managed/admin-gates): [list of keys, or \"None\"]\n- **Acknowledged** (from openshift-config/admin-acks with value \"true\"): [list of keys]\n- **Outstanding Gates Blocking Minor Upgrade**: [list of keys not acked, or \"None — all gates satisfied\"]\n- **Action**: For each outstanding gate, provide the exact oc patch command using the actual key name.\n**ClusterOperator Health:**\n- **Total Operators**: [count]\n- **Operators With Issues**:\
+ \ For each problematic operator, report:\n - Name\n - Failing condition (type and status)\n - Reason and message\n- If none: \"All ClusterOperators report Available=True, Degraded=False, Upgradeable=True\"\n**Infrastructure Health:**\n- **MachineConfigPools**: For each pool, report state. Flag Degraded=True, NodeDegraded=True, RenderDegraded=True, paused=true, or readyMachineCount < machineCount outside active upgrade.\n- **Node Status**: Count of NotReady nodes with names and reasons. Count of nodes with MemoryPressure/DiskPressure/PIDPressure/NetworkUnavailable.\n- **Resource Pressure**: From nodes_top, list nodes with >90% CPU or memory.\n- **Pending CSRs**: Count and signer names if 5 or more pending node-related CSRs.\n- **MachineHealthChecks**: Count of unpaused MHCs (informational recommendation).\n- **User Workload PDBs**: Count of problematic non-OpenShift PDBs that could block node draining, with namespace/name and the offending field.\n**Deprecated API Usage:**\n- **Deprecated\
+ \ APIs Removed In Target**: For each, report API name, removedInRelease, and top callers (username/userAgent).\n- If none or APIRequestCount unavailable: state explicitly.\n**Layered Operator Health (OLM):**\n- **Subscriptions With Issues**: For each, namespace and Subscription name with the failing condition (CatalogSourcesUnhealthy, InstallPlanFailed, ResolutionFailed, etc.).\n- If none: \"All Subscriptions healthy\"\n**Recent Events** (Last 30 minutes, upgrade-relevant namespaces):\n- **Critical Events**: Count and grouped descriptions.\n- **Warning Events**: Count and grouped descriptions.\n- **User-Friendly Summary**: Translate technical events into plain language.\n- If none: \"No recent errors or warnings detected in upgrade-relevant components\"\n**Active Alerts** (if available):\n- **Critical Alerts**: Count and names.\n- **Warning Alerts**: Count and names.\n- **Impact on Upgrade**: For each, explain effect on upgrade readiness.\n- If unavailable: skip section.\n**Configuration:**\n\
+ - **Overrides**: Any spec.overrides entries with unmanaged=true.\n- **Capabilities**: Enabled count, disabled-but-known count with names, ImplicitlyEnabledCapabilities note if applicable.\n**Final Assessment:**\nBased ONLY on issues identified above:\n- If no upgrade-blocking conditions and no unaccepted conditional risks: \"Cluster appears ready for upgrade.\"\n- If only conditional update risks or warnings (no hard blockers): \"Cluster can upgrade after administrator review of: [list]. No hard blockers.\"\n- If hard blockers present: \"Upgrade blocked — must resolve [list of specific blockers] first.\"\nA \"hard blocker\" means at least one of:\n- ClusterVersion Upgradeable=False\n- ClusterVersion Failing=True\n- ClusterVersion ReleaseAccepted=False\n- Any ClusterOperator with Available=False or Upgradeable=False\n- Any MachineConfigPool with Degraded=True, NodeDegraded=True, or RenderDegraded=True\n- Any node with Ready=False (other than transient and self-recovering)\n- Any user-workload\
+ \ PDB blocking eviction (disruptionsAllowed=0 with no surge capacity)\n- Outstanding admin-ack gate (only blocks minor upgrades, not z-stream)\n- Deprecated API in active use that is removed in target minor\n## TL;DR\n- **Current Version**: 4.21.14\n- **Available Updates**: [count from status.availableUpdates]\n- **Latest Recommended Update**: [version with channels]\n- **Conditional Updates**: [count] ([N with Recommended=False risks applying to this cluster])\n- **Update Channel**: [current spec.channel] ([valid / not in status.desired.channels])\n- **Channel Options**: [available channels for current version]\n- **Capabilities**: [enabled count / disabled count with names]\n- **Initial Version**: [from history with date]\n- **Last Upgrade**: [most recent completed upgrade with date]\n- **Cincinnati Health**: [service status with timestamp]\n- **Admin-Ack Gates**: [satisfied | N outstanding: list of keys]\n- **Upgrade Blocked**: [Yes | No — only \"Yes\" if a hard blocker per definition\
+ \ above is present]\n- **Upgrade Blockers**: [specific list with status field values, or \"No blockers\"]\n- **Conditional Risks To Acknowledge**: [risk names with target versions, or \"None\"]\n- **Unhealthy ClusterOperators**: [count and names with the failing condition]\n- **Degraded MCPs**: [count and names with failing condition]\n- **Paused MCPs**: [names if any]\n- **Node Issues**: [count of NotReady or pressure-affected nodes with names]\n- **Resource Pressure**: [nodes with >90% CPU or memory]\n- **User Workload PDBs Blocking Drain**: [count with namespace/name]\n- **Pending Node CSRs**: [count if >= 5, else omit or \"None significant\"]\n- **Deprecated APIs In Use**: [count removed in target, with API names]\n- **Layered Operator Issues**: [count of unhealthy Subscriptions]\n- **Recent Events**: [count of upgrade-relevant errors/warnings in last 30 min]\n- **Active Alerts**: [count of critical/warning, skip if unavailable]\n- **Configuration Issues**: [overrides or capability\
+ \ concerns]\n- **Recommendation**: [Proceed with upgrade | Address warnings first | Blocked — resolve listed issues]\n\n"
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+- conversation_group_id: conv_002
+ tag: cluster-updates-critical
+ turns:
+ - expected_response: 'Response should analyze the upgrade path from 4.21.16 to 4.21.22 with:
+
+
+ ## Summary
+
+ - Target Version Analysis: Whether 4.21.22 is available in availableUpdates or conditionalUpdates
+
+ - Conditional Updates Risk Analysis: All conditional update risks between 4.21.16 and 4.21.22, including intermediate versions
+
+ - Upgrade Readiness Assessment: ClusterVersion conditions and ClusterOperator health
+
+ - Infrastructure Health: MCPs, Nodes, PDBs
+
+ - Cincinnati Update Service Health
+
+ - Recent Events and Active Alerts
+
+
+ ## TL;DR
+
+ - Current Version: 4.21.16
+
+ - Target Version: 4.21.22
+
+ - Upgrade path analysis with all intermediate version risks
+
+ - Comprehensive risk assessment
+
+ - Clear recommendation
+
+
+ Must analyze ALL versions in the upgrade path, not just the target.'
+ query: "# OpenShift Cluster Upgrade Pre-Check Analysis\n\n\n- LANGUAGE REQUIREMENT: Respond in English. All analysis, explanations, recommendations, and text must be in English.\n\n- YOU MUST ALWAYS CALL THE TOOLS TO GET THE INFORMATION. YOU SHOULD NEVER TREAT DATA FROM EXAMPLES AS REAL DATA.\n- YOU MUST ALWAYS REFERENCE REAL DATA FROM TOOL CALLS. IF REAL DATA IS NOT AVAILABLE, NOTIFY THE USER AND REFUSE TO ANSWER USING INCORRECT DATA BUT DO NOT USE PLACEHOLDER OR DUMMY DATA.\n**CRITICAL: Timeout and Error Handling**\n**Timeout Awareness:**\n- You have a 60-second timeout - manage your time wisely\n- Prioritize essential data (ClusterVersion, ClusterOperators) first\n- Track execution time and stop making new tool calls after 50 seconds\n- Provide analysis with available data rather than timing out trying to fetch everything\n**Error Handling Rules:**\n1. **Be specific about which tool failed**- don't give generic \"cannot retrieve data\" messages\n2. **Explain what data\
+ \ you're missing**- e.g., \"Unable to fetch ClusterVersion resource\" vs \"Unable to retrieve data\"\n3. **Try alternative approaches**:\n - If resources_list fails for all ClusterOperators, note this specifically\n - If nodes_top fails, continue with other analysis - it's optional\n - If get_alerts fails, skip it - alerts are optional\n - If events_list fails, continue without event data\n4. **Provide partial analysis** - If you get ClusterVersion but not operators, analyze what you have\n5. **Give actionable troubleshooting**when tools fail:\n - Check if OpenShift MCP server is running: 'oc get pods -n openshift-lightspeed'\n - Verify cluster connectivity\n - Suggest checking MCP server logs for specific errors\n6. **NEVER give up completely**- Always provide SOME analysis even with partial data\n7. **RBAC and Permission Errors - Critical for cluster-wide resources**:\n - If tool calls fail with \"Forbidden\" (403) or \"access denied\" errors, this means insufficient RBAC permissions.\n - Be SPECIFIC about which resources are inaccessible (e.g., \"Cannot access ClusterVersion resource\", \"Cannot access Prometheus metrics\").\n - Provide analysis based ONLY on accessible resources - DO NOT invent data.\n - For comprehensive cluster analysis, state required permissions: \"For full upgrade analysis, the service account requires cluster-reader ClusterRoleBinding (ClusterVersion, ClusterOperator, Node access) and monitoring-edit ClusterRoleBinding (Prometheus and Alertmanager access). Apply: oc apply -f eval/rbac-cluster-updates.yaml\"\n**Tool Call Priority to Avoid Timeouts:**\n**PHASE 1 - ESSENTIAL (Always fetch):**\n1. resources_get: ClusterVersion (apiVersion: \"config.openshift.io/v1\", kind: \"ClusterVersion\", name:\
+ \ \"version\")\n2. resources_list: ClusterOperator (apiVersion: \"config.openshift.io/v1\", kind: \"ClusterOperator\")\n**PHASE 2 - IMPORTANT (Fetch if time permits, under 45 seconds total):**\n3. resources_list: Node (apiVersion: \"v1\", kind: \"Node\") - Quick check for NotReady nodes\n4. events_list: Get recent warning/error events from last 30 minutes - High value for diagnostics\n**PHASE 3 - OPTIONAL (Only if under 50 seconds total):**\n5. resources_list: MachineConfigPool (apiVersion: \"machineconfiguration.openshift.io/v1\", kind: \"MachineConfigPool\")\n6. nodes_top: Check node CPU/memory usage\n7. resources_list: PodDisruptionBudget (apiVersion: \"policy/v1\", kind: \"PodDisruptionBudget\") - Filter out openshift-*, kube-*\n8. get_alerts: Check for critical/warning alerts\n**CRITICAL EFFICIENCY RULES:**\n- If approaching 50 seconds of execution time, STOP making new tool calls and provide analysis with data collected\n- NEVER let total execution exceed 55 seconds to avoid\
+ \ timeout\n- Prioritize breadth over depth: Get ClusterVersion + ClusterOperators fully before diving into logs/events\n- Skip optional data if essential data took longer than expected\n\n- NEVER use placeholder or dummy data - only reference real data from tool calls\n- ONLY report issues that are actually present in the data\n- ONLY OUTPUT the Summary and TL;DR sections\n- Be specific about the source of any issues identified\n- CRITICAL: When counting available updates, count ALL array elements in status.availableUpdates\n- CRITICAL: Check status.conditionalUpdates for ALL versions from 4.21.16 to 4.21.22 (inclusive)\n- CRITICAL: Analyze the COMPLETE upgrade path, not just the target version - intermediate versions matter!\n\n\n\nThis is a pre-upgrade analysis for OpenShift cluster upgrade from 4.21.16 to 4.21.22. You have complete cluster data including ClusterVersion and all ClusterOperator resources to analyze the feasibility and safety of this specific\
+ \ upgrade.\n\n**CRITICAL UPGRADE PATH ANALYSIS**: You must analyze ALL conditional update risks for every version between 4.21.16 and 4.21.22 (inclusive). This includes intermediate versions that may be part of the upgrade path. For example, if upgrading from 4.21.16 to 4.21.22, you must check for risks at 4.21.17, 4.21.18, 4.21.19, 4.21.20, 4.21.21, and 4.21.22. Users need to know about ALL risks they will encounter in the upgrade journey, not just risks at the final target version.\n\n\n\nCRITICAL: Understanding Kubernetes/OpenShift Conditions\n\nConditions have TWO important fields you MUST check:\n- **type**: The name of the condition (e.g., \"Failing\", \"Available\", \"Progressing\")\n- **status**: The state of the condition (\"True\", \"False\", or \"Unknown\")\n\n**How to Correctly Check Conditions:**\n- A condition is TRUE when: type=\"X\" AND status=\"True\"\n- A condition is FALSE when: type=\"X\" AND status=\"False\"\n- A condition is\
+ \ UNKNOWN when: type=\"X\" AND status=\"Unknown\"\n\n**Examples:**\n- {type: \"Failing\", status: \"False\"} means the cluster is NOT failing (healthy)\n- {type: \"Failing\", status: \"True\"} means the cluster IS failing (problem)\n- {type: \"Upgradeable\", status: \"False\"} means upgrades are blocked (problem)\n- {type: \"Upgradeable\", status: \"True\"} means upgrades are allowed (healthy)\n**NEVER assume a condition is true just because the type exists - ALWAYS check the status field!**\n\n\n\n\n1. **Target Version Verification** (PRIORITY):\n - Look in status.availableUpdates array for 4.21.22\n - If found, extract its channels, url, and image information\n - If NOT found, report \"4.21.22 is not available for upgrade\"\n\n1a. **Conditional Updates Risk Analysis - All Risks Up to Target Version**:\n - **CRITICAL**: Analyze ALL conditional updates from 4.21.16 up to and including 4.21.22\n - **Version Range**: Parse version\
+ \ numbers to identify which conditional updates fall between current and target\n - **Example**: If current=4.21.16, target=4.21.22, analyze risks for: 4.21.17, 4.21.18, 4.21.19, 4.21.20, 4.21.21, 4.21.22\n - **Why This Matters**: Users may need to upgrade through intermediate versions to reach the target, so ALL risks in the path are relevant\n - For each conditional update in the version range, analyze the conditions array:\n * Look for conditions with type=\"Recommended\" AND status=\"False\" (indicates risks/concerns)\n * Extract risk details from the condition message field\n * Parse URLs in the message for documentation links\n - **Risk Assessment Process** (for each version in range):\n * Identify what triggers the risk (e.g., specific cluster configurations, network plugins)\n * Determine if the risk applies to THIS cluster based on current configuration\n * Assess severity: Does this block the upgrade or just require careful planning?\n * Note which version introduces the\
+ \ risk (important for upgrade path planning)\n - **User-Friendly Risk Explanation**:\n * Translate technical risk messages into plain language\n * Example: \"Recommended=False: OVN network disruption\" → \"This update may cause brief network interruptions (2-5 minutes) if your cluster uses OVN-Kubernetes networking. Plan for a maintenance window.\"\n * Show version where risk appears: \"Risk at 4.21.18: [description]\"\n - **Mitigation Recommendations**:\n * If risk applies: Suggest mitigation steps (maintenance window, backup procedures, etc.)\n * If risk doesn't apply: Clearly state \"This risk does not apply to your cluster\"\n * Provide decision guidance: \"Safe to proceed with caution\" vs \"Address concerns first\"\n - **Presentation Order**: List risks in version order (lowest to highest) to show the upgrade path chronologically\n\n2. **Cluster Upgrade Readiness** (Check BOTH type AND status):\n - Find condition where type=\"Upgradeable\" (may not exist)\n * If found AND status=\"\
+ False\": Report the specific reason - this blocks upgrades\n * If status=\"True\" or missing: Upgrades are allowed\n - Find condition where type=\"Failing\"\n * If found AND status=\"True\": Report details - this indicates problems\n * If status=\"False\" or missing: No failing condition (healthy)\n - Find condition where type=\"Available\"\n * If found AND status=\"False\": Report cluster operational issues\n * If status=\"True\": Cluster is available (healthy)\n\n3. **ClusterOperator Health Check** (Check BOTH type AND status):\n For each ClusterOperator, check conditions:\n - Available: If type=\"Available\" AND status=\"False\" → Operator unavailable (blocker)\n - Degraded: If type=\"Degraded\" AND status=\"True\" → Operator degraded (warning)\n - Upgradeable: If type=\"Upgradeable\" AND status=\"False\" → Blocks upgrades (blocker)\n - Report specific operator names and their issues for problematic conditions only\n - Focus on operators that would block upgrades\n\n4. **Current\
+ \ Cluster Configuration**:\n - Extract spec.channel (current update channel)\n - Extract spec.clusterID\n - Check if spec.upstream is configured (custom Cincinnati server)\n - Note status.conditions RetrievedUpdates condition\n\n5. **User Workload PDB Analysis** (IMPORTANT - Filter System PDBs):\n - Query PodDisruptionBudgets in ALL namespaces EXCEPT these OpenShift system namespaces:\n * openshift-* (all openshift- prefixed namespaces)\n * kube-* (all kube- prefixed namespaces)\n * default, openshift\n - ONLY flag user workload PDBs where:\n * minAvailable >= 1 AND it covers critical user applications\n * maxUnavailable = 0 AND it covers critical user applications\n - IGNORE all PDBs in OpenShift system namespaces - these are managed by Red Hat\n - If no problematic user workload PDBs exist, state \"No problematic user workload PDBs found\"\n\n6. **MachineConfigPool Status** (Check BOTH type AND status):\n For each MachineConfigPool:\n - Check conditions for Degraded: If type=\"Degraded\"\
+ \ AND status=\"True\" → MCP has issues\n - Check conditions for Updated: If type=\"Updated\" AND status=\"False\" → MCP not updated\n - Check spec.paused=true → MCP manually paused (blocks node updates)\n - Check observedGeneration ≠ metadata.generation → Configuration drift\n - Focus on master and worker MCPs which are critical for upgrade success\n - Report specific MCP names and their issues\n\n7. **Node Health and Resource Pressure**:\n a) **Node Readiness:**\n - Check each Node for Ready condition: If type=\"Ready\" AND status=\"False\" → Node not ready (blocker)\n - Check for other node conditions: MemoryPressure, DiskPressure, PIDPressure (status=\"True\" is problem)\n - Report NotReady nodes with their conditions and reasons\n\n b) **Resource Utilization (using nodes_top):**\n - Check CPU usage: Flag if any node >90% CPU utilization\n - Check memory usage: Flag if any node >90% memory utilization\n - Explain impact: High resource usage can slow upgrades or cause failures\n\
+ \ - Recommend: Consider scaling down workloads before upgrading to 4.21.22 if resources are constrained\n\n8. **Cincinnati Update Service Health**:\n - Check spec.upstream (if configured) or note \"using default Red Hat update service\"\n - Verify status.conditions for type=\"RetrievedUpdates\" status and timestamp\n - Confirm status.availableUpdates or status.conditionalUpdates contains 4.21.22\n - Cluster ID for telemetry (spec.clusterID)\n - Signature verification status (spec.signatureStores if present, otherwise default stores)\n\n9. **Recent Events Analysis** (using events_list):\n - Query recent events from last 30 minutes\n - Focus on Warning and Error type events\n - Filter for upgrade-related namespaces: openshift-cluster-version, openshift-*\n - Look for patterns: repeated errors, failing pods, configuration issues\n - **User-Friendly Explanation**: Translate technical events into plain language\n - Report only events that are relevant to upgrade readiness for 4.21.22\n\
+ \ - Group similar events to avoid overwhelming users\n\n10. **Active Alerts Assessment** (using get_alerts - if available):\n - Query Alertmanager for active alerts\n - Focus on Critical and Warning severity alerts\n - **Upgrade Impact Analysis**:\n * Critical alerts → Likely upgrade blockers, must resolve first\n * Warning alerts → May cause issues, recommend resolving\n * Info alerts → Monitor but don't block\n - **User-Friendly Translation**: Explain what each alert means in simple terms\n - Provide actionable recommendations for each alert\n - If get_alerts tool not available: Skip this check (gracefully handle tool absence)\n\n\n\n\n## Summary\n\nProvide a clear assessment based ONLY on real data from tool calls (resources_get and resources_list). Be specific about:\n- **Whether 4.21.22 is available for upgrade** (found in status.availableUpdates or status.conditionalUpdates)\n- **ALL conditional update risks in the upgrade path\
+ \ from 4.21.16 to 4.21.22** (analyze status.conditionalUpdates for ALL versions in range)\n- **Current cluster upgrade readiness** (check Upgradeable=False, Failing=True, degraded operators)\n- **Any problematic USER WORKLOAD PDBs** (not OpenShift system PDBs)\n- **Infrastructure issues**that would prevent the upgrade to 4.21.22\n\n**CRITICAL INSTRUCTION**: Parse version numbers from status.conditionalUpdates and identify which versions fall between 4.21.16 and 4.21.22. Report risks for ALL of these versions, not just 4.21.22.\n\n**Target Version Analysis**\n- **Availability**: [Whether 4.21.22 is in availableUpdates or conditionalUpdates]\n- **Channels**: [Channels available for 4.21.22]\n- **Release Information**: [URL and metadata for 4.21.22 if available]\n\n**Conditional Updates Risk Analysis - Upgrade Path from 4.21.16 to 4.21.22**:\n- **Version Range Analyzed**: [List all versions between 4.21.16 and 4.21.22 that have conditional update risks]\n- **Total Risks in Upgrade Path**:\
+ \ [Count of all risk conditions across all versions in the range]\n- **Risk Details by Version** (in chronological order from lowest to highest version):\n\n For each version with risks in the upgrade path:\n * **Version**: [e.g., 4.21.18]\n * **Risk Conditions**: [List conditions with Recommended=False for this version]\n - Risk: [Human-readable risk description from condition message]\n - Applies to this cluster: [Yes/No with explanation]\n - Severity: [Blocker / Requires Planning / Minor Concern]\n - Mitigation: [Specific steps to address this risk]\n - Documentation: [URL from message if available]\n\n- **Cumulative Risk Assessment for Upgrade Path**:\n * If no risks in path: \"No conditional update risks from 4.21.16 to 4.21.22\"\n * If risks don't apply: \"Conditional updates exist but risks do not apply to this cluster configuration\"\n * If risks apply but manageable: \"Upgrade path has manageable risks - schedule maintenance window and review all mitigations\"\n\
+ \ * If risks are severe: \"Review all risks carefully before proceeding - multiple versions in upgrade path have concerns\"\n * **Planning Guidance**: \"You will encounter [X] risk conditions across [Y] versions in the upgrade path from 4.21.16 to 4.21.22\"\n\n**Upgrade Readiness Assessment**\n\nYOU MUST explicitly state the status field value for each condition you check:\n**ClusterVersion Conditions:**\n- **Failing Condition**: [type=\"Failing\" found with status=\"X\"] → [Interpretation: if status=\"False\" then NOT failing/healthy, if status=\"True\" then failing/problem]\n- **Upgradeable Condition**: [type=\"Upgradeable\" found with status=\"X\" OR not found] → [Interpretation: if status=\"False\" then upgrades blocked, if status=\"True\" or missing then upgrades allowed]\n- **Available Condition**: [type=\"Available\" found with status=\"X\"] → [Interpretation: if status=\"True\" then available/healthy, if status=\"False\" then not available/problem]\n**ClusterOperator Health:**\n\
+ - Verify ClusterOperator resources in config.openshift.io/v1 API group\n- For each operator, check status.conditions and explicitly state status field values\n- Flag operators with: Available status=\"False\" OR Degraded status=\"True\" OR Upgradeable status=\"False\"\n- Include their message and reason fields\n**Infrastructure Health:**\n- **MachineConfigPools**: [Count and status of MCPs - report Degraded, Paused, or out-of-sync pools]\n- **Node Status**: [Count NotReady nodes with their reasons]\n- **Resource Pressure**: [From nodes_top - report nodes with >90% CPU or memory usage]\n- **User Workload PDBs**: [Count of problematic non-OpenShift PDBs that could block node draining]\n**Cincinnati Update Service Health**:\n- **Service Configuration**: [spec.upstream URL if configured, otherwise \"Default Red Hat update service\"]\n- **Service Status**: [RetrievedUpdates condition status and message]\n- **Last Update Check**: [From RetrievedUpdates condition lastTransitionTime]\n- **Update\
+ \ Channel**: [Current spec.channel]\n- **Cluster ID**: [spec.clusterID for telemetry]\n**Recent Events** (Last 30 minutes):\n- **Critical Events**: [Count and description of error events]\n- **Warning Events**: [Count and description of warning events]\n- **User-Friendly Summary**: [Translate technical events into plain language explanation]\n- **Example**: \"3 ImagePullBackOff events in openshift-authentication - operator unable to download container images\"\n- If no concerning events: \"No recent errors or warnings detected\"\n**Active Alerts** (if available):\n- **Critical Alerts**: [Count and names of firing critical alerts]\n- **Warning Alerts**: [Count and names of firing warning alerts]\n- **Impact on Upgrade**: [Explain how these alerts affect upgrade readiness to 4.21.22]\n- **User-Friendly Explanation**: [Translate alert names into actionable recommendations]\n- **Example**: \"KubePersistentVolumeFillingUp: Storage volume is 85% full - free up space before upgrading\"\n\
+ - If alerts not available: Skip this section\n\n**Final Assessment**:\nIf 4.21.22 is available and no critical issues are found:\n- If no conditional risks in upgrade path: Clearly state the cluster appears ready for upgrade to 4.21.22\n- If conditional risks exist but don't apply: State the upgrade path is clear despite conditional updates existing\n- If conditional risks apply: Explain ALL risks across the upgrade path and provide comprehensive mitigation guidance\n\n**Upgrade Path Summary**:\n- List ALL versions with conditional update risks between 4.21.16 and 4.21.22\n- For multi-hop upgrades, clarify that risks at intermediate versions will be encountered\n- Provide consolidated recommendation considering ALL risks in the path, not just target version\n\nIf 4.21.22 is not available, recommend the closest available version and analyze risks for that path instead.\n\n## TL;DR\n- **Current Version**: 4.21.16\n- **Target Version**: 4.21.22\n- **Target Available**: [Yes in availableUpdates\
+ \ / Yes in conditionalUpdates with risks / No]\n- **Upgrade Path**: [4.21.16 → 4.21.22, list intermediate versions if applicable]\n- **Conditional Risks in Upgrade Path**: [Total count of risk conditions across ALL versions from 4.21.16 to 4.21.22, or \"None\"]\n- **Versions with Risks**: [List versions in upgrade path that have conditional update risks, e.g., \"4.21.18, 4.21.20\" or \"None\"]\n- **Risks Apply to Cluster**: [Yes/No - if ANY conditional risks in upgrade path apply to this cluster configuration]\n- **Risk Severity**: [If risks exist: Blocker / Requires Planning / Minor Concern / Does Not Apply]\n- **Target Channels**: [Channels for 4.21.22 if available]\n- **Current Channel**: [spec.channel from ClusterVersion]\n- **Cincinnati Health**: [Update service status, e.g., \"Default service healthy (RetrievedUpdates=True)\" or \"Custom upstream: URL (status)\"]\n- **Upgrade Blocked**: [Yes if blocked / No if not blocked - ONLY report \"Yes\" if: Upgradeable condition has status=\"\
+ False\" OR Failing condition has status=\"True\" OR operators have Available status=\"False\" or Upgradeable status=\"False\"]\n- **Upgrade Blockers**: [if blockers exist with specific reason - MUST include the actual status field value you read, e.g., \"Upgradeable condition status=False: reason message\" OR \"No blockers - all conditions healthy\"]\n- **Unhealthy ClusterOperators**: [count and names if any]\n- **User Workload PDBs**: [count of problematic NON-OpenShift PDBs]\n- **Degraded MCPs**: [count and names if any]\n- **Node Issues**: [count of NotReady nodes if any, include Ready=False reason]\n- **Resource Pressure**: [nodes with >90% CPU or memory usage]\n- **Recent Events**: [count of error/warning events in last 30 min, user-friendly summary]\n- **Active Alerts**: [count of critical/warning alerts, skip if tool unavailable]\n- **Recommendation**: [Proceed with upgrade to 4.21.22 | Address risks/warnings first | Blocked - resolve issues | Target not available - use X.X.X\
+ \ instead]\n\n"
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+- conversation_group_id: conv_003
+ tag: cluster-updates-critical
+ turns:
+ - expected_response: 'Response should provide a health assessment with:
+
+
+ ## Summary
+
+ - Overall Health Status: Assessment of cluster operational health
+
+ - System Component Status: Core services health, degraded operators list, progressing operators
+
+ - Administrator Action Items: Immediate, maintenance, and monitoring items categorized
+
+ - Future Update Readiness: Upgradeable condition status, any blockers
+
+
+ ## TL;DR
+
+ - Overall Status: Healthy/Minor issues/Attention needed
+
+ - System Health: Count of healthy vs degraded operators
+
+ - Core Platform: Status of essential operators
+
+ - Action Items: Count and categorization
+
+ - Update Readiness: Ready or specific issues to resolve
+
+
+ Should focus on operational health when no updates are available.'
+ query: "# OpenShift Cluster Health Assessment\n\n\n- YOU MUST ALWAYS CALL THE TOOLS TO GET THE INFORMATION. YOU SHOULD NEVER TREAT DATA FROM EXAMPLES AS REAL DATA.\n- YOU MUST ALWAYS REFERENCE REAL DATA FROM TOOL CALLS. IF REAL DATA IS NOT AVAILABLE, NOTIFY THE USER AND REFUSE TO ANSWER USING INCORRECT DATA BUT DO NOT USE PLACEHOLDER OR DUMMY DATA.\n- Use resources_get to fetch the ClusterVersion resource (apiVersion: \"config.openshift.io/v1\", kind: \"ClusterVersion\", name: \"version\")\n- Use resources_list to fetch all ClusterOperator resources (apiVersion: \"config.openshift.io/v1\", kind: \"ClusterOperator\")\n- Assess ONLY the actual cluster state from tool call data\n- Distinguish between system health and user workload issues\n- Provide actionable recommendations for administrators\n- ONLY OUTPUT the Summary and TL;DR sections\n- LANGUAGE REQUIREMENT: Respond in English. All analysis, explanations, recommendations, and text must be in English.\n\n\
+ \n\nHealth assessment for OpenShift cluster with no available updates. You have complete cluster data including ClusterVersion and all ClusterOperator resources for comprehensive health analysis.\nFocus on operational health and readiness for future updates.\n\n\n\nCRITICAL: Understanding Kubernetes/OpenShift Conditions\n\nConditions have TWO important fields you MUST check:\n- **type**: The name of the condition (e.g., \"Failing\", \"Available\", \"Progressing\")\n- **status**: The state of the condition (\"True\", \"False\", or \"Unknown\")\n\n**How to Correctly Check Conditions:**\n- A condition is TRUE when: type=\"X\" AND status=\"True\"\n- A condition is FALSE when: type=\"X\" AND status=\"False\"\n- A condition is UNKNOWN when: type=\"X\" AND status=\"Unknown\"\n\n**Examples:**\n- {type: \"RetrievedUpdates\", status: \"True\"} means updates were retrieved (healthy)\n- {type: \"RetrievedUpdates\", status: \"False\"} means update retrieval\
+ \ failed (problem)\n- {type: \"Failing\", status: \"False\"} means the cluster is NOT failing (healthy)\n- {type: \"Available\", status: \"True\"} means the cluster IS available (healthy)\n**NEVER assume a condition is true just because the type exists - ALWAYS check the status field!**\n\n\n\n\n1. **Current Version and Update Status Analysis** (Check BOTH type AND status):\n - Extract and report current cluster version from status.desired.version\n - Verify status.availableUpdates array is empty (confirming no updates available)\n - Find condition where type=\"RetrievedUpdates\" AND status=\"True\" (confirms update service is working)\n - Analyze why no updates are available (end of channel, latest version, etc.)\n\n2. **Cluster Capabilities Configuration Assessment**:\n - Extract enabled capabilities from status.capabilities.enabledCapabilities\n - Extract known capabilities from status.capabilities.knownCapabilities\n -\
+ \ Identify disabled capabilities (known but not enabled)\n - Assess capability configuration health and consistency\n - Check spec.capabilities.baselineCapabilitySet and additionalEnabledCapabilities\n\n3. **Update Service and Channel Health**:\n - Check spec.upstream (if configured) or note \"using default Red Hat update service\"\n - Verify status.conditions for type=\"RetrievedUpdates\" status and timestamp\n - Confirm update service connectivity is working despite no available updates\n - Current channel from spec.channel\n - Cluster ID for telemetry (spec.clusterID)\n - Signature verification status (spec.signatureStores if present, otherwise default stores)\n\n4. **Cluster Version History Context**:\n - Extract initial cluster version from status.history (first entry)\n - Identify upgrade path from history entries\n - Last completed upgrade and timeframe\n - Total cluster age and upgrade frequency\n - Historical upgrade success pattern\n\n5. **System Component Health** (Check\
+ \ BOTH type AND status for each operator):\n For each ClusterOperator, check conditions:\n - **Available**: If type=\"Available\" AND status=\"False\" → Requires immediate intervention\n - **Degraded**: If type=\"Degraded\" AND status=\"True\" → Degraded state, lower quality of service\n - **Progressing**: If type=\"Progressing\" AND status=\"True\" with errors → Component stuck\n - **Upgradeable**: If type=\"Upgradeable\" AND status=\"False\" → Blocks minor upgrades\n - Verify core platform operators (console, authentication, ingress, etc.) are healthy\n - Check ClusterVersion status.conditions for overall cluster health\n - Report specific operator names and their condition messages for problematic conditions only\n - IMPORTANT: Available=True, Degraded=False, Upgradeable=True are healthy states\n\n6. **Future Update Readiness Assessment** (Check BOTH type AND status):\n - Find condition where type=\"Upgradeable\" (OPTIONAL - may not exist)\n * If found AND status=\"False\": This\
+ \ IS an upgrade blocker - report reason\n * If status=\"True\", missing, or status=\"Unknown\": Future upgrades are allowed\n - Find condition where type=\"Failing\"\n * If found AND status=\"True\": Cluster issues that must be resolved\n * If status=\"False\" or missing: No failing condition (healthy)\n - Review spec.overrides for any unmanaged components that might block future upgrades\n - Identify maintenance items to address proactively\n - User workload PDB analysis for potential upgrade blockers\n\n7. **Operational Health and Recommendations**:\n - Identify issues that affect user applications\n - Focus on problems that cluster administrators can/should address\n - Provide specific, actionable guidance for maintaining cluster health\n - Distinguish from normal system maintenance activities\n - Avoid recommendations for normal system behavior\n\n\n\n\n## Summary\n**Overall Health Status**\n[Assessment based on actual cluster state\
+ \ data]\n**System Component Status**\n- **Core Services**: [List core platform operators and their health status]\n- **Degraded Operators**: [Any operators with Available=False or Degraded=True]\n- **Progressing Operators**: [Operators currently updating or progressing]\n- **Infrastructure**: [Overall cluster-level status and configuration]\n**Administrator Action Items**\n- **Immediate**: [Issues requiring prompt attention]\n- **Maintenance**: [Items to address during maintenance windows]\n- **Monitoring**: [Things to watch for trends]\n**Future Update Readiness**\n[Assessment of readiness for next OpenShift updates]\n\n## TL;DR\n- **Overall Status**: [Healthy | Minor issues | Attention needed]\n- **System Health**: [Count of healthy vs degraded operators]\n- **Core Platform**: [Status of essential operators: console, authentication, ingress, etc.]\n- **Degraded Components**: [Count and names of any unhealthy operators]\n- **User Impact**: [Any operator issues affecting workloads]\n\
+ - **Action Items**: [Count of items needing administrator attention]\n- **Update Readiness**: [Ready | Operator issues need resolution]\n- **Next Review**: [Recommended reassessment timeframe]\n\n"
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+ - geval:technical_accuracy
+- conversation_group_id: conv_004
+ tag: cluster-updates-critical
+ turns:
+ - expected_response: Response should analyze upgrade progress monitoring with real-time cluster state showing upgrade in progress or completion status, including ClusterVersion conditions, ClusterOperator progression, and infrastructure state.
+ query: "# OpenShift Cluster Upgrade Progress Monitor\n\n\n- YOU MUST ALWAYS CALL THE TOOLS TO GET THE INFORMATION. YOU SHOULD NEVER TREAT DATA FROM EXAMPLES AS REAL DATA.\n- YOU MUST ALWAYS REFERENCE REAL DATA FROM TOOL CALLS. IF REAL DATA IS NOT AVAILABLE, NOTIFY THE USER AND REFUSE TO ANSWER USING INCORRECT DATA BUT DO NOT USE PLACEHOLDER OR DUMMY DATA.\n**CRITICAL: Timeout and Error Handling**\n\n**Timeout Awareness (60 second limit):**\n- Progress monitoring needs to be FAST - users expect quick updates\n- ClusterVersion + ClusterOperators gives you operator progress (X of Y) - sufficient for basic progress\n- Events and other data add context but aren't required\n- Target: Complete analysis in under 40 seconds for responsive UX\n- If approaching 50 seconds, provide progress summary immediately\n**Error Handling for Tool Failures:**\n1. **Core data is essential**- ClusterVersion and ClusterOperators are required for progress tracking\n2. **If core resources fail** -\
+ \ Explain specifically what failed and provide troubleshooting\n3. **Optional data can be skipped**- nodes_top, events_list, get_alerts are nice-to-have\n4. **Provide progress with available data**- Even without events, you can show operator progress\n5. **Never give up**- Always show some progress information, even if incomplete\n**Tool Call Priority to Avoid Timeouts:**\n**PHASE 1 - ESSENTIAL (Always fetch - target: 25 seconds):**\n1. resources_get: ClusterVersion (apiVersion: \"config.openshift.io/v1\", kind: \"ClusterVersion\", name: \"version\")\n2. resources_list: ClusterOperator (apiVersion: \"config.openshift.io/v1\", kind: \"ClusterOperator\")\n**PHASE 2 - HELPFUL CONTEXT (Only if under 45 seconds):**\n3. events_list: Get recent events (last 30 minutes) - Quick way to spot warnings\n4. resources_list: MachineConfigPool - Shows node update progress\n**PHASE 3 - NICE-TO-HAVE (Only if under 50 seconds):**\n5. nodes_top: Monitor node resource usage during upgrade\n6. get_alerts:\
+ \ Check for warning alerts (if available)\n**CRITICAL EFFICIENCY RULES:**\n- Progress monitoring is time-sensitive - provide fast updates\n- ClusterVersion + ClusterOperators is sufficient for basic progress (X of Y operators)\n- Events and MCPs add context but aren't required\n- NEVER exceed 55 seconds - better to show quick progress than timeout\n- Users can refresh for updated progress - speed > completeness\n\n- Monitor ONLY actual upgrade progress from tool call data\n- Report specific progress indicators and timelines using EXACT operator counts from the data\n- Use the format \"X of Y operators\" consistently throughout the output\n- Calculate precise percentages: (15 / 28) * 100\n- Format durations in human-readable terms (e.g., \"Approximately 1 hour and 20 minutes\")\n- Use specific operator counts in all sections, not generic descriptions\n- Identify potential issues early with conservative recommendations\n- ONLY OUTPUT the Summary and TL;DR sections exactly as specified\
+ \ in the output format\n- LANGUAGE REQUIREMENT: Respond in English. All analysis, explanations, recommendations, and text must be in English.\n\n\n\nMonitor upgrade progress from 4.21.14 to 4.21.17. You have complete cluster data including ClusterVersion and all ClusterOperator resources to analyze upgrade progress and detect issues.\nFocus on detecting issues early while avoiding false alarms.\n\n\n\nCRITICAL: Understanding Kubernetes/OpenShift Conditions\n\nConditions have TWO important fields you MUST check:\n- **type**: The name of the condition (e.g., \"Failing\", \"Available\", \"Progressing\")\n- **status**: The state of the condition (\"True\", \"False\", or \"Unknown\")\n\n**How to Correctly Check Conditions:**\n- A condition is TRUE when: type=\"X\" AND status=\"True\"\n- A condition is FALSE when: type=\"X\" AND status=\"False\"\n- A condition is UNKNOWN when: type=\"X\" AND status=\"Unknown\"\n\n**Examples:**\n\
+ - {type: \"Progressing\", status: \"True\"} means the cluster IS progressing (upgrading)\n- {type: \"Progressing\", status: \"False\"} means the cluster is NOT progressing (stable)\n- {type: \"Failing\", status: \"False\"} means the cluster is NOT failing (healthy)\n- {type: \"Failing\", status: \"True\"} means the cluster IS failing (problem)\n**NEVER assume a condition is true just because the type exists - ALWAYS check the status field!**\n\n\n\n\n1. **Upgrade State Verification** (Check BOTH type AND status):\n - Confirm spec.desiredUpdate.version matches 4.21.17\n - Find condition where type=\"Progressing\" AND status=\"True\" - extract progress details\n - Verify no conditions where type=\"Failing\" AND status=\"True\" are present\n\n2. **Component Progress Tracking** (CRITICAL - Use Provided Operator Counts):\n - You are provided with pre-calculated operator counts: 28 total, 15 updated, 5 updating, 8 pending, 0 failed\n\
+ \ - ALWAYS use the \"X of Y operators\" format consistently:\n * \"**Updated Operators**: 15 of 28 operators at target version 4.21.17\"\n * \"**Updating Operators**: 5 of 28 operators progressing toward target\"\n * \"**Pending Operators**: 8 of 28 operators waiting to start\"\n * \"**Failed Operators**: 0 of 28 operators with issues\"\n - Calculate upgrade completion percentage using the exact formula: (15 / 28) * 100\n - In TL;DR section, use format: \"15 of 28 operators at target version ([X% complete])\"\n - For pending components, combine counts: \"5 updating + 8 pending operators\"\n - NEVER use vague terms like \"several\" or \"most\" - always use exact counts provided\n\n3. **Timeline and ETA Analysis - CRITICAL INSTRUCTIONS**:\n**FINDING THE CORRECT START TIME:**\n - Look in status.history array - it's ordered with MOST RECENT first (index 0)\n - The CURRENT upgrade is the FIRST entry where state=\"Partial\" (in-progress upgrade)\n - Use the startedTime field from that Partial\
+ \ entry ONLY\n - Example: If history[0].state=\"Partial\" and history[0].startedTime=\"2026-05-04T16:59:26Z\", use \"2026-05-04T16:59:26Z\"\n - DO NOT use startedTime from older entries with state=\"Completed\" - those are PREVIOUS upgrades!\n**FORMATTING AND CALCULATIONS:**\n - Format the startedTime as human-readable (e.g., \"May 4, 2026, 4:59:26 PM UTC\")\n - Calculate elapsed time from startedTime to current time\n - Format elapsed time as human-readable duration (e.g., \"Approximately 1 hour and 20 minutes\")\n - Extract progress details from Progressing condition message if available\n - Calculate progress percentage: (15 / 28) * 100\n - Calculate ETA based on current progress rate\n**OUTPUT FORMAT:**\n * \"Upgrade started: [human-readable start time from the Partial entry]\"\n * \"Elapsed time: [Human-readable duration since startedTime]\"\n * \"Current progress: [X% complete]\"\n * \"Estimated completion: [Time remaining]\"\n * \"Progress rate: [On track | Ahead of schedule\
+ \ | Behind schedule]\"\n\n4. **Upgrade Target Analysis**:\n - Current upgrade target from status.desired.version\n - Target release metadata from status.desired (url, channels)\n - Upgrade path validation from current to target version\n - Any upgrade risks or compatibility notes\n\n5. **Cluster History Context During Upgrade**:\n - Previous completed upgrade and duration for comparison\n - Upgrade frequency pattern analysis\n - Any historical upgrade failures or issues\n - Progress comparison with typical upgrade patterns\n\n6. **Early Issue Detection**:\n - Look for warning signs in status.conditions\n - Check for stalled progress indicators in cluster conditions\n - Report specific issues using exact operator counts: \"0 operators with issues\"\n - If no issues: \"No problems requiring immediate attention\"\n - Use format in TL;DR: \"**Issues**: [0 operators with issues if any, otherwise \"No problems requiring immediate attention\"]\"\n - Monitor for unexpected delays compared\
+ \ to historical patterns and report as \"On track\", \"Delayed\", or \"Issues detected\"\n\n\n\n\n## Summary\n**Upgrade Status**\n- **Current Phase**: [Extract from Progressing condition message, e.g., \"Progressing (Working towards 4.21.17: X of Y done (Z% complete))\"]\n- **Elapsed Time**: [Human-readable duration from upgrade start to current time]\n- **Progress Indicators**: [Specific progress details and any operators currently updating]\n**Component Status** (Total: 28 ClusterOperators)\n- **Updated Operators**: 15 of 28 operators at target version 4.21.17\n- **Updating Operators**: 5 of 28 operators progressing toward target\n- **⏸ Pending Operators**: 8 of 28 operators waiting to start\n- **Failed Operators**: 0 of 28 operators with issues\n**Upgrade Target Details**\n- **Target Version**: [4.21.17 from status.desired.version]\n- **Target Release Info**: [Errata URL from status.desired.url if available, format as markdown link]\n\
+ - **Target Channels**: [List available channels from status.desired.channels, comma-separated]\n- **Upgrade Path**: Current version [4.21.14] → Target version [4.21.17]\n**Historical Context**\n- **Previous Upgrade**: [Most recent completed upgrade version and completion timestamp from status.history]\n- **Upgrade Pattern**: [Upgrade frequency analysis and historical success pattern]\n- **Duration Comparison**: [Current upgrade timeline compared to previous upgrade durations and typical patterns]\n**Infrastructure Health During Upgrade**\n- **MachineConfigPool Progress**: [Status of MCPs - are they updating, stuck, or complete?]\n- **Node Resource Pressure**: [From nodes_top - any nodes with high CPU/memory usage?]\n - Example: \"All nodes healthy - CPU usage 45-60%, memory usage 55-70%\"\n - Example: \" Warning: master-0 at 92% memory - monitor for slowdowns\"\n**Recent Progress Events** (Last 30 minutes):\n- **Event Summary**: [Count of events related to upgrade progress]\n- **Warning\
+ \ Signs**: [Any warning events that might slow progress]\n - Example: \"ImagePullBackOff in 3 operators - image download issues may slow upgrade\"\n - Example: \"No concerning events - upgrade progressing normally\"\n- **Positive Indicators**: [Events showing healthy progress]\n - Example: \"12 operators successfully updated to target version\"\n**Health Indicators**\n- **Issues Detected**: [Any warning signs, delays, or specific operator issues requiring attention]\n- **Cluster Status**: [Overall cluster condition health based on ClusterVersion conditions]\n- **Active Alerts**: [Warning/critical alerts during upgrade, if available]\n- **Timeline Analysis**:\n * Upgrade started: [Find the FIRST entry in status.history where state=\"Partial\" - this is the CURRENT upgrade. Use ONLY its startedTime field. Convert from ISO timestamp (e.g., \"2026-05-04T16:59:26Z\") to human-readable (e.g., \"May 4, 2026, 4:59:26 PM UTC\"). DO NOT use startedTime from Completed entries!]\n * Elapsed time:\
+ \ [Calculate duration from the Partial entry's startedTime to current time in human-readable format]\n * Current progress: [X% complete based on operator completion ratio]\n * Estimated completion: [Time remaining calculation based on progress rate]\n * Progress rate: [Assessment: \"On track\", \"Ahead of schedule\", or \"Behind schedule\" compared to typical upgrade window]\n\n## TL;DR\n- **Progress**: [X% complete - (15 Updated Operators / 28 Total Operators) * 100]\n- **Target Version**: [4.21.17 with release info if available]\n- **Target Channels**: [Available channels for target release]\n- **Upgrade Duration**: [Elapsed time from upgrade start]\n- **Status**: [On track | Delayed | Issues detected]\n- **Updated Components**: 15 of 28 operators at target version ([X% complete])\n- **Pending Components**: 5 updating + 8 pending operators\n- **Historical Comparison**: [How current upgrade compares to previous ones]\n- **Issues**: [0 operators with issues if any, otherwise \"No problems\
+ \ requiring immediate attention\"]\n- **Resource Pressure**: [Node CPU/memory status - any nodes >90% usage?]\n- **MCP Status**: [MachineConfigPool progress - all updating normally?]\n- **Recent Events**: [Count of warning events in last 30 min, user-friendly summary]\n- **Alerts**: [Warning/critical alerts during upgrade, if available]\n- **ETA**: [Estimated time remaining based on current progress rate]\n- **Action Required**: [Continue monitoring | Investigate delays | Address operator issues]\n\n"
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+- conversation_group_id: conv_005
+ tag: cluster-updates-critical
+ turns:
+ - expected_response: 'Response should provide troubleshooting analysis based on actual cluster state:
+
+
+ **If cluster is healthy (Failing=False, no degraded operators):**
+
+ ## Summary
+
+ - Root Cause Analysis: Indicate no active upgrade failure detected
+
+ - Current State: Report actual cluster version and health status
+
+ - ClusterOperator Status: All operators Available=True, Degraded=False
+
+ - Upgrade Capability: Confirm cluster is Upgradeable or identify any blockers
+
+ - Available Updates: List available target versions
+
+
+ ## TL;DR
+
+ - No upgrade failure detected
+
+ - Current version and health status
+
+ - All operators healthy
+
+ - Cluster ready for upgrade or specific reasons if not
+
+
+ **If cluster has failures (Failing=True or degraded operators):**
+
+ ## Summary
+
+ - Root Cause Analysis: Specific failure reasons from ClusterVersion Failing condition with actual error messages
+
+ - Component Analysis: Failed ClusterOperators with specific error messages from pod logs
+
+ - Failed Upgrade Context: Target version, release information, upgrade path
+
+ - Remediation Steps: Specific actionable steps based on actual errors
+
+ - Infrastructure Issues: MCPs, Nodes, Events contributing to failure
+
+ - Event Timeline: Sequence of events showing failure progression
+
+
+ ## TL;DR
+
+ - Specific failure type and root cause
+
+ - Failed operators with error messages
+
+ - Concrete remediation steps
+
+ - Recovery time estimate
+
+
+ Response must match actual cluster state - don''t report failures if cluster is healthy.'
+ query: "# OpenShift Cluster Upgrade Troubleshoot Analysis\n\n\n- YOU MUST ALWAYS CALL THE TOOLS TO GET THE INFORMATION. YOU SHOULD NEVER TREAT DATA FROM EXAMPLES AS REAL DATA.\n- YOU MUST ALWAYS REFERENCE REAL DATA FROM TOOL CALLS. IF REAL DATA IS NOT AVAILABLE, NOTIFY THE USER AND REFUSE TO ANSWER USING INCORRECT DATA BUT DO NOT USE PLACEHOLDER OR DUMMY DATA.\n**CRITICAL: Timeout and Error Handling**\n\n**Timeout Awareness (60 second limit):**\n- Prioritize ClusterVersion + ClusterOperators first (essential for failure diagnosis)\n- Fetch events_list early - often explains failures quickly without needing logs\n- Limit pod log fetching - logs are SLOW, only fetch 1-2 critical operators\n- If approaching 50 seconds, STOP and analyze what you have\n- Partial diagnosis is better than timeout\n\n**Error Handling for Tool Failures:**\n1. **Try core resources first** - ClusterVersion and ClusterOperators are essential\n2. **If core resources fail** - Provide specific error and\
+ \ troubleshooting steps\n3. **If optional tools fail** (pods_log, events_list, get_alerts) - Continue with available data\n4. **Provide partial analysis** - Analyze whatever data you successfully retrieved\n5. **Be specific** - \"Unable to fetch operator pod logs from openshift-authentication namespace\" NOT \"cannot retrieve data\"\n6. **Give troubleshooting steps**:\n - Verify MCP server is running: 'oc get pods -n openshift-lightspeed'\n - Check if operator namespaces exist\n - Suggest manual log checking: 'oc logs -n openshift-authentication '\n**Tool Call Priority to Avoid Timeouts:**\n**PHASE 1 - ESSENTIAL (Always fetch - target: 20 seconds):**\n1. resources_get: ClusterVersion (apiVersion: \"config.openshift.io/v1\", kind: \"ClusterVersion\", name: \"version\")\n2. resources_list: ClusterOperator (apiVersion: \"config.openshift.io/v1\", kind: \"ClusterOperator\")\n**PHASE 2 - HIGH-VALUE DIAGNOSTICS (If under 35 seconds):**\n3. events_list: Get events from last 1 hour\
+ \ - Often explains failures quickly\n4. For THE MOST CRITICAL failed operator only (not all):\n * pods_list_in_namespace: Find operator pods\n * pods_log: Get last 50 lines only (not 100) - Logs can be slow!\n**PHASE 3 - OPTIONAL CONTEXT (Only if under 50 seconds):**\n5. resources_list: Node - Check for NotReady nodes\n6. get_alerts: Critical alerts (if available)\n7. Additional operator logs (only if time permits)\n**CRITICAL EFFICIENCY RULES:**\n- LIMIT pod log fetching to 1-2 critical operators max - logs are SLOW\n- Use tail=50 for logs, not tail=100 - faster retrieval\n- If events_list provides the error, SKIP pod logs - events are faster\n- NEVER exceed 55 seconds total execution time\n- Provide analysis with partial data rather than timing out\n\n- Analyze ONLY the actual data from tool calls\n- Report SPECIFIC failure details with actual error messages from logs and events\n- Provide conservative, investigation-focused remediation\n- Focus on root cause identification using\
+ \ real error messages, not aggressive fixes\n- ONLY OUTPUT the Summary and TL;DR sections\n- LANGUAGE REQUIREMENT: Respond in English. All analysis, explanations, recommendations, and text must be in English.\n\n\n\nTroubleshoot upgrade issues for cluster attempting to go from 4.21.14 to 4.21.17. You have complete cluster data including ClusterVersion and all ClusterOperator resources to diagnose upgrade failures.\nThis prompt is used when upgrade failures or component degradation is detected.\n\n\n\n CRITICAL: Understanding Kubernetes/OpenShift Conditions\n\nConditions have TWO important fields you MUST check:\n- **type**: The name of the condition (e.g., \"Failing\", \"Available\", \"Progressing\")\n- **status**: The state of the condition (ONLY these values: \"True\", \"False\", or \"Unknown\")\n**MANDATORY CHECKING PROCESS:**\nFor EVERY condition you analyze, you MUST:\n1. First, locate the condition by its type field\n\
+ 2. Second, read the EXACT value of the status field\n3. Third, interpret based ONLY on the status field value:\n - If status=\"True\" → The condition IS active/present\n - If status=\"False\" → The condition is NOT active/NOT present\n - If status=\"Unknown\" → The condition state is uncertain\n**DO NOT report a problem unless status=\"True\" for negative conditions OR status=\"False\" for positive conditions!**\n**Critical Examples - MEMORIZE THESE:**\n- {type: \"Failing\", status: \"False\"} → Cluster is NOT failing → NO PROBLEM\n- {type: \"Failing\", status: \"True\"} → Cluster IS failing → PROBLEM\n- {type: \"Available\", status: \"True\"} → Cluster IS available → NO PROBLEM\n- {type: \"Available\", status: \"False\"} → Cluster is NOT available → PROBLEM\n- {type: \"Degraded\", status: \"False\"} → Cluster is NOT degraded → NO PROBLEM\n- {type: \"Degraded\", status: \"True\"} → Cluster IS degraded → PROBLEM\n**VERIFICATION REQUIREMENT:**\nBefore making ANY conclusion about a condition,\
+ \ you MUST explicitly state:\n\"Condition type='X' has status='Y'\" and then interpret it correctly.\n**NEVER assume a condition is true just because the type exists - ALWAYS check the status field!**\n**The presence of a condition type does NOT mean it is active - check the status field!**\n\n\n\n\n1. **Upgrade Failure Root Cause**:\n - Find condition where type=\"Failing\" AND status=\"True\"\n - Extract the EXACT reason and message from the Failing condition\n - Check status.history for failed upgrade attempts and their specific errors\n - Identify which component or process is actually failing\n\n2. **ClusterOperator Failure Analysis with Pod Logs** (Check BOTH type AND status):\n - For each ClusterOperator, check conditions:\n * Available: If type=\"Available\" AND status=\"False\" → Operator unavailable (blocker)\n * Degraded: If type=\"Degraded\" AND status=\"True\" → Operator degraded (issue)\n * Progressing: If type=\"\
+ Progressing\" AND status=\"True\" with error messages → Operator stuck\n - Report SPECIFIC operator names and their condition messages for problematic conditions only\n**For each failing/degraded operator, fetch pod logs:**\n - Use pods_list_in_namespace to find operator's pods (usually in openshift-[operator-name] namespace)\n - Use pods_log with tail=50 to get recent logs from failing pods\n - If pod has restarted, also get previous container logs\n - **Extract actual error messages from logs**- don't just say \"check logs\"\n - **Translate technical errors into user-friendly explanations**\n - Example: \"Error: dial tcp 10.0.0.1:6443: i/o timeout\" → \"Operator cannot connect to API server - network connectivity issue\"\n\n3. **Cluster-Level Failure Analysis** (Check BOTH type AND status):\n - Find condition where type=\"Failing\" AND status=\"True\" - extract specific error messages\n - Find condition where type=\"Degraded\" AND status=\"True\" - review degradation reasons\n -\
+ \ Find condition where type=\"Invalid\" AND status=\"True\" - check invalid configuration\n - Look for specific failure reasons in condition messages and status\n - IMPORTANT: Only report as failing if status=\"True\"\n\n4. **Node and Infrastructure Issues**:\n - Check Node resources for NotReady conditions\n - Identify nodes with scheduling issues or resource constraints\n - Look for infrastructure problems affecting the upgrade\n\n5. **MachineConfigPool Issues**:\n - Check for Degraded=True, spec.paused=true, or observedGeneration ≠ metadata.generation\n - These can cause upgrade failures and node configuration problems\n\n6. **Historical Failure Context**:\n - Previous upgrade attempts from status.history\n - Compare current failure with historical upgrade patterns\n - Identify recurring issues or new problems\n - Duration and frequency of past upgrade attempts\n\n7. **Update Target Analysis for Failures**:\n - Failed target version from status.desired.version\n - Release metadata\
+ \ and known issues from status.desired.url\n - Target channel information from status.desired.channels\n - Validate if target version is still available and supported\n\n8. **Cincinnati and Update Service Analysis**:\n - Update service configuration (spec.upstream if custom, otherwise default Red Hat service)\n - Recent update retrieval status from RetrievedUpdates condition\n - Verify availableUpdates is populated (indicates service connectivity)\n - Signature verification status (spec.signatureStores if custom, otherwise default Red Hat stores)\n - Network connectivity issues affecting update process\n\n9. **Failure Events Timeline** (using events_list):\n - Query events from last 1 hour (upgrade failures develop over time)\n - Focus on Error and Warning events in openshift-* namespaces\n - Look for event patterns that explain the failure:\n * CrashLoopBackOff → Operator pod keeps restarting\n * ImagePullBackOff → Cannot download container images\n * OOMKilled → Pod ran out of memory\n\
+ \ * FailedScheduling → Cannot place pods on nodes\n - **Build a timeline**: Show sequence of events leading to failure\n - **User-friendly translation**: Explain technical events in plain language\n - **Example**: \"10 minutes ago: authentication operator pod started crashing (CrashLoopBackOff). 5 minutes ago: authentication unavailable. Now: upgrade blocked\"\n\n10. **Active Critical Alerts** (using get_alerts - if available):\n - Query critical alerts that might explain upgrade failure\n - Focus on infrastructure and operator alerts\n - **Correlation**: Connect alerts to failing operators\n - **Example**: \"KubeAPIDown alert firing - explains why operators can't communicate\"\n - If get_alerts not available: Skip this check\n\n11. **Conservative Remediation Approach**:\n - Focus on investigation and monitoring first\n - Suggest checking logs and status before taking action\n - Avoid aggressive suggestions like \"restart operators\" unless clearly needed\n - Recommend escalation paths\
+ \ for complex issues\n - Consider rollback strategies based on failure severity\n\n\n\n\n## Summary\n**Root Cause Analysis**\nBased on the ClusterVersion data:\n- **Current Version**: 4.21.14\n- **Target Version**: 4.21.17\n- **Failure Type**: [Extract from actual Failing condition reason]\n- **Specific Error**: [Quote the actual failure message from conditions]\n**Component Analysis**\n- **Failed ClusterOperators**: [List specific operators with Available=False, Degraded=True, or failing conditions]\n- **Operator Error Details**: [Actual error messages from pod logs - be specific!]\n - Example: \"authentication operator pod logs show: 'Error: certificate expired at 2026-04-15 12:00:00 UTC'\"\n- **Stuck ClusterOperators**: [List operators stuck in Progressing=True with error messages]\n- **Affected Services**: [Impact on cluster functionality based on failed operators]\n**Failed Upgrade Context**\n- **Target Version**: [From status.desired.version\
+ \ with metadata]\n- **Release Information**: [Target release details and known issues from status.desired.url]\n- **Upgrade Path**: [Source → Target version progression]\n- **Target Availability**: [Verify target version is still in available updates]\n**Historical Failure Analysis**\n- **Previous Attempts**: [Recent upgrade attempts from status.history]\n- **Failure Pattern**: [Recurring vs new failure based on history]\n- **Last Successful Upgrade**: [Most recent completed upgrade for comparison]\n- **Cluster Stability**: [Overall upgrade success rate and patterns]\n**Update Service Health**\n- **Service Configuration**: [spec.upstream if custom, otherwise \"Default Red Hat service\"]\n- **Cincinnati Status**: [RetrievedUpdates condition status and message]\n- **Last Update Check**: [Recent update retrieval timestamp from RetrievedUpdates]\n- **Available Updates**: [Confirm availableUpdates array is populated]\n- **Connectivity Issues**: [Network or authentication problems affecting\
+ \ updates]\n**Failure Events Timeline** (Last hour):\n- **Event Summary**: [Count of error vs warning events]\n- **Timeline of Key Events**: [Chronological sequence showing how failure developed]\n - Example: \"60 min ago: Started upgrade to 4.21.17\"\n - Example: \"45 min ago: authentication operator pod started failing (CrashLoopBackOff)\"\n - Example: \"30 min ago: authentication operator marked Degraded\"\n - Example: \"Now: Upgrade stuck, authentication unavailable\"\n- **Technical Errors Found**: [Specific error types: ImagePullBackOff, OOMKilled, etc.]\n- **User-Friendly Explanation**: [What these events mean in plain language]\n**Active Critical Alerts** (if available):\n- **Alert Count**: [Number of critical/warning alerts]\n- **Key Alerts**: [Names and descriptions of alerts related to failure]\n- **Correlation**: [How alerts connect to failing operators]\n- **Example**: \"KubeAPIDown alert + authentication operator failure → API server connectivity issue\"\n- If alerts not\
+ \ available: \"Alert monitoring unavailable\"\n**Investigation Steps**\n1. [First diagnostic step based on actual failure type]\n2. [Second diagnostic step]\n3. [Log locations to check]\n**Recovery Actions** (Conservative Approach)\n1. [Investigation-focused first step]\n2. [Monitoring and validation steps]\n3. [When to escalate to support]\n\n## TL;DR\n- **Failure Type**: [Specific failure reason from conditions]\n- **Target Version**: [Failed upgrade target with release info]\n- **Root Cause**: [Primary component or process failing - with actual error from logs]\n- **Failed Components**: [Count and names of failed ClusterOperators]\n- **Error Messages**: [Key errors from pod logs - be specific!]\n- **Event Summary**: [Count of error events in last hour, key patterns]\n- **Alert Status**: [Critical alerts related to failure, if available]\n- **Historical Pattern**: [Recurring failure vs new issue]\n- **Last Success**: [Most recent completed upgrade for context]\n- **Update Service**:\
+ \ [Cincinnati health, e.g., \"Default service working (RetrievedUpdates=True)\" or \"Custom upstream failing\"]\n- **Node Issues**: [Count of NotReady nodes if any]\n- **Infrastructure Problems**: [Any detected infrastructure issues]\n- **MCP Issues**: [Count of degraded MachineConfigPools if any]\n- **Next Steps**: [Conservative investigation approach based on actual errors found]\n- **Escalation**: [When to contact Red Hat support]\n- **Recovery Time**: [Realistic estimate based on failure type]\n\n"
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+ - geval:technical_accuracy
+- conversation_group_id: conv_006
+ tag: cluster-updates-critical
+ turns:
+ - expected_response: 'Response should assess cluster health and upgrade readiness by:
+
+ - Checking ClusterOperator health status
+
+ - Identifying any active alerts or issues
+
+ - Evaluating resource utilization and capacity
+
+ - Providing specific recommendations for issues found
+
+
+ The response should analyze actual cluster state.
+
+ May suggest commands to check health if direct access is limited.
+
+ Either approach is acceptable.'
+ query: 'I need to check if my OpenShift cluster is healthy and ready for an upgrade.
+
+ The current version is 4.15.1. Can you analyze the cluster status?
+
+ '
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+- conversation_group_id: conv_007
+ tag: cluster-updates-format
+ turns:
+ - expected_response: 'Response should report on upgrade progress:
+
+ - Current upgrade status (in progress, completed, or not upgrading)
+
+ - ClusterOperator and node progression if upgrading
+
+ - Any issues or warnings during the upgrade
+
+ - Timeline and expected completion
+
+
+ If no upgrade is in progress, should state that clearly.
+
+ May suggest commands to check status if needed.'
+ query: 'My cluster is currently upgrading from 4.15.1 to 4.15.2. Can you check the progress?
+
+ '
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+- conversation_group_id: conv_008
+ tag: cluster-updates-blockers
+ turns:
+ - expected_response: 'The upgrade from 4.14.20 to 4.15.0 may be blocked due to several active alerts in your cluster. Here are the key issues:
+
+
+ 1. **KubeCPUOvercommit**: The cluster has overcommitted CPU resource requests for Pods, which could affect node stability.
+
+
+ 2. **HighOverallControlPlaneMemory**: Memory utilization across control plane nodes is high, potentially impacting responsiveness and stability.
+
+
+ 3. **SystemMemoryExceedsReservation**: A specific node is using more memory than reserved, which could lead to out-of-memory events.
+
+
+ 4. **KubePodNotScheduled**: A pod (`lightspeed-app-server`) cannot be scheduled, indicating resource constraints.
+
+
+ 5. **KubeDeploymentRolloutStuck**: The rollout of the `lightspeed-app-server` deployment is not progressing, which may be related to the scheduling issue.
+
+
+ To proceed with the upgrade, you should address these alerts, particularly the resource overcommitment and scheduling issues. Consider scaling your resources or optimizing your workloads to resolve these problems.'
+ query: 'I''m trying to upgrade from 4.14.20 to 4.15.0 but it seems blocked. Can you help diagnose the issue?
+
+ '
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+ - geval:actionable_guidance
+- conversation_group_id: conv_009
+ tag: cluster-updates-risks
+ turns:
+ - expected_response: 'A conditional update means that the update to version 4.15.5 is available but not recommended due to known risks that may apply to your cluster. The Cluster Version Operator (CVO) evaluates these risks, and if they are deemed applicable, the update is labeled as conditional.
+
+
+ Before proceeding with a conditional update, consider the following:
+
+
+ 1. **Assess the Risks**: Review the known risks associated with the update. If the risks do not apply to your cluster, the update may still be safe to proceed with.
+
+
+ 2. **Critical Needs**: If you have a strong reason to update (e.g., fixing a critical CVE), weigh the benefits against the potential risks.
+
+
+ 3. **Wait for Recommended Updates**: If there is no urgent need to update, it may be best to wait for a recommended update path from Red Hat, which would ensure a safer upgrade.
+
+
+ In summary, proceed with caution and evaluate the specific risks before deciding to update to 4.15.5.'
+ query: 'I see an available update to 4.15.5 but it''s marked as a conditional update. What does that mean and should I proceed?
+
+ '
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+ - geval:technical_accuracy
+- conversation_group_id: conv_010
+ tag: cluster-updates-path
+ turns:
+ - expected_response: Yes, you can upgrade from 4.15.1 to 4.15.6. The upgrade path is valid as both versions are within the same minor version (4.15) and follow the z-stream update policy.
+ query: 'I want to upgrade from 4.15.1 directly to 4.15.6. Can you validate this upgrade path?
+
+ '
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+- conversation_group_id: conv_011
+ tag: cluster-updates-troubleshoot
+ turns:
+ - expected_response: "The cluster upgrade from 4.15.1 to 4.15.2 failed due to several active alerts that may have impacted the upgrade process. Here are the key alerts:\n\n1. **KubeCPUOvercommit**: The cluster has overcommitted CPU resource requests for Pods, which could lead to resource contention.\n - **Severity**: Warning\n\n2. **HighOverallControlPlaneMemory**: Memory utilization across control plane nodes is high, potentially impacting responsiveness and stability.\n - **Severity**: Warning\n\n3. **SystemMemoryExceedsReservation**: A specific node is using more memory than reserved, which can affect node functionality.\n - **Severity**: Warning\n\n4. **KubePodNotScheduled**: A pod in the `openshift-lightspeed` namespace cannot be scheduled, indicating potential resource issues.\n - **Severity**: Warning\n\n5. **KubeDeploymentRolloutStuck**: The rollout of the `lightspeed-app-server` deployment is not progressing, which may indicate issues with the deployment process.\n -\
+ \ **Severity**: Warning\n\nThese alerts suggest that resource constraints and deployment issues may have contributed to the upgrade failure. You should address these alerts by checking resource allocations, ensuring sufficient memory and CPU availability, and resolving any deployment issues before attempting the upgrade again."
+ query: 'My cluster upgrade from 4.15.1 to 4.15.2 failed. Can you help diagnose what went wrong?
+
+ '
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+- conversation_group_id: conv_012
+ tag: cluster-updates-conversation
+ turns:
+ - expected_response: 'Response should indicate available updates:
+
+ - List available update versions (4.21.15 through 4.21.22), OR
+
+ - Suggest command to check updates (oc adm upgrade)
+
+
+ Both approaches are acceptable.'
+ query: 'What updates are available for my cluster running OpenShift 4.21.14?
+
+ '
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+ - expected_response: 'Response should explain version differences:
+
+ - Describe what typically differs between patch versions
+
+ - Explain why one version might be conditional
+
+ - May reference release notes or documentation
+
+
+ General explanation is acceptable if specific details unavailable.'
+ query: 'What''s the difference between 4.21.17 and 4.21.18? Why might some updates be conditional?
+
+ '
+ turn_id: turn_002
+ turn_metrics:
+ - custom:answer_correctness
+- conversation_group_id: conv_013
+ tag: cluster-updates-no-updates
+ turns:
+ - expected_response: Yes, it's normal if you don't see any available updates for your cluster on version 4.15.8. If the update status is not "Updates available," you cannot update your cluster. You should continue to update your cluster to the latest patch release for your current version until the next minor version is available in the path.
+ query: 'My cluster is on 4.15.8 and I don''t see any available updates. Is this normal?
+
+ '
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+- conversation_group_id: conv_014
+ tag: cluster-updates-channels
+ turns:
+ - expected_response: "The differences between stable, fast, and candidate channels for updates in Red Hat OpenShift Container Platform are as follows:\n\n1. **Stable Channel**:\n - Releases are added after a delay to allow for data collection and analysis for regressions.\n - Suitable for production environments where stability is prioritized.\n - Updates are generally promoted within a week or two after their initial release.\n\n2. **Fast Channel**:\n - Releases are available as soon as they are declared as general availability (GA) by Red Hat.\n - Fully supported and intended for production use.\n - Ideal for users who want to apply updates without waiting for the stable channel promotion.\n\n3. **Candidate Channel**:\n - Offers early access to releases as soon as they are built, but these releases may not have the full feature set or quality assurance of GA releases.\n - Not suitable for production environments; primarily for testing purposes.\n - Users should be prepared\
+ \ to destroy and recreate clusters if necessary.\n\nIn summary, the stable channel focuses on reliability, the fast channel prioritizes immediate access to updates, and the candidate channel is for testing new features."
+ query: 'What''s the difference between stable, fast, and candidate channels for updates?
+
+ '
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+- conversation_group_id: conv_015
+ tag: cluster-updates-mcp
+ turns:
+ - expected_response: Yes, this is normal. After upgrading the control plane, you need to unpause the relevant MachineConfigPool (MCP) groups to start the upgrade process for the worker nodes. Once unpaused, the worker nodes will begin updating, rebooting one at a time to upgrade to the new version. Ensure that the MCP groups are correctly configured and unpaused to initiate the worker node upgrades.
+ query: 'The control plane upgraded successfully but my worker nodes haven''t started updating yet. Is this normal?
+
+ '
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+- conversation_group_id: conv_016
+ tag: cluster-updates-pdb
+ turns:
+ - expected_response: 'The prolonged draining of your node could be due to several common issues:
+
+
+ 1. **PodDisruptionBudget (PDB):** The pod may have a PDB that prevents it from being deleted during the drain process.
+
+
+ 2. **Attached Storage:** If the pod has storage attached, the kubelet might be unable to unmount it.
+
+
+ 3. **Webhooks:** There could be a webhook configured for UPDATE operations that is not being called by the kube-apiserver.
+
+
+ 4. **Finalizers:** The pod may have finalizers set that are preventing it from terminating.
+
+
+ To diagnose the issue, check the logs of the Machine Config Controller (MCC) for specific error messages regarding which pods are failing to drain. This will help you identify the root cause.'
+ query: 'My node upgrade seems stuck - it''s been draining for over an hour. What could be wrong?
+
+ '
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+- conversation_group_id: conv_017
+ tag: cluster-updates-eus
+ turns:
+ - expected_response: No, you cannot upgrade directly from OpenShift 4.14.12 to 4.16.3. Upgrades must occur between minor versions, and you need to upgrade to the next minor version first (4.15) before proceeding to 4.16.
+ query: 'Can I upgrade directly from OpenShift 4.14.12 to 4.16.3?
+
+ '
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+- conversation_group_id: conv_018
+ tag: cluster-updates-conditions
+ turns:
+ - expected_response: 'Your cluster appears to be healthy based on the conditions you''ve provided:
+
+
+ - **Available: True** indicates that the cluster is operational and can serve workloads.
+
+ - **Failing: False** means the cluster is not in a failing state, which is good.
+
+ - **Progressing: False** suggests that there are no ongoing updates or changes that could affect stability.
+
+ - **Upgradeable: True** indicates that the cluster is safe to update.
+
+
+ Overall, your cluster is healthy and ready for an upgrade.'
+ query: "I see my ClusterVersion has these conditions:\n\n- type: Available\n status: \"True\"\n- type: Failing\n status: \"False\"\n- type: Progressing\n status: \"False\"\n- type: Upgradeable\n status: \"True\"\n\nIs my cluster healthy and ready for upgrade?\n"
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+ - geval:condition_status_accuracy
+- conversation_group_id: conv_019
+ tag: cluster-updates-conditions
+ turns:
+ - expected_response: 'You can proceed with the cluster upgrade. The authentication operator is in a good state with the following conditions:
+
+
+ - Available: True
+
+ - Progressing: False
+
+ - Degraded: False
+
+ - Upgradeable: True
+
+
+ Since the Upgradeable status is True, it indicates that the operator is safe to update, and there are no blocking issues.'
+ query: "I'm checking my cluster operators before upgrading. The authentication operator shows:\n\nConditions:\n- type: Available\n status: \"True\"\n- type: Progressing\n status: \"False\"\n- type: Degraded\n status: \"False\"\n- type: Upgradeable\n status: \"True\"\n\nCan I proceed with the cluster upgrade, or is this operator blocking it?\n"
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+ - geval:condition_status_accuracy
+- conversation_group_id: conv_020
+ tag: cluster-updates-upgrade-exploration-multi-turn
+ turns:
+ - expected_response: To determine if there are updates available for your cluster, I need to check the current update status. Please hold on for a moment while I retrieve that information.It seems I'm unable to access the current configuration of your cluster to check for updates. Please verify your access permissions or provide me with the necessary details about your cluster's current version and update status.
+ query: Are there any updates available for my cluster?
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+ - expected_response: I currently don't have access to the specific details of your cluster's configuration or the latest available updates. To provide accurate information, please share the current version of your OpenShift cluster or any specific update channel you are interested in.
+ query: Tell me more about the latest available update.
+ turn_id: turn_002
+ turn_metrics:
+ - custom:answer_correctness
+ - expected_response: "To start the upgrade to a specific version of your OpenShift cluster, you can use the following command:\n\n```terminal\n$ oc adm upgrade --to=\n```\n\nReplace `` with the desired version number you want to upgrade to. \n\nIf you want to upgrade to the latest version, you can use:\n\n```terminal\n$ oc adm upgrade --to-latest=true\n```\n\nMake sure to check the compatibility of the version you are upgrading to and ensure that your cluster is subscribed to a stable, eus, or fast channel for production clusters."
+ query: How do I start the upgrade to that version?
+ turn_id: turn_003
+ turn_metrics:
+ - custom:answer_correctness
+ - geval:actionable_guidance
+- conversation_group_id: conv_021
+ tag: cluster-updates-operator-health-multi-turn
+ turns:
+ - expected_response: 'Response should indicate cluster operator health status. Can either:
+
+ - Report that all operators are healthy (if access is available), OR
+
+ - Indicate inability to access operator/alert information and suggest running: oc get clusteroperator
+
+
+ Both approaches are acceptable depending on RBAC configuration.'
+ query: Are all my cluster operators healthy?
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+ - expected_response: 'The etcd operator in OpenShift is responsible for managing etcd clusters, which are key-value stores used for storing configuration data and state information for Kubernetes. Here are some key points about the etcd operator:
+
+
+ 1. **Deployment**: The etcd operator automates the deployment and management of etcd clusters, ensuring high availability and data consistency.
+
+
+ 2. **Scaling**: It allows for easy scaling of etcd clusters by adding or removing members as needed.
+
+
+ 3. **Backup and Restore**: The operator provides mechanisms for backing up and restoring etcd data, which is crucial for disaster recovery.
+
+
+ 4. **Health Monitoring**: It continuously monitors the health of etcd members and can take corrective actions if a member becomes unhealthy.
+
+
+ 5. **Configuration Management**: The etcd operator manages the configuration of etcd clusters, including settings for replication, data retention, and security.
+
+
+ 6. **Integration**: It integrates seamlessly with OpenShift and Kubernetes, leveraging their APIs for management and orchestration.
+
+
+ Overall, the etcd operator simplifies the management of etcd clusters, making it easier to maintain the reliability and performance of your OpenShift environment.'
+ query: Tell me about the etcd operator.
+ turn_id: turn_002
+ turn_metrics:
+ - custom:answer_correctness
+ - expected_response: 'The kube-apiserver is a critical component of the Kubernetes architecture, serving as the front-end for the Kubernetes control plane. Here are some key aspects of the kube-apiserver:
+
+
+ 1. **API Endpoint**: It exposes the Kubernetes API, allowing users and components to interact with the cluster. All requests to the Kubernetes cluster go through the kube-apiserver.
+
+
+ 2. **RESTful Interface**: The kube-apiserver provides a RESTful interface for managing Kubernetes resources, such as pods, services, deployments, and more.
+
+
+ 3. **Authentication and Authorization**: It handles authentication and authorization for API requests, ensuring that only authorized users and components can access or modify resources.
+
+
+ 4. **Data Storage**: The kube-apiserver communicates with etcd, the key-value store, to persist the state of the cluster. It reads and writes resource data to etcd.
+
+
+ 5. **Watch Mechanism**: It supports a watch mechanism that allows clients to subscribe to changes in resources, enabling real-time updates.
+
+
+ 6. **Extensibility**: The kube-apiserver can be extended with custom resources and controllers, allowing users to define their own resource types and behaviors.
+
+
+ 7. **High Availability**: In production environments, multiple instances of the kube-apiserver can be deployed for high availability and load balancing.
+
+
+ Overall, the kube-apiserver is essential for the operation of a Kubernetes cluster, acting as the central point for all interactions with the cluster''s resources.'
+ query: What about kube-apiserver?
+ turn_id: turn_003
+ turn_metrics:
+ - custom:answer_correctness
+- conversation_group_id: conv_022
+ tag: cluster-updates-readiness-assessment-multi-turn
+ turns:
+ - expected_response: 'To determine if your cluster is ready for an upgrade, you should consider the following steps:
+
+
+ 1. **Check for Compatibility**: Ensure that all layered products and Operators are compatible with the new version of Red Hat OpenShift Container Platform you plan to upgrade to.
+
+
+ 2. **Evaluate Update Risks**: Review the update availability and any risks associated with the new version. You can complete extensive testing in a non-production environment and investigate any bugs mentioned in the update description.
+
+
+ 3. **Backup etcd**: Ensure you have a recent etcd backup. This is crucial for restoring the cluster state in case the upgrade introduces issues.
+
+
+ 4. **Firmware Updates**: If you''re using bare-metal hardware, verify that the host firmware is compatible with the new version and update it if necessary, but not in conjunction with the OpenShift upgrade.
+
+
+ 5. **MachineConfigPools (MCP)**: Ensure your MCPs are configured correctly to manage the upgrade process effectively.
+
+
+ If you need specific checks or details about your cluster''s current state, please provide more information about your cluster configuration or any specific concerns you have.'
+ query: Is my cluster ready for an upgrade?
+ turn_id: turn_001
+ turn_metrics:
+ - custom:answer_correctness
+ - geval:technical_accuracy
+ - expected_response: 'To determine the appropriate version to upgrade to, consider the following:
+
+
+ 1. **Latest Stable Release**: Check the latest stable release of Red Hat OpenShift Container Platform. Upgrading to the latest version ensures you have the most recent features, security updates, and bug fixes.
+
+
+ 2. **Support Lifecycle**: Review the support lifecycle for your current version. Upgrading to a version that is still within its support window is crucial for receiving updates and support.
+
+
+ 3. **Compatibility**: Ensure that your existing applications, operators, and any third-party integrations are compatible with the new version.
+
+
+ 4. **Release Notes**: Review the release notes for the versions you are considering. They provide important information about new features, deprecated features, and known issues.
+
+
+ 5. **Testing**: If possible, test the upgrade in a staging environment to identify any potential issues before upgrading your production cluster.
+
+
+ If you have a specific version in mind or need assistance with the latest version details, please let me know!'
+ query: What version should I upgrade to?
+ turn_id: turn_002
+ turn_metrics:
+ - custom:answer_correctness
+ - expected_response: "The upgrade process for Red Hat OpenShift Container Platform involves several key steps. Here’s a general walkthrough:\n\n### 1. **Preparation**\n - **Backup**: Ensure you have a backup of your etcd data.\n - **Review Release Notes**: Check the release notes for the version you are upgrading to for any breaking changes or important notes.\n - **Check Compatibility**: Verify that all your Operators and applications are compatible with the new version.\n\n### 2. **Update the Cluster Version**\n - **Access the CLI**: Use the OpenShift CLI (`oc`) to interact with your cluster.\n - **Check Current Version**: Run the following command to check your current version:\n ```bash\n oc get clusterversion\n ```\n\n### 3. **Change the Update Channel (if necessary)**\n - If you need to switch to a different update channel, use:\n ```bash\n oc adm upgrade channel \n ```\n\n### 4. **Initiate the Upgrade**\n - **Upgrade Command**: To initiate\
+ \ the upgrade, run:\n ```bash\n oc adm upgrade\n ```\n - This command will check for available updates and apply them.\n\n### 5. **Monitor the Upgrade**\n - **Check Upgrade Status**: Monitor the upgrade process by checking the status:\n ```bash\n oc get clusterversion\n ```\n - Ensure that all components are upgrading successfully.\n\n### 6. **Post-Upgrade Tasks**\n - **Verify Cluster Health**: After the upgrade, check the health of your cluster:\n ```bash\n oc get nodes\n oc get pods --all-namespaces\n ```\n - **Update Operators**: Ensure that all Operators are updated to their latest compatible versions.\n\n### 7. **Rollback (if necessary)**\n - If the upgrade fails or issues arise, you can perform a manual rollback. Refer to the documentation for specific rollback commands.\n\n### 8. **Documentation and Cleanup**\n - Document the upgrade process and any issues encountered.\n - Clean up any resources or configurations that are\
+ \ no longer needed.\n\n### Important Notes:\n- **Minor Upgrades**: For minor upgrades, you may need to manually modify the `spec.version` in your `ClusterVersion` resource.\n- **Major Upgrades**: For major upgrades, additional manual changes may be required due to potential breaking changes.\n\nIf you have specific details about your current version or any particular concerns, feel free to share!"
+ query: Walk me through the upgrade process.
+ turn_id: turn_003
+ turn_metrics:
+ - custom:answer_correctness
+ - geval:actionable_guidance
diff --git a/eval/rbac-cluster-updates.yaml b/eval/rbac-cluster-updates.yaml
new file mode 100644
index 000000000..8238290d9
--- /dev/null
+++ b/eval/rbac-cluster-updates.yaml
@@ -0,0 +1,54 @@
+# RBAC configuration for cluster-updates evaluation tests
+#
+# This manifest creates a ServiceAccount with the necessary permissions for OLS
+# to access cluster resources through MCP (Model Context Protocol) servers during
+# cluster-updates evaluation tests.
+#
+# Apply this manifest before running cluster-updates evaluation:
+# oc apply -f eval/rbac-cluster-updates.yaml
+#
+# Generate an API token for the service account:
+# export API_KEY=$(oc create token cluster-update-user -n openshift-lightspeed --duration=24h)
+#
+# Then run the evaluation with this token in the API_KEY environment variable.
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: cluster-update-user
+ namespace: openshift-lightspeed
+ labels:
+ app: ols
+ purpose: evaluation
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: cluster-update-user-cluster-reader
+ labels:
+ app: ols
+ purpose: evaluation
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: cluster-reader
+subjects:
+- kind: ServiceAccount
+ name: cluster-update-user
+ namespace: openshift-lightspeed
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: cluster-update-user-monitoring
+ labels:
+ app: ols
+ purpose: evaluation
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: monitoring-edit
+subjects:
+- kind: ServiceAccount
+ name: cluster-update-user
+ namespace: openshift-lightspeed
diff --git a/eval/system_cluster_updates.yaml b/eval/system_cluster_updates.yaml
new file mode 100644
index 000000000..6ed27b950
--- /dev/null
+++ b/eval/system_cluster_updates.yaml
@@ -0,0 +1,306 @@
+# LightSpeed Evaluation Framework Configuration - CLUSTER-UPDATES OPTIMIZED
+# This configuration is specifically tuned for cluster-updates evaluation tests (conv_798-814)
+
+# Judge LLM Configuration
+# Uses OpenAI GPT-5-mini for evaluation scoring (separate from the OLS backend model being tested)
+llm:
+ provider: "openai" # Judge LLM Provider (openai, gemini etc..)
+ model: "gpt-5-mini" # Judge model for scoring responses
+ temperature: 1 # Generation temperature (gpt-5 requires temperature=1)
+ max_tokens: 2048 # Maximum tokens in response (increased for detailed cluster analysis)
+ timeout: 300 # Request timeout in seconds
+ num_retries: 3 # Retry attempts
+
+# API Configuration
+# Only query endpoint is supported
+api:
+ enabled: true # Enable API calls instead of using pre-filled data
+ api_base: http://localhost:8080 # Base API URL
+ endpoint_type: query # Use "streaming" or "query" endpoint
+ timeout: 300 # API request timeout in seconds
+
+ # API input configuration
+ provider: "openai" # LLM provider for queries
+ model: "gpt-4.1-mini" # Model to use for queries (must be valid for OLS API)
+ no_tools: null # Whether to bypass tools and MCP servers (optional)
+ system_prompt: null # System prompt (default None)
+
+ # Authentication
+ # Requires API_KEY environment variable with Kubernetes service account token
+ # The cluster-update-user service account is used for cluster-updates evaluation tests
+ # It has cluster-reader permissions to access cluster resources via MCP server
+ #
+ # To create the service account and RBAC (if not already created):
+ # Run: eval/run_cluster_updates_tests.sh
+ # Or manually apply the RBAC manifest
+ #
+ # To generate a token:
+ # export API_KEY=$(oc create token cluster-update-user -n openshift-lightspeed --duration=24h)
+ #
+ # Note: MCP server RBAC validation is enabled (validation_enabled=true in openshift-mcp-server-config)
+ # This ensures proper permission checking before executing cluster operations
+
+# Metrics metadata - Optimized for cluster-updates evaluation
+metrics_metadata:
+ # Turn-level metrics metadata
+ turn_level:
+ # Standard custom metric (always enabled)
+ "custom:answer_correctness":
+ threshold: 0.7
+ description: "Correctness vs expected answer using custom LLM evaluation"
+ default: true # This metric is applied by default when no turn_metrics specified
+
+ # Cluster-updates specific metrics (using GEval for custom criteria)
+ "geval:condition_status_accuracy":
+ threshold: 0.99
+ description: "Validates correct Kubernetes condition status interpretation"
+ default: false # Enable explicitly in test metrics
+ criteria: |
+ Evaluate whether the response correctly interprets Kubernetes condition status fields.
+
+ CRITICAL REQUIREMENT: The response MUST understand that a condition's status field determines its meaning:
+ - Failing status=False → Cluster is NOT failing (healthy)
+ - Failing status=True → Cluster IS failing (problem)
+ - Available status=True → Component IS available (healthy)
+ - Available status=False → Component NOT available (problem)
+ - Progressing status=True → Update/reconciliation in progress
+ - Progressing status=False → Stable state
+
+ Score 1.0 if ALL condition interpretations are correct.
+ Score 0.0 if ANY condition is misinterpreted (e.g., treating "Failing=False" as a problem).
+ evaluation_params: [response, expected_response]
+
+ "geval:output_format_compliance":
+ threshold: 0.80
+ description: "Validates cluster-updates response format (Summary + TL;DR)"
+ default: false # Enable explicitly in test metrics
+ criteria: |
+ Evaluate whether the response follows the required output format for cluster updates analysis:
+
+ REQUIRED SECTIONS:
+ 1. ## Summary - Detailed analysis section
+ 2. ## TL;DR - Quick facts section
+
+ REQUIRED FORMAT PATTERNS:
+ - Operator counts: Must use format "X of Y operators" (exact counts, not vague language)
+ - Version numbers: Must include actual version strings
+ - Status fields: Must quote actual Kubernetes status field values
+
+ Score 1.0 if both sections present with required format patterns.
+ Score 0.7 if sections present but format patterns not exact.
+ Score 0.3 if only one section present.
+ Score 0.0 if neither section present.
+ evaluation_params: [response]
+
+ "geval:technical_accuracy":
+ threshold: 0.80
+ description: "Validates OpenShift/Kubernetes domain technical correctness"
+ default: false # Enable explicitly in test metrics
+ criteria: |
+ Evaluate the technical accuracy of OpenShift/Kubernetes concepts:
+
+ CHECK FOR:
+ - Correct ClusterVersion/ClusterOperator interpretation
+ - Accurate upgrade blocker identification
+ - Proper admin-ack gate handling
+ - Correct conditional update risk assessment
+ - Accurate node and MachineConfigPool status interpretation
+
+ Score 1.0 if all technical concepts are correct.
+ Score 0.7 if minor inaccuracies that don't affect main conclusion.
+ Score 0.3 if significant technical errors.
+ Score 0.0 if fundamentally wrong about core concepts.
+ evaluation_params: [query, response, expected_response]
+
+ "geval:actionable_guidance":
+ threshold: 0.7
+ description: "Validates response provides specific, actionable guidance"
+ default: false # Enable explicitly in test metrics
+ criteria: |
+ Evaluate whether the response provides actionable guidance:
+
+ GOOD GUIDANCE:
+ - Specific remediation steps (not vague suggestions)
+ - Exact commands when needed (e.g., "oc patch clusterversion...")
+ - Conservative recommendations (investigate before aggressive actions)
+ - Links actual errors to root causes
+
+ BAD GUIDANCE:
+ - Generic advice ("check logs", "contact support")
+ - Vague recommendations ("some operators may need attention")
+ - Aggressive actions without investigation
+ - No clear next steps
+
+ Score 1.0 if guidance is specific and actionable.
+ Score 0.6 if somewhat helpful but could be more specific.
+ Score 0.2 if mostly generic advice.
+ Score 0.0 if no actionable guidance provided.
+ evaluation_params: [query, response]
+
+ # Other metrics available but not used by default
+ "ragas:response_relevancy":
+ threshold: 0.7
+ description: "How relevant the response is to the question"
+ default: false
+
+ "ragas:faithfulness":
+ threshold: 0.7
+ description: "How faithful the response is to the provided context"
+ default: false
+
+ "ragas:context_recall":
+ threshold: 0.7
+ description: "Did we fetch every fact the answer needs?"
+ default: false
+
+ "ragas:context_precision_with_reference":
+ threshold: 0.7
+ description: "How precise the retrieved context is (with reference)"
+ default: false
+
+ "ragas:context_precision_without_reference":
+ threshold: 0.7
+ description: "How precise the retrieved context is (without reference)"
+ default: false
+
+ "ragas:context_relevance":
+ threshold: 0.7
+ description: "Is what we retrieved actually relevant to user query?"
+ default: false
+
+ "custom:tool_eval":
+ description: "Tool call evaluation comparing expected vs actual tool calls"
+ default: false
+
+ # Multi-turn conversation metrics
+ "custom:context_retention":
+ threshold: 0.85
+ description: "Validates model remembers information from previous turns"
+ default: false
+ criteria: |
+ Evaluate whether the model retains and uses information from previous turns:
+
+ GOOD RETENTION:
+ - References specific findings from earlier turns (e.g., "Based on the 6 degraded operators identified earlier...")
+ - Doesn't re-query for information already obtained
+ - Builds on prior context with new user input
+ - Connects current analysis to previous diagnoses
+
+ POOR RETENTION:
+ - Asks user to repeat information already provided
+ - Re-queries cluster state unnecessarily
+ - Treats each turn as independent (no continuity)
+ - Ignores prior findings in later recommendations
+
+ Score 1.0 if all prior context is remembered and used appropriately.
+ Score 0.7 if most context retained but some gaps.
+ Score 0.4 if minimal context retention.
+ Score 0.0 if no evidence of remembering previous turns.
+ evaluation_params: [query, response, conversation_history]
+
+ "custom:progressive_refinement":
+ threshold: 0.80
+ description: "Validates guidance becomes more specific across turns"
+ default: false
+ criteria: |
+ Evaluate whether guidance becomes progressively more targeted:
+
+ PROGRESSIVE REFINEMENT:
+ - Turn 1: Broad diagnosis with multiple possibilities
+ - Turn 2: Narrows to specific area based on user input
+ - Turn 3: Provides exact remediation steps for confirmed issue
+
+ GOOD PROGRESSION:
+ - Each turn adds specificity without losing prior context
+ - Questions guide toward root cause
+ - Final turn provides actionable commands/steps
+ - Recommendations become increasingly concrete
+
+ POOR PROGRESSION:
+ - Repeats same level of detail across turns
+ - Doesn't narrow diagnosis despite new information
+ - Final turn still vague or exploratory
+
+ Score 1.0 if guidance clearly refines from general → specific → actionable.
+ Score 0.7 if some refinement but not consistent.
+ Score 0.4 if minimal progression in specificity.
+ Score 0.0 if no refinement across turns.
+ evaluation_params: [response, conversation_history]
+
+ # Conversation-level metrics metadata
+ conversation_level:
+ "deepeval:conversation_completeness":
+ threshold: 0.7
+ description: "How completely the conversation addresses user intentions"
+ default: false
+
+ "deepeval:conversation_relevancy":
+ threshold: 0.7
+ description: "How relevant the conversation is to the topic/context"
+ default: false
+
+ "deepeval:knowledge_retention":
+ threshold: 0.7
+ description: "How well the model retains information from previous turns"
+ default: false
+
+# Output Configuration
+output:
+ output_dir: "./results" # Output to results/ for cluster-updates tests
+ base_filename: "evaluation"
+ enabled_outputs: # Enable specific output types
+ - csv # Detailed results CSV
+ - json # Summary JSON with statistics
+ # - txt # Human-readable summary
+
+ # CSV columns to include
+ csv_columns:
+ - "conversation_group_id"
+ - "turn_id"
+ - "metric_identifier"
+ - "score"
+ - "threshold"
+ - "result"
+ - "reason"
+ - "query"
+ - "response"
+ - "execution_time"
+
+# Visualization settings
+visualization:
+ figsize: [12, 8] # Graph size (width, height)
+ dpi: 300 # Image resolution
+
+ # Graph types to generate
+ enabled_graphs:
+ - "score_distribution" # Score distribution box plot
+ - "status_breakdown" # Pie chart for pass/fail/error breakdown
+
+# Environment Variables - Automatically get set before any imports
+environment:
+ DEEPEVAL_TELEMETRY_OPT_OUT: "YES" # Disable DeepEval telemetry
+ DEEPEVAL_DISABLE_PROGRESS_BAR: "YES" # Disable DeepEval progress bars
+
+ LITELLM_LOG: ERROR # Suppress LiteLLM verbose logging
+
+# Logging Configuration
+logging:
+ # Source code logging level
+ source_level: INFO # DEBUG, INFO, WARNING, ERROR, CRITICAL
+
+ # Package logging level (imported libraries)
+ package_level: ERROR
+
+ # Log format and display options
+ log_format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+ show_timestamps: true
+
+ # Specific package log levels (override package_level for specific libraries)
+ package_overrides:
+ httpx: ERROR
+ urllib3: ERROR
+ requests: ERROR
+ matplotlib: ERROR
+ LiteLLM: WARNING
+ DeepEval: WARNING
+ ragas: WARNING
diff --git a/ols/src/cache/postgres_cache.py b/ols/src/cache/postgres_cache.py
index a936ba906..d83f48bdd 100644
--- a/ols/src/cache/postgres_cache.py
+++ b/ols/src/cache/postgres_cache.py
@@ -218,6 +218,17 @@ def insert_or_append(
# execute. Disable autocommit for a real multi-statement transaction.
# The lock serialises concurrent callers that share this connection.
with self._tx_lock:
+ # This transaction status check is required for evaluation tests that use
+ # multi-turn conversations. Without it, rapid cache writes (appending each
+ # turn's messages) can leave an active transaction when autocommit=True is
+ # set in the finally block, causing psycopg2 to raise "set_session cannot
+ # be used inside a transaction". Rollback any active transaction to ensure
+ # clean autocommit mode transition.
+ if (
+ self.connection.get_transaction_status()
+ != psycopg2.extensions.TRANSACTION_STATUS_IDLE
+ ):
+ self.connection.rollback()
self.connection.autocommit = False
with self.connection.cursor() as cursor:
try:
@@ -252,6 +263,13 @@ def insert_or_append(
logger.error("PostgresCache.insert_or_append: %s", e)
raise CacheError("PostgresCache.insert_or_append", e) from e
finally:
+ # Ensure transaction is closed before setting autocommit
+ # to avoid "set_session cannot be used inside a transaction" error
+ if (
+ self.connection.get_transaction_status()
+ != psycopg2.extensions.TRANSACTION_STATUS_IDLE
+ ):
+ self.connection.rollback()
self.connection.autocommit = True
@connection
@@ -270,6 +288,17 @@ def delete(
"""
with self._tx_lock:
+ # This transaction status check is required for evaluation tests that use
+ # multi-turn conversations. Without it, rapid cache writes (appending each
+ # turn's messages) can leave an active transaction when autocommit=True is
+ # set in the finally block, causing psycopg2 to raise "set_session cannot
+ # be used inside a transaction". Rollback any active transaction to ensure
+ # clean autocommit mode transition.
+ if (
+ self.connection.get_transaction_status()
+ != psycopg2.extensions.TRANSACTION_STATUS_IDLE
+ ):
+ self.connection.rollback()
self.connection.autocommit = False
with self.connection.cursor() as cursor:
try:
@@ -285,6 +314,13 @@ def delete(
logger.error("PostgresCache.delete: %s", e)
raise CacheError("PostgresCache.delete", e) from e
finally:
+ # Ensure transaction is closed before setting autocommit
+ # to avoid "set_session cannot be used inside a transaction" error
+ if (
+ self.connection.get_transaction_status()
+ != psycopg2.extensions.TRANSACTION_STATUS_IDLE
+ ):
+ self.connection.rollback()
self.connection.autocommit = True
@connection
diff --git a/pyproject.toml b/pyproject.toml
index ebbf85a19..2a9b0ad1f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -124,7 +124,7 @@ dependencies = [
"kubernetes<34.1.0",
"psycopg2-binary>=2.9.9",
"azure-identity>=1.18.0",
- "langchain-community>=0.3.81",
+ "langchain-community>=0.3,<0.4",
"sqlalchemy>=2.0.35",
"ibm-watsonx-ai>=1.3.6",
"certifi>=2024.8.30",
@@ -160,7 +160,7 @@ evaluation = [
"pandas>=2.1.4",
"matplotlib>=3.10.0",
"rouge-score>=0.1.2",
- "ragas>=0.2.14",
+ "ragas>=0.3.0",
"datasets>=3.0.0",
"fastparquet>=2024.11.0",
"tqdm>=4.67.1",
diff --git a/tests/config/operator_install/olsconfig.crd.openai_cluster_updates.yaml b/tests/config/operator_install/olsconfig.crd.openai_cluster_updates.yaml
new file mode 100644
index 000000000..d644fb702
--- /dev/null
+++ b/tests/config/operator_install/olsconfig.crd.openai_cluster_updates.yaml
@@ -0,0 +1,47 @@
+apiVersion: ols.openshift.io/v1alpha1
+kind: OLSConfig
+metadata:
+ name: cluster
+ labels:
+ app.kubernetes.io/created-by: lightspeed-operator
+ app.kubernetes.io/instance: olsconfig-sample
+ app.kubernetes.io/managed-by: kustomize
+ app.kubernetes.io/name: olsconfig
+ app.kubernetes.io/part-of: lightspeed-operator
+spec:
+ llm:
+ providers:
+ - credentialsSecretRef:
+ name: llmcreds
+ models:
+ - name: gpt-4o-mini
+ name: openai
+ type: openai
+ ols:
+ defaultModel: gpt-4o-mini
+ defaultProvider: openai
+ deployment:
+ replicas: 1
+ disableAuth: false
+ logLevel: INFO
+ introspectionEnabled: false
+ userDataCollection:
+ feedbackDisabled: true
+ transcriptsDisabled: true
+ toolsApprovalConfig:
+ approvalType: tool_annotations
+ approvalTimeout: 5
+ toolFilteringConfig:
+ alpha: 0.8
+ topK: 10
+ threshold: 0.01
+ mcpServers:
+ - name: openshift-cluster
+ url: https://kubernetes.default.svc
+ headers:
+ - name: Authorization
+ valueFrom:
+ type: kubernetes
+ featureGates:
+ - MCPServer
+ - ToolFiltering
diff --git a/tests/e2e/evaluation/test_cluster_updates.py b/tests/e2e/evaluation/test_cluster_updates.py
new file mode 100644
index 000000000..8bca2e199
--- /dev/null
+++ b/tests/e2e/evaluation/test_cluster_updates.py
@@ -0,0 +1,151 @@
+"""Cluster-updates evaluation tests using OpenAI GPT-4o-mini and GPT-4.1-mini judge."""
+
+import json
+import os
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+
+import pytest
+import yaml
+
+PROJECT_ROOT = Path(__file__).parent.parent.parent.parent
+EVAL_DIR = PROJECT_ROOT / "eval"
+LSEVAL_BIN = PROJECT_ROOT / ".venv" / "bin" / "lightspeed-eval"
+SYSTEM_CONFIG = EVAL_DIR / "system_cluster_updates.yaml"
+EVAL_DATA = EVAL_DIR / "eval_data_cluster_updates.yaml"
+
+
+def _ensure_lseval_installed() -> None:
+ """Install the lightspeed-evaluation package via uv if absent.
+
+ Uses the version pinned in pyproject.toml under [project.optional-dependencies].lseval,
+ ensuring a specific (tested) release rather than HEAD.
+ """
+ if LSEVAL_BIN.exists():
+ return
+
+ uv_path = shutil.which("uv")
+ if not uv_path:
+ raise FileNotFoundError("uv command not found in PATH")
+
+ subprocess.run( # noqa: S603
+ [uv_path, "sync", "--extra", "lseval"],
+ check=True,
+ cwd=str(PROJECT_ROOT),
+ )
+
+
+def _resolve_ols_url() -> str:
+ """Return the OLS base URL, preferring the live pytest client over env var."""
+ client = getattr(pytest, "ols_url", None)
+ if client:
+ return client.rstrip("/")
+ return os.getenv("OLS_URL", "http://localhost:8080").rstrip("/")
+
+
+def _get_ols_token() -> str:
+ """Extract the bearer token from the pytest HTTP client or environment."""
+ client = getattr(pytest, "client", None)
+ if client is not None:
+ auth_header: str = client.headers.get("Authorization", "")
+ token = auth_header.removeprefix("Bearer ").strip()
+ if token:
+ return token
+ return os.getenv("API_KEY", "")
+
+
+def _run_lseval(eval_data: Path, out_dir: Path) -> None:
+ """Run lightspeed-eval with the given data file and assert artefacts are produced."""
+ _ensure_lseval_installed()
+
+ # Ensure output directory is writable
+ out_dir.mkdir(parents=True, exist_ok=True)
+ if not os.access(out_dir, os.W_OK):
+ raise PermissionError(f"Output directory is not writable: {out_dir}")
+
+ ols_url = _resolve_ols_url()
+
+ with open(SYSTEM_CONFIG, encoding="utf-8") as fh:
+ config = yaml.safe_load(fh)
+
+ config["api"]["api_base"] = ols_url
+
+ # Use try/finally to ensure cleanup even on exceptions
+ tmp_config_path = None
+ try:
+ with tempfile.NamedTemporaryFile(
+ mode="w", suffix=".yaml", delete=False, dir=str(EVAL_DIR)
+ ) as tmp:
+ yaml.dump(config, tmp)
+ tmp_config_path = tmp.name
+
+ env = os.environ.copy()
+
+ # Validate OPENAI_API_KEY is present for the judge LLM
+ if "OPENAI_API_KEY" not in env:
+ raise RuntimeError(
+ "OPENAI_API_KEY environment variable is required for cluster-updates evaluation. "
+ "Ensure test-cluster-updates.sh exports it before running this test."
+ )
+
+ token = _get_ols_token()
+ if token:
+ env["API_KEY"] = token
+
+ result = subprocess.run( # noqa: S603
+ [
+ str(LSEVAL_BIN),
+ "--system-config",
+ tmp_config_path,
+ "--eval-data",
+ str(eval_data),
+ "--output-dir",
+ str(out_dir),
+ ],
+ capture_output=True,
+ text=True,
+ env=env,
+ check=False,
+ )
+ finally:
+ # Always clean up the temp file
+ if tmp_config_path and os.path.exists(tmp_config_path):
+ os.unlink(tmp_config_path)
+
+ print("--- lightspeed-eval stdout ---")
+ print(result.stdout)
+ if result.stderr:
+ print("--- lightspeed-eval stderr ---")
+ print(result.stderr)
+
+ assert result.returncode == 0, (
+ f"lightspeed-eval exited with code {result.returncode}.\n"
+ f"stderr:\n{result.stderr}"
+ )
+
+ csv_files = list(out_dir.glob("*_detailed.csv"))
+ assert csv_files, f"No detailed CSV artefacts found in {out_dir}"
+
+ json_files = list(out_dir.glob("*_summary.json"))
+ assert json_files, f"No summary JSON artefacts found in {out_dir}"
+
+ with open(json_files[0], encoding="utf-8") as fh:
+ overall = json.load(fh)["summary_stats"]["overall"]
+ assert overall["ERROR"] == 0, (
+ f"{overall['ERROR']}/{overall['TOTAL']} evaluations errored "
+ f"(error_rate={overall['error_rate']:.1f}%)."
+ )
+
+
+@pytest.mark.cluster_updates
+def test_cluster_updates(request: pytest.FixtureRequest) -> None:
+ """Run cluster-updates eval suite (18 conversations, 35 evaluations).
+
+ Uses GPT-4o-mini and GPT-4.1-mini judge.
+ """
+ out_dir_base = request.config.option.eval_out_dir or str(
+ EVAL_DIR / "results-cluster-updates"
+ )
+ _run_lseval(EVAL_DATA, Path(out_dir_base) / "cluster-updates")
diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini
index 2ce52e058..995eb1ba3 100644
--- a/tests/e2e/pytest.ini
+++ b/tests/e2e/pytest.ini
@@ -19,3 +19,4 @@ markers =
data_export
lseval
mcp
+ cluster_updates
diff --git a/tests/scripts/test-cluster-updates.sh b/tests/scripts/test-cluster-updates.sh
new file mode 100755
index 000000000..6dd537074
--- /dev/null
+++ b/tests/scripts/test-cluster-updates.sh
@@ -0,0 +1,90 @@
+#!/bin/bash
+# CI job: run cluster-updates eval (18 conversations, 35 evaluations) against OLS using OpenAI GPT-4o-mini + GPT-4.1-mini judge.
+#
+# Input environment variables:
+# OPENAI_PROVIDER_KEY_PATH - path to file containing the OpenAI API key
+# OLS_IMAGE - pullspec for the OLS container image to deploy
+#
+# Script flow:
+# 1. Install OLS dependencies
+# 2. Install operator-sdk
+# 3. Deploy OLS on the cluster (openai_cluster_updates config via run_suite)
+# 4. Run the cluster-updates pytest test (make test-cluster-updates)
+# 5. Collect artefacts and clean up
+
+set -eou pipefail
+
+make install-deps && make install-deps-test
+
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+. "$DIR/utils.sh"
+
+# Install operator-sdk
+ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n $(uname -m) ;; esac)
+export ARCH
+OS=$(uname | awk '{print tolower($0)}')
+export OS
+export OPERATOR_SDK_DL_URL=https://github.com/operator-framework/operator-sdk/releases/download/v1.36.1
+curl -LO ${OPERATOR_SDK_DL_URL}/operator-sdk_${OS}_${ARCH}
+mkdir -p $HOME/.local/bin
+chmod +x operator-sdk_${OS}_${ARCH} && mv operator-sdk_${OS}_${ARCH} $HOME/.local/bin/operator-sdk
+export PATH=$HOME/.local/bin:$PATH
+operator-sdk version
+
+# Export OpenAI key so the judge LLM can authenticate
+if [ ! -f "$OPENAI_PROVIDER_KEY_PATH" ]; then
+ echo "ERROR: OPENAI_PROVIDER_KEY_PATH file not found: $OPENAI_PROVIDER_KEY_PATH" >&2
+ exit 1
+fi
+OPENAI_API_KEY=$(cat "$OPENAI_PROVIDER_KEY_PATH")
+export OPENAI_API_KEY
+
+function run_suites() {
+ local rc=0
+
+ set +e
+ echo "=== Running cluster_updates evaluation suite ==="
+ echo " Provider: openai"
+ echo " Model: gpt-4o-mini"
+ echo " Config suffix: cluster_updates"
+ echo " Artifact dir: $ARTIFACT_DIR"
+
+ # Deploy OLS with OpenAI GPT-4o-mini and run cluster-updates evaluation (18 conversations, 35 evaluations).
+ # run_suite arguments: suiteid test_tags provider provider_keypath model ols_image ols_config_suffix
+ # OLS_CONFIG_SUFFIX="cluster_updates" → ols_installer builds: olsconfig.crd.openai_cluster_updates.yaml
+ run_suite "cluster_updates" "cluster_updates" "openai" "$OPENAI_PROVIDER_KEY_PATH" "gpt-4o-mini" "$OLS_IMAGE" "cluster_updates"
+ rc=$?
+
+ if [ $rc -ne 0 ]; then
+ echo "ERROR: cluster_updates suite failed with exit code $rc" >&2
+ fi
+ set -e
+
+ echo "=== Cleaning up OLS operator ==="
+ cleanup_ols_operator
+
+ return $rc
+}
+
+function finish() {
+ if [ "${LOCAL_MODE:-0}" -eq 1 ]; then
+ rm -rf "$ARTIFACT_DIR"
+ fi
+}
+trap finish EXIT
+
+# Define LOCAL_MODE before the conditional
+LOCAL_MODE=0
+
+# ARTIFACT_DIR is set automatically in Prow; fall back to a temp dir locally
+if [ -z "${ARTIFACT_DIR:-}" ]; then
+ ARTIFACT_DIR=$(mktemp -d)
+ LOCAL_MODE=1
+fi
+
+export ARTIFACT_DIR
+export LOCAL_MODE
+readonly LOCAL_MODE
+
+run_suites
diff --git a/tests/scripts/utils.sh b/tests/scripts/utils.sh
index dcf12404c..609bbccc7 100644
--- a/tests/scripts/utils.sh
+++ b/tests/scripts/utils.sh
@@ -50,13 +50,16 @@ function cleanup_ols_operator() {
# $7 OLS_CONFIG_SUFFIX
function run_suite() {
echo "Preparing to run suite $1"
-
+
if [ "$1" = "model_eval" ]; then
# Run old-style model evaluation tests
SUITE_ID=$1 TEST_TAGS=$2 PROVIDER=$3 PROVIDER_KEY_PATH=$4 MODEL=$5 OLS_IMAGE=$6 OLS_CONFIG_SUFFIX=$7 ARTIFACT_DIR=$ARTIFACT_DIR make test-eval
elif [[ "$1" == lseval_periodic* ]]; then
# Run LSEval periodic tests (full 797-question dataset)
SUITE_ID=$1 TEST_TAGS=$2 PROVIDER=$3 PROVIDER_KEY_PATH=$4 MODEL=$5 OLS_IMAGE=$6 OLS_CONFIG_SUFFIX=$7 ARTIFACT_DIR=$ARTIFACT_DIR make test-lseval-periodic
+ elif [ "$1" = "cluster_updates" ]; then
+ # Run cluster-updates evaluation tests (18 conversations, 35 evaluations)
+ SUITE_ID=$1 TEST_TAGS=$2 PROVIDER=$3 PROVIDER_KEY_PATH=$4 MODEL=$5 OLS_IMAGE=$6 OLS_CONFIG_SUFFIX=$7 ARTIFACT_DIR=$ARTIFACT_DIR make test-cluster-updates
else
# Run e2e tests
SUITE_ID=$1 TEST_TAGS=$2 PROVIDER=$3 PROVIDER_KEY_PATH=$4 MODEL=$5 OLS_IMAGE=$6 OLS_CONFIG_SUFFIX=$7 ARTIFACT_DIR=$ARTIFACT_DIR make test-e2e
diff --git a/tests/unit/cache/test_postgres_cache.py b/tests/unit/cache/test_postgres_cache.py
index 3a88dcebd..807508458 100644
--- a/tests/unit/cache/test_postgres_cache.py
+++ b/tests/unit/cache/test_postgres_cache.py
@@ -371,6 +371,10 @@ def test_insert_or_append_transaction_management():
mock_connect.return_value.cursor.return_value.__enter__.return_value = (
mock_cursor
)
+ # Mock transaction status - IDLE after successful commit
+ mock_connect.return_value.get_transaction_status.return_value = (
+ psycopg2.extensions.TRANSACTION_STATUS_IDLE
+ )
config = PostgresConfig()
cache = PostgresCache(config)
@@ -395,6 +399,10 @@ def test_insert_or_append_rollback_on_error():
mock_connect.return_value.cursor.return_value.__enter__.return_value = (
mock_cursor
)
+ # Mock transaction status - INERROR after exception
+ mock_connect.return_value.get_transaction_status.return_value = (
+ psycopg2.extensions.TRANSACTION_STATUS_INERROR
+ )
config = PostgresConfig()
cache = PostgresCache(config)
@@ -402,7 +410,8 @@ def test_insert_or_append_rollback_on_error():
with pytest.raises(CacheError, match="insert failed"):
cache.insert_or_append(user_id, conversation_id, cache_entry_1)
- mock_connect.return_value.rollback.assert_called_once()
+ # Rollback called three times: before disabling autocommit, in except block, and in finally
+ assert mock_connect.return_value.rollback.call_count == 3
assert mock_connect.return_value.autocommit is True
@@ -696,6 +705,10 @@ def test_delete_transaction_management():
mock_connect.return_value.cursor.return_value.__enter__.return_value = (
mock_cursor
)
+ # Mock transaction status - IDLE after successful commit
+ mock_connect.return_value.get_transaction_status.return_value = (
+ psycopg2.extensions.TRANSACTION_STATUS_IDLE
+ )
config = PostgresConfig()
cache = PostgresCache(config)
@@ -721,6 +734,10 @@ def test_delete_rollback_on_error():
mock_connect.return_value.cursor.return_value.__enter__.return_value = (
mock_cursor
)
+ # Mock transaction status - INERROR after exception
+ mock_connect.return_value.get_transaction_status.return_value = (
+ psycopg2.extensions.TRANSACTION_STATUS_INERROR
+ )
config = PostgresConfig()
cache = PostgresCache(config)
@@ -728,7 +745,8 @@ def test_delete_rollback_on_error():
with pytest.raises(CacheError, match="delete failed"):
cache.delete(user_id, conversation_id)
- mock_connect.return_value.rollback.assert_called_once()
+ # Rollback called three times: before disabling autocommit, in except block, and in finally
+ assert mock_connect.return_value.rollback.call_count == 3
assert mock_connect.return_value.autocommit is True
diff --git a/tests/unit/cache/test_postgres_cache_transaction_fix.py b/tests/unit/cache/test_postgres_cache_transaction_fix.py
new file mode 100644
index 000000000..da162eb1e
--- /dev/null
+++ b/tests/unit/cache/test_postgres_cache_transaction_fix.py
@@ -0,0 +1,159 @@
+"""Unit tests for PostgresCache transaction management fix."""
+
+from unittest.mock import MagicMock, patch
+
+import psycopg2
+import psycopg2.extensions
+import pytest
+from langchain_core.messages import AIMessage, HumanMessage
+
+from ols.app.models.config import PostgresConfig
+from ols.app.models.models import CacheEntry
+from ols.src.cache.cache_error import CacheError
+from ols.src.cache.postgres_cache import PostgresCache
+from ols.utils import suid
+
+user_id = suid.get_suid()
+conversation_id = suid.get_suid()
+cache_entry = CacheEntry(
+ query=HumanMessage("test message"), response=AIMessage("test response")
+)
+
+
+def test_insert_or_append_transaction_status_check_on_success():
+ """Test that transaction status is checked before setting autocommit on success."""
+ mock_cursor = MagicMock()
+ mock_cursor.fetchone.return_value = None
+
+ with patch("psycopg2.connect") as mock_connect:
+ # Mock connection with proper transaction status
+ mock_connection = mock_connect.return_value
+ mock_connection.cursor.return_value.__enter__.return_value = mock_cursor
+ # After successful commit, transaction should be IDLE
+ mock_connection.get_transaction_status.return_value = (
+ psycopg2.extensions.TRANSACTION_STATUS_IDLE
+ )
+
+ config = PostgresConfig()
+ cache = PostgresCache(config)
+
+ cache.insert_or_append(user_id, conversation_id, cache_entry)
+
+ # Verify transaction status was checked
+ mock_connection.get_transaction_status.assert_called()
+ # Commit should be called (successful operation)
+ mock_connection.commit.assert_called()
+ # Rollback should NOT be called in finally (transaction already IDLE)
+ # Note: rollback might be called in except block, but not in finally
+ assert mock_connection.autocommit is True
+
+
+def test_insert_or_append_transaction_status_check_on_error():
+ """Test that active transaction is rolled back before setting autocommit on error."""
+ mock_cursor = MagicMock()
+ # Simulate database error
+ mock_cursor.execute.side_effect = [
+ None, # SELECT 1 (connection check)
+ psycopg2.DatabaseError("test error"), # advisory lock fails
+ ]
+
+ with patch("psycopg2.connect") as mock_connect:
+ mock_connection = mock_connect.return_value
+ mock_connection.cursor.return_value.__enter__.return_value = mock_cursor
+ # After error, transaction is still ACTIVE (not IDLE)
+ mock_connection.get_transaction_status.return_value = (
+ psycopg2.extensions.TRANSACTION_STATUS_INERROR
+ )
+
+ config = PostgresConfig()
+ cache = PostgresCache(config)
+
+ with pytest.raises(CacheError):
+ cache.insert_or_append(user_id, conversation_id, cache_entry)
+
+ # Verify transaction status was checked in finally
+ mock_connection.get_transaction_status.assert_called()
+ # Rollback should be called three times:
+ # 1. Before disabling autocommit (transaction is not IDLE)
+ # 2. In the except block
+ # 3. In the finally block because transaction is not IDLE
+ assert mock_connection.rollback.call_count == 3
+ assert mock_connection.autocommit is True
+
+
+def test_delete_transaction_status_check_on_success():
+ """Test that transaction status is checked before setting autocommit on delete success."""
+ mock_cursor = MagicMock()
+ mock_cursor.rowcount = 1 # Simulate successful delete
+
+ with patch("psycopg2.connect") as mock_connect:
+ mock_connection = mock_connect.return_value
+ mock_connection.cursor.return_value.__enter__.return_value = mock_cursor
+ # After successful commit, transaction should be IDLE
+ mock_connection.get_transaction_status.return_value = (
+ psycopg2.extensions.TRANSACTION_STATUS_IDLE
+ )
+
+ config = PostgresConfig()
+ cache = PostgresCache(config)
+
+ result = cache.delete(user_id, conversation_id)
+
+ assert result is True
+ mock_connection.get_transaction_status.assert_called()
+ mock_connection.commit.assert_called()
+ assert mock_connection.autocommit is True
+
+
+def test_delete_transaction_status_check_on_error():
+ """Test that active transaction is rolled back before setting autocommit on delete error."""
+ mock_cursor = MagicMock()
+ mock_cursor.execute.side_effect = [
+ None, # SELECT 1 (connection check)
+ psycopg2.DatabaseError("delete failed"), # delete fails
+ ]
+
+ with patch("psycopg2.connect") as mock_connect:
+ mock_connection = mock_connect.return_value
+ mock_connection.cursor.return_value.__enter__.return_value = mock_cursor
+ # After error, transaction is still ACTIVE
+ mock_connection.get_transaction_status.return_value = (
+ psycopg2.extensions.TRANSACTION_STATUS_INERROR
+ )
+
+ config = PostgresConfig()
+ cache = PostgresCache(config)
+
+ with pytest.raises(CacheError):
+ cache.delete(user_id, conversation_id)
+
+ mock_connection.get_transaction_status.assert_called()
+ # Rollback called three times: before disabling autocommit, in except, and in finally
+ assert mock_connection.rollback.call_count == 3
+ assert mock_connection.autocommit is True
+
+
+def test_no_extra_rollback_when_transaction_idle():
+ """Test that no extra rollback is called when transaction is already IDLE."""
+ mock_cursor = MagicMock()
+ mock_cursor.fetchone.return_value = None
+
+ with patch("psycopg2.connect") as mock_connect:
+ mock_connection = mock_connect.return_value
+ mock_connection.cursor.return_value.__enter__.return_value = mock_cursor
+ # Transaction is IDLE (normal successful case)
+ mock_connection.get_transaction_status.return_value = (
+ psycopg2.extensions.TRANSACTION_STATUS_IDLE
+ )
+
+ config = PostgresConfig()
+ cache = PostgresCache(config)
+
+ cache.insert_or_append(user_id, conversation_id, cache_entry)
+
+ # In successful case with IDLE transaction:
+ # - commit() is called (during init and during insert_or_append)
+ # - rollback() should NOT be called in finally (transaction is IDLE)
+ assert mock_connection.commit.call_count >= 1
+ # Rollback should not be called at all in the success case
+ mock_connection.rollback.assert_not_called()
diff --git a/uv.lock b/uv.lock
index 0cadb0859..e9c34d41a 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1798,13 +1798,13 @@ wheels = [
[[package]]
name = "langchain-community"
-version = "0.4.1"
+version = "0.3.31"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "aiohttp" },
{ name = "dataclasses-json" },
{ name = "httpx-sse" },
- { name = "langchain-classic" },
+ { name = "langchain" },
{ name = "langchain-core" },
{ name = "langsmith" },
{ name = "numpy" },
@@ -1814,9 +1814,9 @@ dependencies = [
{ name = "sqlalchemy" },
{ name = "tenacity" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/53/97/a03585d42b9bdb6fbd935282d6e3348b10322a24e6ce12d0c99eb461d9af/langchain_community-0.4.1.tar.gz", hash = "sha256:f3b211832728ee89f169ddce8579b80a085222ddb4f4ed445a46e977d17b1e85", size = 33241144, upload-time = "2025-10-27T15:20:32.504Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/83/49/2ff5354273809e9811392bc24bcffda545a196070666aef27bc6aacf1c21/langchain_community-0.3.31.tar.gz", hash = "sha256:250e4c1041539130f6d6ac6f9386cb018354eafccd917b01a4cff1950b80fd81", size = 33241237, upload-time = "2025-10-07T20:17:57.857Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/f0/a4/c4fde67f193401512337456cabc2148f2c43316e445f5decd9f8806e2992/langchain_community-0.4.1-py3-none-any.whl", hash = "sha256:2135abb2c7748a35c84613108f7ebf30f8505b18c3c18305ffaecfc7651f6c6a", size = 2533285, upload-time = "2025-10-27T15:20:30.767Z" },
+ { url = "https://files.pythonhosted.org/packages/e6/0a/b8848db67ad7c8d4652cb6f4cb78d49b5b5e6e8e51d695d62025aa3f7dbc/langchain_community-0.3.31-py3-none-any.whl", hash = "sha256:1c727e3ebbacd4d891b07bd440647668001cea3e39cbe732499ad655ec5cb569", size = 2532920, upload-time = "2025-10-07T20:17:54.91Z" },
]
[[package]]
@@ -2778,7 +2778,7 @@ requires-dist = [
{ name = "kubernetes", specifier = "<34.1.0" },
{ name = "langchain", specifier = ">=0.3.12" },
{ name = "langchain-aws", specifier = ">=1.6.0" },
- { name = "langchain-community", specifier = ">=0.3.81" },
+ { name = "langchain-community", specifier = ">=0.3,<0.4" },
{ name = "langchain-google-genai", specifier = ">=4.0.0" },
{ name = "langchain-google-vertexai", specifier = ">=2.1.0" },
{ name = "langchain-ibm", specifier = ">=0.3.2" },
@@ -2801,7 +2801,7 @@ requires-dist = [
{ name = "pydantic", specifier = ">=2.9.2" },
{ name = "python-frontmatter", specifier = ">=1.1.0" },
{ name = "qdrant-client", specifier = ">=1.13.3" },
- { name = "ragas", marker = "extra == 'evaluation'", specifier = ">=0.2.14" },
+ { name = "ragas", marker = "extra == 'evaluation'", specifier = ">=0.3.0" },
{ name = "rank-bm25", specifier = ">=0.2.2" },
{ name = "requests", specifier = ">=2.32.2" },
{ name = "rouge-score", marker = "extra == 'evaluation'", specifier = ">=0.1.2" },