From fda3ff4760f49560f628874df9f7e017247d3c4c Mon Sep 17 00:00:00 2001 From: Douglas Hensel Date: Wed, 1 Jul 2026 21:42:28 -0400 Subject: [PATCH 1/4] Add TNF two-node fencing diagnostics toolset Extends the OpenShift MCP server with a new `tnf` toolset for Two-Node Fencing (TNF) cluster diagnostics. Tools: - tnf_check_fencing_config: validates cluster topology, operator health, Machine/Node/BMH correlation, BMC credentials, FenceAgentsRemediation, and NodeHealthCheck configuration - tnf_check_stonith_status: runs pcs diagnostics via debug pod to check pacemaker cluster state, STONITH devices, quorum, and fencing history Prompt: - tnf-troubleshoot: structured troubleshooting workflow collecting all diagnostic data with API-down fallback to out-of-band recovery guide Resource: - tnf://domain-knowledge/fencing: domain knowledge covering quorum rules, split-brain risk matrix, and recovery procedures Co-Authored-By: Claude Opus 4.6 --- docs/README.md | 1 + docs/tnf.md | 114 ++++ pkg/mcp/modules.go | 1 + pkg/toolsets/tnf/fencing/api_check.go | 254 ++++++++ pkg/toolsets/tnf/fencing/api_check_test.go | 106 ++++ pkg/toolsets/tnf/fencing/fencing.go | 706 +++++++++++++++++++++ pkg/toolsets/tnf/fencing/fencing_test.go | 36 ++ pkg/toolsets/tnf/fencing/stonith.go | 463 ++++++++++++++ pkg/toolsets/tnf/fencing/stonith_test.go | 180 ++++++ pkg/toolsets/tnf/mcp_resources.go | 100 +++ pkg/toolsets/tnf/mcp_resources_test.go | 37 ++ pkg/toolsets/tnf/toolset.go | 44 ++ pkg/toolsets/tnf/troubleshoot.go | 561 ++++++++++++++++ pkg/toolsets/tnf/troubleshoot_test.go | 371 +++++++++++ 14 files changed, 2974 insertions(+) create mode 100644 docs/tnf.md create mode 100644 pkg/toolsets/tnf/fencing/api_check.go create mode 100644 pkg/toolsets/tnf/fencing/api_check_test.go create mode 100644 pkg/toolsets/tnf/fencing/fencing.go create mode 100644 pkg/toolsets/tnf/fencing/fencing_test.go create mode 100644 pkg/toolsets/tnf/fencing/stonith.go create mode 100644 pkg/toolsets/tnf/fencing/stonith_test.go create mode 100644 pkg/toolsets/tnf/mcp_resources.go create mode 100644 pkg/toolsets/tnf/mcp_resources_test.go create mode 100644 pkg/toolsets/tnf/toolset.go create mode 100644 pkg/toolsets/tnf/troubleshoot.go create mode 100644 pkg/toolsets/tnf/troubleshoot_test.go diff --git a/docs/README.md b/docs/README.md index afbe367c0..e7f721ec1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -29,6 +29,7 @@ Choose the guide that matches your needs: - **[OADP](OADP.md)** - Tools for OpenShift API for Data Protection (Velero backups, restores, schedules) - **[Kiali](KIALI.md)** - Tools for Kiali ServiceMesh with Istio - **[KubeVirt](kubevirt.md)** - KubeVirt virtual machine management tools +- **[TNF](tnf.md)** - Two-Node Fencing diagnostics (pacemaker, STONITH, BMC health) ## Feature Specifications diff --git a/docs/tnf.md b/docs/tnf.md new file mode 100644 index 000000000..f92cf5b2d --- /dev/null +++ b/docs/tnf.md @@ -0,0 +1,114 @@ +# TNF (Two-Node Fencing) Support + +The `tnf` toolset extends the OpenShift MCP server with diagnostics for Two-Node Fencing clusters. These are 2-node bare metal OpenShift clusters that use pacemaker and STONITH for fencing to prevent split-brain scenarios. + +## Tools + +### tnf_check_fencing_config + +Validates fencing configuration and readiness for a TNF cluster. Checks cluster topology (platform, node count, TNF profile), critical operator health (etcd, machine-api, baremetal), Machine/Node/BareMetalHost correlation, BMC addresses and credential secrets, FenceAgentsRemediation templates and active remediations, and NodeHealthCheck resources. Returns a diagnostic summary identifying configuration issues that could prevent fencing from functioning correctly. + +**Arguments:** +- `namespace` (optional): Namespace containing BareMetalHost resources (default: searches all namespaces) + +### tnf_check_stonith_status + +Creates a temporary privileged debug pod on a control-plane node to run `pcs` diagnostic commands. Returns pacemaker cluster state, STONITH device configuration, quorum status, and recent fencing history. The debug pod is automatically cleaned up after execution. + +**Arguments:** +- `node` (optional): Name of the node to run diagnostics on (default: auto-detects the first control-plane node) +- `namespace` (optional): Namespace to create the temporary debug pod in (default: `default`) +- `timeout_seconds` (optional): Maximum time in seconds to wait for the diagnostic commands to complete (default: 120) + +## Prompt + +### tnf-troubleshoot + +Runs a full TNF fencing diagnostic workflow that collects cluster topology, node health, critical operator status, BareMetalHost/BMC health, pacemaker/STONITH status, and remediation operator status into a single structured report. Guides the LLM through analysis using the domain knowledge resource to assess split-brain risk and recommend actions. + +**Arguments:** +- `node` (optional): Preferred node to run STONITH diagnostics from + +**Invoke in Claude Code:** +```text +/mcp__kubernetes-mcp-server__tnf-troubleshoot +``` + +## Resource + +### tnf://domain-knowledge/fencing + +Static reference material covering: +- How two-node fencing works (corosync, pacemaker, STONITH) +- Two-node quorum rules and `wait_for_all` behavior +- Split-brain risk assessment matrix +- Common issues and recovery procedures (STONITH disabled, BMC unreachable, node won't rejoin, quorum lost, fence race) + +## Enable the TNF Toolset + +### Option 1: Command Line + +```bash +kubernetes-mcp-server --toolsets core,config,tnf +``` + +### Option 2: Configuration File + +```toml +toolsets = ["core", "config", "tnf"] +``` + +### Option 3: MCP Client Configuration + +```json +{ + "mcpServers": { + "kubernetes": { + "command": "kubernetes-mcp-server", + "args": ["--toolsets", "core,config,tnf"] + } + } +} +``` + +## Prerequisites + +TNF support requires: + +1. **Two-node bare metal OpenShift cluster** with pacemaker/STONITH fencing configured +2. **BareMetalHost CRDs** installed (standard on bare metal clusters via the baremetal-operator) +3. **Proper RBAC** for reading Nodes, BareMetalHosts, Secrets, and creating debug pods + +### Verify TNF Setup + +```bash +# Check for 2-node bare metal cluster +oc get nodes +oc get infrastructure cluster -o jsonpath='{.status.platform}' + +# Check BareMetalHosts exist +oc get baremetalhosts -A + +# Check pacemaker is running +oc debug node/ -- chroot /host pcs status +``` + +## What It Diagnoses + +| Area | What's Checked | +|------|---------------| +| Cluster topology | Platform, node count, TNF profile detection | +| Node health | Kubernetes Ready status, conditions | +| BareMetalHost | Provisioning state, power status, BMC address | +| BMC credentials | Secret existence, username/password keys present | +| Pacemaker | Cluster name, online/offline/standby nodes | +| STONITH devices | Configuration, agent type, target node, started status | +| Quorum | Quorate status, vote count, two-node flags | +| Fencing history | Recent fencing events, success/failure | +| Remediation operators | FenceAgentsRemediation and NodeHealthCheck CRD status | + +## Limitations + +- **API must be reachable**: All tools depend on the Kubernetes API. When the API is unreachable (e.g. during a fencing event that takes down the API server), the tools detect the failure and return an out-of-band recovery guide with BMC access and manual diagnostic procedures instead of raw cluster data. The `tnf-troubleshoot` prompt integrates this fallback automatically. +- **Read-only diagnostics**: The tools do not trigger, stop, or modify fencing operations. +- **Debug pod privileges**: The STONITH check requires creating a privileged debug pod with host access (same as `oc debug node/`). diff --git a/pkg/mcp/modules.go b/pkg/mcp/modules.go index a62373f0a..e0190fb5c 100644 --- a/pkg/mcp/modules.go +++ b/pkg/mcp/modules.go @@ -13,5 +13,6 @@ import ( _ "github.com/containers/kubernetes-mcp-server/pkg/toolsets/oadp" _ "github.com/containers/kubernetes-mcp-server/pkg/toolsets/openshift" _ "github.com/containers/kubernetes-mcp-server/pkg/toolsets/tekton" + _ "github.com/containers/kubernetes-mcp-server/pkg/toolsets/tnf" _ "github.com/rhobs/obs-mcp/pkg/toolset" ) diff --git a/pkg/toolsets/tnf/fencing/api_check.go b/pkg/toolsets/tnf/fencing/api_check.go new file mode 100644 index 000000000..2ec11f190 --- /dev/null +++ b/pkg/toolsets/tnf/fencing/api_check.go @@ -0,0 +1,254 @@ +package fencing + +import ( + "errors" + "net" + "strings" + + "github.com/containers/kubernetes-mcp-server/pkg/api" +) + +// IsAPIUnreachable probes the Kubernetes API server via the discovery +// client. It returns true only when the API server cannot be reached +// at the network level (connection refused, no route, timeout). API-level +// errors such as Forbidden or Unauthorized indicate the server IS +// reachable and return false. +func IsAPIUnreachable(client api.KubernetesClient) bool { + if client == nil { + return false + } + _, err := client.DiscoveryClient().ServerVersion() + if err == nil { + return false + } + return isConnectionError(err) +} + +func isConnectionError(err error) bool { + var netErr net.Error + if errors.As(err, &netErr) { + return true + } + var opErr *net.OpError + if errors.As(err, &opErr) { + return true + } + msg := err.Error() + return strings.Contains(msg, "connection refused") || + strings.Contains(msg, "no route to host") || + strings.Contains(msg, "no such host") || + strings.Contains(msg, "i/o timeout") || + strings.Contains(msg, "dial tcp") || + strings.Contains(msg, "context deadline exceeded") || + strings.Contains(msg, "connection reset by peer") +} + +// APIUnreachableGuide returns a structured troubleshooting guide for when +// the Kubernetes API server is unreachable. It covers three scenarios: +// failed install, upgrade/reboot, and crash/degraded. +func APIUnreachableGuide() string { + return `# TNF Diagnostic: Kubernetes API Unreachable + +The Kubernetes API server is not responding. This prevents all cluster-level +diagnostics. Use out-of-band access (BMC, SSH) to determine what happened. + +--- + +## Determine Your Scenario + +| Clue | Likely Scenario | +|------|----------------| +| Install was recently started, cluster never came up | **Scenario 1: Install failure** | +| Upgrade or MachineConfig change was in progress | **Scenario 2: Upgrade/reboot** | +| Cluster was running normally, then stopped | **Scenario 3: Crash/degraded** | + +--- + +## Common First Steps (All Scenarios) + +### 1. Check Node Power State via BMC + +Use IPMI or Redfish to verify nodes are powered on: + +` + "```" + ` +# IPMI +ipmitool -I lanplus -H -U -P power status + +# Redfish +curl -k -u : https:///redfish/v1/Systems/1 | jq '.PowerState' +` + "```" + ` + +### 2. SSH to Nodes + +If nodes booted far enough, SSH access may be available: + +` + "```" + ` +ssh core@ +` + "```" + ` + +### 3. Check API VIP Assignment + +From any node, verify the API VIP is assigned to a network interface: + +` + "```" + ` +ip addr show | grep +` + "```" + ` + +If the VIP is not assigned, the API server or keepalived is not running. + +### 4. Check DNS Resolution + +` + "```" + ` +dig api.. +` + "```" + ` + +--- + +## Scenario 1: Failed or In-Progress Install + +The cluster has never fully bootstrapped. The assisted-installer agent +or bootstrap process may still be running or may have failed. + +### Check Agent-Based Installer Status + +` + "```" + ` +# Agent service logs (ABI installs) +journalctl -u agent --no-pager -n 100 + +# Assisted-installer local API (if agent is running) +curl -s http://localhost:8090/api/assisted-install/v2/clusters | jq '.[].status' +` + "```" + ` + +### Check Bootstrap Progression + +` + "```" + ` +# Has bootstrap completed? +ls -la /opt/openshift/.bootkube.done + +# Are static pod manifests present? +ls /etc/kubernetes/manifests/ + +# Is kubelet running? +systemctl status kubelet + +# etcd status +sudo crictl ps --name etcd +sudo crictl logs $(sudo crictl ps --name etcd -q) 2>&1 | tail -20 +` + "```" + ` + +### Common Install Failures + +- **DNS misconfiguration** — api/api-int/*.apps must resolve correctly +- **Pull secret invalid** — check /root/.docker/config.json or /var/lib/kubelet/config.json +- **BMC unreachable during install** — needed for host provisioning +- **Certificate issues** — clock skew between nodes causes TLS failures +- **Disk/storage** — insufficient space on /var or /sysroot + +--- + +## Scenario 2: Upgrade or Reboot in Progress + +A node rebooted for an OS update (MCO) or configuration change. The API +should return within 5-15 minutes on a two-node cluster. + +### Check OS Update Status + +` + "```" + ` +# Current and pending OS deployments +rpm-ostree status + +# Was a reboot triggered by MCO? +journalctl -u machine-config-daemon --no-pager -n 50 +` + "```" + ` + +### Check Kubelet Recovery + +` + "```" + ` +systemctl status kubelet +journalctl -u kubelet --no-pager -n 50 +` + "```" + ` + +### Two-Node Etcd Considerations + +In a 2-node cluster, if one node reboots, etcd loses quorum until it +returns. The API will be unavailable until both etcd members are healthy: + +` + "```" + ` +sudo crictl ps --name etcd +sudo crictl exec $(sudo crictl ps --name etcd -q) etcdctl endpoint health \ + --cluster --cacert /etc/kubernetes/static-pod-certs/configmaps/etcd-serving-ca/ca-bundle.crt \ + --cert /etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-master-0.crt \ + --key /etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-master-0.key +` + "```" + ` + +### If the Upgrade Appears Stuck + +- Check if the node completed its reboot: ` + "`uptime`" + ` +- Check if the other node is still running and healthy +- Check pacemaker: has fencing been triggered during the upgrade? + +--- + +## Scenario 3: Crash or Degraded State + +The cluster was running and stopped unexpectedly. A node may have crashed, +or a critical service may have failed. + +### Check What Happened + +` + "```" + ` +# Recent kubelet activity +journalctl -u kubelet --since "30 min ago" --no-pager + +# Recent system events +journalctl --since "30 min ago" --priority err --no-pager + +# Are control-plane containers running? +sudo crictl ps --name kube-apiserver +sudo crictl ps --name etcd +sudo crictl ps --name kube-controller-manager +` + "```" + ` + +### Check Pacemaker and Fencing + +On a TNF cluster, the surviving node may have fenced the failed node: + +` + "```" + ` +pcs status +pcs stonith history +` + "```" + ` + +If fencing occurred: +- Identify which node was fenced and which survived +- Check BMC power state of the fenced node +- The fenced node may need manual power-on and cluster rejoin + +### Etcd Recovery + +` + "```" + ` +# Check etcd member list and health +sudo crictl exec $(sudo crictl ps --name etcd -q) etcdctl member list \ + --cacert /etc/kubernetes/static-pod-certs/configmaps/etcd-serving-ca/ca-bundle.crt \ + --cert /etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-master-0.crt \ + --key /etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-master-0.key +` + "```" + ` + +### If Both Nodes Are Down + +This is a critical situation. Use BMC to: +1. Check power state of both nodes +2. Power on nodes via IPMI/Redfish +3. Access BMC virtual console to observe boot process +4. Once nodes boot, follow Scenario 1 or 2 steps depending on whether + the cluster was previously installed + +--- + +## Next Steps + +Once the API server is reachable again, re-run the TNF diagnostic tools: +- ` + "`tnf_check_fencing_config`" + ` — validate fencing configuration +- ` + "`tnf_check_stonith_status`" + ` — check pacemaker and STONITH state +- ` + "`tnf-troubleshoot`" + ` — full troubleshooting guide with live data +` +} diff --git a/pkg/toolsets/tnf/fencing/api_check_test.go b/pkg/toolsets/tnf/fencing/api_check_test.go new file mode 100644 index 000000000..2ae19d90d --- /dev/null +++ b/pkg/toolsets/tnf/fencing/api_check_test.go @@ -0,0 +1,106 @@ +package fencing + +import ( + "errors" + "fmt" + "net" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsConnectionError(t *testing.T) { + tests := []struct { + name string + err error + expected bool + }{ + { + name: "connection refused", + err: fmt.Errorf("dial tcp 192.168.111.5:6443: connect: connection refused"), + expected: true, + }, + { + name: "no route to host", + err: fmt.Errorf("dial tcp 192.168.111.5:6443: connect: no route to host"), + expected: true, + }, + { + name: "no such host", + err: fmt.Errorf("dial tcp: lookup api.cluster.example.com: no such host"), + expected: true, + }, + { + name: "i/o timeout", + err: fmt.Errorf("dial tcp 192.168.111.5:6443: i/o timeout"), + expected: true, + }, + { + name: "net.OpError", + err: &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")}, + expected: true, + }, + { + name: "wrapped dial error", + err: fmt.Errorf("get server version: %w", fmt.Errorf("dial tcp 10.0.0.1:6443: connect: connection refused")), + expected: true, + }, + { + name: "context deadline exceeded", + err: fmt.Errorf("Get \"https://127.0.0.1:6443/version?timeout=5s\": context deadline exceeded"), + expected: true, + }, + { + name: "connection reset by peer", + err: fmt.Errorf("read tcp 127.0.0.1:47428->127.0.0.1:6443: read: connection reset by peer"), + expected: true, + }, + { + name: "k8s wrapPreviousError style", + err: fmt.Errorf("Get \"https://127.0.0.1:16443/version?timeout=5s\": context deadline exceeded - error from a previous attempt: read tcp 127.0.0.1:47428->127.0.0.1:16443: read: connection reset by peer"), + expected: true, + }, + { + name: "forbidden (API reachable)", + err: fmt.Errorf("the server has asked for the client to provide credentials"), + expected: false, + }, + { + name: "not found (API reachable)", + err: fmt.Errorf("the server could not find the requested resource"), + expected: false, + }, + { + name: "generic error", + err: errors.New("something went wrong"), + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := isConnectionError(tt.err) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestIsAPIUnreachableNilClient(t *testing.T) { + assert.False(t, IsAPIUnreachable(nil)) +} + +func TestAPIUnreachableGuide(t *testing.T) { + guide := APIUnreachableGuide() + + assert.Contains(t, guide, "API Unreachable") + assert.Contains(t, guide, "Scenario 1") + assert.Contains(t, guide, "Scenario 2") + assert.Contains(t, guide, "Scenario 3") + assert.Contains(t, guide, "BMC") + assert.Contains(t, guide, "ssh core@") + assert.Contains(t, guide, "journalctl -u agent") + assert.Contains(t, guide, "rpm-ostree status") + assert.Contains(t, guide, "pcs status") + assert.Contains(t, guide, "etcdctl") + assert.Contains(t, guide, "tnf_check_fencing_config") +} diff --git a/pkg/toolsets/tnf/fencing/fencing.go b/pkg/toolsets/tnf/fencing/fencing.go new file mode 100644 index 000000000..38363e0a5 --- /dev/null +++ b/pkg/toolsets/tnf/fencing/fencing.go @@ -0,0 +1,706 @@ +package fencing + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + + "github.com/google/jsonschema-go/jsonschema" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/utils/ptr" + + "github.com/containers/kubernetes-mcp-server/pkg/api" +) + +var ( + BareMetalHostGVR = schema.GroupVersionResource{ + Group: "metal3.io", Version: "v1alpha1", Resource: "baremetalhosts", + } + ClusterOperatorGVR = schema.GroupVersionResource{ + Group: "config.openshift.io", Version: "v1", Resource: "clusteroperators", + } + InfrastructureGVR = schema.GroupVersionResource{ + Group: "config.openshift.io", Version: "v1", Resource: "infrastructures", + } + MachineGVR = schema.GroupVersionResource{ + Group: "machine.openshift.io", Version: "v1beta1", Resource: "machines", + } + FenceAgentsRemediationGVR = schema.GroupVersionResource{ + Group: "fence-agents-remediation.medik8s.io", Version: "v1alpha1", + Resource: "fenceagentsremediations", + } + FenceAgentsRemediationTemplateGVR = schema.GroupVersionResource{ + Group: "fence-agents-remediation.medik8s.io", Version: "v1alpha1", + Resource: "fenceagentsremediationtemplates", + } + NodeHealthCheckGVR = schema.GroupVersionResource{ + Group: "remediation.medik8s.io", Version: "v1alpha1", + Resource: "nodehealthchecks", + } +) + +func InitFencing() []api.ServerTool { + return []api.ServerTool{ + { + Tool: api.Tool{ + Name: "tnf_check_fencing_config", + Description: "Check fencing configuration and readiness for a Two-Node Fencing (TNF) cluster. " + + "Validates cluster topology, critical operator health (etcd, machine-api, baremetal), " + + "Machine/Node/BareMetalHost correlation, BMC addresses and credentials, " + + "FenceAgentsRemediation templates and active remediations, and NodeHealthCheck resources. " + + "Returns a diagnostic summary identifying configuration issues that could " + + "prevent fencing from functioning correctly.", + InputSchema: &jsonschema.Schema{ + Type: "object", + Properties: map[string]*jsonschema.Schema{ + "namespace": { + Type: "string", + Description: "Namespace containing BareMetalHost resources (e.g. 'openshift-machine-api'). If omitted, searches all namespaces.", + }, + }, + }, + Annotations: api.ToolAnnotations{ + Title: "TNF: Check Fencing Config", + ReadOnlyHint: ptr.To(true), + DestructiveHint: ptr.To(false), + IdempotentHint: ptr.To(true), + OpenWorldHint: ptr.To(false), + }, + }, + Handler: checkFencingHealth, + }, + } +} + +func checkFencingHealth(params api.ToolHandlerParams) (*api.ToolCallResult, error) { + if IsAPIUnreachable(params.KubernetesClient) { + return api.NewToolCallResult(APIUnreachableGuide(), nil), nil + } + + p := api.WrapParams(params) + namespace := p.OptionalString("namespace", "") + if err := p.Err(); err != nil { + return api.NewToolCallResult("", fmt.Errorf("failed to check fencing health: %w", err)), nil + } + + dynamicClient := params.DynamicClient() + coreClient := params.CoreV1() + + var report strings.Builder + var issues []string + + report.WriteString("# TNF Fencing Health Check\n\n") + + topoIssues := checkInfrastructureTopology(params.Context, dynamicClient, coreClient, &report) + issues = append(issues, topoIssues...) + + coIssues := checkClusterOperatorHealth(params.Context, dynamicClient, &report) + issues = append(issues, coIssues...) + + bmhToNode, corrIssues := correlateMachinesWithHosts(params.Context, dynamicClient, &report) + issues = append(issues, corrIssues...) + + hosts, err := listBareMetalHosts(params.Context, dynamicClient, namespace) + if err != nil { + return api.NewToolCallResult("", fmt.Errorf("failed to list BareMetalHost resources: %w", err)), nil + } + + if len(hosts) == 0 { + msg := "No BareMetalHost resources found" + if namespace != "" { + msg += fmt.Sprintf(" in namespace %q", namespace) + } + report.WriteString(msg + ". This cluster may not be a bare metal deployment.\n\n") + issues = append(issues, msg) + } else { + fmt.Fprintf(&report, "BareMetalHosts found: %d\n\n", len(hosts)) + for _, host := range hosts { + hostIssues := writeHostSection(params.Context, coreClient, &report, host, bmhToNode) + issues = append(issues, hostIssues...) + } + } + + farIssues := checkFenceAgentsRemediation(params.Context, dynamicClient, &report) + issues = append(issues, farIssues...) + + nhcIssues := checkNodeHealthChecks(params.Context, dynamicClient, &report) + issues = append(issues, nhcIssues...) + + report.WriteString("## Summary\n\n") + if len(issues) == 0 { + report.WriteString("No fencing health issues detected.\n") + } else { + fmt.Fprintf(&report, "Found %d issue(s):\n\n", len(issues)) + for i, issue := range issues { + fmt.Fprintf(&report, "%d. %s\n", i+1, issue) + } + } + + return api.NewToolCallResult(report.String(), nil), nil +} + +func checkInfrastructureTopology(ctx context.Context, client dynamic.Interface, coreClient corev1.CoreV1Interface, report *strings.Builder) []string { + var issues []string + report.WriteString("## Cluster Topology\n\n") + + infra, err := client.Resource(InfrastructureGVR).Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + slog.Debug("could not get Infrastructure CR", "error", err) + if isCRDNotInstalled(err) { + report.WriteString("- Infrastructure CR: not available (may not be an OpenShift cluster)\n\n") + } else { + fmt.Fprintf(report, "- Infrastructure CR: **error** — %v\n\n", err) + issues = append(issues, fmt.Sprintf("failed to get Infrastructure CR: %v", err)) + } + return issues + } + + platform, _, err := unstructured.NestedString(infra.Object, "status", "platform") + if err != nil { + slog.Debug("malformed Infrastructure CR field", "field", "status.platform", "error", err) + } + infraTopology, _, err := unstructured.NestedString(infra.Object, "status", "infrastructureTopology") + if err != nil { + slog.Debug("malformed Infrastructure CR field", "field", "status.infrastructureTopology", "error", err) + } + cpTopology, _, err := unstructured.NestedString(infra.Object, "status", "controlPlaneTopology") + if err != nil { + slog.Debug("malformed Infrastructure CR field", "field", "status.controlPlaneTopology", "error", err) + } + + fmt.Fprintf(report, "- Platform: %s\n", ValueOrNA(platform)) + fmt.Fprintf(report, "- Infrastructure Topology: %s\n", ValueOrNA(infraTopology)) + fmt.Fprintf(report, "- Control Plane Topology: %s\n", ValueOrNA(cpTopology)) + + nodes, err := coreClient.Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + slog.Debug("could not list nodes for topology check", "error", err) + } else { + cpCount := 0 + for _, node := range nodes.Items { + if _, ok := node.Labels["node-role.kubernetes.io/control-plane"]; ok { + cpCount++ + } else if _, ok := node.Labels["node-role.kubernetes.io/master"]; ok { + cpCount++ + } + } + fmt.Fprintf(report, "- Total nodes: %d\n", len(nodes.Items)) + fmt.Fprintf(report, "- Control-plane nodes: %d\n", cpCount) + + isTNF := strings.EqualFold(platform, "BareMetal") && cpCount == 2 + if isTNF { + report.WriteString("- TNF Profile: **Yes** (2-node bare metal)\n") + } else { + report.WriteString("- TNF Profile: No\n") + if !strings.EqualFold(platform, "BareMetal") { + issues = append(issues, fmt.Sprintf("platform is %q, expected BareMetal for TNF", platform)) + } + if cpCount != 2 { + issues = append(issues, fmt.Sprintf("found %d control-plane nodes, expected 2 for TNF", cpCount)) + } + } + } + + report.WriteString("\n") + return issues +} + +func checkClusterOperatorHealth(ctx context.Context, client dynamic.Interface, report *strings.Builder) []string { + var issues []string + report.WriteString("## Cluster Operator Health\n\n") + + tnfOperators := map[string]bool{ + "etcd": true, + "machine-api": true, + "baremetal": true, + } + + list, err := client.Resource(ClusterOperatorGVR).List(ctx, metav1.ListOptions{}) + if err != nil { + slog.Debug("could not list ClusterOperators", "error", err) + if isCRDNotInstalled(err) { + report.WriteString("- ClusterOperators: not available (may not be an OpenShift cluster)\n\n") + } else { + fmt.Fprintf(report, "- ClusterOperators: **error** — %v\n\n", err) + issues = append(issues, fmt.Sprintf("failed to list ClusterOperators: %v", err)) + } + return issues + } + + foundOps := make(map[string]bool) + + for _, item := range list.Items { + name := item.GetName() + if !tnfOperators[name] { + continue + } + foundOps[name] = true + + conditions, _, _ := unstructured.NestedSlice(item.Object, "status", "conditions") + available := "Unknown" + degraded := "Unknown" + progressing := "Unknown" + var opIssues []string + + for _, cond := range conditions { + condMap, ok := cond.(map[string]interface{}) + if !ok { + continue + } + condType, _ := condMap["type"].(string) + condStatus, _ := condMap["status"].(string) + message, _ := condMap["message"].(string) + + switch condType { + case "Available": + available = condStatus + if condStatus != "True" { + opIssues = append(opIssues, fmt.Sprintf("not available: %s", message)) + } + case "Degraded": + degraded = condStatus + if condStatus == "True" { + opIssues = append(opIssues, fmt.Sprintf("degraded: %s", message)) + } + case "Progressing": + progressing = condStatus + } + } + + fmt.Fprintf(report, "- %s: Available=%s, Degraded=%s, Progressing=%s\n", + name, available, degraded, progressing) + for _, oi := range opIssues { + fmt.Fprintf(report, " - **Issue**: %s\n", oi) + issues = append(issues, fmt.Sprintf("operator %s: %s", name, oi)) + } + } + + for op := range tnfOperators { + if !foundOps[op] { + fmt.Fprintf(report, "- %s: **not found**\n", op) + issues = append(issues, fmt.Sprintf("operator %s not found", op)) + } + } + + report.WriteString("\n") + return issues +} + +// correlateMachinesWithHosts writes a Machine/Node/BMH correlation table and +// returns a mapping from "namespace/bmhName" to the Node name resolved via +// Machine.status.nodeRef.name, along with any issues found. +func correlateMachinesWithHosts(ctx context.Context, client dynamic.Interface, report *strings.Builder) (map[string]string, []string) { + bmhToNode := make(map[string]string) + var issues []string + report.WriteString("## Machine/Node/BMH Correlation\n\n") + + list, err := client.Resource(MachineGVR).Namespace("openshift-machine-api").List(ctx, metav1.ListOptions{}) + if err != nil { + slog.Debug("could not list Machines", "error", err) + if isCRDNotInstalled(err) { + report.WriteString("- Machines: not available (Machine API not installed)\n\n") + } else { + fmt.Fprintf(report, "- Machines: **error** — %v\n\n", err) + issues = append(issues, fmt.Sprintf("failed to list Machines: %v", err)) + } + return bmhToNode, issues + } + + if len(list.Items) == 0 { + report.WriteString("No Machine resources found in openshift-machine-api.\n\n") + return bmhToNode, issues + } + + report.WriteString("| Machine | Node | BMH | Phase |\n") + report.WriteString("|---------|------|-----|-------|\n") + + for _, machine := range list.Items { + machineName := machine.GetName() + annotations := machine.GetAnnotations() + + nodeRef, _, _ := unstructured.NestedString(machine.Object, "status", "nodeRef", "name") + phase, _, _ := unstructured.NestedString(machine.Object, "status", "phase") + bmhRef := annotations["metal3.io/BareMetalHost"] + + nodeDisplay := ValueOrNA(nodeRef) + bmhDisplay := ValueOrNA(bmhRef) + + fmt.Fprintf(report, "| %s | %s | %s | %s |\n", + machineName, nodeDisplay, bmhDisplay, ValueOrNA(phase)) + + if nodeRef != "" && bmhRef != "" { + bmhToNode[bmhRef] = nodeRef + } + + if nodeRef == "" { + issues = append(issues, fmt.Sprintf("machine %s has no nodeRef (not yet provisioned or failed)", machineName)) + } + if bmhRef == "" { + issues = append(issues, fmt.Sprintf("machine %s has no metal3.io/BareMetalHost annotation", machineName)) + } + } + + report.WriteString("\n") + return bmhToNode, issues +} + +func writeHostSection(ctx context.Context, coreClient corev1.CoreV1Interface, report *strings.Builder, host unstructured.Unstructured, bmhToNode map[string]string) []string { + var issues []string + hostName := host.GetName() + hostNamespace := host.GetNamespace() + + fmt.Fprintf(report, "## Host: %s/%s\n\n", hostNamespace, hostName) + + provisioningState, _, _ := unstructured.NestedString(host.Object, "status", "provisioning", "state") + operationalStatus, _, _ := unstructured.NestedString(host.Object, "status", "operationalStatus") + poweredOn, _, _ := unstructured.NestedBool(host.Object, "status", "poweredOn") + errorMessage, _, _ := unstructured.NestedString(host.Object, "status", "errorMessage") + errorType, _, _ := unstructured.NestedString(host.Object, "status", "errorType") + bmcAddress, _, _ := unstructured.NestedString(host.Object, "spec", "bmc", "address") + credentialsName, _, _ := unstructured.NestedString(host.Object, "spec", "bmc", "credentialsName") + online, _, _ := unstructured.NestedBool(host.Object, "spec", "online") + goodCredsName, _, _ := unstructured.NestedString(host.Object, "status", "goodCredentials", "credentials", "name") + + consumerKind, _, _ := unstructured.NestedString(host.Object, "spec", "consumerRef", "kind") + consumerName, _, _ := unstructured.NestedString(host.Object, "spec", "consumerRef", "name") + + manufacturer, _, _ := unstructured.NestedString(host.Object, "status", "hardware", "systemVendor", "manufacturer") + productName, _, _ := unstructured.NestedString(host.Object, "status", "hardware", "systemVendor", "productName") + + isUnmanaged := provisioningState == "" || provisioningState == "unmanaged" + + if isUnmanaged && bmcAddress == "" { + report.WriteString("- State: **Unmanaged** (not controlled by baremetal-operator)\n") + report.WriteString("- BMC/credentials: not applicable for unmanaged hosts\n") + if operationalStatus != "" { + fmt.Fprintf(report, "- Operational Status: %s\n", operationalStatus) + } + if consumerKind != "" { + fmt.Fprintf(report, "- Consumer: %s/%s\n", consumerKind, consumerName) + } + } else { + fmt.Fprintf(report, "- Provisioning State: %s\n", ValueOrNA(provisioningState)) + fmt.Fprintf(report, "- Operational Status: %s\n", ValueOrNA(operationalStatus)) + fmt.Fprintf(report, "- Online (desired): %t\n", online) + fmt.Fprintf(report, "- Powered On (actual): %t\n", poweredOn) + fmt.Fprintf(report, "- BMC Address: %s\n", ValueOrNA(bmcAddress)) + fmt.Fprintf(report, "- Credentials Secret: %s\n", ValueOrNA(credentialsName)) + + if goodCredsName != "" { + fmt.Fprintf(report, "- Good Credentials: %s (verified by BMO)\n", goodCredsName) + } + + if consumerKind != "" { + fmt.Fprintf(report, "- Consumer: %s/%s\n", consumerKind, consumerName) + } + + if manufacturer != "" || productName != "" { + fmt.Fprintf(report, "- Hardware: %s %s\n", manufacturer, productName) + } + + if errorMessage != "" { + fmt.Fprintf(report, "- **Error**: [%s] %s\n", errorType, errorMessage) + issues = append(issues, fmt.Sprintf("%s/%s: BMH error — [%s] %s", hostNamespace, hostName, errorType, errorMessage)) + } + + if bmcAddress == "" { + issues = append(issues, fmt.Sprintf("%s/%s: no BMC address configured", hostNamespace, hostName)) + } + + if credentialsName != "" { + credIssues := checkCredentials(ctx, coreClient, hostNamespace, credentialsName, hostName) + issues = append(issues, credIssues...) + if len(credIssues) == 0 { + report.WriteString("- Credentials: present and valid\n") + } else { + for _, ci := range credIssues { + fmt.Fprintf(report, "- **Credentials Issue**: %s\n", ci) + } + } + } else if !isUnmanaged { + issues = append(issues, fmt.Sprintf("%s/%s: no BMC credentials secret configured", hostNamespace, hostName)) + } + } + + nodeName := hostName + bmhKey := fmt.Sprintf("%s/%s", hostNamespace, hostName) + if resolved, ok := bmhToNode[bmhKey]; ok { + nodeName = resolved + } + + nodeIssues := checkNodeHealth(ctx, coreClient, nodeName) + issues = append(issues, nodeIssues...) + for _, ni := range nodeIssues { + fmt.Fprintf(report, "- **Node Issue**: %s\n", ni) + } + if len(nodeIssues) == 0 { + report.WriteString("- Node: healthy\n") + } + + report.WriteString("\n") + return issues +} + +func checkFenceAgentsRemediation(ctx context.Context, client dynamic.Interface, report *strings.Builder) []string { + var issues []string + report.WriteString("## Fencing Remediation\n\n") + + report.WriteString("### Templates\n\n") + templates, err := client.Resource(FenceAgentsRemediationTemplateGVR).List(ctx, metav1.ListOptions{}) + if err != nil { + slog.Debug("could not list FenceAgentsRemediationTemplates", "error", err) + if isCRDNotInstalled(err) { + report.WriteString("FenceAgentsRemediationTemplate CRD not installed. Cluster may use traditional pacemaker/STONITH fencing instead.\n\n") + } else { + fmt.Fprintf(report, "FenceAgentsRemediationTemplates: **error** — %v\n\n", err) + issues = append(issues, fmt.Sprintf("failed to list FenceAgentsRemediationTemplates: %v", err)) + } + } else if len(templates.Items) == 0 { + report.WriteString("No FenceAgentsRemediationTemplates found. Cluster may use traditional pacemaker/STONITH fencing instead.\n\n") + } else { + for _, tmpl := range templates.Items { + name := tmpl.GetName() + ns := tmpl.GetNamespace() + agent, _, _ := unstructured.NestedString(tmpl.Object, "spec", "template", "spec", "agent") + sharedParams, _, _ := unstructured.NestedMap(tmpl.Object, "spec", "template", "spec", "sharedparameters") + retryCount, _, _ := unstructured.NestedInt64(tmpl.Object, "spec", "template", "spec", "retrycount") + + location := name + if ns != "" { + location = fmt.Sprintf("%s/%s", ns, name) + } + fmt.Fprintf(report, "- **%s**: agent=%s", location, ValueOrNA(agent)) + if retryCount > 0 { + fmt.Fprintf(report, ", retryCount=%d", retryCount) + } + if len(sharedParams) > 0 { + var paramKeys []string + for k := range sharedParams { + paramKeys = append(paramKeys, k) + } + fmt.Fprintf(report, ", sharedParams={%s}", strings.Join(paramKeys, ", ")) + } + report.WriteString("\n") + } + } + + report.WriteString("\n### Active Remediations\n\n") + remediations, err := client.Resource(FenceAgentsRemediationGVR).List(ctx, metav1.ListOptions{}) + if err != nil { + slog.Debug("could not list FenceAgentsRemediations", "error", err) + if isCRDNotInstalled(err) { + report.WriteString("FenceAgentsRemediation CRD not installed.\n\n") + } else { + fmt.Fprintf(report, "Active Remediations: **error** — %v\n\n", err) + issues = append(issues, fmt.Sprintf("failed to list FenceAgentsRemediations: %v", err)) + } + return issues + } + + if len(remediations.Items) == 0 { + report.WriteString("No active fencing remediations.\n\n") + } else { + for _, rem := range remediations.Items { + remName := rem.GetName() + remNs := rem.GetNamespace() + agent, _, _ := unstructured.NestedString(rem.Object, "spec", "agent") + conditions, _, _ := unstructured.NestedSlice(rem.Object, "status", "conditions") + + fmt.Fprintf(report, "- **%s/%s**: agent=%s\n", remNs, remName, ValueOrNA(agent)) + for _, cond := range conditions { + condMap, ok := cond.(map[string]interface{}) + if !ok { + continue + } + condType, _ := condMap["type"].(string) + condStatus, _ := condMap["status"].(string) + message, _ := condMap["message"].(string) + fmt.Fprintf(report, " - %s=%s: %s\n", condType, condStatus, message) + } + issues = append(issues, fmt.Sprintf("active fencing remediation in progress: %s/%s", remNs, remName)) + } + } + + report.WriteString("\n") + return issues +} + +func checkNodeHealthChecks(ctx context.Context, client dynamic.Interface, report *strings.Builder) []string { + var issues []string + report.WriteString("## Node Health Checks\n\n") + + list, err := client.Resource(NodeHealthCheckGVR).List(ctx, metav1.ListOptions{}) + if err != nil { + slog.Debug("could not list NodeHealthChecks", "error", err) + if isCRDNotInstalled(err) { + report.WriteString("NodeHealthCheck CRD not installed. Cluster may use traditional pacemaker-based health monitoring.\n\n") + } else { + fmt.Fprintf(report, "NodeHealthChecks: **error** — %v\n\n", err) + issues = append(issues, fmt.Sprintf("failed to list NodeHealthChecks: %v", err)) + } + return issues + } + + if len(list.Items) == 0 { + report.WriteString("No NodeHealthCheck resources found. Cluster may use traditional pacemaker-based health monitoring.\n\n") + return issues + } + + for _, nhc := range list.Items { + nhcName := nhc.GetName() + phase, _, _ := unstructured.NestedString(nhc.Object, "status", "phase") + minHealthy, _, _ := unstructured.NestedString(nhc.Object, "spec", "minHealthy") + + selectorLabels, _, _ := unstructured.NestedStringMap(nhc.Object, "spec", "selector", "matchLabels") + + var selectorStr string + if len(selectorLabels) > 0 { + var parts []string + for k, v := range selectorLabels { + if v == "" { + parts = append(parts, k) + } else { + parts = append(parts, fmt.Sprintf("%s=%s", k, v)) + } + } + selectorStr = strings.Join(parts, ", ") + } else { + selectorStr = "(all nodes)" + } + + fmt.Fprintf(report, "- **%s**: selector={%s}\n", nhcName, selectorStr) + fmt.Fprintf(report, " - Phase: %s\n", ValueOrNA(phase)) + if minHealthy != "" { + fmt.Fprintf(report, " - Min Healthy: %s\n", minHealthy) + } + + escalating, found, _ := unstructured.NestedSlice(nhc.Object, "spec", "escalatingRemediations") + if found && len(escalating) > 0 { + report.WriteString(" - Escalating Remediations:\n") + for i, rem := range escalating { + remMap, ok := rem.(map[string]interface{}) + if !ok { + continue + } + remRef, _ := remMap["remediationTemplate"].(map[string]interface{}) + kind, _ := remRef["kind"].(string) + name, _ := remRef["name"].(string) + ns, _ := remRef["namespace"].(string) + timeout, _ := remMap["timeout"].(string) + + location := name + if ns != "" { + location = fmt.Sprintf("%s/%s", ns, name) + } + fmt.Fprintf(report, " %d. %s/%s", i+1, kind, location) + if timeout != "" { + fmt.Fprintf(report, " (timeout=%s)", timeout) + } + report.WriteString("\n") + } + } + + unhealthyConditions, found, _ := unstructured.NestedSlice(nhc.Object, "spec", "unhealthyConditions") + if found && len(unhealthyConditions) > 0 { + report.WriteString(" - Unhealthy Conditions:\n") + for _, uc := range unhealthyConditions { + ucMap, ok := uc.(map[string]interface{}) + if !ok { + continue + } + ucType, _ := ucMap["type"].(string) + ucStatus, _ := ucMap["status"].(string) + ucDuration, _ := ucMap["duration"].(string) + fmt.Fprintf(report, " - %s=%s (duration=%s)\n", ucType, ucStatus, ucDuration) + } + } + } + + report.WriteString("\n") + return issues +} + +func listBareMetalHosts(ctx context.Context, client dynamic.Interface, namespace string) ([]unstructured.Unstructured, error) { + var list *unstructured.UnstructuredList + var err error + + if namespace != "" { + list, err = client.Resource(BareMetalHostGVR).Namespace(namespace).List(ctx, metav1.ListOptions{}) + } else { + list, err = client.Resource(BareMetalHostGVR).List(ctx, metav1.ListOptions{}) + } + + if err != nil { + return nil, err + } + return list.Items, nil +} + +func checkCredentials(ctx context.Context, client corev1.CoreV1Interface, namespace, secretName, hostName string) []string { + var issues []string + + secret, err := client.Secrets(namespace).Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + issues = append(issues, fmt.Sprintf("%s/%s: credentials secret %q not found — %v", namespace, hostName, secretName, err)) + return issues + } + + if v, ok := secret.Data["username"]; !ok || len(v) == 0 { + issues = append(issues, fmt.Sprintf("%s/%s: credentials secret %q missing or empty 'username' key", namespace, hostName, secretName)) + } + if v, ok := secret.Data["password"]; !ok || len(v) == 0 { + issues = append(issues, fmt.Sprintf("%s/%s: credentials secret %q missing or empty 'password' key", namespace, hostName, secretName)) + } + + return issues +} + +func checkNodeHealth(ctx context.Context, client corev1.CoreV1Interface, hostName string) []string { + var issues []string + + node, err := client.Nodes().Get(ctx, hostName, metav1.GetOptions{}) + if err != nil { + slog.Debug("could not find Node matching BareMetalHost", "host", hostName, "error", err) + issues = append(issues, fmt.Sprintf("%s: no matching Node resource found", hostName)) + return issues + } + + for _, condition := range node.Status.Conditions { + switch condition.Type { + case "Ready": + if condition.Status != "True" { + issues = append(issues, fmt.Sprintf("node %s: NotReady — %s", hostName, condition.Message)) + } + case "MemoryPressure", "DiskPressure", "PIDPressure": + if condition.Status == "True" { + issues = append(issues, fmt.Sprintf("node %s: %s — %s", hostName, condition.Type, condition.Message)) + } + } + } + + return issues +} + +// ValueOrNA returns s if non-empty, otherwise "N/A". +func ValueOrNA(s string) string { + if s == "" { + return "N/A" + } + return s +} + +func isCRDNotInstalled(err error) bool { + if apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err) { + return true + } + var ve *api.ValidationError + return errors.As(err, &ve) && ve.Code == api.ErrorCodeResourceNotFound +} diff --git a/pkg/toolsets/tnf/fencing/fencing_test.go b/pkg/toolsets/tnf/fencing/fencing_test.go new file mode 100644 index 000000000..a39f496f8 --- /dev/null +++ b/pkg/toolsets/tnf/fencing/fencing_test.go @@ -0,0 +1,36 @@ +package fencing + +import ( + "testing" + + "github.com/containers/kubernetes-mcp-server/pkg/api" + "github.com/stretchr/testify/suite" +) + +type FencingHandlerSuite struct { + suite.Suite +} + +type staticRequest struct { + args map[string]any +} + +func (s staticRequest) GetArguments() map[string]any { + return s.args +} + +func (s *FencingHandlerSuite) TestInvalidNamespaceType() { + params := api.ToolHandlerParams{ + ToolCallRequest: staticRequest{args: map[string]any{ + "namespace": 12345, + }}, + } + result, err := checkFencingHealth(params) + s.Require().NoError(err) + s.Require().NotNil(result.Error) + s.Contains(result.Error.Error(), "failed to check fencing health") +} + +func TestFencingHandler(t *testing.T) { + suite.Run(t, new(FencingHandlerSuite)) +} diff --git a/pkg/toolsets/tnf/fencing/stonith.go b/pkg/toolsets/tnf/fencing/stonith.go new file mode 100644 index 000000000..e4e63969a --- /dev/null +++ b/pkg/toolsets/tnf/fencing/stonith.go @@ -0,0 +1,463 @@ +package fencing + +import ( + "context" + "errors" + "fmt" + "log/slog" + "math" + "strings" + "time" + + "github.com/google/jsonschema-go/jsonschema" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/utils/ptr" + + "github.com/containers/kubernetes-mcp-server/pkg/api" + "github.com/containers/kubernetes-mcp-server/pkg/cluster-diagnostics/nodesdebug" +) + +const defaultSTONITHTimeout = 120 * time.Second + +var stonithDiagnosticScript = strings.Join([]string{ + "echo '===PCS_STATUS==='", + "pcs status 2>&1", + "echo '===PCS_STONITH_CONFIG==='", + "pcs stonith config 2>&1", + "echo '===PCS_STONITH_STATUS==='", + "pcs stonith status 2>&1", + "echo '===PCS_STONITH_HISTORY==='", + "pcs stonith history 2>&1", + "echo '===PCS_QUORUM_STATUS==='", + "corosync-quorumtool 2>&1", + "echo '===PCS_PROPERTY==='", + "pcs property list 2>&1", + "echo '===END==='", +}, "; ") + +func InitSTONITH() []api.ServerTool { + return []api.ServerTool{ + { + Tool: api.Tool{ + Name: "tnf_check_stonith_status", + Description: "Check STONITH and pacemaker fencing status on a Two-Node Fencing (TNF) cluster. " + + "Creates a temporary privileged debug pod on a control-plane node to run pcs diagnostic " + + "commands. Returns pacemaker cluster state, STONITH device configuration, quorum status, " + + "and recent fencing history. The debug pod is automatically cleaned up after execution.", + InputSchema: &jsonschema.Schema{ + Type: "object", + Properties: map[string]*jsonschema.Schema{ + "node": { + Type: "string", + Description: "Name of the node to run diagnostics on. If omitted, auto-detects the first control-plane node.", + }, + "namespace": { + Type: "string", + Description: "Namespace to create the temporary debug pod in (optional, defaults to 'default').", + }, + "timeout_seconds": { + Type: "integer", + Description: "Maximum time in seconds to wait for the diagnostic commands to complete (optional, defaults to 120).", + Minimum: ptr.To(float64(1)), + }, + }, + }, + Annotations: api.ToolAnnotations{ + Title: "TNF: Check STONITH Status", + ReadOnlyHint: ptr.To(true), + DestructiveHint: ptr.To(false), + IdempotentHint: ptr.To(true), + OpenWorldHint: ptr.To(false), + }, + }, + Handler: checkSTONITHStatus, + }, + } +} + +func checkSTONITHStatus(params api.ToolHandlerParams) (*api.ToolCallResult, error) { + if IsAPIUnreachable(params.KubernetesClient) { + return api.NewToolCallResult(APIUnreachableGuide(), nil), nil + } + + p := api.WrapParams(params) + nodeName := p.OptionalString("node", "") + namespace := p.OptionalString("namespace", "") + if err := p.Err(); err != nil { + return api.NewToolCallResult("", fmt.Errorf("failed to check STONITH status: %w", err)), nil + } + + timeout := defaultSTONITHTimeout + if timeoutRaw, exists := params.GetArguments()["timeout_seconds"]; exists && timeoutRaw != nil { + t, err := parseTimeout(timeoutRaw) + if err != nil { + return api.NewToolCallResult("", fmt.Errorf("failed to check STONITH status: %w", err)), nil + } + timeout = t + } + + report, issues, err := RunSTONITHDiagnostics(params.Context, params.KubernetesClient, params.CoreV1(), nodeName, namespace, timeout) + if err != nil { + return api.NewToolCallResult(report, err), nil + } + + var reportBuilder strings.Builder + reportBuilder.WriteString(report) + + reportBuilder.WriteString("\n## Summary\n\n") + if len(issues) == 0 { + reportBuilder.WriteString("No STONITH issues detected.\n") + } else { + fmt.Fprintf(&reportBuilder, "Found %d issue(s):\n\n", len(issues)) + for i, issue := range issues { + fmt.Fprintf(&reportBuilder, "%d. %s\n", i+1, issue) + } + } + + return api.NewToolCallResult(reportBuilder.String(), nil), nil +} + +// RunSTONITHDiagnostics runs pcs diagnostics via a debug pod and returns a +// structured report and list of issues. If nodeName is empty, auto-detects +// the first control-plane node. +func RunSTONITHDiagnostics(ctx context.Context, k8sClient api.KubernetesClient, coreClient corev1client.CoreV1Interface, nodeName, namespace string, timeout time.Duration) (string, []string, error) { + if nodeName == "" { + detected, err := DetectControlPlaneNode(ctx, coreClient) + if err != nil { + return "", nil, fmt.Errorf("failed to auto-detect control-plane node: %w", err) + } + nodeName = detected + } + + client := nodesdebug.NewNodeDebug(k8sClient) + command := []string{"chroot", "/host", "sh", "-c", stonithDiagnosticScript} + + stdout, stderr, execErr := client.NodesDebugExec( + ctx, namespace, nodeName, "", command, "", "", timeout, + ) + + if execErr != nil { + output := stderr + if stdout != "" { + output = fmt.Sprintf("Partial output:\n%s\n\nError: %v", stdout, execErr) + } + return output, nil, execErr + } + + report, issues := buildSTONITHReport(nodeName, stdout) + + if stderr != "" { + slog.Debug("STONITH diagnostic stderr", "node", nodeName, "stderr", stderr) + } + + return report, issues, nil +} + +// DetectControlPlaneNode finds the first control-plane node by label. +// Checks both node-role.kubernetes.io/control-plane and the legacy +// node-role.kubernetes.io/master label. +func DetectControlPlaneNode(ctx context.Context, client corev1client.CoreV1Interface) (string, error) { + for _, label := range []string{"node-role.kubernetes.io/control-plane", "node-role.kubernetes.io/master"} { + nodes, err := client.Nodes().List(ctx, metav1.ListOptions{ + LabelSelector: label, + }) + if err != nil { + return "", fmt.Errorf("failed to list control-plane nodes: %w", err) + } + if len(nodes.Items) > 0 { + return nodes.Items[0].Name, nil + } + } + return "", errors.New("no control-plane nodes found") +} + +func buildSTONITHReport(nodeName, rawOutput string) (string, []string) { + sections := parseSections(rawOutput) + var report strings.Builder + var issues []string + + fmt.Fprintf(&report, "# TNF STONITH Status (from node: %s)\n\n", nodeName) + + pcsIssues := writePCSStatusSection(&report, sections["PCS_STATUS"]) + issues = append(issues, pcsIssues...) + + propIssues := writePropertySection(&report, sections["PCS_PROPERTY"]) + issues = append(issues, propIssues...) + + deviceIssues := writeStonithDeviceSection(&report, sections["PCS_STONITH_CONFIG"], sections["PCS_STONITH_STATUS"]) + issues = append(issues, deviceIssues...) + + quorumIssues := writeQuorumSection(&report, sections["PCS_QUORUM_STATUS"]) + issues = append(issues, quorumIssues...) + + writeHistorySection(&report, sections["PCS_STONITH_HISTORY"]) + + return report.String(), issues +} + +func parseSections(raw string) map[string]string { + sections := make(map[string]string) + markers := []string{"PCS_STATUS", "PCS_STONITH_CONFIG", "PCS_STONITH_STATUS", + "PCS_STONITH_HISTORY", "PCS_QUORUM_STATUS", "PCS_PROPERTY", "END"} + + for i, marker := range markers { + start := strings.Index(raw, "==="+marker+"===") + if start == -1 { + continue + } + contentStart := start + len("==="+marker+"===") + contentEnd := len(raw) + for j := i + 1; j < len(markers); j++ { + nextStart := strings.Index(raw, "==="+markers[j]+"===") + if nextStart != -1 { + contentEnd = nextStart + break + } + } + if contentStart < contentEnd { + sections[marker] = strings.TrimSpace(raw[contentStart:contentEnd]) + } + } + + return sections +} + +func writePCSStatusSection(report *strings.Builder, section string) []string { + var issues []string + report.WriteString("## Pacemaker Cluster\n\n") + + if section == "" { + report.WriteString("- pcs status: not available\n\n") + issues = append(issues, "pcs status command returned no output") + return issues + } + + var onlineNodes, offlineNodes, standbyNodes []string + + for _, line := range strings.Split(section, "\n") { + trimmed := strings.TrimSpace(line) + + if strings.HasPrefix(trimmed, "Cluster name:") { + fmt.Fprintf(report, "- %s\n", trimmed) + } + + if strings.Contains(trimmed, "Online:") && strings.Contains(trimmed, "[") { + nodes := extractBracketedList(trimmed) + onlineNodes = nodes + if len(nodes) > 0 { + fmt.Fprintf(report, "- Nodes Online: %s\n", strings.Join(nodes, ", ")) + } + } + if strings.Contains(trimmed, "OFFLINE:") && strings.Contains(trimmed, "[") { + nodes := extractBracketedList(trimmed) + offlineNodes = nodes + if len(nodes) > 0 { + fmt.Fprintf(report, "- Nodes Offline: %s\n", strings.Join(nodes, ", ")) + issues = append(issues, fmt.Sprintf("nodes offline: %s", strings.Join(nodes, ", "))) + } + } + if strings.Contains(trimmed, "Standby:") && strings.Contains(trimmed, "[") { + nodes := extractBracketedList(trimmed) + standbyNodes = append(standbyNodes, nodes...) + } + if strings.HasPrefix(trimmed, "* Node") && strings.Contains(trimmed, "standby") { + parts := strings.Fields(trimmed) + if len(parts) >= 3 { + name := strings.TrimSuffix(parts[2], ":") + standbyNodes = append(standbyNodes, name) + } + } + + if strings.Contains(trimmed, "STONITH") && strings.Contains(trimmed, "enabled") { + fmt.Fprintf(report, "- %s\n", trimmed) + } + + if strings.Contains(trimmed, "Failed") && strings.Contains(trimmed, "Resource Actions") { + fmt.Fprintf(report, "- **%s**\n", trimmed) + } + } + + if len(standbyNodes) > 0 { + fmt.Fprintf(report, "- Nodes Standby: %s\n", strings.Join(standbyNodes, ", ")) + issues = append(issues, fmt.Sprintf("nodes in standby: %s", strings.Join(standbyNodes, ", "))) + } + + if len(onlineNodes) == 0 && len(offlineNodes) == 0 && len(standbyNodes) == 0 { + report.WriteString("- (could not parse node status from pcs output)\n") + } + + report.WriteString("\n") + + if strings.Contains(section, "Failed Resource Actions") || strings.Contains(section, "Failed Actions") { + failedSection := extractAfter(section, "Failed") + if failedSection != "" && !strings.Contains(failedSection, "0 resource") { + issues = append(issues, "pcs reports failed resource actions") + } + } + + return issues +} + +func writePropertySection(report *strings.Builder, section string) []string { + var issues []string + + if section == "" { + return issues + } + + for _, line := range strings.Split(section, "\n") { + trimmed := strings.TrimSpace(line) + lower := strings.ToLower(trimmed) + + if strings.HasPrefix(lower, "stonith-enabled:") { + val := strings.TrimSpace(strings.SplitN(trimmed, ":", 2)[1]) + if strings.EqualFold(val, "false") { + issues = append(issues, "STONITH is disabled (stonith-enabled: false)") + } + } + if strings.HasPrefix(lower, "no-quorum-policy:") { + val := strings.TrimSpace(strings.SplitN(trimmed, ":", 2)[1]) + fmt.Fprintf(report, "- No-Quorum Policy: %s\n", val) + } + } + + return issues +} + +func writeStonithDeviceSection(report *strings.Builder, configSection, statusSection string) []string { + var issues []string + report.WriteString("## STONITH Devices\n\n") + + if configSection == "" { + report.WriteString("No STONITH device configuration found.\n\n") + issues = append(issues, "no STONITH devices configured") + return issues + } + + if strings.Contains(configSection, "NO stonith devices") || strings.Contains(configSection, "No stonith") { + report.WriteString("No STONITH devices configured.\n\n") + issues = append(issues, "no STONITH devices configured") + return issues + } + + report.WriteString("### Configuration\n\n") + report.WriteString("```\n") + report.WriteString(configSection) + report.WriteString("\n```\n\n") + + if statusSection != "" { + report.WriteString("### Status\n\n") + for _, line := range strings.Split(statusSection, "\n") { + trimmed := strings.TrimSpace(line) + if trimmed == "" { + continue + } + fmt.Fprintf(report, "- %s\n", trimmed) + + lower := strings.ToLower(trimmed) + if strings.Contains(lower, "stopped") { + issues = append(issues, fmt.Sprintf("STONITH device stopped: %s", trimmed)) + } + if strings.Contains(lower, "failed") { + issues = append(issues, fmt.Sprintf("STONITH device failed: %s", trimmed)) + } + } + report.WriteString("\n") + } + + return issues +} + +func writeQuorumSection(report *strings.Builder, section string) []string { + var issues []string + report.WriteString("## Quorum\n\n") + + if section == "" { + report.WriteString("- corosync-quorumtool: not available\n\n") + return issues + } + + for _, line := range strings.Split(section, "\n") { + trimmed := strings.TrimSpace(line) + lower := strings.ToLower(trimmed) + + if strings.HasPrefix(lower, "quorate:") { + fmt.Fprintf(report, "- %s\n", trimmed) + val := strings.TrimSpace(strings.SplitN(trimmed, ":", 2)[1]) + if strings.EqualFold(val, "No") { + issues = append(issues, "cluster is NOT quorate") + } + } + if strings.HasPrefix(lower, "expected votes:") || strings.HasPrefix(lower, "total votes:") { + fmt.Fprintf(report, "- %s\n", trimmed) + } + if strings.HasPrefix(lower, "flags:") || strings.Contains(lower, "two_node") || strings.Contains(lower, "two node") { + fmt.Fprintf(report, "- %s\n", trimmed) + } + } + + report.WriteString("\n") + return issues +} + +func writeHistorySection(report *strings.Builder, section string) { + report.WriteString("## Fencing History\n\n") + + if section == "" || strings.Contains(section, "No fencing") || strings.Contains(section, "no fencing") { + report.WriteString("No fencing events recorded.\n\n") + return + } + + for _, line := range strings.Split(section, "\n") { + trimmed := strings.TrimSpace(line) + if trimmed == "" { + continue + } + fmt.Fprintf(report, "- %s\n", trimmed) + } + report.WriteString("\n") +} + +func extractBracketedList(line string) []string { + start := strings.Index(line, "[") + end := strings.Index(line, "]") + if start == -1 || end == -1 || end <= start { + return nil + } + inner := strings.TrimSpace(line[start+1 : end]) + if inner == "" { + return nil + } + return strings.Fields(inner) +} + +func extractAfter(text, marker string) string { + idx := strings.Index(text, marker) + if idx == -1 { + return "" + } + return text[idx:] +} + +func parseTimeout(raw interface{}) (time.Duration, error) { + switch v := raw.(type) { + case float64: + if v < 1 || v != math.Trunc(v) { + return 0, errors.New("timeout_seconds must be an integer >= 1") + } + return time.Duration(int64(v)) * time.Second, nil + case int: + if v < 1 { + return 0, errors.New("timeout_seconds must be >= 1") + } + return time.Duration(v) * time.Second, nil + case int64: + if v < 1 { + return 0, errors.New("timeout_seconds must be >= 1") + } + return time.Duration(v) * time.Second, nil + default: + return 0, errors.New("timeout_seconds must be a numeric value") + } +} diff --git a/pkg/toolsets/tnf/fencing/stonith_test.go b/pkg/toolsets/tnf/fencing/stonith_test.go new file mode 100644 index 000000000..d549e8fd2 --- /dev/null +++ b/pkg/toolsets/tnf/fencing/stonith_test.go @@ -0,0 +1,180 @@ +package fencing + +import ( + "testing" + + "github.com/containers/kubernetes-mcp-server/pkg/api" + "github.com/stretchr/testify/suite" +) + +type STONITHHandlerSuite struct { + suite.Suite +} + +func (s *STONITHHandlerSuite) TestInvalidNodeType() { + params := api.ToolHandlerParams{ + ToolCallRequest: staticRequest{args: map[string]any{ + "node": 12345, + }}, + } + result, err := checkSTONITHStatus(params) + s.Require().NoError(err) + s.Require().NotNil(result.Error) + s.Contains(result.Error.Error(), "failed to check STONITH status") +} + +func (s *STONITHHandlerSuite) TestInvalidTimeoutType() { + params := api.ToolHandlerParams{ + ToolCallRequest: staticRequest{args: map[string]any{ + "timeout_seconds": "not-a-number", + }}, + } + result, err := checkSTONITHStatus(params) + s.Require().NoError(err) + s.Require().NotNil(result.Error) + s.Contains(result.Error.Error(), "timeout_seconds must be a numeric value") +} + +func (s *STONITHHandlerSuite) TestInvalidTimeoutValue() { + params := api.ToolHandlerParams{ + ToolCallRequest: staticRequest{args: map[string]any{ + "timeout_seconds": float64(0), + }}, + } + result, err := checkSTONITHStatus(params) + s.Require().NoError(err) + s.Require().NotNil(result.Error) + s.Contains(result.Error.Error(), "timeout_seconds must be an integer >= 1") +} + +func (s *STONITHHandlerSuite) TestParseSectionsBasic() { + raw := `===PCS_STATUS=== +Cluster name: ostest +Online: [ master-0 master-1 ] +===PCS_STONITH_CONFIG=== + Resource: fence_master_0 (class=stonith type=fence_ipmilan) + Attributes: fence_master_0-instance_attributes + ipaddr=192.168.1.10 + lanplus=1 +===PCS_STONITH_STATUS=== + * fence_master_0 (stonith:fence_ipmilan): Started master-1 +===PCS_STONITH_HISTORY=== +No fencing actions recorded. +===PCS_QUORUM_STATUS=== +Quorate: Yes +Expected votes: 2 +===PCS_PROPERTY=== +stonith-enabled: true +no-quorum-policy: stop +===END===` + + sections := parseSections(raw) + s.Contains(sections["PCS_STATUS"], "Cluster name: ostest") + s.Contains(sections["PCS_STONITH_CONFIG"], "fence_master_0") + s.Contains(sections["PCS_STONITH_STATUS"], "Started master-1") + s.Contains(sections["PCS_QUORUM_STATUS"], "Quorate") + s.Contains(sections["PCS_PROPERTY"], "stonith-enabled: true") +} + +func (s *STONITHHandlerSuite) TestBuildReportHealthy() { + raw := `===PCS_STATUS=== +Cluster name: ostest +Online: [ master-0 master-1 ] +OFFLINE: [ ] +===PCS_STONITH_CONFIG=== + Resource: fence_master_0 (class=stonith type=fence_ipmilan) +===PCS_STONITH_STATUS=== + * fence_master_0 (stonith:fence_ipmilan): Started master-1 + * fence_master_1 (stonith:fence_ipmilan): Started master-0 +===PCS_STONITH_HISTORY=== +No fencing actions recorded. +===PCS_QUORUM_STATUS=== +Quorate: Yes +Expected votes: 2 +===PCS_PROPERTY=== +stonith-enabled: true +no-quorum-policy: stop +===END===` + + report, issues := buildSTONITHReport("master-0", raw) + s.Contains(report, "TNF STONITH Status") + s.Contains(report, "master-0, master-1") + s.Contains(report, "fence_master_0") + s.Contains(report, "Quorate") + s.Empty(issues) +} + +func (s *STONITHHandlerSuite) TestBuildReportStonithDisabled() { + raw := `===PCS_STATUS=== +Cluster name: ostest +Online: [ master-0 master-1 ] +===PCS_STONITH_CONFIG=== + Resource: fence_master_0 (class=stonith type=fence_ipmilan) +===PCS_STONITH_STATUS=== + * fence_master_0 (stonith:fence_ipmilan): Started master-1 +===PCS_STONITH_HISTORY=== +===PCS_QUORUM_STATUS=== +Quorate: Yes +===PCS_PROPERTY=== +stonith-enabled: false +no-quorum-policy: stop +===END===` + + _, issues := buildSTONITHReport("master-0", raw) + s.Contains(issues, "STONITH is disabled (stonith-enabled: false)") +} + +func (s *STONITHHandlerSuite) TestBuildReportNodeOffline() { + raw := `===PCS_STATUS=== +Cluster name: ostest +Online: [ master-0 ] +OFFLINE: [ master-1 ] +===PCS_STONITH_CONFIG=== + Resource: fence_master_0 (class=stonith type=fence_ipmilan) +===PCS_STONITH_STATUS=== + * fence_master_0 (stonith:fence_ipmilan): Stopped +===PCS_STONITH_HISTORY=== +===PCS_QUORUM_STATUS=== +Quorate: No +===PCS_PROPERTY=== +stonith-enabled: true +===END===` + + _, issues := buildSTONITHReport("master-0", raw) + s.Contains(issues, "nodes offline: master-1") + s.Contains(issues, "STONITH device stopped: * fence_master_0\t(stonith:fence_ipmilan):\t Stopped") + s.Contains(issues, "cluster is NOT quorate") +} + +func (s *STONITHHandlerSuite) TestBuildReportNoDevices() { + raw := `===PCS_STATUS=== +Cluster name: ostest +Online: [ master-0 master-1 ] +===PCS_STONITH_CONFIG=== +NO stonith devices configured +===PCS_STONITH_STATUS=== +===PCS_STONITH_HISTORY=== +===PCS_QUORUM_STATUS=== +Quorate: Yes +===PCS_PROPERTY=== +stonith-enabled: true +===END===` + + _, issues := buildSTONITHReport("master-0", raw) + s.Contains(issues, "no STONITH devices configured") +} + +func (s *STONITHHandlerSuite) TestExtractBracketedList() { + nodes := extractBracketedList("Online: [ master-0 master-1 ]") + s.Equal([]string{"master-0", "master-1"}, nodes) + + empty := extractBracketedList("OFFLINE: [ ]") + s.Nil(empty) + + none := extractBracketedList("no brackets here") + s.Nil(none) +} + +func TestSTONITHHandler(t *testing.T) { + suite.Run(t, new(STONITHHandlerSuite)) +} diff --git a/pkg/toolsets/tnf/mcp_resources.go b/pkg/toolsets/tnf/mcp_resources.go new file mode 100644 index 000000000..9244fc65a --- /dev/null +++ b/pkg/toolsets/tnf/mcp_resources.go @@ -0,0 +1,100 @@ +package tnf + +import ( + "context" + + "github.com/containers/kubernetes-mcp-server/pkg/api" +) + +func initMCPResources() []api.ServerResource { + return []api.ServerResource{ + { + Resource: api.Resource{ + URI: "tnf://domain-knowledge/fencing", + Name: "tnf-fencing-domain-knowledge", + Description: "TNF two-node fencing domain knowledge: quorum rules, split-brain risk matrix, and recovery procedures", + MIMEType: "text/markdown", + }, + Handler: resourceFencingDomainKnowledge, + }, + } +} + +func resourceFencingDomainKnowledge(_ context.Context) (*api.ResourceContent, error) { + return &api.ResourceContent{Text: fencingDomainKnowledge}, nil +} + +const fencingDomainKnowledge = `# TNF Fencing Domain Knowledge + +## How Two-Node Fencing Works + +TNF clusters use **pacemaker** and **corosync** for high availability: + +1. **Corosync** provides cluster communication and membership between the 2 nodes +2. **Pacemaker** manages cluster resources (services, virtual IPs) and decides placement +3. **STONITH** (Shoot The Other Node In The Head) is the fencing mechanism: + - Each node has a fence device (IPMI via fence_ipmilan or Redfish via fence_redfish) that targets the OTHER node + - When a node fails or loses communication, the surviving node uses the fence device to power-cycle the failed node via its BMC + - This prevents **split-brain**: a condition where both nodes think they own shared resources, causing data corruption + +**Two-node quorum** is special: +- Normal quorum requires a majority (>50%) of votes. With 2 nodes, losing 1 node means only 50% — no majority +- Corosync uses the two_node: 1 and wait_for_all: 1 settings to handle this +- wait_for_all: Both nodes must be present at initial startup before quorum is granted +- Once running, if one node fails, the remaining node retains quorum (provided it successfully fences the other) +- The no-quorum-policy property controls what happens when quorum is lost (usually "stop" — stop all resources) + +## Split-Brain Risk Assessment + +Analyze collected data for these risk conditions: + +| Condition | Risk Level | Action Required | +|-----------|-----------|-----------------| +| STONITH disabled | **CRITICAL** | Re-enable immediately — cluster has no fencing protection | +| Fence device stopped/failed | **HIGH** | Fencing will fail on demand — check BMC connectivity | +| BMC credentials missing | **HIGH** | Fence agent cannot authenticate to BMC | +| BMC address not configured | **HIGH** | Fence agent has no target to fence | +| One node offline (pacemaker) | **MEDIUM** | Check if fencing occurred — verify other node has quorum | +| Quorum lost | **HIGH** | Resources may be stopped depending on no-quorum-policy | +| Both nodes online, healthy | **LOW** | Normal operation | + +## Common Issues and Recovery Procedures + +### 1. STONITH Disabled + +- **Symptom:** stonith-enabled: false in pcs property list +- **Risk:** No fencing protection — split-brain can occur +- **Recovery:** Verify fence devices are configured and working, then run: pcs property set stonith-enabled=true +- **Verify:** pcs stonith status shows devices Started + +### 2. BMC Unreachable / Credentials Invalid + +- **Symptom:** Fence device fails to operate; BMH shows credential errors +- **Risk:** Fencing will fail when needed — node cannot be power-cycled +- **Recovery:** Check BMC network connectivity, update the credential Secret referenced by the BareMetalHost +- **Verify:** fence_ -a -l -p -o status (from the node) + +### 3. Node Won't Rejoin After Fencing + +- **Symptom:** Node was fenced but pacemaker shows it OFFLINE +- **Recovery:** On the fenced node after it boots: check pcs status, clear stale fencing state with pcs stonith history cleanup , then pcs cluster start if needed +- **Check:** BMC power state via fence_ -o status, verify node booted successfully + +### 4. Quorum Lost (Single Node Running) + +- **Symptom:** Quorate: No in corosync-quorumtool; resources may be stopped +- **Context:** With no-quorum-policy: stop, pacemaker stops all resources when quorum is lost +- **Recovery:** If the other node is truly down and fenced, the remaining node can be given quorum with pcs quorum unblock or by using --force flags (use with caution) + +### 5. Fence Device Misconfigured + +- **Symptom:** Wrong agent type, incorrect BMC IP, or wrong parameters +- **Recovery:** pcs stonith update = to correct parameters +- **Verify:** pcs stonith config to review, fence_ -o status to test + +### 6. Fence Race (Both Nodes Fencing Each Other) + +- **Symptom:** Both nodes tried to fence each other simultaneously +- **Diagnosis:** Check pcs stonith history on both nodes — one should have won +- **Recovery:** The node that lost the fence race should have been power-cycled. If both are up, check which one has quorum and resources +` diff --git a/pkg/toolsets/tnf/mcp_resources_test.go b/pkg/toolsets/tnf/mcp_resources_test.go new file mode 100644 index 000000000..348ec7d09 --- /dev/null +++ b/pkg/toolsets/tnf/mcp_resources_test.go @@ -0,0 +1,37 @@ +package tnf + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInitMCPResources(t *testing.T) { + resources := initMCPResources() + require.Len(t, resources, 1) + + r := resources[0] + assert.Equal(t, "tnf://domain-knowledge/fencing", r.Resource.URI) + assert.Equal(t, "tnf-fencing-domain-knowledge", r.Resource.Name) + assert.Equal(t, "text/markdown", r.Resource.MIMEType) + assert.NotNil(t, r.Handler) +} + +func TestFencingDomainKnowledgeResource(t *testing.T) { + resources := initMCPResources() + require.Len(t, resources, 1) + + content, err := resources[0].Handler(context.Background()) + require.NoError(t, err) + require.NotNil(t, content) + assert.NotEmpty(t, content.Text) + + assert.Contains(t, content.Text, "Split-Brain Risk Assessment") + assert.Contains(t, content.Text, "STONITH") + assert.Contains(t, content.Text, "quorum") + assert.Contains(t, content.Text, "fence_ipmilan") + assert.Contains(t, content.Text, "Recovery") + assert.Contains(t, content.Text, "pcs property set stonith-enabled=true") +} diff --git a/pkg/toolsets/tnf/toolset.go b/pkg/toolsets/tnf/toolset.go new file mode 100644 index 000000000..1e79818d7 --- /dev/null +++ b/pkg/toolsets/tnf/toolset.go @@ -0,0 +1,44 @@ +package tnf + +import ( + "slices" + + "github.com/containers/kubernetes-mcp-server/pkg/api" + "github.com/containers/kubernetes-mcp-server/pkg/toolsets" + "github.com/containers/kubernetes-mcp-server/pkg/toolsets/tnf/fencing" +) + +type Toolset struct{} + +var _ api.Toolset = (*Toolset)(nil) + +func (t *Toolset) GetName() string { + return "tnf" +} + +func (t *Toolset) GetDescription() string { + return "Two-Node Fencing (TNF) cluster diagnostics" +} + +func (t *Toolset) GetTools(_ api.Openshift) []api.ServerTool { + return slices.Concat( + fencing.InitFencing(), + fencing.InitSTONITH(), + ) +} + +func (t *Toolset) GetPrompts() []api.ServerPrompt { + return initTNFTroubleshoot() +} + +func (t *Toolset) GetResources() []api.ServerResource { + return initMCPResources() +} + +func (t *Toolset) GetResourceTemplates() []api.ServerResourceTemplate { + return nil +} + +func init() { + toolsets.Register(&Toolset{}) +} diff --git a/pkg/toolsets/tnf/troubleshoot.go b/pkg/toolsets/tnf/troubleshoot.go new file mode 100644 index 000000000..c2ee40792 --- /dev/null +++ b/pkg/toolsets/tnf/troubleshoot.go @@ -0,0 +1,561 @@ +package tnf + +import ( + "context" + "fmt" + "log/slog" + "strings" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/dynamic" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + + "github.com/containers/kubernetes-mcp-server/pkg/api" + "github.com/containers/kubernetes-mcp-server/pkg/toolsets/tnf/fencing" +) + +const defaultTroubleshootTimeout = 120 * time.Second + +func initTNFTroubleshoot() []api.ServerPrompt { + return []api.ServerPrompt{ + { + Prompt: api.Prompt{ + Name: "tnf-troubleshoot", + Title: "TNF Fencing Troubleshoot", + Description: "Generate a step-by-step troubleshooting guide for diagnosing " + + "Two-Node Fencing (TNF) cluster issues including STONITH configuration, " + + "quorum status, BMC health, and split-brain risk assessment", + Arguments: []api.PromptArgument{ + { + Name: "node", + Description: "Node to run STONITH diagnostics on (auto-detects if omitted)", + Required: false, + }, + { + Name: "namespace", + Description: "Namespace for BareMetalHost resources (default: openshift-machine-api)", + Required: false, + }, + }, + }, + Handler: tnfTroubleshootHandler, + }, + } +} + +func tnfTroubleshootHandler(params api.PromptHandlerParams) (*api.PromptCallResult, error) { + if fencing.IsAPIUnreachable(params.KubernetesClient) { + guide := fencing.APIUnreachableGuide() + return api.NewPromptCallResult( + "Kubernetes API is unreachable — returning out-of-band troubleshooting guide", + []api.PromptMessage{ + { + Role: "user", + Content: api.PromptContent{ + Type: "text", + Text: guide, + }, + }, + { + Role: "assistant", + Content: api.PromptContent{ + Type: "text", + Text: "The Kubernetes API is unreachable. I'll walk through the out-of-band troubleshooting guide to help determine whether this is a failed install, an upgrade/reboot, or a crash scenario.", + }, + }, + }, + nil, + ), nil + } + + args := params.GetArguments() + nodeName := args["node"] + namespace := args["namespace"] + if namespace == "" { + namespace = "openshift-machine-api" + } + + ctx := params.Context + + dynamicClient := params.DynamicClient() + coreClient := params.CoreV1() + + topologyData := fetchClusterTopology(ctx, dynamicClient, coreClient) + nodeHealthData := fetchNodeHealth(ctx, coreClient) + operatorData := fetchOperatorHealth(ctx, dynamicClient) + bmhData := fetchBMHStatus(ctx, dynamicClient, coreClient, namespace) + + stonithData := fetchSTONITHData(ctx, params.KubernetesClient, coreClient, nodeName) + + remediationData := fetchRemediationStatus(ctx, dynamicClient) + + guideText := fmt.Sprintf(`# TNF Fencing Troubleshooting Guide + +Use this guide to diagnose issues with Two-Node Fencing (TNF) clusters. All relevant +cluster data has been collected below. + +--- + +## Step 1: Cluster Topology + +Verify this is a TNF cluster (2-node bare metal with pacemaker fencing). +- Expect: Platform=BareMetal, 2 control-plane nodes, TNF Profile=Yes +- If not TNF: this guide may not apply + +%s + +--- + +## Step 2: Node Health + +Both nodes must be Ready for the cluster to function. A NotReady node may trigger fencing. +- Both nodes Ready = healthy +- One node NotReady = potential fencing trigger (check pacemaker) +- Both nodes NotReady = critical — cluster is down + +%s + +--- + +## Step 3: Critical Operators + +These operators are essential for TNF cluster health: +- **etcd**: Stores all cluster state; losing quorum is fatal for 2-node clusters +- **machine-api**: Manages BareMetalHost lifecycle +- **baremetal**: Manages BMC communication and provisioning + +%s + +--- + +## Step 4: BareMetalHost & BMC Health + +BareMetalHost resources represent the physical nodes. BMC credentials must be valid +for STONITH fencing to work — the fence agent uses BMC to power-cycle nodes. +- Check: BMH provisioning state, BMC address configured, credentials secret exists +- Missing or invalid BMC credentials = fencing will fail when needed + +%s + +--- + +## Step 5: Pacemaker & STONITH Status + +This section shows the live pacemaker cluster state gathered from the node via a +debug pod. Key things to check: +- **STONITH enabled**: Must be true for production. Disabling STONITH is a split-brain risk. +- **Nodes online**: Both nodes should be in the pacemaker cluster +- **Fence devices**: Each node needs a working fence device targeting the OTHER node +- **Quorum**: Cluster must be quorate. Two-node mode uses special quorum rules. + +%s + +--- + +## Step 6: Remediation Operators + +Optional operators that automate fencing response: +- **FenceAgentsRemediation (FAR)**: Operator-driven fencing using fence agents +- **NodeHealthCheck (NHC)**: Monitors node conditions and triggers remediation + +These are complementary to pacemaker STONITH — not all clusters use them. + +%s + +--- + +## TNF Domain Knowledge + +For detailed domain knowledge about two-node fencing mechanics, split-brain +risk assessment, and common recovery procedures, read the MCP resource: + + tnf://domain-knowledge/fencing + +Reference this resource to interpret the data collected above and assess +split-brain risk. + +--- + +## Troubleshooting Analysis + +Based on the data collected above (and the domain knowledge resource), analyze: + +1. **Is this a TNF cluster?** Check topology — 2-node bare metal with pacemaker +2. **Are both nodes healthy?** Check node Ready status and pacemaker online list +3. **Is STONITH properly configured?** Devices exist, started, targeting correct nodes +4. **Are BMC credentials valid?** Secrets exist with username/password keys +5. **Is the cluster quorate?** Check corosync quorum status +6. **Any active issues?** Failed resources, active remediations, fencing events +7. **Split-brain risk level?** Use the risk matrix from the domain knowledge resource + +--- + +## Report Findings + +After analysis, report: +- **Status:** Healthy / At Risk / Degraded / Critical +- **Split-Brain Risk:** None / Low / Medium / High / Critical +- **Issues Found:** List of specific problems +- **Root Cause:** Description of the primary issue (or "None found") +- **Recommended Actions:** Specific steps to resolve issues +- **Verification:** How to confirm the fix worked +`, topologyData, nodeHealthData, operatorData, bmhData, stonithData, remediationData) + + return api.NewPromptCallResult( + "TNF fencing troubleshooting guide generated", + []api.PromptMessage{ + { + Role: "user", + Content: api.PromptContent{ + Type: "text", + Text: guideText, + }, + }, + { + Role: "assistant", + Content: api.PromptContent{ + Type: "text", + Text: "I'll analyze the collected TNF cluster data to assess fencing health, identify split-brain risks, and provide specific remediation guidance.", + }, + }, + }, + nil, + ), nil +} + +func fetchClusterTopology(ctx context.Context, dynamicClient dynamic.Interface, coreClient corev1client.CoreV1Interface) string { + var result strings.Builder + result.WriteString("### Cluster Topology\n\n") + + infra, err := dynamicClient.Resource(fencing.InfrastructureGVR).Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + slog.Debug("could not get Infrastructure CR", "error", err) + result.WriteString("*Infrastructure CR not available*\n") + return result.String() + } + + platform, _, _ := unstructured.NestedString(infra.Object, "status", "platform") + infraTopology, _, _ := unstructured.NestedString(infra.Object, "status", "infrastructureTopology") + cpTopology, _, _ := unstructured.NestedString(infra.Object, "status", "controlPlaneTopology") + + fmt.Fprintf(&result, "- **Platform:** %s\n", fencing.ValueOrNA(platform)) + fmt.Fprintf(&result, "- **Infrastructure Topology:** %s\n", fencing.ValueOrNA(infraTopology)) + fmt.Fprintf(&result, "- **Control Plane Topology:** %s\n", fencing.ValueOrNA(cpTopology)) + + nodes, err := coreClient.Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + slog.Debug("could not list nodes", "error", err) + return result.String() + } + + cpCount := 0 + for _, node := range nodes.Items { + if _, ok := node.Labels["node-role.kubernetes.io/control-plane"]; ok { + cpCount++ + } else if _, ok := node.Labels["node-role.kubernetes.io/master"]; ok { + cpCount++ + } + } + fmt.Fprintf(&result, "- **Total Nodes:** %d\n", len(nodes.Items)) + fmt.Fprintf(&result, "- **Control-Plane Nodes:** %d\n", cpCount) + + isTNF := strings.EqualFold(platform, "BareMetal") && cpCount == 2 + if isTNF { + result.WriteString("- **TNF Profile:** Yes (2-node bare metal)\n") + } else { + result.WriteString("- **TNF Profile:** No\n") + } + + return result.String() +} + +func fetchNodeHealth(ctx context.Context, coreClient corev1client.CoreV1Interface) string { + var result strings.Builder + result.WriteString("### Node Status\n\n") + + nodes, err := coreClient.Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + fmt.Fprintf(&result, "*Error listing nodes: %v*\n", err) + return result.String() + } + + if len(nodes.Items) == 0 { + result.WriteString("*No nodes found*\n") + return result.String() + } + + result.WriteString("| Node | Ready | Roles | Conditions |\n") + result.WriteString("|------|-------|-------|------------|\n") + + for _, node := range nodes.Items { + ready := "Unknown" + var conditions []string + + for _, cond := range node.Status.Conditions { + switch cond.Type { + case "Ready": + if cond.Status == "True" { + ready = "Yes" + } else { + ready = "**No**" + } + case "MemoryPressure", "DiskPressure", "PIDPressure": + if cond.Status == "True" { + conditions = append(conditions, string(cond.Type)) + } + } + } + + var roles []string + for label := range node.Labels { + if strings.HasPrefix(label, "node-role.kubernetes.io/") { + role := strings.TrimPrefix(label, "node-role.kubernetes.io/") + roles = append(roles, role) + } + } + + condStr := "None" + if len(conditions) > 0 { + condStr = strings.Join(conditions, ", ") + } + + fmt.Fprintf(&result, "| %s | %s | %s | %s |\n", + node.Name, ready, strings.Join(roles, ", "), condStr) + } + + return result.String() +} + +func fetchOperatorHealth(ctx context.Context, dynamicClient dynamic.Interface) string { + var result strings.Builder + result.WriteString("### Cluster Operators\n\n") + + tnfOperators := []string{"etcd", "machine-api", "baremetal"} + + list, err := dynamicClient.Resource(fencing.ClusterOperatorGVR).List(ctx, metav1.ListOptions{}) + if err != nil { + slog.Debug("could not list ClusterOperators", "error", err) + result.WriteString("*ClusterOperators not available (may not be an OpenShift cluster)*\n") + return result.String() + } + + result.WriteString("| Operator | Available | Degraded | Progressing |\n") + result.WriteString("|----------|-----------|----------|-------------|\n") + + found := make(map[string]bool) + for _, item := range list.Items { + name := item.GetName() + isRelevant := false + for _, op := range tnfOperators { + if name == op { + isRelevant = true + break + } + } + if !isRelevant { + continue + } + found[name] = true + + conditions, _, _ := unstructured.NestedSlice(item.Object, "status", "conditions") + available, degraded, progressing := "Unknown", "Unknown", "Unknown" + + for _, cond := range conditions { + condMap, ok := cond.(map[string]interface{}) + if !ok { + continue + } + condType, _ := condMap["type"].(string) + condStatus, _ := condMap["status"].(string) + + switch condType { + case "Available": + available = condStatus + case "Degraded": + degraded = condStatus + case "Progressing": + progressing = condStatus + } + } + + fmt.Fprintf(&result, "| %s | %s | %s | %s |\n", + name, available, degraded, progressing) + } + + for _, op := range tnfOperators { + if !found[op] { + fmt.Fprintf(&result, "| %s | **NOT FOUND** | - | - |\n", op) + } + } + + return result.String() +} + +func fetchBMHStatus(ctx context.Context, dynamicClient dynamic.Interface, coreClient corev1client.CoreV1Interface, namespace string) string { + var result strings.Builder + result.WriteString("### BareMetalHost Status\n\n") + + var list *unstructured.UnstructuredList + var err error + if namespace != "" { + list, err = dynamicClient.Resource(fencing.BareMetalHostGVR).Namespace(namespace).List(ctx, metav1.ListOptions{}) + } else { + list, err = dynamicClient.Resource(fencing.BareMetalHostGVR).List(ctx, metav1.ListOptions{}) + } + if err != nil { + fmt.Fprintf(&result, "*Error listing BareMetalHosts: %v*\n", err) + return result.String() + } + + if len(list.Items) == 0 { + result.WriteString("*No BareMetalHost resources found — this may not be a bare metal cluster*\n") + return result.String() + } + + fmt.Fprintf(&result, "Found %d BareMetalHost(s):\n\n", len(list.Items)) + + for _, host := range list.Items { + hostName := host.GetName() + hostNS := host.GetNamespace() + + provState, _, _ := unstructured.NestedString(host.Object, "status", "provisioning", "state") + opStatus, _, _ := unstructured.NestedString(host.Object, "status", "operationalStatus") + poweredOn, _, _ := unstructured.NestedBool(host.Object, "status", "poweredOn") + errorMsg, _, _ := unstructured.NestedString(host.Object, "status", "errorMessage") + bmcAddr, _, _ := unstructured.NestedString(host.Object, "spec", "bmc", "address") + credName, _, _ := unstructured.NestedString(host.Object, "spec", "bmc", "credentialsName") + + fmt.Fprintf(&result, "#### %s/%s\n\n", hostNS, hostName) + + isUnmanaged := provState == "" || provState == "unmanaged" + if isUnmanaged && bmcAddr == "" { + result.WriteString("- State: Unmanaged (not controlled by baremetal-operator)\n") + } else { + fmt.Fprintf(&result, "- Provisioning State: %s\n", fencing.ValueOrNA(provState)) + fmt.Fprintf(&result, "- Operational Status: %s\n", fencing.ValueOrNA(opStatus)) + fmt.Fprintf(&result, "- Powered On: %t\n", poweredOn) + fmt.Fprintf(&result, "- BMC Address: %s\n", fencing.ValueOrNA(bmcAddr)) + fmt.Fprintf(&result, "- Credentials Secret: %s\n", fencing.ValueOrNA(credName)) + + if errorMsg != "" { + fmt.Fprintf(&result, "- **Error:** %s\n", errorMsg) + } + + if credName != "" { + credStatus := checkCredentialSecret(ctx, coreClient, hostNS, credName) + fmt.Fprintf(&result, "- Credentials Check: %s\n", credStatus) + } else if !isUnmanaged { + result.WriteString("- Credentials Check: **No credentials secret configured**\n") + } + } + result.WriteString("\n") + } + + return result.String() +} + +func checkCredentialSecret(ctx context.Context, coreClient corev1client.CoreV1Interface, namespace, secretName string) string { + secret, err := coreClient.Secrets(namespace).Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + return fmt.Sprintf("**Secret %q not found** — %v", secretName, err) + } + + userVal, hasUser := secret.Data["username"] + passVal, hasPass := secret.Data["password"] + userValid := hasUser && len(userVal) > 0 + passValid := hasPass && len(passVal) > 0 + + if !userValid && !passValid { + return fmt.Sprintf("**Secret %q missing or empty 'username' and 'password'**", secretName) + } + if !userValid { + return fmt.Sprintf("**Secret %q missing or empty 'username'**", secretName) + } + if !passValid { + return fmt.Sprintf("**Secret %q missing or empty 'password'**", secretName) + } + return "Valid (username and password present)" +} + +func fetchSTONITHData(ctx context.Context, k8sClient api.KubernetesClient, coreClient corev1client.CoreV1Interface, nodeName string) string { + report, issues, err := fencing.RunSTONITHDiagnostics(ctx, k8sClient, coreClient, nodeName, "", defaultTroubleshootTimeout) + if err != nil { + return fmt.Sprintf("### STONITH Diagnostics\n\n*Error running STONITH diagnostics: %v*\n\n"+ + "This may indicate:\n- No control-plane nodes found\n- Debug pod could not be created\n"+ + "- pcs/corosync not installed (not a pacemaker-based cluster)\n", err) + } + + if len(issues) > 0 { + var issueSection strings.Builder + issueSection.WriteString("\n### Detected Issues\n\n") + for i, issue := range issues { + fmt.Fprintf(&issueSection, "%d. **%s**\n", i+1, issue) + } + return report + issueSection.String() + } + + return report +} + +func fetchRemediationStatus(ctx context.Context, dynamicClient dynamic.Interface) string { + var result strings.Builder + result.WriteString("### Remediation Operators\n\n") + + // FenceAgentsRemediation Templates + result.WriteString("#### FenceAgentsRemediation Templates\n\n") + templates, err := dynamicClient.Resource(fencing.FenceAgentsRemediationTemplateGVR).List(ctx, metav1.ListOptions{}) + if err != nil { + slog.Debug("could not list FAR templates", "error", err) + result.WriteString("*FenceAgentsRemediationTemplate CRD not installed — cluster may use traditional pacemaker/STONITH fencing*\n\n") + } else if len(templates.Items) == 0 { + result.WriteString("*No FenceAgentsRemediationTemplates configured*\n\n") + } else { + for _, tmpl := range templates.Items { + name := tmpl.GetName() + ns := tmpl.GetNamespace() + agent, _, _ := unstructured.NestedString(tmpl.Object, "spec", "template", "spec", "agent") + location := name + if ns != "" { + location = fmt.Sprintf("%s/%s", ns, name) + } + fmt.Fprintf(&result, "- %s: agent=%s\n", location, fencing.ValueOrNA(agent)) + } + result.WriteString("\n") + } + + // Active remediations + result.WriteString("#### Active Remediations\n\n") + remediations, err := dynamicClient.Resource(fencing.FenceAgentsRemediationGVR).List(ctx, metav1.ListOptions{}) + if err != nil { + slog.Debug("could not list FAR remediations", "error", err) + result.WriteString("*Could not check active remediations*\n\n") + } else if len(remediations.Items) == 0 { + result.WriteString("*No active fencing remediations*\n\n") + } else { + for _, rem := range remediations.Items { + agent, _, _ := unstructured.NestedString(rem.Object, "spec", "agent") + fmt.Fprintf(&result, "- **%s/%s**: agent=%s\n", rem.GetNamespace(), rem.GetName(), fencing.ValueOrNA(agent)) + } + result.WriteString("\n") + } + + // NodeHealthChecks + result.WriteString("#### NodeHealthCheck\n\n") + nhcList, err := dynamicClient.Resource(fencing.NodeHealthCheckGVR).List(ctx, metav1.ListOptions{}) + if err != nil { + slog.Debug("could not list NHC", "error", err) + result.WriteString("*NodeHealthCheck CRD not installed*\n\n") + } else if len(nhcList.Items) == 0 { + result.WriteString("*No NodeHealthCheck resources configured*\n\n") + } else { + for _, nhc := range nhcList.Items { + phase, _, _ := unstructured.NestedString(nhc.Object, "status", "phase") + fmt.Fprintf(&result, "- %s: phase=%s\n", nhc.GetName(), fencing.ValueOrNA(phase)) + } + result.WriteString("\n") + } + + return result.String() +} diff --git a/pkg/toolsets/tnf/troubleshoot_test.go b/pkg/toolsets/tnf/troubleshoot_test.go new file mode 100644 index 000000000..ad5cff0fe --- /dev/null +++ b/pkg/toolsets/tnf/troubleshoot_test.go @@ -0,0 +1,371 @@ +package tnf + +import ( + "context" + "testing" + + "github.com/containers/kubernetes-mcp-server/pkg/api" + "github.com/containers/kubernetes-mcp-server/pkg/toolsets/tnf/fencing" + "github.com/stretchr/testify/suite" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic/fake" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" +) + +type mockPromptCallRequest struct { + args map[string]string +} + +func (m *mockPromptCallRequest) GetArguments() map[string]string { + return m.args +} + +var _ api.PromptCallRequest = (*mockPromptCallRequest)(nil) + +type fakeCoreV1 struct { + corev1client.CoreV1Interface + nodes []*corev1.Node + secrets []*corev1.Secret +} + +func (f *fakeCoreV1) Nodes() corev1client.NodeInterface { + return &fakeNodes{nodes: f.nodes} +} + +func (f *fakeCoreV1) Secrets(namespace string) corev1client.SecretInterface { + return &fakeSecrets{namespace: namespace, secrets: f.secrets} +} + +type fakeNodes struct { + corev1client.NodeInterface + nodes []*corev1.Node +} + +func (f *fakeNodes) List(_ context.Context, _ metav1.ListOptions) (*corev1.NodeList, error) { + items := make([]corev1.Node, len(f.nodes)) + for i, n := range f.nodes { + items[i] = *n + } + return &corev1.NodeList{Items: items}, nil +} + +type fakeSecrets struct { + corev1client.SecretInterface + namespace string + secrets []*corev1.Secret +} + +func (f *fakeSecrets) Get(_ context.Context, name string, _ metav1.GetOptions) (*corev1.Secret, error) { + for _, s := range f.secrets { + if s.Name == name && s.Namespace == f.namespace { + return s, nil + } + } + return nil, apierrors.NewNotFound(schema.GroupResource{Resource: "secrets"}, name) +} + +func newFakeCoreV1(objects ...runtime.Object) corev1client.CoreV1Interface { + f := &fakeCoreV1{} + for _, obj := range objects { + switch o := obj.(type) { + case *corev1.Node: + f.nodes = append(f.nodes, o) + case *corev1.Secret: + f.secrets = append(f.secrets, o) + } + } + return f +} + +type TNFTroubleshootSuite struct { + suite.Suite +} + +func (s *TNFTroubleshootSuite) TestPromptRegistration() { + prompts := initTNFTroubleshoot() + s.Require().Len(prompts, 1) + s.Equal("tnf-troubleshoot", prompts[0].Prompt.Name) + s.Equal("TNF Fencing Troubleshoot", prompts[0].Prompt.Title) + s.Len(prompts[0].Prompt.Arguments, 2) + s.Equal("node", prompts[0].Prompt.Arguments[0].Name) + s.False(prompts[0].Prompt.Arguments[0].Required) + s.Equal("namespace", prompts[0].Prompt.Arguments[1].Name) + s.False(prompts[0].Prompt.Arguments[1].Required) + s.NotNil(prompts[0].Handler) +} + +func (s *TNFTroubleshootSuite) TestToolsetGetPrompts() { + toolset := &Toolset{} + prompts := toolset.GetPrompts() + s.Require().Len(prompts, 1) + s.Equal("tnf-troubleshoot", prompts[0].Prompt.Name) +} + +func (s *TNFTroubleshootSuite) TestFetchClusterTopology() { + ctx := context.Background() + gvrToListKind := map[schema.GroupVersionResource]string{ + fencing.InfrastructureGVR: "InfrastructureList", + } + + s.Run("detects platform from infrastructure CR", func() { + infra := &unstructured.Unstructured{} + infra.SetUnstructuredContent(map[string]interface{}{ + "apiVersion": "config.openshift.io/v1", + "kind": "Infrastructure", + "metadata": map[string]interface{}{ + "name": "cluster", + }, + "status": map[string]interface{}{ + "platform": "BareMetal", + "infrastructureTopology": "HighlyAvailable", + "controlPlaneTopology": "HighlyAvailable", + }, + }) + + node1 := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "master-0", + Labels: map[string]string{"node-role.kubernetes.io/master": ""}, + }, + } + node2 := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "master-1", + Labels: map[string]string{"node-role.kubernetes.io/master": ""}, + }, + } + + dynamicClient := fake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gvrToListKind, infra) + coreClient := newFakeCoreV1(node1, node2) + + result := fetchClusterTopology(ctx, dynamicClient, coreClient) + s.Contains(result, "BareMetal") + s.Contains(result, "TNF Profile:** Yes") + s.Contains(result, "Total Nodes:** 2") + }) + + s.Run("handles missing infrastructure CR", func() { + dynamicClient := fake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gvrToListKind) + coreClient := newFakeCoreV1() + + result := fetchClusterTopology(ctx, dynamicClient, coreClient) + s.Contains(result, "not available") + }) +} + +func (s *TNFTroubleshootSuite) TestFetchNodeHealth() { + ctx := context.Background() + + s.Run("reports node status table", func() { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "master-0", + Labels: map[string]string{ + "node-role.kubernetes.io/master": "", + "node-role.kubernetes.io/control-plane": "", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + {Type: "Ready", Status: "True"}, + }, + }, + } + + coreClient := newFakeCoreV1(node) + result := fetchNodeHealth(ctx, coreClient) + s.Contains(result, "master-0") + s.Contains(result, "Yes") + s.Contains(result, "master") + }) + + s.Run("reports NotReady node", func() { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "master-0", + Labels: map[string]string{"node-role.kubernetes.io/master": ""}, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + {Type: "Ready", Status: "False"}, + }, + }, + } + + coreClient := newFakeCoreV1(node) + result := fetchNodeHealth(ctx, coreClient) + s.Contains(result, "**No**") + }) +} + +func (s *TNFTroubleshootSuite) TestFetchOperatorHealth() { + ctx := context.Background() + gvrToListKind := map[schema.GroupVersionResource]string{ + fencing.ClusterOperatorGVR: "ClusterOperatorList", + } + + s.Run("reports operator status", func() { + etcdOp := &unstructured.Unstructured{} + etcdOp.SetUnstructuredContent(map[string]interface{}{ + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterOperator", + "metadata": map[string]interface{}{ + "name": "etcd", + }, + "status": map[string]interface{}{ + "conditions": []interface{}{ + map[string]interface{}{ + "type": "Available", + "status": "True", + }, + map[string]interface{}{ + "type": "Degraded", + "status": "False", + }, + }, + }, + }) + + dynamicClient := fake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gvrToListKind, etcdOp) + result := fetchOperatorHealth(ctx, dynamicClient) + s.Contains(result, "etcd") + s.Contains(result, "True") + s.Contains(result, "machine-api") + s.Contains(result, "NOT FOUND") + }) +} + +func (s *TNFTroubleshootSuite) TestFetchBMHStatus() { + ctx := context.Background() + gvrToListKind := map[schema.GroupVersionResource]string{ + fencing.BareMetalHostGVR: "BareMetalHostList", + } + + s.Run("reports BMH details", func() { + bmh := &unstructured.Unstructured{} + bmh.SetUnstructuredContent(map[string]interface{}{ + "apiVersion": "metal3.io/v1alpha1", + "kind": "BareMetalHost", + "metadata": map[string]interface{}{ + "name": "master-0", + "namespace": "openshift-machine-api", + }, + "spec": map[string]interface{}{ + "bmc": map[string]interface{}{ + "address": "redfish://192.168.1.10/redfish/v1/Systems/1", + "credentialsName": "master-0-bmc-secret", + }, + "online": true, + }, + "status": map[string]interface{}{ + "provisioning": map[string]interface{}{ + "state": "externally provisioned", + }, + "operationalStatus": "OK", + "poweredOn": true, + }, + }) + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "master-0-bmc-secret", + Namespace: "openshift-machine-api", + }, + Data: map[string][]byte{ + "username": []byte("admin"), + "password": []byte("secret"), + }, + } + + dynamicClient := fake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gvrToListKind, bmh) + coreClient := newFakeCoreV1(secret) + + result := fetchBMHStatus(ctx, dynamicClient, coreClient, "openshift-machine-api") + s.Contains(result, "master-0") + s.Contains(result, "redfish://192.168.1.10") + s.Contains(result, "externally provisioned") + s.Contains(result, "Valid") + }) + + s.Run("handles no BMH resources", func() { + dynamicClient := fake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gvrToListKind) + coreClient := newFakeCoreV1() + + result := fetchBMHStatus(ctx, dynamicClient, coreClient, "openshift-machine-api") + s.Contains(result, "No BareMetalHost resources found") + }) +} + +func (s *TNFTroubleshootSuite) TestFetchRemediationStatus() { + ctx := context.Background() + gvrToListKind := map[schema.GroupVersionResource]string{ + fencing.FenceAgentsRemediationTemplateGVR: "FenceAgentsRemediationTemplateList", + fencing.FenceAgentsRemediationGVR: "FenceAgentsRemediationList", + fencing.NodeHealthCheckGVR: "NodeHealthCheckList", + } + + s.Run("reports empty state", func() { + dynamicClient := fake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gvrToListKind) + result := fetchRemediationStatus(ctx, dynamicClient) + s.Contains(result, "No FenceAgentsRemediationTemplates configured") + s.Contains(result, "No active fencing remediations") + s.Contains(result, "No NodeHealthCheck resources configured") + }) +} + +func (s *TNFTroubleshootSuite) TestCheckCredentialSecret() { + ctx := context.Background() + + s.Run("valid credentials", func() { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bmc-secret", + Namespace: "ns", + }, + Data: map[string][]byte{ + "username": []byte("admin"), + "password": []byte("pass"), + }, + } + + coreClient := newFakeCoreV1(secret) + result := checkCredentialSecret(ctx, coreClient, "ns", "bmc-secret") + s.Contains(result, "Valid") + }) + + s.Run("missing username", func() { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bmc-secret", + Namespace: "ns", + }, + Data: map[string][]byte{ + "password": []byte("pass"), + }, + } + + coreClient := newFakeCoreV1(secret) + result := checkCredentialSecret(ctx, coreClient, "ns", "bmc-secret") + s.Contains(result, "missing or empty 'username'") + }) + + s.Run("missing secret", func() { + coreClient := newFakeCoreV1() + result := checkCredentialSecret(ctx, coreClient, "ns", "nonexistent") + s.Contains(result, "not found") + }) +} + +func (s *TNFTroubleshootSuite) TestValueOrNA() { + s.Equal("N/A", fencing.ValueOrNA("")) + s.Equal("test", fencing.ValueOrNA("test")) +} + +func TestTNFTroubleshoot(t *testing.T) { + suite.Run(t, new(TNFTroubleshootSuite)) +} From 0e1e2efac9ad3908db1a86d0db4fc3fcb836957c Mon Sep 17 00:00:00 2001 From: Douglas Hensel Date: Tue, 7 Jul 2026 11:54:09 -0400 Subject: [PATCH 2/4] Add TNF eval tasks and snapshot test Add eval tasks for the TNF toolset covering all three difficulty levels: - check-fencing-config (easy): validates BMC credentials and cluster topology - check-stonith-status (medium): checks pacemaker/STONITH device status - diagnose-fencing-health (hard): full fencing health assessment with split-brain risk analysis Register TNF taskSet in both claude-code and openai-agent eval configs. Add TNF to TestGranularToolsetsTools snapshot test with generated snapshot JSON. Co-Authored-By: Claude Opus 4.6 --- evals/claude-code/eval.yaml | 10 ++++ evals/openai-agent/eval.yaml | 10 ++++ .../tasks/tnf/check-fencing-config/task.yaml | 15 ++++++ .../tasks/tnf/check-stonith-status/task.yaml | 15 ++++++ .../tnf/diagnose-fencing-health/task.yaml | 15 ++++++ pkg/mcp/testdata/toolsets-tnf-tools.json | 53 +++++++++++++++++++ pkg/mcp/toolsets_test.go | 2 + 7 files changed, 120 insertions(+) create mode 100644 evals/tasks/tnf/check-fencing-config/task.yaml create mode 100644 evals/tasks/tnf/check-stonith-status/task.yaml create mode 100644 evals/tasks/tnf/diagnose-fencing-health/task.yaml create mode 100644 pkg/mcp/testdata/toolsets-tnf-tools.json diff --git a/evals/claude-code/eval.yaml b/evals/claude-code/eval.yaml index 3584ae02c..dfcc7a508 100644 --- a/evals/claude-code/eval.yaml +++ b/evals/claude-code/eval.yaml @@ -102,3 +102,13 @@ config: toolPattern: ".*" minToolCalls: 1 maxToolCalls: 20 + # TNF (Two-Node Fencing) tasks - requires 2-node bare metal cluster + - glob: ../tasks/tnf/*/*.yaml + labelSelector: + suite: tnf + assertions: + toolsUsed: + - server: kubernetes + toolPattern: "tnf_.*" + minToolCalls: 1 + maxToolCalls: 10 diff --git a/evals/openai-agent/eval.yaml b/evals/openai-agent/eval.yaml index d3745d45d..1eed6e0fe 100644 --- a/evals/openai-agent/eval.yaml +++ b/evals/openai-agent/eval.yaml @@ -102,3 +102,13 @@ config: toolPattern: ".*" minToolCalls: 1 maxToolCalls: 20 + # TNF (Two-Node Fencing) tasks - requires 2-node bare metal cluster + - glob: ../tasks/tnf/*/*.yaml + labelSelector: + suite: tnf + assertions: + toolsUsed: + - server: kubernetes + toolPattern: "tnf_.*" + minToolCalls: 1 + maxToolCalls: 10 diff --git a/evals/tasks/tnf/check-fencing-config/task.yaml b/evals/tasks/tnf/check-fencing-config/task.yaml new file mode 100644 index 000000000..fc3f11ef8 --- /dev/null +++ b/evals/tasks/tnf/check-fencing-config/task.yaml @@ -0,0 +1,15 @@ +kind: Task +metadata: + labels: + suite: tnf + name: check-fencing-config + difficulty: easy +steps: + prompt: + inline: Check the fencing configuration for this Two-Node Fencing cluster. Verify that BareMetalHost resources exist, BMC credentials are valid, and the cluster topology matches a TNF profile. + verify: + contains: "BMC" + assertions: + toolsUsed: + - server: kubernetes + toolPattern: "tnf_check_fencing_config" diff --git a/evals/tasks/tnf/check-stonith-status/task.yaml b/evals/tasks/tnf/check-stonith-status/task.yaml new file mode 100644 index 000000000..d37501caf --- /dev/null +++ b/evals/tasks/tnf/check-stonith-status/task.yaml @@ -0,0 +1,15 @@ +kind: Task +metadata: + labels: + suite: tnf + name: check-stonith-status + difficulty: medium +steps: + prompt: + inline: Check the pacemaker and STONITH status on this cluster. Report which STONITH devices are configured, whether they are started, and the current quorum status. + verify: + contains: "STONITH" + assertions: + toolsUsed: + - server: kubernetes + toolPattern: "tnf_check_stonith_status" diff --git a/evals/tasks/tnf/diagnose-fencing-health/task.yaml b/evals/tasks/tnf/diagnose-fencing-health/task.yaml new file mode 100644 index 000000000..b532390c1 --- /dev/null +++ b/evals/tasks/tnf/diagnose-fencing-health/task.yaml @@ -0,0 +1,15 @@ +kind: Task +metadata: + labels: + suite: tnf + name: diagnose-fencing-health + difficulty: hard +steps: + prompt: + inline: Perform a full fencing health assessment for this Two-Node Fencing cluster. Check the fencing configuration, STONITH status, and analyze whether there is any split-brain risk. Provide a summary with an overall health status and any recommended actions. + verify: + contains: "fencing" + assertions: + toolsUsed: + - server: kubernetes + toolPattern: "tnf_.*" diff --git a/pkg/mcp/testdata/toolsets-tnf-tools.json b/pkg/mcp/testdata/toolsets-tnf-tools.json new file mode 100644 index 000000000..0f68619f3 --- /dev/null +++ b/pkg/mcp/testdata/toolsets-tnf-tools.json @@ -0,0 +1,53 @@ +[ + { + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": true, + "title": "TNF: Check Fencing Config" + }, + "description": "Check fencing configuration and readiness for a Two-Node Fencing (TNF) cluster. Validates cluster topology, critical operator health (etcd, machine-api, baremetal), Machine/Node/BareMetalHost correlation, BMC addresses and credentials, FenceAgentsRemediation templates and active remediations, and NodeHealthCheck resources. Returns a diagnostic summary identifying configuration issues that could prevent fencing from functioning correctly.", + "inputSchema": { + "properties": { + "namespace": { + "description": "Namespace containing BareMetalHost resources (e.g. 'openshift-machine-api'). If omitted, searches all namespaces.", + "type": "string" + } + }, + "type": "object" + }, + "name": "tnf_check_fencing_config", + "title": "TNF: Check Fencing Config" + }, + { + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": true, + "title": "TNF: Check STONITH Status" + }, + "description": "Check STONITH and pacemaker fencing status on a Two-Node Fencing (TNF) cluster. Creates a temporary privileged debug pod on a control-plane node to run pcs diagnostic commands. Returns pacemaker cluster state, STONITH device configuration, quorum status, and recent fencing history. The debug pod is automatically cleaned up after execution.", + "inputSchema": { + "properties": { + "namespace": { + "description": "Namespace to create the temporary debug pod in (optional, defaults to 'default').", + "type": "string" + }, + "node": { + "description": "Name of the node to run diagnostics on. If omitted, auto-detects the first control-plane node.", + "type": "string" + }, + "timeout_seconds": { + "description": "Maximum time in seconds to wait for the diagnostic commands to complete (optional, defaults to 120).", + "minimum": 1, + "type": "integer" + } + }, + "type": "object" + }, + "name": "tnf_check_stonith_status", + "title": "TNF: Check STONITH Status" + } +] diff --git a/pkg/mcp/toolsets_test.go b/pkg/mcp/toolsets_test.go index cd7264fa8..19cd170fb 100644 --- a/pkg/mcp/toolsets_test.go +++ b/pkg/mcp/toolsets_test.go @@ -23,6 +23,7 @@ import ( mgToolset "github.com/containers/kubernetes-mcp-server/pkg/toolsets/mustgather" "github.com/containers/kubernetes-mcp-server/pkg/toolsets/openshift" "github.com/containers/kubernetes-mcp-server/pkg/toolsets/tekton" + "github.com/containers/kubernetes-mcp-server/pkg/toolsets/tnf" "github.com/modelcontextprotocol/go-sdk/mcp" "github.com/stretchr/testify/suite" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" @@ -182,6 +183,7 @@ func (s *ToolsetsSuite) TestGranularToolsetsTools() { &kubevirt.Toolset{}, &tekton.Toolset{}, &clusterDiagnosticsToolset.Toolset{}, + &tnf.Toolset{}, } for _, testCase := range testCases { s.Run("Toolset "+testCase.GetName(), func() { From b7300400f7072f684d7bfdbd38023c5e2c0b0ee6 Mon Sep 17 00:00:00 2001 From: Douglas Hensel Date: Tue, 14 Jul 2026 13:38:22 -0400 Subject: [PATCH 3/4] Skip FAR/NHC checks on TNF clusters and port eval knobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TNF clusters use Pacemaker/STONITH directly — skip FenceAgentsRemediation and NodeHealthCheck checks that only apply to non-TNF bare metal clusters. Also falls back from pcs property config to pcs property list for compatibility. Ports claude-agent-acp, AGENT/SUITE/MODEL knobs, and run-server fixes from containers/kubernetes-mcp-server. Co-Authored-By: Claude Opus 4.6 --- build/evals.mk | 44 +++++++++++++++++++-------- pkg/toolsets/tnf/fencing/fencing.go | 23 ++++++++------ pkg/toolsets/tnf/fencing/stonith.go | 2 +- pkg/toolsets/tnf/troubleshoot.go | 23 +++++++------- pkg/toolsets/tnf/troubleshoot_test.go | 6 ++-- 5 files changed, 61 insertions(+), 37 deletions(-) diff --git a/build/evals.mk b/build/evals.mk index abe8679ee..ff15cdf3d 100644 --- a/build/evals.mk +++ b/build/evals.mk @@ -7,8 +7,21 @@ MCP_CONFIG_DIR ?= dev/config/mcp-configs MCPCHECKER = $(shell pwd)/_output/tools/bin/mcpchecker MCPCHECKER_VERSION ?= latest -EVAL_CONFIG ?= evals/openai-agent/eval.yaml -EVAL_LABEL_SELECTOR ?= suite=kubernetes +CLAUDE_AGENT_ACP_VERSION ?= latest + +# High-level knobs for local single-suite runs, e.g.: +# make run-evals SUITE=kubevirt AGENT=claude-code MODEL=sonnet +# AGENT selects the eval config, SUITE selects the task suite label, and MODEL +# sets ANTHROPIC_MODEL for the claude-agent-acp adapter (the openai-agent ignores it). +AGENT ?= openai-agent +SUITE ?= core +MODEL ?= + +# Prefer a per-suite eval config when one exists: those carry no llmJudge, so a +# local run needs no OpenAI key. Otherwise fall back to the agent's top-level +# config (the one CI uses; its llmJudge requires an OpenAI key). +EVAL_CONFIG ?= $(or $(wildcard evals/tasks/$(SUITE)/$(AGENT)/eval.yaml),evals/$(AGENT)/eval.yaml) +EVAL_LABEL_SELECTOR ?= suite=$(SUITE) EVAL_TASK_FILTER ?= EVAL_VERBOSE ?= false @@ -24,9 +37,20 @@ mcpchecker: ##@ Evals +# Install the claude-agent-acp adapter, required by the claude-code eval agent +# (evals/claude-code/agent.yaml runs `claude-agent-acp`). No-op if already present. +.PHONY: claude-agent-acp +claude-agent-acp: ## Install the claude-agent-acp adapter for the claude-code eval agent + @command -v claude-agent-acp >/dev/null 2>&1 || { \ + set -e ;\ + echo "Installing claude-agent-acp@$(CLAUDE_AGENT_ACP_VERSION) (npm install -g)..." ;\ + npm install -g @agentclientprotocol/claude-agent-acp@$(CLAUDE_AGENT_ACP_VERSION) ;\ + echo "✅ claude-agent-acp installed" ;\ + } + .PHONY: run-evals -run-evals: mcpchecker ## Run mcpchecker evaluations against the MCP server - $(MCPCHECKER) check $(EVAL_CONFIG) \ +run-evals: mcpchecker $(if $(filter claude-code,$(AGENT)),claude-agent-acp) ## Run mcpchecker evals (knobs: SUITE, AGENT, MODEL; see evals/README.md) + $(if $(MODEL),ANTHROPIC_MODEL=$(MODEL) )$(MCPCHECKER) check $(EVAL_CONFIG) \ $(if $(EVAL_LABEL_SELECTOR),--label-selector $(EVAL_LABEL_SELECTOR),) \ $(if $(EVAL_TASK_FILTER),--run "$(EVAL_TASK_FILTER)",) \ $(if $(filter true,$(EVAL_VERBOSE)),--verbose,) \ @@ -52,20 +76,16 @@ diff-evals: mcpchecker ## Diff latest mcpchecker results against baseline .PHONY: run-server run-server: build ## Start MCP server in background and wait for health check @echo "Starting MCP server on port $(MCP_PORT)..." - @if [ -n "$(MCP_LOG_FILE)" ]; then \ - echo "Redirecting server logs to $(MCP_LOG_FILE)"; \ - REDIRECT="> $(MCP_LOG_FILE) 2>&1"; \ - fi; \ - if [ -n "$(TOOLSETS)" ]; then \ - eval "./$(BINARY_NAME) --port $(MCP_PORT) --toolsets $(TOOLSETS) --config-dir $(MCP_CONFIG_DIR) $$REDIRECT &" echo $$! > .mcp-server.pid; \ + @if [ -n "$(TOOLSETS)" ]; then \ + ./$(BINARY_NAME) --port $(MCP_PORT) --toolsets $(TOOLSETS) --config-dir $(MCP_CONFIG_DIR) & echo $$! > .mcp-server.pid; \ else \ - eval "./$(BINARY_NAME) --port $(MCP_PORT) $$REDIRECT &" echo $$! > .mcp-server.pid; \ + ./$(BINARY_NAME) --port $(MCP_PORT) & echo $$! > .mcp-server.pid; \ fi @echo "MCP server started with PID $$(cat .mcp-server.pid)" @echo "Waiting for MCP server to be ready..." @elapsed=0; \ while [ $$elapsed -lt $(MCP_HEALTH_TIMEOUT) ]; do \ - if curl -s http://localhost:$(MCP_PORT)/health > /dev/null 2>&1; then \ + if curl -fsS http://localhost:$(MCP_PORT)/healthz > /dev/null 2>&1; then \ echo "MCP server is ready"; \ exit 0; \ fi; \ diff --git a/pkg/toolsets/tnf/fencing/fencing.go b/pkg/toolsets/tnf/fencing/fencing.go index 38363e0a5..fdcbb5aa7 100644 --- a/pkg/toolsets/tnf/fencing/fencing.go +++ b/pkg/toolsets/tnf/fencing/fencing.go @@ -55,7 +55,7 @@ func InitFencing() []api.ServerTool { Description: "Check fencing configuration and readiness for a Two-Node Fencing (TNF) cluster. " + "Validates cluster topology, critical operator health (etcd, machine-api, baremetal), " + "Machine/Node/BareMetalHost correlation, BMC addresses and credentials, " + - "FenceAgentsRemediation templates and active remediations, and NodeHealthCheck resources. " + + "pacemaker/STONITH fencing status via Kubernetes resources. " + "Returns a diagnostic summary identifying configuration issues that could " + "prevent fencing from functioning correctly.", InputSchema: &jsonschema.Schema{ @@ -99,7 +99,7 @@ func checkFencingHealth(params api.ToolHandlerParams) (*api.ToolCallResult, erro report.WriteString("# TNF Fencing Health Check\n\n") - topoIssues := checkInfrastructureTopology(params.Context, dynamicClient, coreClient, &report) + isTNF, topoIssues := checkInfrastructureTopology(params.Context, dynamicClient, coreClient, &report) issues = append(issues, topoIssues...) coIssues := checkClusterOperatorHealth(params.Context, dynamicClient, &report) @@ -128,11 +128,13 @@ func checkFencingHealth(params api.ToolHandlerParams) (*api.ToolCallResult, erro } } - farIssues := checkFenceAgentsRemediation(params.Context, dynamicClient, &report) - issues = append(issues, farIssues...) + if !isTNF { + farIssues := checkFenceAgentsRemediation(params.Context, dynamicClient, &report) + issues = append(issues, farIssues...) - nhcIssues := checkNodeHealthChecks(params.Context, dynamicClient, &report) - issues = append(issues, nhcIssues...) + nhcIssues := checkNodeHealthChecks(params.Context, dynamicClient, &report) + issues = append(issues, nhcIssues...) + } report.WriteString("## Summary\n\n") if len(issues) == 0 { @@ -147,8 +149,9 @@ func checkFencingHealth(params api.ToolHandlerParams) (*api.ToolCallResult, erro return api.NewToolCallResult(report.String(), nil), nil } -func checkInfrastructureTopology(ctx context.Context, client dynamic.Interface, coreClient corev1.CoreV1Interface, report *strings.Builder) []string { +func checkInfrastructureTopology(ctx context.Context, client dynamic.Interface, coreClient corev1.CoreV1Interface, report *strings.Builder) (bool, []string) { var issues []string + isTNF := false report.WriteString("## Cluster Topology\n\n") infra, err := client.Resource(InfrastructureGVR).Get(ctx, "cluster", metav1.GetOptions{}) @@ -160,7 +163,7 @@ func checkInfrastructureTopology(ctx context.Context, client dynamic.Interface, fmt.Fprintf(report, "- Infrastructure CR: **error** — %v\n\n", err) issues = append(issues, fmt.Sprintf("failed to get Infrastructure CR: %v", err)) } - return issues + return isTNF, issues } platform, _, err := unstructured.NestedString(infra.Object, "status", "platform") @@ -195,7 +198,7 @@ func checkInfrastructureTopology(ctx context.Context, client dynamic.Interface, fmt.Fprintf(report, "- Total nodes: %d\n", len(nodes.Items)) fmt.Fprintf(report, "- Control-plane nodes: %d\n", cpCount) - isTNF := strings.EqualFold(platform, "BareMetal") && cpCount == 2 + isTNF = strings.EqualFold(platform, "BareMetal") && cpCount == 2 if isTNF { report.WriteString("- TNF Profile: **Yes** (2-node bare metal)\n") } else { @@ -210,7 +213,7 @@ func checkInfrastructureTopology(ctx context.Context, client dynamic.Interface, } report.WriteString("\n") - return issues + return isTNF, issues } func checkClusterOperatorHealth(ctx context.Context, client dynamic.Interface, report *strings.Builder) []string { diff --git a/pkg/toolsets/tnf/fencing/stonith.go b/pkg/toolsets/tnf/fencing/stonith.go index e4e63969a..fd4ca9576 100644 --- a/pkg/toolsets/tnf/fencing/stonith.go +++ b/pkg/toolsets/tnf/fencing/stonith.go @@ -32,7 +32,7 @@ var stonithDiagnosticScript = strings.Join([]string{ "echo '===PCS_QUORUM_STATUS==='", "corosync-quorumtool 2>&1", "echo '===PCS_PROPERTY==='", - "pcs property list 2>&1", + "{ pcs property config 2>&1 || pcs property list 2>&1; }", "echo '===END==='", }, "; ") diff --git a/pkg/toolsets/tnf/troubleshoot.go b/pkg/toolsets/tnf/troubleshoot.go index c2ee40792..a0096e12f 100644 --- a/pkg/toolsets/tnf/troubleshoot.go +++ b/pkg/toolsets/tnf/troubleshoot.go @@ -82,14 +82,19 @@ func tnfTroubleshootHandler(params api.PromptHandlerParams) (*api.PromptCallResu dynamicClient := params.DynamicClient() coreClient := params.CoreV1() - topologyData := fetchClusterTopology(ctx, dynamicClient, coreClient) + topologyData, isTNF := fetchClusterTopology(ctx, dynamicClient, coreClient) nodeHealthData := fetchNodeHealth(ctx, coreClient) operatorData := fetchOperatorHealth(ctx, dynamicClient) bmhData := fetchBMHStatus(ctx, dynamicClient, coreClient, namespace) stonithData := fetchSTONITHData(ctx, params.KubernetesClient, coreClient, nodeName) - remediationData := fetchRemediationStatus(ctx, dynamicClient) + var remediationData string + if isTNF { + remediationData = "TNF clusters use Pacemaker/STONITH for fencing — see STONITH data above.\n" + } else { + remediationData = fetchRemediationStatus(ctx, dynamicClient) + } guideText := fmt.Sprintf(`# TNF Fencing Troubleshooting Guide @@ -156,12 +161,6 @@ debug pod. Key things to check: ## Step 6: Remediation Operators -Optional operators that automate fencing response: -- **FenceAgentsRemediation (FAR)**: Operator-driven fencing using fence agents -- **NodeHealthCheck (NHC)**: Monitors node conditions and triggers remediation - -These are complementary to pacemaker STONITH — not all clusters use them. - %s --- @@ -225,7 +224,7 @@ After analysis, report: ), nil } -func fetchClusterTopology(ctx context.Context, dynamicClient dynamic.Interface, coreClient corev1client.CoreV1Interface) string { +func fetchClusterTopology(ctx context.Context, dynamicClient dynamic.Interface, coreClient corev1client.CoreV1Interface) (string, bool) { var result strings.Builder result.WriteString("### Cluster Topology\n\n") @@ -233,7 +232,7 @@ func fetchClusterTopology(ctx context.Context, dynamicClient dynamic.Interface, if err != nil { slog.Debug("could not get Infrastructure CR", "error", err) result.WriteString("*Infrastructure CR not available*\n") - return result.String() + return result.String(), false } platform, _, _ := unstructured.NestedString(infra.Object, "status", "platform") @@ -247,7 +246,7 @@ func fetchClusterTopology(ctx context.Context, dynamicClient dynamic.Interface, nodes, err := coreClient.Nodes().List(ctx, metav1.ListOptions{}) if err != nil { slog.Debug("could not list nodes", "error", err) - return result.String() + return result.String(), false } cpCount := 0 @@ -268,7 +267,7 @@ func fetchClusterTopology(ctx context.Context, dynamicClient dynamic.Interface, result.WriteString("- **TNF Profile:** No\n") } - return result.String() + return result.String(), isTNF } func fetchNodeHealth(ctx context.Context, coreClient corev1client.CoreV1Interface) string { diff --git a/pkg/toolsets/tnf/troubleshoot_test.go b/pkg/toolsets/tnf/troubleshoot_test.go index ad5cff0fe..8c6eeb884 100644 --- a/pkg/toolsets/tnf/troubleshoot_test.go +++ b/pkg/toolsets/tnf/troubleshoot_test.go @@ -143,18 +143,20 @@ func (s *TNFTroubleshootSuite) TestFetchClusterTopology() { dynamicClient := fake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gvrToListKind, infra) coreClient := newFakeCoreV1(node1, node2) - result := fetchClusterTopology(ctx, dynamicClient, coreClient) + result, isTNF := fetchClusterTopology(ctx, dynamicClient, coreClient) s.Contains(result, "BareMetal") s.Contains(result, "TNF Profile:** Yes") s.Contains(result, "Total Nodes:** 2") + s.True(isTNF) }) s.Run("handles missing infrastructure CR", func() { dynamicClient := fake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gvrToListKind) coreClient := newFakeCoreV1() - result := fetchClusterTopology(ctx, dynamicClient, coreClient) + result, isTNF := fetchClusterTopology(ctx, dynamicClient, coreClient) s.Contains(result, "not available") + s.False(isTNF) }) } From 04d0ddeee2fd330af677c1c918858e25649d3752 Mon Sep 17 00:00:00 2001 From: Douglas Hensel Date: Thu, 16 Jul 2026 14:43:36 -0400 Subject: [PATCH 4/4] Regenerate TNF toolset snapshot to match updated description The tnf_check_fencing_config description was updated to reflect the actual checks performed but the test snapshot was not regenerated. Co-Authored-By: Claude Opus 4.6 --- pkg/mcp/testdata/toolsets-tnf-tools.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/mcp/testdata/toolsets-tnf-tools.json b/pkg/mcp/testdata/toolsets-tnf-tools.json index 0f68619f3..af7012ef9 100644 --- a/pkg/mcp/testdata/toolsets-tnf-tools.json +++ b/pkg/mcp/testdata/toolsets-tnf-tools.json @@ -7,7 +7,7 @@ "readOnlyHint": true, "title": "TNF: Check Fencing Config" }, - "description": "Check fencing configuration and readiness for a Two-Node Fencing (TNF) cluster. Validates cluster topology, critical operator health (etcd, machine-api, baremetal), Machine/Node/BareMetalHost correlation, BMC addresses and credentials, FenceAgentsRemediation templates and active remediations, and NodeHealthCheck resources. Returns a diagnostic summary identifying configuration issues that could prevent fencing from functioning correctly.", + "description": "Check fencing configuration and readiness for a Two-Node Fencing (TNF) cluster. Validates cluster topology, critical operator health (etcd, machine-api, baremetal), Machine/Node/BareMetalHost correlation, BMC addresses and credentials, pacemaker/STONITH fencing status via Kubernetes resources. Returns a diagnostic summary identifying configuration issues that could prevent fencing from functioning correctly.", "inputSchema": { "properties": { "namespace": {