CORENET-7134: Add ovn-kubernetes toolset skeleton

evals/claude-code/eval.yaml

@@ -102,3 +102,13 @@ config:
             toolPattern: ".*"
         minToolCalls: 1
         maxToolCalls: 20
+    # OVN-Kubernetes tasks
+    - glob: ../tasks/ovn-kubernetes/*/*.yaml
+      labelSelector:
+        suite: ovn-kubernetes
+      assertions:
+        toolsUsed:
+          - server: kubernetes
+            toolPattern: ".*"
+        minToolCalls: 1
+        maxToolCalls: 20

evals/gemini-agent/eval.yaml

@@ -36,3 +36,13 @@ config:
             toolPattern: ".*"
         minToolCalls: 1
         maxToolCalls: 20
+    # OVN-Kubernetes tasks
+    - glob: ../tasks/ovn-kubernetes/*/*.yaml
+      labelSelector:
+        suite: ovn-kubernetes
+      assertions:
+        toolsUsed:
+          - server: kubernetes
+            toolPattern: ".*"
+        minToolCalls: 1
+        maxToolCalls: 20

evals/openai-agent/eval.yaml

@@ -102,3 +102,13 @@ config:
             toolPattern: ".*"
         minToolCalls: 1
         maxToolCalls: 20
+    # OVN-Kubernetes tasks
+    - glob: ../tasks/ovn-kubernetes/*/*.yaml
+      labelSelector:
+        suite: ovn-kubernetes
+      assertions:
+        toolsUsed:
+          - server: kubernetes
+            toolPattern: ".*"
+        minToolCalls: 1
+        maxToolCalls: 20

evals/tasks/ovn-kubernetes/ovn-get/ovn-get-list-logical-routers.yaml

@@ -0,0 +1,27 @@
+kind: Task
+apiVersion: mcpchecker/v1alpha2
+metadata:
+  name: "ovn-get-list-logical-routers"
+  difficulty: easy
+  category: "OVN Database Query"
+  description: "List all Logical_Router records from OVN Northbound database"
+  labels:
+    suite: ovn-kubernetes
+    requires: ovn-kubernetes
+spec:
+  limits:
+    timeout: "5m"
+  setup:
+    - script:
+        file: ./setup.sh
+  prompt:
+    inline: |
+      List all Logical_Router records from the OVN Northbound database.
+  verify:
+    - llmJudge:
+        contains: "Shows all the Logical_Router records from the OVN Northbound database. Should include a distributed router handling inter-node routing and a local gateway router handling external connectivity"
+        reasoning: "Output should show all Logical_Router records including distributed router and gateway router"
+  assertions:
+    toolsUsed:
+      - server: kubernetes
+        toolPattern: "ovn_get"

evals/tasks/ovn-kubernetes/ovn-get/ovn-get-pattern-with-columns.yaml

@@ -0,0 +1,28 @@
+kind: Task
+apiVersion: mcpchecker/v1alpha2
+metadata:
+  name: "ovn-get-pattern-with-columns"
+  difficulty: medium
+  category: "OVN Database Query"
+  description: "Filter records with pattern and show specific columns"
+  labels:
+    suite: ovn-kubernetes
+    requires: ovn-kubernetes
+spec:
+  limits:
+    timeout: "5m"
+  setup:
+    - script:
+        file: ./setup.sh
+  prompt:
+    inline: |
+      From the Logical_Router table in the OVN Northbound database, show me only the name and nat
+      columns for entries matching the pattern "GR_".
+  verify:
+    - llmJudge:
+        contains: "Shows Logical_Router entry matching the pattern \"GR_\" with name and nat columns."
+        reasoning: "Output should show Logical_Router entries matching pattern GR_ with name and nat columns"
+  assertions:
+    toolsUsed:
+      - server: kubernetes
+        toolPattern: "ovn_get"

evals/tasks/ovn-kubernetes/ovn-get/ovn-get-southbound-chassis.yaml

@@ -0,0 +1,27 @@
+kind: Task
+apiVersion: mcpchecker/v1alpha2
+metadata:
+  name: "ovn-get-southbound-chassis"
+  difficulty: easy
+  category: "OVN Database Query"
+  description: "Query Chassis table from OVN Southbound database"
+  labels:
+    suite: ovn-kubernetes
+    requires: ovn-kubernetes
+spec:
+  limits:
+    timeout: "5m"
+  setup:
+    - script:
+        file: ./setup.sh
+  prompt:
+    inline: |
+      Show me all Chassis records from the OVN Southbound database.
+  verify:
+    - llmJudge:
+        contains: "Shows Chassis records corresponding to all cluster nodes from the OVN Southbound database. Indicates whether a chassis is local or remote"
+        reasoning: "Output should show Chassis records for all cluster nodes with local/remote indication"
+  assertions:
+    toolsUsed:
+      - server: kubernetes
+        toolPattern: "ovn_get"

evals/tasks/ovn-kubernetes/ovn-get/ovn-get-specific-router.yaml

@@ -0,0 +1,27 @@
+kind: Task
+apiVersion: mcpchecker/v1alpha2
+metadata:
+  name: "ovn-get-specific-router"
+  difficulty: easy
+  category: "OVN Database Query"
+  description: "Get ovn_cluster_router record from Logical_Router table"
+  labels:
+    suite: ovn-kubernetes
+    requires: ovn-kubernetes
+spec:
+  limits:
+    timeout: "5m"
+  setup:
+    - script:
+        file: ./setup.sh
+  prompt:
+    inline: |
+      Get the ovn_cluster_router record from the Logical_Router table in the OVN Northbound database.
+  verify:
+    - llmJudge:
+        contains: "Shows ovn_cluster_router record from the Logical_Router table. Should include list of ports, routing policies and static routes configured for the cluster router(k8s-cluster-router)"
+        reasoning: "Output should show ovn_cluster_router record with ports, routing policies, and static routes"
+  assertions:
+    toolsUsed:
+      - server: kubernetes
+        toolPattern: "ovn_get"

evals/tasks/ovn-kubernetes/ovn-get/ovn-get-switch-with-columns.yaml

@@ -0,0 +1,27 @@
+kind: Task
+apiVersion: mcpchecker/v1alpha2
+metadata:
+  name: "ovn-get-switch-with-columns"
+  difficulty: medium
+  category: "OVN Database Query"
+  description: "Get specific columns from Logical_Switch table"
+  labels:
+    suite: ovn-kubernetes
+    requires: ovn-kubernetes
+spec:
+  limits:
+    timeout: "5m"
+  setup:
+    - script:
+        file: ./setup.sh
+  prompt:
+    inline: |
+      From the Logical_Switch table in the OVN Northbound database, show me only the name and ports columns.
+  verify:
+    - llmJudge:
+        contains: "Shows Logical_Switch table in the OVN Northbound database showing only the name and ports columns indicating switches with their associated ports. Should contain join, transit and node specific switches in the list"
+        reasoning: "Output should show Logical_Switch entries with name and ports columns including join, transit, and node specific switches"
+  assertions:
+    toolsUsed:
+      - server: kubernetes
+        toolPattern: "ovn_get"

evals/tasks/ovn-kubernetes/ovn-get/setup.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+kubectl get pods -n openshift-ovn-kubernetes -l app=ovnkube-node --field-selector=status.phase=Running -o name | grep -q . || {
+  echo "ERROR: No running ovnkube-node pods found"
+  exit 1
+}

evals/tasks/ovn-kubernetes/ovn-lflow-list/ovn-lflow-list-datapath-with-pattern.yaml

@@ -0,0 +1,30 @@
+kind: Task
+apiVersion: mcpchecker/v1alpha2
+metadata:
+  name: "ovn-lflow-list-datapath-with-pattern"
+  difficulty: medium
+  category: "OVN Logical Flows"
+  description: "List flows for specific datapath with pattern filter"
+  labels:
+    suite: ovn-kubernetes
+    requires: ovn-kubernetes
+spec:
+  limits:
+    timeout: "5m"
+  setup:
+    - script:
+        file: ./setup.sh
+  prompt:
+    inline: |
+      From the ovn_cluster_router datapath, show me logical flows from the routing table.
+  verify:
+    - llmJudge:
+        contains: "Shows logical flows equivalent to routes in routing table for the ovn_cluster_router datapath"
+        reasoning: "Output should show logical flows representing the routing table for ovn_cluster_router"
+    - llmJudge:
+        contains: "Comprised of routing information from ovn_cluster_router to other switches"
+        reasoning: "Output should describe routing information between ovn_cluster_router and connected switches"
+  assertions:
+    toolsUsed:
+      - server: kubernetes
+        toolPattern: "ovn_lflow_list"

evals/tasks/ovn-kubernetes/ovn-lflow-list/ovn-lflow-list-for-datapath.yaml

@@ -0,0 +1,27 @@
+kind: Task
+apiVersion: mcpchecker/v1alpha2
+metadata:
+  name: "ovn-lflow-list-for-datapath"
+  difficulty: medium
+  category: "OVN Logical Flows"
+  description: "List logical flows for a specific datapath"
+  labels:
+    suite: ovn-kubernetes
+    requires: ovn-kubernetes
+spec:
+  limits:
+    timeout: "5m"
+  setup:
+    - script:
+        file: ./setup.sh
+  prompt:
+    inline: |
+      List logical flows for the join datapath from the OVN Southbound database.
+  verify:
+    - llmJudge:
+        contains: "Shows data from OVN Southbound database. Contains logical flows from both ingress and egress pipeline for the join datapath"
+        reasoning: "Output should show logical flows from both ingress and egress pipelines for the join datapath"
+  assertions:
+    toolsUsed:
+      - server: kubernetes
+        toolPattern: "ovn_lflow_list"

evals/tasks/ovn-kubernetes/ovn-lflow-list/setup.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+kubectl get pods -n openshift-ovn-kubernetes -l app=ovnkube-node --field-selector=status.phase=Running -o name | grep -q . || {
+  echo "ERROR: No running ovnkube-node pods found"
+  exit 1
+}

evals/tasks/ovn-kubernetes/ovn-show/ovn-show-northbound.yaml

@@ -0,0 +1,30 @@
+kind: Task
+apiVersion: mcpchecker/v1alpha2
+metadata:
+  name: "ovn-show-northbound"
+  difficulty: easy
+  category: "OVN Configuration"
+  description: "Display OVN Northbound database configuration using ovn_show tool"
+  labels:
+    suite: ovn-kubernetes
+    requires: ovn-kubernetes
+spec:
+  limits:
+    timeout: "5m"
+  setup:
+    - script:
+        file: ./setup.sh
+  prompt:
+    inline: |
+      Show me the OVN Northbound database configuration from one of the ovnkube-node pods. show me last 1000 lines.
+  verify:
+    - llmJudge:
+        contains: "Shows logical topology comprised of node specific switches, join switch, transit switch, gateway router and ovn_cluster_router"
+        reasoning: "Output should describe the logical topology including switches and routers"
+    - llmJudge:
+        contains: "Includes logical router ports, logical switch ports, IP address, MAC address of ports and SNAT rules"
+        reasoning: "Output should include details about ports, addresses, and SNAT rules"
+  assertions:
+    toolsUsed:
+      - server: kubernetes
+        toolPattern: "ovn_show"

evals/tasks/ovn-kubernetes/ovn-show/ovn-show-southbound.yaml

@@ -0,0 +1,30 @@
+kind: Task
+apiVersion: mcpchecker/v1alpha2
+metadata:
+  name: "ovn-show-southbound"
+  difficulty: easy
+  category: "OVN Configuration"
+  description: "Display OVN Southbound database configuration using ovn_show tool"
+  labels:
+    suite: ovn-kubernetes
+    requires: ovn-kubernetes
+spec:
+  limits:
+    timeout: "5m"
+  setup:
+    - script:
+        file: ./setup.sh
+  prompt:
+    inline: |
+      Show me the OVN Southbound database configuration from one of the ovnkube-node pods.
+  verify:
+    - llmJudge:
+        contains: "Shows Southbound database configuration from a node comprised of chassis information and port bindings"
+        reasoning: "Output should describe chassis information and port bindings"
+    - llmJudge:
+        contains: "Shows a overlay network using Geneve encapsulation with all nodes participating as chassis and connected ports"
+        reasoning: "Output should describe the Geneve overlay network with chassis and ports"
+  assertions:
+    toolsUsed:
+      - server: kubernetes
+        toolPattern: "ovn_show"

evals/tasks/ovn-kubernetes/ovn-show/setup.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+kubectl get pods -n openshift-ovn-kubernetes -l app=ovnkube-node --field-selector=status.phase=Running -o name | grep -q . || {
+  echo "ERROR: No running ovnkube-node pods found"
+  exit 1
+}