From a434a16c8f6bca078cbe41cfa6049fd9b260cef7 Mon Sep 17 00:00:00 2001 From: grokspawn Date: Thu, 25 Jun 2026 16:08:08 -0500 Subject: [PATCH 1/3] add anthropic/gemini vertex jobs --- .../openshift-openshift-mcp-server-main.yaml | 82 ++++++++++ ...-openshift-mcp-server-main-presubmits.yaml | 152 ++++++++++++++++++ 2 files changed, 234 insertions(+) diff --git a/ci-operator/config/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main.yaml b/ci-operator/config/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main.yaml index 6d64f49ed9b2e..e05d282905dbd 100644 --- a/ci-operator/config/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main.yaml +++ b/ci-operator/config/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main.yaml @@ -38,6 +38,88 @@ tests: skip_if_only_changed: ^\.tekton/|\.md$|^(LICENSE|OWNERS)$ steps: workflow: openshift-ci-security +- as: mcpchecker-eval-google + optional: true + steps: + test: + - as: mcpchecker-eval-google + commands: | + set -euo pipefail + + export GOOGLE_APPLICATION_CREDENTIALS="/var/run/ocp-mcp/service-account.json" + export GOOGLE_CLOUD_LOCATION="us-east1" + export GEMINI_USE_VERTEX="1" + + # Extract project ID from service account JSON + GOOGLE_CLOUD_PROJECT="$(python3 -c "import json; print(json.load(open('${GOOGLE_APPLICATION_CREDENTIALS}'))['project_id'])")" + export GOOGLE_CLOUD_PROJECT + + # Build MCP server binary + GOFLAGS='-mod=readonly' make build + + # Install mcpchecker + make mcpchecker + + # Start MCP server in background + make run-server TOOLSETS=core,config + trap 'make stop-server || true' EXIT + + # Run mcpchecker evaluation with Google Gemini agent + _output/tools/bin/mcpchecker check \ + evals/core-eval-testing/builtin-google/eval-core.yaml \ + --label-selector suite=core \ + --output json + credentials: + - mount_path: /var/run/ocp-mcp + name: ocp-mcp + namespace: test-credentials + from: src + grace_period: 1m0s + resources: + requests: + cpu: 500m + memory: 200Mi +- as: mcpchecker-eval-anthropic + optional: true + steps: + test: + - as: mcpchecker-eval-anthropic + commands: | + set -euo pipefail + + export GOOGLE_APPLICATION_CREDENTIALS="/var/run/ocp-mcp/service-account.json" + export GOOGLE_CLOUD_LOCATION="us-east1" + export ANTHROPIC_USE_VERTEX="1" + + # Extract project ID from service account JSON + GOOGLE_CLOUD_PROJECT="$(python3 -c "import json; print(json.load(open('${GOOGLE_APPLICATION_CREDENTIALS}'))['project_id'])")" + export GOOGLE_CLOUD_PROJECT + + # Build MCP server binary + GOFLAGS='-mod=readonly' make build + + # Install mcpchecker + make mcpchecker + + # Start MCP server in background + make run-server TOOLSETS=core,config + trap 'make stop-server || true' EXIT + + # Run mcpchecker evaluation with Anthropic Claude agent + _output/tools/bin/mcpchecker check \ + evals/core-eval-testing/builtin-anthropic/eval-core.yaml \ + --label-selector suite=core \ + --output json + credentials: + - mount_path: /var/run/ocp-mcp + name: ocp-mcp + namespace: test-credentials + from: src + grace_period: 1m0s + resources: + requests: + cpu: 500m + memory: 200Mi zz_generated_metadata: branch: main org: openshift diff --git a/ci-operator/jobs/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main-presubmits.yaml b/ci-operator/jobs/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main-presubmits.yaml index e59d2f74652b6..b92e5b818da2e 100644 --- a/ci-operator/jobs/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main-presubmits.yaml +++ b/ci-operator/jobs/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main-presubmits.yaml @@ -199,6 +199,158 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )lint,?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - ^main$ + - ^main- + cluster: build13 + context: ci/prow/mcpchecker-eval-anthropic + decorate: true + decoration_config: + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile.ci + labels: + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-openshift-mcp-server-main-mcpchecker-eval-anthropic + optional: true + rerun_command: /test mcpchecker-eval-anthropic + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --target=mcpchecker-eval-anthropic + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )mcpchecker-eval-anthropic,?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - ^main$ + - ^main- + cluster: build13 + context: ci/prow/mcpchecker-eval-google + decorate: true + decoration_config: + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile.ci + labels: + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-openshift-mcp-server-main-mcpchecker-eval-google + optional: true + rerun_command: /test mcpchecker-eval-google + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --target=mcpchecker-eval-google + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )mcpchecker-eval-google,?($|\s.*) - agent: kubernetes always_run: false branches: From 120baa021e0ca234d51d34dffb59027aa4bce046 Mon Sep 17 00:00:00 2001 From: grokspawn Date: Mon, 29 Jun 2026 11:31:54 -0500 Subject: [PATCH 2/3] experimenting with semi-hermetic build env Signed-off-by: grokspawn --- .../openshift-openshift-mcp-server-main.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci-operator/config/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main.yaml b/ci-operator/config/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main.yaml index e05d282905dbd..87882c31e86d9 100644 --- a/ci-operator/config/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main.yaml +++ b/ci-operator/config/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main.yaml @@ -57,8 +57,8 @@ tests: # Build MCP server binary GOFLAGS='-mod=readonly' make build - # Install mcpchecker - make mcpchecker + # Install mcpchecker (clear -mod=vendor set by CI environment) + GOFLAGS='' make mcpchecker # Start MCP server in background make run-server TOOLSETS=core,config @@ -98,8 +98,8 @@ tests: # Build MCP server binary GOFLAGS='-mod=readonly' make build - # Install mcpchecker - make mcpchecker + # Install mcpchecker (clear -mod=vendor set by CI environment) + GOFLAGS='' make mcpchecker # Start MCP server in background make run-server TOOLSETS=core,config From a34d6706f2f7220db2c38cff072dc5c09e0a1065 Mon Sep 17 00:00:00 2001 From: grokspawn Date: Thu, 16 Jul 2026 10:12:55 -0500 Subject: [PATCH 3/3] switch to step registry and parameterized workflows Signed-off-by: grokspawn --- .../openshift-openshift-mcp-server-main.yaml | 94 ++++----------- ...-openshift-mcp-server-main-presubmits.yaml | 4 + .../step-registry/openshift/mcp-server/OWNERS | 15 +++ .../mcp-server/mcpchecker-eval/OWNERS | 15 +++ .../mcp-server/mcpchecker-eval/README.md | 112 ++++++++++++++++++ .../mcpchecker-eval/credentials-setup/OWNERS | 15 +++ ...checker-eval-credentials-setup-commands.sh | 12 ++ ...r-eval-credentials-setup-ref.metadata.json | 22 ++++ ...mcpchecker-eval-credentials-setup-ref.yaml | 17 +++ .../mcpchecker-eval/model-api/OWNERS | 15 +++ ...checker-eval-model-api-chain.metadata.json | 22 ++++ ...erver-mcpchecker-eval-model-api-chain.yaml | 10 ++ ...ift-mcp-server-mcpchecker-eval-commands.sh | 46 +++++++ ...p-server-mcpchecker-eval-ref.metadata.json | 22 ++++ ...nshift-mcp-server-mcpchecker-eval-ref.yaml | 44 +++++++ .../mcpchecker-eval/vertex-setup/OWNERS | 15 +++ ...r-mcpchecker-eval-vertex-setup-commands.sh | 14 +++ ...hecker-eval-vertex-setup-ref.metadata.json | 22 ++++ ...rver-mcpchecker-eval-vertex-setup-ref.yaml | 18 +++ .../mcp-server/mcpchecker-eval/vertex/OWNERS | 15 +++ ...mcpchecker-eval-vertex-chain.metadata.json | 22 ++++ ...p-server-mcpchecker-eval-vertex-chain.yaml | 10 ++ 22 files changed, 507 insertions(+), 74 deletions(-) create mode 100644 ci-operator/step-registry/openshift/mcp-server/OWNERS create mode 100644 ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/OWNERS create mode 100644 ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/README.md create mode 100644 ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/credentials-setup/OWNERS create mode 100644 ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/credentials-setup/openshift-mcp-server-mcpchecker-eval-credentials-setup-commands.sh create mode 100644 ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/credentials-setup/openshift-mcp-server-mcpchecker-eval-credentials-setup-ref.metadata.json create mode 100644 ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/credentials-setup/openshift-mcp-server-mcpchecker-eval-credentials-setup-ref.yaml create mode 100644 ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/model-api/OWNERS create mode 100644 ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/model-api/openshift-mcp-server-mcpchecker-eval-model-api-chain.metadata.json create mode 100644 ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/model-api/openshift-mcp-server-mcpchecker-eval-model-api-chain.yaml create mode 100644 ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/openshift-mcp-server-mcpchecker-eval-commands.sh create mode 100644 ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/openshift-mcp-server-mcpchecker-eval-ref.metadata.json create mode 100644 ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/openshift-mcp-server-mcpchecker-eval-ref.yaml create mode 100644 ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/vertex-setup/OWNERS create mode 100644 ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/vertex-setup/openshift-mcp-server-mcpchecker-eval-vertex-setup-commands.sh create mode 100644 ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/vertex-setup/openshift-mcp-server-mcpchecker-eval-vertex-setup-ref.metadata.json create mode 100644 ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/vertex-setup/openshift-mcp-server-mcpchecker-eval-vertex-setup-ref.yaml create mode 100644 ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/vertex/OWNERS create mode 100644 ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/vertex/openshift-mcp-server-mcpchecker-eval-vertex-chain.metadata.json create mode 100644 ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/vertex/openshift-mcp-server-mcpchecker-eval-vertex-chain.yaml diff --git a/ci-operator/config/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main.yaml b/ci-operator/config/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main.yaml index 87882c31e86d9..300a9694694e7 100644 --- a/ci-operator/config/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main.yaml +++ b/ci-operator/config/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main.yaml @@ -41,85 +41,31 @@ tests: - as: mcpchecker-eval-google optional: true steps: + cluster_profile: openshift-org-aws + env: + ANTHROPIC_USE_VERTEX: "" + EVAL_CONFIG: evals/core-eval-testing/builtin-google/eval-core.yaml + EVAL_LABEL_SELECTOR: suite=core + GEMINI_USE_VERTEX: "1" + GOOGLE_CLOUD_LOCATION: us-east5 + GOOGLE_CLOUD_PROJECT: ocp-mcp-server-team test: - - as: mcpchecker-eval-google - commands: | - set -euo pipefail - - export GOOGLE_APPLICATION_CREDENTIALS="/var/run/ocp-mcp/service-account.json" - export GOOGLE_CLOUD_LOCATION="us-east1" - export GEMINI_USE_VERTEX="1" - - # Extract project ID from service account JSON - GOOGLE_CLOUD_PROJECT="$(python3 -c "import json; print(json.load(open('${GOOGLE_APPLICATION_CREDENTIALS}'))['project_id'])")" - export GOOGLE_CLOUD_PROJECT - - # Build MCP server binary - GOFLAGS='-mod=readonly' make build - - # Install mcpchecker (clear -mod=vendor set by CI environment) - GOFLAGS='' make mcpchecker - - # Start MCP server in background - make run-server TOOLSETS=core,config - trap 'make stop-server || true' EXIT - - # Run mcpchecker evaluation with Google Gemini agent - _output/tools/bin/mcpchecker check \ - evals/core-eval-testing/builtin-google/eval-core.yaml \ - --label-selector suite=core \ - --output json - credentials: - - mount_path: /var/run/ocp-mcp - name: ocp-mcp - namespace: test-credentials - from: src - grace_period: 1m0s - resources: - requests: - cpu: 500m - memory: 200Mi + - chain: openshift-mcp-server-mcpchecker-eval-vertex + workflow: ipi-aws - as: mcpchecker-eval-anthropic optional: true steps: + cluster_profile: openshift-org-aws + env: + ANTHROPIC_USE_VERTEX: "1" + EVAL_CONFIG: evals/core-eval-testing/builtin-anthropic/eval-core.yaml + EVAL_LABEL_SELECTOR: suite=core + GEMINI_USE_VERTEX: "" + GOOGLE_CLOUD_LOCATION: us-east5 + GOOGLE_CLOUD_PROJECT: ocp-mcp-server-team test: - - as: mcpchecker-eval-anthropic - commands: | - set -euo pipefail - - export GOOGLE_APPLICATION_CREDENTIALS="/var/run/ocp-mcp/service-account.json" - export GOOGLE_CLOUD_LOCATION="us-east1" - export ANTHROPIC_USE_VERTEX="1" - - # Extract project ID from service account JSON - GOOGLE_CLOUD_PROJECT="$(python3 -c "import json; print(json.load(open('${GOOGLE_APPLICATION_CREDENTIALS}'))['project_id'])")" - export GOOGLE_CLOUD_PROJECT - - # Build MCP server binary - GOFLAGS='-mod=readonly' make build - - # Install mcpchecker (clear -mod=vendor set by CI environment) - GOFLAGS='' make mcpchecker - - # Start MCP server in background - make run-server TOOLSETS=core,config - trap 'make stop-server || true' EXIT - - # Run mcpchecker evaluation with Anthropic Claude agent - _output/tools/bin/mcpchecker check \ - evals/core-eval-testing/builtin-anthropic/eval-core.yaml \ - --label-selector suite=core \ - --output json - credentials: - - mount_path: /var/run/ocp-mcp - name: ocp-mcp - namespace: test-credentials - from: src - grace_period: 1m0s - resources: - requests: - cpu: 500m - memory: 200Mi + - chain: openshift-mcp-server-mcpchecker-eval-vertex + workflow: ipi-aws zz_generated_metadata: branch: main org: openshift diff --git a/ci-operator/jobs/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main-presubmits.yaml b/ci-operator/jobs/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main-presubmits.yaml index b92e5b818da2e..593ca9d4386d5 100644 --- a/ci-operator/jobs/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main-presubmits.yaml +++ b/ci-operator/jobs/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main-presubmits.yaml @@ -212,6 +212,8 @@ presubmits: - .ci-operator.yaml - Dockerfile.ci labels: + ci-operator.openshift.io/cloud: aws + ci-operator.openshift.io/cloud-cluster-profile: openshift-org-aws ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" name: pull-ci-openshift-openshift-mcp-server-main-mcpchecker-eval-anthropic @@ -288,6 +290,8 @@ presubmits: - .ci-operator.yaml - Dockerfile.ci labels: + ci-operator.openshift.io/cloud: aws + ci-operator.openshift.io/cloud-cluster-profile: openshift-org-aws ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" name: pull-ci-openshift-openshift-mcp-server-main-mcpchecker-eval-google diff --git a/ci-operator/step-registry/openshift/mcp-server/OWNERS b/ci-operator/step-registry/openshift/mcp-server/OWNERS new file mode 100644 index 0000000000000..3c93b553e96a1 --- /dev/null +++ b/ci-operator/step-registry/openshift/mcp-server/OWNERS @@ -0,0 +1,15 @@ +approvers: +- Cali0707 +- manusa +- matzew +- Kaustubh-pande +reviewers: +- 2uasimojo +- bentito +- cajieh +- Cali0707 +- dlom +- grokspawn +- manusa +- matzew +- Kaustubh-pande diff --git a/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/OWNERS b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/OWNERS new file mode 100644 index 0000000000000..3c93b553e96a1 --- /dev/null +++ b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/OWNERS @@ -0,0 +1,15 @@ +approvers: +- Cali0707 +- manusa +- matzew +- Kaustubh-pande +reviewers: +- 2uasimojo +- bentito +- cajieh +- Cali0707 +- dlom +- grokspawn +- manusa +- matzew +- Kaustubh-pande diff --git a/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/README.md b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/README.md new file mode 100644 index 0000000000000..eb46862c82514 --- /dev/null +++ b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/README.md @@ -0,0 +1,112 @@ +# openshift-mcp-server-mcpchecker-eval step registry + +Reusable CI steps for running [mcpchecker](https://github.com/mcpchecker/mcpchecker) +evaluations against the [openshift-mcp-server](https://github.com/openshift/openshift-mcp-server) +on a live OCP cluster. Designed to support both pre-merge (presubmit) and scheduled +(periodic) workflows via env-var and eval-target parameterization. + +## Structure + +``` +mcpchecker-eval/ +├── openshift-mcp-server-mcpchecker-eval # core eval step (shared) +├── vertex-setup/ # credential setup: Vertex AI +├── credentials-setup/ # credential setup: model API key +├── vertex/ # chain: vertex-setup → eval +└── model-api/ # chain: credentials-setup → eval +``` + +## Design intent + +All evaluation logic lives in the single **`openshift-mcp-server-mcpchecker-eval`** step, +parameterized entirely by environment variables. Credential sourcing is separated into +provider-specific **setup steps** that write exports to `${SHARED_DIR}/mcpchecker-creds.env`, +which the eval step sources at startup. This keeps the eval step provider-agnostic and +makes adding a new provider a matter of adding a new setup step and chain. + +The two chains are: + +| Chain | Credential source | Typical use | +|---|---|---| +| `openshift-mcp-server-mcpchecker-eval-vertex` | `ocp-mcp` secret (GCP service account) | Gemini or Anthropic via Vertex AI | +| `openshift-mcp-server-mcpchecker-eval-model-api` | `openai-token` secret | Any non-Vertex provider | + +Agent, model, and judge selection are specified entirely within the in-repo eval config +pointed to by `EVAL_CONFIG`. The credential setup steps exist only to surface authentication +material (service account files, API keys) that the eval config's agent definition requires; +they do not drive model routing themselves. + +Both chains run against a live OCP cluster provisioned by the `ipi-aws` workflow. +`kubectl` is unavailable in the CI image; the eval step symlinks `oc` (provided via +`cli: latest`) to satisfy eval setup scripts that call `kubectl`. + +## Eval step parameters + +All set via `steps.env` in the calling CI config: + +| Variable | Default | Purpose | +|---|---|---| +| `EVAL_CONFIG` | `evals/openai-agent/eval.yaml` | Repo-relative path to the eval YAML | +| `EVAL_LABEL_SELECTOR` | `suite=core` | Label filter passed to `mcpchecker check` | +| `TOOLSETS` | `core,config` | MCP server toolsets to start | +| `TASK_PASS_RATE` | `0.0` | Minimum fraction of tasks that must pass (0.0 = no gate) | +| `ASSERTION_PASS_RATE` | `0.0` | Minimum fraction of assertions that must pass (0.0 = no gate) | +| `GOOGLE_CLOUD_LOCATION` | `` | Vertex AI region (e.g. `us-east5`); Vertex chain only | +| `GOOGLE_CLOUD_PROJECT` | `` | GCP project ID; Vertex chain only | +| `GEMINI_USE_VERTEX` | `` | Set to `"1"` for Google Gemini via Vertex AI | +| `ANTHROPIC_USE_VERTEX` | `` | Set to `"1"` for Anthropic Claude via Vertex AI | + +Artifacts written to `${ARTIFACT_DIR}`: `mcpchecker-out.json` and `junit_mcpchecker.xml`. + +## Usage + +### Presubmit — Vertex AI provider (Gemini or Anthropic) + +```yaml +- as: mcpchecker-eval-google + optional: true + steps: + cluster_profile: openshift-org-aws + env: + EVAL_CONFIG: evals/core-eval-testing/builtin-google/eval-core.yaml + EVAL_LABEL_SELECTOR: suite=core + GEMINI_USE_VERTEX: "1" + GOOGLE_CLOUD_LOCATION: us-east5 + GOOGLE_CLOUD_PROJECT: ocp-mcp-server-team + test: + - chain: openshift-mcp-server-mcpchecker-eval-vertex + workflow: ipi-aws +``` + +Switch to Anthropic by setting `ANTHROPIC_USE_VERTEX: "1"` (and clearing `GEMINI_USE_VERTEX`) +and pointing `EVAL_CONFIG` at the Anthropic eval YAML. The chain is identical. + +### Periodic — non-Vertex provider + +```yaml +- as: periodic-mcpchecker-eval + cron: 0 9 * * 1 + steps: + cluster_profile: openshift-org-aws + env: + EVAL_CONFIG: evals/openai-agent/eval.yaml + EVAL_LABEL_SELECTOR: suite=core + TASK_PASS_RATE: "0.8" + ASSERTION_PASS_RATE: "0.8" + test: + - chain: openshift-mcp-server-mcpchecker-eval-model-api + workflow: ipi-aws +``` + +The agent, model, and judge are configured inside the eval YAML (`EVAL_CONFIG`). The +`openai-token` secret in the `test-credentials` namespace must contain whatever +authentication keys that eval config's agent definition requires. + +### Adding a new provider + +1. Create a new `-setup/` directory with a ref that mounts the relevant secret + and appends exports to `${SHARED_DIR}/mcpchecker-creds.env`. +2. Create a `/` directory with a chain that runs your setup step followed by + `openshift-mcp-server-mcpchecker-eval`. +3. Add an `OWNERS` file to both directories. +4. Run `make registry-metadata` to regenerate metadata. diff --git a/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/credentials-setup/OWNERS b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/credentials-setup/OWNERS new file mode 100644 index 0000000000000..3c93b553e96a1 --- /dev/null +++ b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/credentials-setup/OWNERS @@ -0,0 +1,15 @@ +approvers: +- Cali0707 +- manusa +- matzew +- Kaustubh-pande +reviewers: +- 2uasimojo +- bentito +- cajieh +- Cali0707 +- dlom +- grokspawn +- manusa +- matzew +- Kaustubh-pande diff --git a/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/credentials-setup/openshift-mcp-server-mcpchecker-eval-credentials-setup-commands.sh b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/credentials-setup/openshift-mcp-server-mcpchecker-eval-credentials-setup-commands.sh new file mode 100644 index 0000000000000..2156a5ba3036a --- /dev/null +++ b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/credentials-setup/openshift-mcp-server-mcpchecker-eval-credentials-setup-commands.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +# Disable tracing while handling credential values +[[ $- == *x* ]] && WAS_TRACING=true || WAS_TRACING=false +set +x + +cat >> "${SHARED_DIR}/mcpchecker-creds.env" </dev/null 2>&1; then + mkdir -p /tmp/bin + ln -s "$(command -v oc)" /tmp/bin/kubectl + export PATH="/tmp/bin:${PATH}" +fi + +PATH="${PATH}:$(pwd)/_output/tools/bin" +export PATH +export MCP_CONFIG_DIR=dev/config/mcp-configs + +trap 'make stop-server || true' EXIT + +GOFLAGS='-mod=readonly' make build +GOFLAGS='' make mcpchecker + +make run-server TOOLSETS="${TOOLSETS}" +make run-evals EVAL_CONFIG="${EVAL_CONFIG}" EVAL_LABEL_SELECTOR="${EVAL_LABEL_SELECTOR}" + +RESULTS_FILE="$(find . -maxdepth 1 -name 'mcpchecker-*-out.json' | sort | tail -1)" +if [[ -z "${RESULTS_FILE}" ]]; then + echo "ERROR: no mcpchecker results file found" >&2 + exit 1 +fi + +MCPCHECKER="$(pwd)/_output/tools/bin/mcpchecker" +cp "${RESULTS_FILE}" "${ARTIFACT_DIR}/mcpchecker-out.json" + +"${MCPCHECKER}" result convert junit "${ARTIFACT_DIR}/mcpchecker-out.json" \ + --output-file "${ARTIFACT_DIR}/junit_mcpchecker.xml" + +if [[ "${TASK_PASS_RATE}" != "0.0" || "${ASSERTION_PASS_RATE}" != "0.0" ]]; then + "${MCPCHECKER}" result verify "${ARTIFACT_DIR}/mcpchecker-out.json" \ + --task "${TASK_PASS_RATE}" --assertion "${ASSERTION_PASS_RATE}" +fi diff --git a/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/openshift-mcp-server-mcpchecker-eval-ref.metadata.json b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/openshift-mcp-server-mcpchecker-eval-ref.metadata.json new file mode 100644 index 0000000000000..2e115e738c289 --- /dev/null +++ b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/openshift-mcp-server-mcpchecker-eval-ref.metadata.json @@ -0,0 +1,22 @@ +{ + "path": "openshift/mcp-server/mcpchecker-eval/openshift-mcp-server-mcpchecker-eval-ref.yaml", + "owners": { + "approvers": [ + "Cali0707", + "manusa", + "matzew", + "Kaustubh-pande" + ], + "reviewers": [ + "2uasimojo", + "bentito", + "cajieh", + "Cali0707", + "dlom", + "grokspawn", + "manusa", + "matzew", + "Kaustubh-pande" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/openshift-mcp-server-mcpchecker-eval-ref.yaml b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/openshift-mcp-server-mcpchecker-eval-ref.yaml new file mode 100644 index 0000000000000..3e9a4932eb99f --- /dev/null +++ b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/openshift-mcp-server-mcpchecker-eval-ref.yaml @@ -0,0 +1,44 @@ +ref: + as: openshift-mcp-server-mcpchecker-eval + from: src + cli: latest + commands: openshift-mcp-server-mcpchecker-eval-commands.sh + env: + - name: EVAL_CONFIG + default: evals/openai-agent/eval.yaml + documentation: Repo-relative path to the mcpchecker eval YAML + - name: EVAL_LABEL_SELECTOR + default: suite=core + documentation: Label selector for task filtering + - name: TOOLSETS + default: core,config + documentation: MCP server toolsets to enable + - name: TASK_PASS_RATE + default: "0.0" + documentation: Minimum task pass rate (0.0 disables the gate) + - name: ASSERTION_PASS_RATE + default: "0.0" + documentation: Minimum assertion pass rate (0.0 disables the gate) + - name: GOOGLE_CLOUD_LOCATION + default: "" + documentation: Vertex AI region (e.g. us-east5); leave empty for non-Vertex runs + - name: GOOGLE_CLOUD_PROJECT + default: "" + documentation: GCP project ID for Vertex AI; leave empty for non-Vertex runs + - name: GEMINI_USE_VERTEX + default: "" + documentation: Set to "1" to route Google Gemini calls through Vertex AI + - name: ANTHROPIC_USE_VERTEX + default: "" + documentation: Set to "1" to route Anthropic Claude calls through Vertex AI + resources: + requests: + cpu: "2" + memory: 4Gi + grace_period: 5m0s + timeout: 4h0m0s + documentation: |- + Runs mcpchecker evaluations against the openshift-mcp-server on a live OCP cluster. + Credentials must be written to ${SHARED_DIR}/mcpchecker-creds.env by a preceding + setup step. Produces JSON and JUnit artifacts under ${ARTIFACT_DIR}. + An optional pass-rate gate exits non-zero if results fall below the threshold. diff --git a/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/vertex-setup/OWNERS b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/vertex-setup/OWNERS new file mode 100644 index 0000000000000..3c93b553e96a1 --- /dev/null +++ b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/vertex-setup/OWNERS @@ -0,0 +1,15 @@ +approvers: +- Cali0707 +- manusa +- matzew +- Kaustubh-pande +reviewers: +- 2uasimojo +- bentito +- cajieh +- Cali0707 +- dlom +- grokspawn +- manusa +- matzew +- Kaustubh-pande diff --git a/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/vertex-setup/openshift-mcp-server-mcpchecker-eval-vertex-setup-commands.sh b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/vertex-setup/openshift-mcp-server-mcpchecker-eval-vertex-setup-commands.sh new file mode 100644 index 0000000000000..ad4bc8406c5c4 --- /dev/null +++ b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/vertex-setup/openshift-mcp-server-mcpchecker-eval-vertex-setup-commands.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -euo pipefail + +# Copy the service account JSON into SHARED_DIR so it is accessible to the +# eval step, which runs in a separate container where /var/run/ocp-mcp is not +# mounted. GOOGLE_CLOUD_LOCATION, GOOGLE_CLOUD_PROJECT, and GEMINI_USE_VERTEX / +# ANTHROPIC_USE_VERTEX are passed via steps.env and flow automatically into all +# subsequent steps. +cp /var/run/ocp-mcp/service-account.json "${SHARED_DIR}/service-account.json" +chmod 0600 "${SHARED_DIR}/service-account.json" + +cat >> "${SHARED_DIR}/mcpchecker-creds.env" <