diff --git a/ci-operator/config/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main.yaml b/ci-operator/config/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main.yaml index 6d64f49ed9b2e..59de2e34ca3c9 100644 --- a/ci-operator/config/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main.yaml +++ b/ci-operator/config/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main.yaml @@ -38,6 +38,98 @@ tests: skip_if_only_changed: ^\.tekton/|\.md$|^(LICENSE|OWNERS)$ steps: workflow: openshift-ci-security +- as: periodic-mcpchecker-eval + cron: 0 9 * * 1 + steps: + cluster_profile: openshift-org-aws + test: + - as: mcpchecker-eval + cli: latest + commands: | + set -euo pipefail + + export TOOLSETS=core,config + export MCP_CONFIG_DIR=dev/config/mcp-configs + export PATH="${PATH}:$(pwd)/_output/tools/bin" + + # Live cluster kubeconfig is provided by the ipi-aws workflow. + export KUBECONFIG="${SHARED_DIR}/kubeconfig" + if ! command -v kubectl >/dev/null 2>&1; then + mkdir -p /tmp/bin + ln -s "$(command -v oc)" /tmp/bin/kubectl + export PATH="/tmp/bin:${PATH}" + fi + + # OpenAI token from selfservice vault (ocp-mcp/openai-credentials). + export OPENAI_API_KEY="$(cat /var/run/ocp-mcp/openai-token)" + + cleanup() { + make stop-server || true + } + trap cleanup EXIT + + GOFLAGS='-mod=readonly' make build + GOFLAGS='' make mcpchecker + + # CI pods expose in-cluster env (build farm). Without forcing kubeconfig, + # the MCP server auto-selects in-cluster and gets RBAC denials. Unset the + # env signals and pass --kubeconfig/--cluster-provider for the IPI cluster. + unset KUBERNETES_SERVICE_HOST KUBERNETES_SERVICE_PORT + oc whoami + oc auth can-i list pods -A + oc auth can-i '*' '*' --all-namespaces + ls -la ./kubernetes-mcp-server + + MCP_PORT=8080 + ./kubernetes-mcp-server \ + --port "${MCP_PORT}" \ + --toolsets "${TOOLSETS}" \ + --config-dir "${MCP_CONFIG_DIR}" \ + --kubeconfig "${KUBECONFIG}" \ + --cluster-provider kubeconfig \ + >"${ARTIFACT_DIR}/mcp-server.log" 2>&1 \ + & echo $! > .mcp-server.pid + + elapsed=0 + while [ "${elapsed}" -lt 60 ]; do + if curl -fsS "http://localhost:${MCP_PORT}/healthz" >/dev/null 2>&1; then + break + fi + sleep 2 + elapsed=$((elapsed + 2)) + done + if ! curl -fsS "http://localhost:${MCP_PORT}/healthz" >/dev/null 2>&1; then + echo "ERROR: MCP server failed to become ready" + exit 1 + fi + + MCPCHECKER="$(pwd)/_output/tools/bin/mcpchecker" + "${MCPCHECKER}" check evals/core-eval-testing/builtin-openai/eval-core.yaml \ + --label-selector suite=core \ + --output json + + RESULTS_FILE="$(ls -t mcpchecker-*-out.json | head -1)" + cp "${RESULTS_FILE}" "${ARTIFACT_DIR}/mcpchecker-out.json" + + # Emit JUnit before verify so Spyglass/Sippy still get artifacts when the + # 80% gate fails. + "${MCPCHECKER}" result convert junit "${ARTIFACT_DIR}/mcpchecker-out.json" \ + --output-file "${ARTIFACT_DIR}/junit_mcpchecker.xml" + + "${MCPCHECKER}" result verify "${ARTIFACT_DIR}/mcpchecker-out.json" \ + --task 0.8 --assertion 0.8 + credentials: + - mount_path: /var/run/ocp-mcp + name: ocp-mcp + namespace: test-credentials + from: src + grace_period: 5m0s + resources: + requests: + cpu: "2" + memory: 4Gi + timeout: 4h0m0s + workflow: ipi-aws zz_generated_metadata: branch: main org: openshift diff --git a/ci-operator/jobs/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main-periodics.yaml b/ci-operator/jobs/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main-periodics.yaml new file mode 100644 index 0000000000000..53fb5d28b29ad --- /dev/null +++ b/ci-operator/jobs/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main-periodics.yaml @@ -0,0 +1,79 @@ +periodics: +- agent: kubernetes + cluster: build05 + cron: 0 9 * * 1 + decorate: true + decoration_config: + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile.ci + extra_refs: + - base_ref: main + org: openshift + repo: openshift-mcp-server + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile.ci + labels: + ci-operator.openshift.io/cloud: aws + ci-operator.openshift.io/cloud-cluster-profile: openshift-org-aws + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-openshift-mcp-server-main-periodic-mcpchecker-eval + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --target=periodic-mcpchecker-eval + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator