Operator: full reconciler implementation with status reporting and tests

.github/workflows/release.yml

@@ -74,3 +74,46 @@ jobs:
         if: startsWith(github.ref, 'refs/tags/v')
         run: |
           helm push *.tgz oci://ghcr.io/${{ github.repository_owner }}/charts
+
+  operator:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: write
+
+    steps:
+      - name: Checkout
+        uses: 'actions/checkout@v6'
+        with:
+          fetch-depth: 0
+
+      - name: Set VERSION from tag
+        run: echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker Build & Push
+        working-directory: operator
+        run: |
+          make docker-buildx
+
+      - name: Build Installer
+        working-directory: operator
+        run: |
+          make build-installer
+
+      - name: Create Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          tag: ${{ github.ref_name }}
+        run: |
+          gh release create "$tag" \
+              --repo="$GITHUB_REPOSITORY" \
+              --title="${tag#v}" \
+              --generate-notes \
+              operator/dist/install.yaml

Tiltfile

@@ -7,7 +7,7 @@ update_settings(max_parallel_updates=10, k8s_upsert_timeout_secs=600)
 load('ext://dotenv', 'dotenv')
 dotenv()
 
-v1alpha1.extension_repo(name='agentic-layer', url='https://github.com/agentic-layer/tilt-extensions', ref='v0.6.0')
+v1alpha1.extension_repo(name='agentic-layer', url='https://github.com/agentic-layer/tilt-extensions', ref='v0.13.0')
 
 v1alpha1.extension(name='cert-manager', repo_name='agentic-layer', repo_path='cert-manager')
 load('ext://cert-manager', 'cert_manager_install')
@@ -56,7 +56,7 @@ k8s_yaml(helm(
 # Apply local development manifests
 k8s_yaml(kustomize('deploy/local'))
 
-k8s_resource('ai-gateway-litellm', port_forwards=['11001:4000'])
+k8s_resource('ai-gateway', port_forwards=['11001:4000'])
 k8s_resource('weather-agent', port_forwards='11010:8000', labels=['agents'], resource_deps=['agent-runtime'])
 k8s_resource('lgtm', port_forwards=['11000:3000', '4318:4318'])
 

chart/templates/evaluate-template.yaml

@@ -12,6 +12,7 @@ spec:
     openApiBasePath:
       type: string
       description: "Base path for OpenAI API"
+      default: ""
 
   # Steps to execute
   steps:

deploy/local/testkube/values.yaml

@@ -1,6 +1,3 @@
 global:
   testWorkflows:
     createOfficialTemplates: false
-testkube-operator:
-  ## deploy Operator chart
-  enabled: enable

operator/.golangci.yml

@@ -1,33 +1,20 @@
+version: "2"
+
 run:
   timeout: 5m
   allow-parallel-runners: true
+  # go version is set to 1.25 for compatibility with golangci-lint v2.10.1
+  # which was built with go1.25; update when a newer linter release is available.
+  go: "1.25"
 
-issues:
-  # don't skip warning about doc comments
-  # don't exclude the default set of lint
-  exclude-use-default: false
-  # restore some of the defaults
-  # (fill in the rest as needed)
-  exclude-rules:
-    - path: "api/*"
-      linters:
-        - lll
-    - path: "internal/*"
-      linters:
-        - dupl
-        - lll
 linters:
   disable-all: true
   enable:
     - dupl
     - errcheck
-    - exportloopref
     - ginkgolinter
     - goconst
     - gocyclo
-    - gofmt
-    - goimports
-    - gosimple
     - govet
     - ineffassign
     - lll
@@ -36,10 +23,24 @@ linters:
     - prealloc
     - revive
     - staticcheck
-    - typecheck
     - unconvert
     - unparam
     - unused
+  exclusions:
+    rules:
+      - path: "^api/"
+        linters:
+          - lll
+      - path: "^internal/"
+        linters:
+          - dupl
+          - lll
+      - path: "(^internal/|^test/|^cmd/)"
+        linters:
+          - revive
+      - path: "^test/"
+        linters:
+          - staticcheck
 
 linters-settings:
   revive:

operator/Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.22 AS builder
+FROM golang:1.26 AS builder
 ARG TARGETOS
 ARG TARGETARCH
 

operator/Makefile

@@ -29,7 +29,7 @@ BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL)
 #
 # For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both
 # agentic-layer.ai/operator-bundle:$VERSION and agentic-layer.ai/operator-catalog:$VERSION.
-IMAGE_TAG_BASE ?= agentic-layer.ai/operator
+IMAGE_TAG_BASE ?= ghcr.io/agentic-layer/testbench/operator
 
 # BUNDLE_IMG defines the image:tag used for the bundle.
 # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=<some-registry>/<project-name-bundle>:<tag>)
@@ -50,7 +50,7 @@ endif
 # This is useful for CI or a project to utilize a specific version of the operator-sdk toolkit.
 OPERATOR_SDK_VERSION ?= v1.41.1
 # Image URL to use all building/pushing image targets
-IMG ?= controller:latest
+IMG ?= $(IMAGE_TAG_BASE):$(VERSION)
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -152,15 +152,18 @@ docker-push: ## Push docker image with the manager.
 # - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/
 # - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=<myregistry/image:<tag>> then the export will fail)
 # To adequately provide solutions that are compatible with multiple platforms, you should consider using this option.
-PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le
+PLATFORMS ?= linux/arm64,linux/amd64
 .PHONY: docker-buildx
 docker-buildx: ## Build and push docker image for the manager for cross-platform support
 	# copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile
 	sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross
-	- $(CONTAINER_TOOL) buildx create --name operator-builder
-	$(CONTAINER_TOOL) buildx use operator-builder
-	- $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross .
-	- $(CONTAINER_TOOL) buildx rm operator-builder
+	- $(CONTAINER_TOOL) buildx create --name testbench-operator-builder
+	$(CONTAINER_TOOL) buildx use testbench-operator-builder
+	$(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) \
+		  --tag $(IMG) \
+		  --tag $(IMAGE_TAG_BASE):latest \
+		  -f Dockerfile.cross .
+	- $(CONTAINER_TOOL) buildx rm testbench-operator-builder
 	rm Dockerfile.cross
 
 .PHONY: build-installer

operator/api/v1alpha1/experiment_types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 )
@@ -46,7 +47,27 @@ type S3Source struct {
 	Key string `json:"key"`
 }
 
-// DatasetSource defines where to load the test dataset from
+// InlineDataset defines an inline experiment dataset with scenarios, model, and threshold.
+type InlineDataset struct {
+	// LLM model used for evaluation (e.g., "gemini-2.5-flash-lite")
+	// +optional
+	LLMAsAJudgeModel string `json:"llmAsAJudgeModel,omitempty"`
+
+	// Default threshold for all metrics (0.0-1.0)
+	// +optional
+	// +kubebuilder:validation:Minimum=0.0
+	// +kubebuilder:validation:Maximum=1.0
+	DefaultThreshold *float64 `json:"defaultThreshold,omitempty"`
+
+	// Test scenarios
+	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:MinItems=1
+	Scenarios []Scenario `json:"scenarios"`
+}
+
+// DatasetSource defines where to load the test dataset from.
+// Exactly one of s3, url, or inline must be set.
+// +kubebuilder:validation:XValidation:rule="(has(self.s3) ? 1 : 0) + (has(self.url) ? 1 : 0) + (has(self.inline) ? 1 : 0) == 1",message="exactly one of s3, url, or inline must be set"
 type DatasetSource struct {
 	// S3 source configuration
 	// +optional
@@ -55,6 +76,10 @@ type DatasetSource struct {
 	// URL source (HTTP/HTTPS)
 	// +optional
 	URL string `json:"url,omitempty"`
+
+	// Inline dataset with scenarios
+	// +optional
+	Inline *InlineDataset `json:"inline,omitempty"`
 }
 
 // ToolCall represents an expected tool invocation
@@ -154,30 +179,23 @@ type TriggerSpec struct {
 }
 
 // ExperimentSpec defines the desired state of Experiment
-// +kubebuilder:validation:XValidation:rule="!(has(self.dataset) && has(self.scenarios))",message="dataset and scenarios are mutually exclusive"
 type ExperimentSpec struct {
 	// Reference to the Agent to evaluate
 	// +kubebuilder:validation:Required
 	AgentRef AgentRef `json:"agentRef"`
 
-	// Source of the test dataset (mutually exclusive with scenarios)
+	// AiGatewayRef references an AiGateway resource for LLM access during evaluation.
+	// Only Name and Namespace fields are used.
 	// +optional
-	Dataset *DatasetSource `json:"dataset,omitempty"`
-
-	// LLM model used for evaluation (e.g., "gemini-2.5-flash-lite", "gpt-4o")
-	// +optional
-	LLMAsAJudgeModel string `json:"llmAsAJudgeModel,omitempty"`
+	AiGatewayRef *corev1.ObjectReference `json:"aiGatewayRef,omitempty"`
 
-	// Default threshold for all metrics (0.0-1.0)
-	// +optional
-	// +kubebuilder:validation:Minimum=0.0
-	// +kubebuilder:validation:Maximum=1.0
-	// +kubebuilder:default=0.9
-	DefaultThreshold float64 `json:"defaultThreshold,omitempty"`
+	// Source of the test dataset
+	// +kubebuilder:validation:Required
+	Dataset DatasetSource `json:"dataset"`
 
-	// Inline test scenarios (mutually exclusive with dataset)
+	// OTLP endpoint URL for publishing metrics (e.g., "http://lgtm.monitoring.svc.cluster.local:4318")
 	// +optional
-	Scenarios []Scenario `json:"scenarios,omitempty"`
+	OTLPEndpoint string `json:"otlpEndpoint,omitempty"`
 
 	// Trigger configuration
 	// +optional