test: migrate from bespoke e2e to go-workflow

Makefile

@@ -640,6 +640,9 @@ run-perf-test:
 run-e2e-test:
 	go test -v ./test/e2e/ -timeout 1h -tags=e2e -count=1  -args -image-tag=${TAG} -image-registry=${IMAGE_REGISTRY} -image-namespace=${IMAGE_NAMESPACE}
 
+run-e2ev3-test:
+	go test -v ./test/e2ev3/ -timeout 1h -tags=e2e -count=1  -args -image-tag=${TAG} -image-registry=${IMAGE_REGISTRY} -image-namespace=${IMAGE_NAMESPACE}
+
 .PHONY: update-hubble
 update-hubble:
 	@echo "Checking for Hubble updates..."

controller/Dockerfile

@@ -18,7 +18,8 @@ ARG GOARCH=amd64 # default to amd64
 ARG GOOS=linux # default to linux
 ENV GOARCH=${GOARCH}
 ENV GOOS=${GOOS}
-RUN if [ "$GOOS" = "linux" ] ; then \
+RUN --mount=type=cache,target=/var/cache/tdnf \
+    if [ "$GOOS" = "linux" ] ; then \
     tdnf install -y clang lld bpftool libbpf-devel; \
     fi
 COPY ./pkg/plugin /go/src/github.com/microsoft/retina/pkg/plugin
@@ -68,7 +69,7 @@ RUN --mount=type=cache,target="/root/.cache/go-build" go build -v -o /go/bin/ret
 
 # tools image
 FROM azurelinux-core AS tools
-RUN tdnf install -y \
+RUN --mount=type=cache,target=/var/cache/tdnf tdnf install -y \
     clang \
     iproute \
     iptables \

go.mod

@@ -361,6 +361,7 @@ require (
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 // indirect
 	github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect
 	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
+	github.com/Azure/go-workflow v0.1.13 // indirect
 	github.com/Azure/msi-dataplane v0.4.3 // indirect
 	github.com/Crocmagnon/fatcontext v0.7.1 // indirect
 	github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 // indirect
@@ -391,6 +392,7 @@ require (
 	github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20240514230400-03fa26f5508f // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
+	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/bkielbasa/cyclop v1.2.3 // indirect
 	github.com/blacktop/go-dwarf v1.0.9 // indirect
 	github.com/blacktop/go-macho v1.1.162 // indirect

go.sum

@@ -142,6 +142,8 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z
 github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
 github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
+github.com/Azure/go-workflow v0.1.13 h1:tqYmmKsw068Uu/OKRIctQEe1H6BZyXs9mJtzocc7jtE=
+github.com/Azure/go-workflow v0.1.13/go.mod h1:gOt4hadDnP+SzV+ywWJRcM1BSopJ+1rfGlhrTIR040I=
 github.com/Azure/msi-dataplane v0.4.3 h1:dWPWzY4b54tLIR9T1Q014Xxd/1DxOsMIp6EjRFAJlQY=
 github.com/Azure/msi-dataplane v0.4.3/go.mod h1:yAfxdJyvcnvSDfSyOFV9qm4fReEQDl+nZLGeH2ZWSmw=
 github.com/Azure/perf-tests/network/benchmarks/netperf v0.0.0-20241008140716-395a79947d2c h1:TMXh4Z1Z98o4Ob7JYhiNHocITQXHqJSsrB5ts4uwKl8=
@@ -295,6 +297,8 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ
 github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
 github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o=
+github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=

operator/Dockerfile

@@ -5,9 +5,14 @@ ARG VERSION
 ARG APP_INSIGHTS_ID
 
 WORKDIR /workspace
+
+# Cache module download separately from source changes.
+COPY go.mod go.sum ./
+RUN go mod download
+
 COPY . .
 
-RUN tdnf install -y jq
+RUN --mount=type=cache,target=/var/cache/tdnf tdnf install -y jq
 
 # Default linux/architecture.
 ARG GOOS=linux

test/e2e/framework/prometheus/prometheus.go

@@ -81,23 +81,55 @@ func CheckMetricFromBuffer(prometheusMetricData []byte, metricName string, valid
 	return nil
 }
 
+func formatMetricDetail(name string, mf *promclient.MetricFamily, m *promclient.Metric) string {
+	var sb strings.Builder
+	sb.WriteString(name)
+	sb.WriteString("{")
+	for i, label := range m.GetLabel() {
+		if i > 0 {
+			sb.WriteString(", ")
+		}
+		fmt.Fprintf(&sb, "%s=%q", label.GetName(), label.GetValue())
+	}
+	sb.WriteString("}")
+
+	switch mf.GetType() {
+	case promclient.MetricType_COUNTER:
+		fmt.Fprintf(&sb, " counter:%v", m.GetCounter().GetValue())
+	case promclient.MetricType_GAUGE:
+		fmt.Fprintf(&sb, " gauge:%v", m.GetGauge().GetValue())
+	case promclient.MetricType_HISTOGRAM:
+		h := m.GetHistogram()
+		fmt.Fprintf(&sb, " histogram:count=%v sum=%v", h.GetSampleCount(), h.GetSampleSum())
+	case promclient.MetricType_SUMMARY:
+		s := m.GetSummary()
+		fmt.Fprintf(&sb, " summary:count=%v sum=%v", s.GetSampleCount(), s.GetSampleSum())
+	case promclient.MetricType_UNTYPED:
+		fmt.Fprintf(&sb, " untyped:%v", m.GetUntyped().GetValue())
+	}
+
+	return sb.String()
+}
+
 func verifyValidMetricPresent(metricName string, data map[string]*promclient.MetricFamily, validMetric map[string]string) error {
-	for _, metric := range data {
-		if metric.GetName() == metricName {
-			for _, metric := range metric.GetMetric() {
+	for _, mf := range data {
+		if mf.GetName() == metricName {
+			for _, m := range mf.GetMetric() {
 
 				// get all labels and values on the metric
 				metricLabels := map[string]string{}
-				for _, label := range metric.GetLabel() {
+				for _, label := range m.GetLabel() {
 					metricLabels[label.GetName()] = label.GetValue()
 				}
 
 				// if valid metric is empty, then we just need to make sure the metric and value is present
 				if len(validMetric) == 0 && len(metricLabels) > 0 {
+					log.Printf("found matching metric: %s", formatMetricDetail(metricName, mf, m))
 					return nil
 				}
 
 				if reflect.DeepEqual(metricLabels, validMetric) {
+					log.Printf("found matching metric: %s", formatMetricDetail(metricName, mf, m))
 					return nil
 				}
 			}
@@ -130,18 +162,19 @@ func getAllPrometheusMetricsFromURL(url string) (map[string]*promclient.MetricFa
 // verifyValidMetricPresentPartial checks if a metric exists with labels that contain
 // all the key-value pairs in validMetric (partial matching - the metric can have additional labels)
 func verifyValidMetricPresentPartial(metricName string, data map[string]*promclient.MetricFamily, validMetric map[string]string) error {
-	for _, metric := range data {
-		if metric.GetName() == metricName {
-			for _, metric := range metric.GetMetric() {
+	for _, mf := range data {
+		if mf.GetName() == metricName {
+			for _, m := range mf.GetMetric() {
 
 				// get all labels and values on the metric
 				metricLabels := map[string]string{}
-				for _, label := range metric.GetLabel() {
+				for _, label := range m.GetLabel() {
 					metricLabels[label.GetName()] = label.GetValue()
 				}
 
 				// if valid metric is empty, then we just need to make sure the metric and value is present
 				if len(validMetric) == 0 && len(metricLabels) > 0 {
+					log.Printf("found matching metric: %s", formatMetricDetail(metricName, mf, m))
 					return nil
 				}
 
@@ -155,6 +188,7 @@ func verifyValidMetricPresentPartial(metricName string, data map[string]*promcli
 				}
 
 				if allMatch {
+					log.Printf("found matching metric: %s", formatMetricDetail(metricName, mf, m))
 					return nil
 				}
 			}

test/e2e/scenarios/dns/validate-advanced-dns-metric.go

@@ -4,7 +4,6 @@ package dns
 
 import (
 	"fmt"
-	"log"
 
 	"github.com/microsoft/retina/test/e2e/framework/constants"
 	"github.com/microsoft/retina/test/e2e/framework/kubernetes"
@@ -50,8 +49,6 @@ func (v *ValidateAdvancedDNSRequestMetrics) Run() error {
 	if err != nil {
 		return errors.Wrapf(err, "failed to verify advance dns request metrics %s", dnsAdvRequestCountMetricName)
 	}
-	log.Printf("found metrics matching %+v\n", dnsAdvRequestCountMetricName)
-
 	return nil
 }
 
@@ -106,8 +103,6 @@ func (v *ValidateAdvanceDNSResponseMetrics) Run() error {
 	if err != nil {
 		return errors.Wrapf(err, "failed to verify advance dns response metrics %s", dnsAdvRequestCountMetricName)
 	}
-	log.Printf("found metrics matching %+v\n", dnsAdvResponseCountMetricName)
-
 	return nil
 }
 

test/e2e/scenarios/dns/validate-basic-dns-metric.go

@@ -4,7 +4,6 @@ package dns
 
 import (
 	"fmt"
-	"log"
 
 	"github.com/microsoft/retina/test/e2e/framework/constants"
 	prom "github.com/microsoft/retina/test/e2e/framework/prometheus"
@@ -30,8 +29,6 @@ func (v *validateBasicDNSRequestMetrics) Run() error {
 	if err != nil {
 		return errors.Wrapf(err, "failed to verify basic dns request metrics %s", dnsBasicRequestCountMetricName)
 	}
-	log.Printf("found metrics matching %+v\n", dnsBasicRequestCountMetricName)
-
 	return nil
 }
 
@@ -64,8 +61,6 @@ func (v *validateBasicDNSResponseMetrics) Run() error {
 	if err != nil {
 		return errors.Wrapf(err, "failed to verify basic dns response metrics %s", dnsBasicResponseCountMetricName)
 	}
-	log.Printf("found metrics matching %+v\n", dnsBasicResponseCountMetricName)
-
 	return nil
 }
 

test/e2e/scenarios/drop/validate-drop-metric.go

@@ -2,7 +2,6 @@ package drop
 
 import (
 	"fmt"
-	"log"
 
 	prom "github.com/microsoft/retina/test/e2e/framework/prometheus"
 )
@@ -45,7 +44,6 @@ func (v *ValidateRetinaDropMetric) Run() error {
 		return fmt.Errorf("failed to verify prometheus metrics %s: %w", dropBytesMetricName, err)
 	}
 
-	log.Printf("found metrics matching %+v\n", metric)
 	return nil
 }
 

test/e2e/scenarios/latency/validate-latency-metric.go

@@ -2,7 +2,6 @@ package latency
 
 import (
 	"fmt"
-	"log"
 
 	"github.com/microsoft/retina/test/e2e/framework/constants"
 	prom "github.com/microsoft/retina/test/e2e/framework/prometheus"
@@ -25,8 +24,6 @@ func (v *ValidateAPIServerLatencyMetric) Run() error {
 	if err != nil {
 		return errors.Wrapf(err, "failed to verify latency metrics %s", latencyBucketMetricName)
 	}
-
-	log.Printf("found metrics matching %s\n", latencyBucketMetricName)
 	return nil
 }
 

test/e2e/scenarios/tcp/validate-flow-metric.go

@@ -2,7 +2,6 @@ package flow
 
 import (
 	"fmt"
-	"log"
 
 	prom "github.com/microsoft/retina/test/e2e/framework/prometheus"
 )
@@ -37,7 +36,6 @@ func (v *ValidateRetinaTCPStateMetric) Run() error {
 		}
 	}
 
-	log.Printf("found metrics matching %+v\n", validMetrics)
 	return nil
 }
 

test/e2e/scenarios/tcp/validate-tcp-connection-remote.go

@@ -2,7 +2,6 @@ package flow
 
 import (
 	"fmt"
-	"log"
 
 	prom "github.com/microsoft/retina/test/e2e/framework/prometheus"
 )
@@ -30,7 +29,6 @@ func (v *ValidateRetinaTCPConnectionRemoteMetric) Run() error {
 		}
 	}
 
-	log.Printf("found metrics matching %+v\n", tcpConnectionRemoteMetricName)
 	return nil
 }
 

test/e2ev3/Makefile

@@ -0,0 +1,50 @@
+TIMEOUT_KIND   ?= 60m
+TIMEOUT_AZURE  ?= 120m
+PROVIDER       ?= kind
+KUBECONFIG     ?=
+CREATE_INFRA   ?= true
+DELETE_INFRA   ?= true
+
+REPO_ROOT := $(shell git rev-parse --show-toplevel)
+
+GO_TEST = cd $(REPO_ROOT) && go test -v -tags e2e ./test/e2ev3/ -timeout
+
+# base flags, computed from variables above
+FLAGS = -provider=$(PROVIDER) -create-infra=$(CREATE_INFRA) -delete-infra=$(DELETE_INFRA)
+ifdef KUBECONFIG
+FLAGS += -kubeconfig=$(KUBECONFIG)
+endif
+
+# timeout picked by provider
+ifeq ($(PROVIDER),kind)
+TIMEOUT = $(TIMEOUT_KIND)
+else
+TIMEOUT = $(TIMEOUT_AZURE)
+endif
+
+.PHONY: test-e2e test-basic-metrics test-advanced-metrics test-hubble-metrics test-capture \
+        test-basic-metrics-exp test-advanced-metrics-exp help
+
+test-e2e: ## run all e2e scenarios, defaults to kind
+	$(GO_TEST) $(TIMEOUT) $(FLAGS)
+
+test-basic-metrics: ## run basic metrics scenarios (drop, tcp, dns)
+	$(GO_TEST) $(TIMEOUT) -run TestE2ERetina/BasicMetrics$$ $(FLAGS)
+
+test-advanced-metrics: ## run advanced metrics scenarios (dns, latency)
+	$(GO_TEST) $(TIMEOUT) -run TestE2ERetina/AdvancedMetrics$$ $(FLAGS)
+
+test-hubble-metrics: ## run hubble metrics scenarios (drop, tcp, dns, flows)
+	$(GO_TEST) $(TIMEOUT) -run TestE2ERetina/HubbleMetrics$$ $(FLAGS)
+
+test-capture: ## run packet capture scenarios
+	$(GO_TEST) $(TIMEOUT) -run TestE2ERetina/Capture$$ $(FLAGS)
+
+test-basic-metrics-exp: ## run experimental basic metrics scenarios
+	$(GO_TEST) $(TIMEOUT) -run TestE2ERetina/BasicMetricsExperimental$$ $(FLAGS)
+
+test-advanced-metrics-exp: ## run experimental advanced metrics scenarios
+	$(GO_TEST) $(TIMEOUT) -run TestE2ERetina/AdvancedMetricsExperimental$$ $(FLAGS)
+
+help: ## show this help
+	@grep -E '^[a-z-]+:.*## ' $(MAKEFILE_LIST) | awk -F ':.*## ' '{printf "  %-26s %s\n", $$1, $$2}'