fix(test): attempt at fixing metrics profile e2e test flake

test/e2e/collection/metrics/profile_test.go

@@ -21,7 +21,7 @@ import (
 
 const (
 	allowlistedMetric    = `vector_open_files`
-	nonAllowlistedMetric = `vector_started_total`
+	nonAllowlistedMetric = `vector_uptime_seconds`
 )
 
 var _ = Describe("[e2e][collection][metrics] Metrics Collection Profiles", Ordered, func() {

test/e2e/flowcontrol/utils.go

@@ -16,18 +16,21 @@ import (
 
 const (
 	VectorCompSentEvents = `rate(vector_component_sent_events_total{component_name="%s"}[30s])`
-	VectorUpTotal        = `vector_started_total`
+	VectorUpTotal        = `vector_uptime_seconds`
 )
 
 func WaitForMetricsToShow() bool {
+	var lastErr error
 	if err := wait.PollUntilContextTimeout(context.TODO(), 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) {
 		response, err := prometheus.Query(VectorUpTotal)
 		if err != nil {
-			return false, err
+			lastErr = err
+			log.V(3).Error(err, "Transient error querying metrics, will retry")
+			return false, nil
 		}
 		return prometheus.HasResults(response), nil
 	}); err != nil {
-		log.V(0).Error(err, "Error waiting for metrics to be available")
+		log.V(0).Error(err, "Error waiting for metrics to be available", "lastQueryError", lastErr)
 		return false
 	}
 	return true

test/helpers/prometheus/prometheus.go

@@ -33,9 +33,7 @@ func GetThanosHost() (string, error) {
 	return strings.TrimSpace(host), err
 }
 
-func QueryPrometheus(host, token, query string) map[string]interface{} {
-	empty := map[string]any{}
-
+func QueryPrometheus(host, token, query string) (map[string]interface{}, error) {
 	tr := &http.Transport{
 		TLSClientConfig: &tls.Config{
 			InsecureSkipVerify: true, //nolint:gosec
@@ -44,8 +42,7 @@ func QueryPrometheus(host, token, query string) map[string]interface{} {
 	client := &http.Client{Transport: tr, Timeout: 30 * time.Second}
 	request, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://%s/api/v1/query", host), nil)
 	if err != nil {
-		log.V(0).Error(err, "Failed to create request", "host", host)
-		return empty
+		return nil, fmt.Errorf("failed to create request for host %s: %w", host, err)
 	}
 	request.Header.Add("Authorization", "Bearer "+token)
 	request.Header.Add("Accept", "application/json")
@@ -56,8 +53,7 @@ func QueryPrometheus(host, token, query string) map[string]interface{} {
 
 	response, err := client.Do(request)
 	if err != nil {
-		log.V(0).Error(err, "Error sending request to Thanos")
-		return empty
+		return nil, fmt.Errorf("error sending request to Thanos: %w", err)
 	}
 	defer func() {
 		if err := response.Body.Close(); err != nil {
@@ -66,23 +62,20 @@ func QueryPrometheus(host, token, query string) map[string]interface{} {
 	}()
 
 	if response.StatusCode != http.StatusOK {
-		log.V(0).Info("Unexpected status from Thanos", "status", response.StatusCode, "query", query)
-		return empty
+		return nil, fmt.Errorf("unexpected status %d from Thanos for query %q", response.StatusCode, query)
 	}
 
 	body, err := io.ReadAll(response.Body)
 	if err != nil {
-		log.V(0).Error(err, "Failed to read response body")
-		return empty
+		return nil, fmt.Errorf("failed to read Thanos response body: %w", err)
 	}
 
 	var result map[string]interface{}
 	if err := json.Unmarshal(body, &result); err != nil {
-		log.V(0).Error(err, "Failed to unmarshal Thanos response")
-		return empty
+		return nil, fmt.Errorf("failed to unmarshal Thanos response: %w", err)
 	}
 
-	return result
+	return result, nil
 }
 
 func Query(query string) (map[string]interface{}, error) {
@@ -94,7 +87,7 @@ func Query(query string) (map[string]interface{}, error) {
 	if err != nil {
 		return nil, err
 	}
-	return QueryPrometheus(host, token, query), nil
+	return QueryPrometheus(host, token, query)
 }
 
 func HasResults(response map[string]interface{}) bool {