tests: add goleak leak detection and advisory race-detector CI (CON-179)

.claude/agents/tester.md

@@ -592,6 +592,29 @@ t.Cleanup(func() {
 - Do not use `tc := tc` in loop bodies. Go 1.22+ fixed loop variable scoping.
 - Use `t.Context()` for test contexts. Exception: in `t.Cleanup()` functions, use `context.Background()` because `t.Context()` is already canceled during cleanup.
 
+# Goroutine Leak Detection (goleak)
+
+Connector packages opt in to goroutine-leak detection by wiring `go.uber.org/goleak` into a package-level `TestMain` (one `TestMain` per package — check for an existing one first). See `internal/impl/protobuf` and `internal/impl/sql` for reference.
+
+```go
+// main_test.go (with the package's usual license header)
+func TestMain(m *testing.M) {
+    goleak.VerifyTestMain(m,
+        goleak.IgnoreCurrent(),
+        // Narrow, commented ignores for known-benign goroutines only, e.g.:
+        // database/sql keeps a connection-pool opener alive per *sql.DB.
+        goleak.IgnoreTopFunction("database/sql.(*DB).connectionOpener"),
+    )
+}
+```
+
+Adding it to a new package:
+
+1. Create `main_test.go` with the pattern above and the package's license header.
+2. Run `go test -count=1 ./internal/impl/<pkg>/...` at least twice. For every goleak failure, decide: real leak in the component (fix it) or a benign third-party goroutine (add a narrow `goleak.IgnoreTopFunction` with a comment naming the library).
+3. `VerifyTestMain` also runs after the package's integration tests in nightly CI, so verify those too (or tune ignores) before merging — the package must be green on a clean baseline, never aspirationally.
+4. Prefer `IgnoreTopFunction` over broad ignores; an ignore list that swallows everything defeats the check.
+
 # Running Tests
 
 ```bash

.github/race-blocking-packages.txt

@@ -0,0 +1,11 @@
+# Packages promoted to BLOCKING in the race-detector CI job (CON-179 R3).
+#
+# The race_test.yml workflow runs `go test -race` on every internal/impl
+# package a PR touches. For packages listed here (one directory name per
+# line, e.g. `sql`), a race failure fails the job; for everything else the
+# failure is an advisory warning only.
+#
+# Promotion rule: add a package here only after it holds a green -race
+# baseline (repeated clean runs of `task test:unit-race` scoped to the
+# package, plus a green advisory run in CI). A blocking gate must never be
+# able to fail on a pre-existing race the PR didn't touch.

.github/workflows/race_test.yml

@@ -0,0 +1,158 @@
+name: Race Detector Tests
+
+# Advisory-first race-detector job (CON-179 R3).
+#
+# Runs `go test -race` scoped to the internal/impl/<component> packages a PR
+# touches, mirroring the auto-scoping in integration_test.yml to keep the
+# 2-10x race-detector runtime cost bounded.
+#
+# Failures are advisory (a warning annotation, not a red check) unless the
+# failing package is listed in .github/race-blocking-packages.txt. Packages
+# are promoted to that list only after they hold a green -race baseline, so
+# this job can never fail a PR on a pre-existing race the PR didn't touch.
+
+on:
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      filter:
+        description: 'Package filter (e.g. sql kafka). Required for manual runs.'
+        required: true
+        type: string
+
+jobs:
+  race-test:
+    runs-on: ubuntu-latest
+    # Bounded comfortably above the worst case of the "Run race-detector
+    # tests" step: that step caps itself at MAX_PACKAGES=8 packages tested
+    # serially, each under its own `go test -timeout 10m` budget (80m worst
+    # case), plus headroom for per-package compilation, checkout with
+    # fetch-depth: 0, and the Go setup step. Packages beyond MAX_PACKAGES are
+    # skipped with a logged advisory warning rather than run, so the job
+    # should never actually approach this ceiling; it exists purely so a
+    # slow-but-not-hanging run can never be killed by the job timeout, which
+    # would produce a red check with no actual race (violating the
+    # advisory-first design).
+    timeout-minutes: 120
+    env:
+      # The Go race detector requires cgo.
+      CGO_ENABLED: 1
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Detect changed packages
+        if: ${{ github.event_name == 'pull_request' }}
+        id: detect
+        env:
+          BASE_REF: ${{ github.base_ref }}
+        run: |
+          set -euo pipefail
+          # Each of these runs as its own statement (rather than nested
+          # inside another command's argument list) so that `set -e` catches
+          # a git failure directly, instead of silently yielding an empty
+          # FILTERS that would be indistinguishable from "no packages
+          # touched".
+          MERGE_BASE=$(git merge-base HEAD "origin/${BASE_REF}")
+          CHANGED=$(git diff --name-only "${MERGE_BASE}"...HEAD)
+          # grep exits 1 when nothing matches "^internal/impl/" - that IS the
+          # legitimate "no connector packages touched" case, so only this
+          # command is allowed to fail non-fatally.
+          MATCHED=$(printf '%s\n' "${CHANGED}" | { grep '^internal/impl/' || true; })
+          # xargs (not tr) joins the package names: on empty/blank input it
+          # emits a genuinely empty string, whereas tr '\n' ' ' would emit a
+          # lone space that defeats the `filters != ''` step guards below and
+          # runs the job (checkout, Go install, empty summary table) on every
+          # PR that touches no connector packages.
+          FILTERS=$(printf '%s\n' "${MATCHED}" | cut -d/ -f3 | sort -u | xargs)
+          echo "filters=${FILTERS}" >> "$GITHUB_OUTPUT"
+
+      - name: Install Go
+        if: ${{ steps.detect.outputs.filters != '' || github.event.inputs.filter != '' }}
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Run race-detector tests
+        if: ${{ steps.detect.outputs.filters != '' || github.event.inputs.filter != '' }}
+        env:
+          FILTER: ${{ steps.detect.outputs.filters || github.event.inputs.filter }}
+        run: |
+          set -uo pipefail
+          BLOCKING_FILE=.github/race-blocking-packages.txt
+          # Cap the number of packages tested per run so the job's own
+          # worst-case duration is bounded (see the job-level timeout-minutes
+          # comment). Packages beyond the cap are skipped with a logged
+          # advisory warning rather than risking a job timeout kill, which
+          # would produce a red check with no actual race.
+          MAX_PACKAGES=8
+          PKG_TIMEOUT=10m
+
+          ALL_PKGS=()
+          for pkg in ${FILTER}; do
+            [ -d "internal/impl/${pkg}" ] && ALL_PKGS+=("${pkg}")
+          done
+
+          PKGS=("${ALL_PKGS[@]}")
+          if [ "${#ALL_PKGS[@]}" -gt "${MAX_PACKAGES}" ]; then
+            PKGS=("${ALL_PKGS[@]:0:${MAX_PACKAGES}}")
+            SKIPPED=("${ALL_PKGS[@]:${MAX_PACKAGES}}")
+            echo "::warning::Race detector job only tests the first ${MAX_PACKAGES} changed packages per run to bound job duration. Skipped this run (not tested, advisory only): ${SKIPPED[*]}"
+            echo "> ⚠️ Race detector: skipped ${#SKIPPED[@]} package(s) to bound job duration: \`${SKIPPED[*]}\`" >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+          {
+            echo "## Race detector results"
+            echo ""
+            echo "| Package | Result |"
+            echo "| --- | --- |"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          BLOCKING_FAILED=()
+
+          for pkg in "${PKGS[@]}"; do
+            outfile=$(mktemp)
+            echo "::group::go test -race ./internal/impl/${pkg}/..."
+            # `set -e` is on by default for GitHub Actions run steps; the
+            # trailing `|| status=$?` keeps a non-zero go test exit from
+            # aborting the script so every package gets classified and its
+            # summary row written, instead of the loop dying on the first
+            # failure.
+            status=0
+            go test -count=1 -race -timeout "${PKG_TIMEOUT}" -shuffle=on "./internal/impl/${pkg}/..." >"${outfile}" 2>&1 || status=$?
+            cat "${outfile}"
+            echo "::endgroup::"
+
+            # Only an actual data race counts as a race for blocking
+            # purposes - build errors, flakes, and per-package timeouts must
+            # not be classified as races, and must never land in the
+            # blocking bucket even for a package on the blocking list (the
+            # ordinary unit-test workflow already gates plain failures).
+            is_race=false
+            grep -q "WARNING: DATA RACE" "${outfile}" && is_race=true
+            is_blocking=false
+            grep -qxF "${pkg}" "${BLOCKING_FILE}" 2>/dev/null && is_blocking=true
+
+            if [ "${status}" -eq 0 ]; then
+              result="✅ pass"
+            elif [ "${is_race}" = true ] && [ "${is_blocking}" = true ]; then
+              result="❌ RACE (blocking)"
+              BLOCKING_FAILED+=("${pkg}")
+            elif [ "${is_race}" = true ]; then
+              result="⚠️ RACE (advisory)"
+              echo "::warning::Data race detected in ${pkg} (advisory, non-blocking). See the job log. Packages get promoted to blocking via ${BLOCKING_FILE} once their -race baseline is green."
+            else
+              result="⚠️ failed (non-race)"
+              echo "::warning::go test failed in ${pkg} without a detected data race (build error, flake, or timeout). Not a race-detector finding, so this stays advisory-only; the ordinary unit-test workflow gates plain test failures."
+            fi
+
+            echo "| \`${pkg}\` | ${result} |" >> "$GITHUB_STEP_SUMMARY"
+            rm -f "${outfile}"
+          done
+
+          if [ "${#BLOCKING_FAILED[@]}" -gt 0 ]; then
+            echo "::error::Race detector failures in blocking packages: ${BLOCKING_FAILED[*]}. These packages have a green -race baseline; this failure was introduced by the PR."
+            exit 1
+          fi

CONTRIBUTING.md

@@ -169,6 +169,7 @@ The rule throughout is **conformance to the existing fleet**: mirror the shape t
 ## 6. Before You Open a PR
 
 - Run `task fmt`, `task lint`, and `task test` locally — all green.
+- Run `task test:unit-race` scoped to the packages you touched, e.g. `task test:unit-race -- ./internal/impl/<component>/...`. CI runs an advisory race-detector job on changed connector packages; packages listed in `.github/race-blocking-packages.txt` treat race failures as blocking.
 - Run `task docs` and commit the result: the generated component pages **and** the `internal/plugins/info.csv` row. CI fails on stale docs.
 - Every new component has an `internal/plugins/info.csv` entry with the correct distribution and cloud classification.
 - A license header on **every** new `.go` file (including test and benchmark helpers), matching the component's distribution.

go.mod

@@ -200,6 +200,7 @@ require (
 	go.opentelemetry.io/otel/sdk/metric v1.43.0
 	go.opentelemetry.io/otel/trace v1.44.0
 	go.starlark.net v0.0.0-20260210143700-b62fd896b91b
+	go.uber.org/goleak v1.3.0
 	go.uber.org/multierr v1.11.0
 	golang.org/x/crypto v0.54.0
 	golang.org/x/net v0.56.0

internal/impl/mysql/main_test.go

@@ -0,0 +1,26 @@
+// Copyright 2026 Redpanda Data, Inc.
+//
+// Licensed as a Redpanda Enterprise file under the Redpanda Community
+// License (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md
+
+package mysql
+
+import (
+	"testing"
+
+	"go.uber.org/goleak"
+)
+
+// TestMain verifies that no goroutines are leaked by the tests in this
+// package (CON-179 R2).
+func TestMain(m *testing.M) {
+	goleak.VerifyTestMain(m,
+		goleak.IgnoreCurrent(),
+		// internal/license: InjectTestService starts an hourly expiry-metric
+		// loop whose cancel func is not reachable from tests.
+		goleak.IgnoreTopFunction("github.com/redpanda-data/connect/v4/internal/license.(*Service).updateExpiryMetricLoop"),
+	)
+}