bench(soak): add mysql_cdc to the nightly soak rotation (CON-179 R6)

.github/workflows/soak_nightly.yml

@@ -49,25 +49,80 @@ permissions:
   contents: read
 
 jobs:
+  # Resolve the scenario matrix BEFORE strategy evaluation ever touches the
+  # dispatch input. Splicing the raw input into fromJSON(format(...)) would
+  # let a crafted value (`a","b`) fan one dispatch into N paid soak jobs,
+  # and a stray quote would kill the run with an opaque strategy error
+  # instead of the allowlist's friendly one — so the input is allowlisted
+  # here (quotes and backslashes can't pass the regex) and the soak job
+  # consumes only this job's output.
+  plan:
+    runs-on: ubuntu-latest
+    outputs:
+      scenarios: ${{ steps.set.outputs.scenarios }}
+    steps:
+      - name: Resolve scenario matrix
+        id: set
+        env:
+          SCENARIO: ${{ github.event.inputs.scenario || 'postgres/orders-soak' }}
+        run: |
+          if [ "${{ github.event_name }}" = "schedule" ]; then
+            # THE ROTATION lives here: the schedule soaks every listed
+            # scenario; adding a connector to the rotation = one more entry
+            # (see SOAK.md "Adding a connector to the rotation").
+            echo 'scenarios=["postgres/orders-soak", "mysql/orders-soak"]' >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          # bash [[ =~ ]], NOT `echo | grep`: grep matches per line, so a
+          # multi-line value (reachable via the REST dispatch API) would pass
+          # on its first line and then write attacker-controlled extra lines
+          # into the line-oriented $GITHUB_OUTPUT below — re-assigning
+          # `scenarios` and fanning one dispatch into N paid jobs. Bash's =~
+          # anchors $ to end-of-STRING and no class here admits a newline,
+          # so any embedded newline fails the match.
+          re='^[A-Za-z0-9][A-Za-z0-9_-]*(/[A-Za-z0-9][A-Za-z0-9_-]*)*$'
+          if ! [[ "${SCENARIO}" =~ $re ]]; then
+            echo "::error::Invalid scenario — expected a single path of [A-Za-z0-9_-] segments, e.g. postgres/orders-soak"
+            exit 1
+          fi
+          echo "scenarios=[\"${SCENARIO}\"]" >> "$GITHUB_OUTPUT"
+
   soak:
+    needs: plan
+    strategy:
+      # max-parallel: 1 because one bench at a time, ever — all sessions
+      # share (and destroy) the same shared Terraform stack and fixed-name
+      # session resources. The workflow-level concurrency group serializes
+      # across runs; this serializes within one.
+      #
+      # fail-fast: false so one connector's red night still soaks the rest.
+      max-parallel: 1
+      fail-fast: false
+      matrix:
+        scenario: ${{ fromJSON(needs.plan.outputs.scenarios) }}
     runs-on: ubuntu-latest
     # Provisioning (~15m) + 95m window + teardown (~15m) + slack. Must stay
-    # under the 4h credential session below.
+    # under the 4h credential session below. Per matrix job, so the full
+    # scheduled rotation may take rotation-size × this.
     timeout-minutes: 220
     env:
       REDPANDA_LICENSE_SECRET: redpanda-connect-bench/license
     steps:
       # Before anything credentialed: the dispatch input is available to
       # the whole write-access population (same as soak_pr.yml's /soak
-      # comment), and Task splices {{.scenario}} raw into a shell command —
-      # so it gets the same allowlist soak_pr.yml applies, and an invalid
-      # value kills the job before the provisioner role is even assumed.
+      # comment), and Task splices {{.scenario}} raw into a shell command.
+      # The plan job already allowlisted it, so this is defense in depth —
+      # it re-checks the value each job actually received, and still kills
+      # the job before the provisioner role is even assumed.
       - name: Validate scenario input
         env:
-          SCENARIO: ${{ github.event.inputs.scenario || 'postgres/orders-soak' }}
+          SCENARIO: ${{ matrix.scenario }}
         run: |
-          if ! echo "${SCENARIO}" | grep -Eq '^[A-Za-z0-9][A-Za-z0-9_-]*(/[A-Za-z0-9][A-Za-z0-9_-]*)*$'; then
-            echo "::error::Invalid scenario '${SCENARIO}' — expected a path of [A-Za-z0-9_-] segments, e.g. postgres/orders-soak"
+          # Full-string match, same as the plan job (grep would pass a
+          # multi-line value on its first matching line).
+          re='^[A-Za-z0-9][A-Za-z0-9_-]*(/[A-Za-z0-9][A-Za-z0-9_-]*)*$'
+          if ! [[ "${SCENARIO}" =~ $re ]]; then
+            echo "::error::Invalid scenario — expected a single path of [A-Za-z0-9_-] segments, e.g. postgres/orders-soak"
             exit 1
           fi
 
@@ -91,7 +146,7 @@ jobs:
         id: gate
         shell: bash
         env:
-          SCENARIO: ${{ github.event.inputs.scenario || 'postgres/orders-soak' }}
+          SCENARIO: ${{ matrix.scenario }}
         run: |
           if [ "${{ github.event_name }}" != "schedule" ]; then
             echo "run=true" >> "$GITHUB_OUTPUT"
@@ -156,7 +211,7 @@ jobs:
         env:
           # Environment passing, not ${{ }} substitution into the script —
           # same reasoning as the gate step above and soak_pr.yml.
-          SCENARIO: ${{ github.event.inputs.scenario || 'postgres/orders-soak' }}
+          SCENARIO: ${{ matrix.scenario }}
         run: |
           task aws:bench scenario="$SCENARIO" 2>&1 | tee "$RUNNER_TEMP/soak-run.log"
 
@@ -182,6 +237,7 @@ jobs:
         if: always() && steps.gate.outputs.run == 'true'
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
-          name: soak-run-log
+          # job-index, not the scenario path: artifact names reject `/`.
+          name: soak-run-log-${{ strategy.job-index }}
           path: ${{ runner.temp }}/soak-run.log
           retention-days: 30

benchmarking/aws/README.md

@@ -3,11 +3,11 @@
 Production-shaped benchmarks and soak tests for Redpanda Connect connectors,
 run on real AWS infrastructure in a dedicated, disposable account.
 
-This tree contains the framework core plus the **postgres_cdc** stack — the
-subset needed by the soak pipeline (CON-179 R6). Further connector stacks
-(mysql, sqlserver, oracle, mongodb, dynamodb, iceberg) exist on the original
-development fork and land here with their own PRs, each bringing its
-scenarios and tests.
+This tree contains the framework core plus the **postgres_cdc** and
+**mysql_cdc** stacks — the subset needed by the soak pipeline (CON-179 R6).
+Further connector stacks (sqlserver, oracle, mongodb, dynamodb, iceberg)
+exist on the original development fork and land here with their own PRs,
+each bringing its scenarios and tests.
 
 ## What a run does
 
@@ -90,19 +90,20 @@ and re-enabled after.
 | Path | Role |
 |---|---|
 | `runner/` | Go orchestrator: provision → stage → seed → sweep/soak → results → teardown |
-| `scenarios/postgres/` | bench + soak + PR-comparison scenarios |
-| `seeders/cdc-rows-postgres/` | write-workload generator |
+| `scenarios/<engine>/` | bench + soak + PR-comparison scenarios (postgres, mysql) |
+| `seeders/cdc-rows-<engine>/` | write-workload generators |
 | `terraform/shared/` | per-session VPC, hosts, brokers, results bucket |
-| `terraform/stacks/postgres/` | per-session RDS Postgres |
+| `terraform/stacks/<engine>/` | per-session RDS Postgres / RDS MySQL |
 | `terraform/persistent/` | applied once: dashboards, alarms, OIDC, reaper, archive |
 | `cleanup-lambda/` | the orphan reaper (own Go module) |
 | `SOAK.md` | soak operations runbook |
 
 ## Known limitations
 
 - postgres_cdc IAM auth cannot work against vanilla RDS (replication
-  connections reject IAM tokens); the credential-rotation soak window is
-  covered by mysql_cdc when its stack lands.
+  connections reject IAM tokens); the credential-rotation soak window
+  belongs to the mysql_cdc soak, which runs password auth today — the IAM
+  increment is still open (see SOAK.md).
 - One-lane serialization: soaks and benches queue on the shared stack.
   Session-scoped isolation is the tracked scaling path.
 - The weekly 24h soak needs a reaper exemption tag + a non-GitHub conductor

benchmarking/aws/SOAK.md

@@ -15,7 +15,7 @@ files named below.
 | Runner soak mode | `runner/` (main.go, matrix.go, cloudwatch.go) | scaled cadences, 10-min S3 checkpoints, per-minute CloudWatch emission, backlog series |
 | Dashboards + alarms | `terraform/persistent/` (`main.tf` `soak_scenarios` var, `alarms.tf`) | one dashboard + three alarms (stall / rss-slope / backlog) per scenario → SNS `redpanda-connect-bench-soak-alerts` |
 | Archive + baseline | `redpanda-connect-bench-soak-archive` bucket | result.json + raw artifacts per run; `soak-index/` feeds the rolling-baseline comparator (advisory < 3 runs, then fails the job on throughput < 85% / RSS > 130% of baseline) |
-| Nightly workflow | `.github/workflows/soak_nightly.yml` | 08:10 UTC cron (arms only from the default branch) + manual dispatch; OIDC creds (4h), license from Secrets Manager, teardown verified against AWS |
+| Nightly workflow | `.github/workflows/soak_nightly.yml` | 08:10 UTC cron over the rotation matrix (postgres, mysql; serialized, arms only from the default branch) + manual dispatch; OIDC creds (4h), license from Secrets Manager, teardown verified against AWS |
 | PR comparison | `.github/workflows/soak_pr.yml` | `/soak` comment (write-access gated) → base-vs-PR binaries, same infra, sticky comparison comment |
 
 ## Adding a connector to the rotation
@@ -32,14 +32,23 @@ files named below.
    in `terraform/persistent/variables.tf` (key → connector + scenario
    name), then `task aws:persistent`. Alarms and the dashboard are
    generated per entry; Slack delivery is on by default via `slack.tf`'s
-   committed IDs — no extra vars needed.
-4. **First runs**: dispatch the nightly workflow manually with the
+   committed IDs — no extra vars needed. **Apply this BEFORE the rotation
+   entry merges** — the cron arms itself the moment the workflow lands on
+   the default branch, and until the persistent apply runs, nothing
+   watches the new connector's metrics.
+4. **Add it to the nightly rotation**: append the scenario path to the
+   schedule list in `soak_nightly.yml`'s plan job. The matrix is
+   serialized (`max-parallel: 1` — one bench at a time) and
+   `fail-fast: false`, so each rotation entry soaks even when another is
+   red; a manual dispatch still runs only the scenario named in the input.
+5. **First runs**: dispatch the nightly workflow manually with the
    scenario input. The baseline comparator stays advisory until three
    soak-index entries exist.
-5. **Optionally add a PR variant** (`*-soak-pr.yaml`): same scenario with
+6. **Optionally add a PR variant** (`*-soak-pr.yaml`): same scenario with
    30m duration and `arms: [{id: base, binary: base}, {id: pr, binary:
    pr}]`. The scenario NAME must differ from the nightly's so its metrics
-   land outside the alarm dimensions.
+   land outside the alarm dimensions. `/soak <engine>/<name>-pr` selects it
+   on a PR (the default remains postgres/orders-soak-pr).
 
 ## Operating it
 
@@ -106,7 +115,7 @@ files named below.
   backend now uses S3-native `use_lockfile` locking), disarmed the orphan
   reaper's schedule rule **every night**, and deleted the stall + backlog
   alarms. The exemption is the `cloud-nuke-excluded = true` tag, applied via
-  `default_tags` in all three stacks — the persistent stack so the reaper
+  `default_tags` in every stack — the persistent stack so the reaper
   schedule and alarms survive, the session stacks so a live bench crossing
   02:25 UTC isn't terminated mid-run. Our OWN reaper keys on `Project`, not
   this tag, so bench cleanup at the 4h TTL is unaffected. Any new resource
@@ -115,8 +124,11 @@ files named below.
 
 - postgres_cdc IAM auth cannot work against vanilla RDS (replication-
   protocol connections reject IAM tokens — verified live 2026-08-12), so
-  the credential-rotation window is covered by a future mysql_cdc soak or
-  Aurora, not the postgres soak.
+  the postgres soak cannot cover the credential-rotation window. The mysql
+  soak is its designated home (RDS IAM tokens ride normal-protocol MySQL
+  connections), but it runs password auth today — the IAM increment (the
+  AWSAuthenticationPlugin DB user in reset, the rds-db:connect policy in
+  the mysql stack, a live validation pass) is still open.
 - The nightly cron only arms once `soak_nightly.yml` is on the repo's
   default branch; until then, manual dispatch.
 - The weekly 24h soak needs two prerequisites before it can exist: a

benchmarking/aws/go.mod

@@ -18,13 +18,15 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.44.6
 	github.com/aws/aws-sdk-go-v2/service/ssm v1.73.6
 	github.com/aws/smithy-go v1.27.8
+	github.com/go-sql-driver/mysql v1.10.0
 	github.com/jackc/pgx/v5 v5.10.0
 	github.com/quasilyte/go-ruleguard/dsl v0.3.23
 	github.com/stretchr/testify v1.12.1
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
+	filippo.io/edwards25519 v1.2.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.19.36 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.37 // indirect

benchmarking/aws/go.sum

@@ -1,3 +1,5 @@
+filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
+filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
 github.com/aws/aws-sdk-go-v2 v1.43.6 h1:RrmFcqCBxkJuf7g1axVo5krB4jM/AO8r5e5oujrgdoQ=
 github.com/aws/aws-sdk-go-v2 v1.43.6/go.mod h1:tXpPM+v0D1lndmga+HqqLDIzUFJlEeR21aspVklHF00=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18 h1:LAfOuhAH331fmOjTQpAaOlH+Ftn7RzSDJ2VFwjdMMy4=
@@ -48,6 +50,8 @@ github.com/aws/smithy-go v1.27.8 h1:FR0dxZfIlV7Z8eh2iHfIofdunw382XsDV3Mxt9nUvRY=
 github.com/aws/smithy-go v1.27.8/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
+github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=

benchmarking/aws/runner/scenario.go

@@ -207,14 +207,27 @@ type engineSpec struct {
 }
 
 // engineSpecs is the registry mechanism a connector's stack PR extends —
-// see the type doc above. mysql_cdc, oracledb_cdc, microsoft_sql_server_cdc,
+// see the type doc above. oracledb_cdc, microsoft_sql_server_cdc,
 // mongodb_cdc, and aws_dynamodb_cdc were trimmed out of this scope-reduced
 // tree (postgres_cdc soak testing only); each returns with its own stack PR.
 var engineSpecs = map[string]engineSpec{
 	"postgres_cdc": {
 		DSNOutputKey: "postgres_dsn",
 		DSNEnvVar:    "POSTGRES_DSN",
 	},
+	// mysql_cdc connects via a go-sql-driver DSN, but the mariadb CLI the
+	// reset runs through takes discrete -h/-P/-u/-p flags rather than a DSN
+	// URL, so the Reset*OutputKey fields point at the mysql stack's
+	// split-out outputs.
+	"mysql_cdc": {
+		DSNOutputKey:       "mysql_dsn",
+		DSNEnvVar:          "MYSQL_DSN",
+		ResetHostOutputKey: "mysql_host",
+		ResetPortOutputKey: "mysql_port",
+		ResetUserOutputKey: "mysql_user",
+		ResetPassOutputKey: "mysql_password",
+		ResetDBOutputKey:   "mysql_db",
+	},
 }
 
 func engineSpecFor(connector string) (engineSpec, bool) {

benchmarking/aws/runner/scenario_test.go

@@ -97,14 +97,40 @@ func TestEngineSpecFor_Postgres(t *testing.T) {
 	}
 }
 
+func TestEngineSpecFor_MySQL(t *testing.T) {
+	es, ok := engineSpecFor("mysql_cdc")
+	if !ok {
+		t.Fatalf("mysql_cdc should be registered")
+	}
+	if es.DSNOutputKey != "mysql_dsn" {
+		t.Errorf("DSNOutputKey = %q, want mysql_dsn", es.DSNOutputKey)
+	}
+	if es.DSNEnvVar != "MYSQL_DSN" {
+		t.Errorf("DSNEnvVar = %q, want MYSQL_DSN", es.DSNEnvVar)
+	}
+	// The mysql CLI takes discrete flags, not a DSN URL, so the reset
+	// builder needs every split-out output key populated.
+	for name, got := range map[string]string{
+		"ResetHostOutputKey": es.ResetHostOutputKey,
+		"ResetPortOutputKey": es.ResetPortOutputKey,
+		"ResetUserOutputKey": es.ResetUserOutputKey,
+		"ResetPassOutputKey": es.ResetPassOutputKey,
+		"ResetDBOutputKey":   es.ResetDBOutputKey,
+	} {
+		if got == "" {
+			t.Errorf("%s must be set for mysql's discrete-flags reset form", name)
+		}
+	}
+}
+
 func TestEngineSpecFor_Unknown(t *testing.T) {
 	if _, ok := engineSpecFor("kafka_franz_in_disguise"); ok {
 		t.Error("unknown connector should not resolve")
 	}
-	// mysql_cdc, oracledb_cdc, microsoft_sql_server_cdc, mongodb_cdc, and
+	// oracledb_cdc, microsoft_sql_server_cdc, mongodb_cdc, and
 	// aws_dynamodb_cdc were trimmed from the registry in this scope-reduced
-	// (postgres_cdc-only) tree; each returns with its own stack PR.
-	for _, trimmed := range []string{"mysql_cdc", "oracledb_cdc", "microsoft_sql_server_cdc", "mongodb_cdc", "aws_dynamodb_cdc"} {
+	// tree; each returns with its own stack PR.
+	for _, trimmed := range []string{"oracledb_cdc", "microsoft_sql_server_cdc", "mongodb_cdc", "aws_dynamodb_cdc"} {
 		if _, ok := engineSpecFor(trimmed); ok {
 			t.Errorf("%s should not be registered in this scope-reduced tree", trimmed)
 		}
@@ -390,6 +416,41 @@ func TestLoadScenario_OrdersSoak(t *testing.T) {
 	require.NoError(t, s.Validate())
 }
 
+// TestLoadScenario_MySQLOrdersSoak is the same validity gate for the mysql
+// rotation entry: the shipped scenarios/mysql/orders-soak.yaml must load and
+// validate as a soak scenario, not merely parse.
+func TestLoadScenario_MySQLOrdersSoak(t *testing.T) {
+	s, err := LoadScenario("../scenarios/mysql/orders-soak.yaml")
+	require.NoError(t, err)
+	require.True(t, s.Soak)
+	require.Equal(t, "mysql_cdc", s.Connector)
+	require.Equal(t, "mysql", s.Stack)
+	require.Equal(t, []int{2}, s.Matrix.CPUPoints)
+	require.Empty(t, s.Matrix.Arms)
+	require.Equal(t, "c8g.xlarge", s.Infra.Runner.InstanceType)
+	require.NotNil(t, s.Workload)
+	require.Equal(t, 90*time.Minute, s.Workload.Duration)
+	require.Equal(t, 5*time.Minute, s.Workload.Warmup)
+	require.Equal(t, 10000, s.Workload.WriteRatePerSec)
+	// mysql_cdc requires a checkpoint cache resource; the scenario must ship
+	// one or the rendered config fails lint on the runner host, an hour of
+	// provisioning too late.
+	require.Contains(t, s.Pipeline, "cache_resources")
+	require.NoError(t, s.Validate())
+}
+
+// TestLoadScenario_MySQLOrdersSoakPR pins the /soak A/B variant: binary-only
+// arms on the same single-point soak profile as the nightly, under a
+// deliberately different scenario name so its metrics dodge the alarms.
+func TestLoadScenario_MySQLOrdersSoakPR(t *testing.T) {
+	s, err := LoadScenario("../scenarios/mysql/orders-soak-pr.yaml")
+	require.NoError(t, err)
+	require.True(t, s.Soak)
+	require.True(t, s.IsBinaryArmScenario())
+	require.Equal(t, 30*time.Minute, s.Workload.Duration)
+	require.NotEqual(t, "mysql-orders-soak", s.Name)
+}
+
 func TestScenarioValidate_RejectsSoakWithMultipleCPUPoints(t *testing.T) {
 	s := &Scenario{
 		Name: "soak-x", Connector: "postgres_cdc", Stack: "postgres", Soak: true,