diff --git a/.github/workflows/soak_nightly.yml b/.github/workflows/soak_nightly.yml index a9af81a760..7862cdd0e6 100644 --- a/.github/workflows/soak_nightly.yml +++ b/.github/workflows/soak_nightly.yml @@ -15,11 +15,13 @@ name: Nightly Soak # reaches the default branch. # # One-time setup (operator): -# 1. TF_VAR_soak_alert_email= task aws:persistent -# (OIDC provider, provisioner role, -# archive bucket, dashboards, reaper; -# the alert email has no default — -# use a monitored team alias) +# 1. task aws:persistent — creates the OIDC provider, provisioner role, +# archive bucket, dashboards, reaper, and Slack alert delivery (the +# workspace/channel IDs are committed defaults in +# terraform/persistent/slack.tf; a NEW account/workspace needs the +# one-time Chatbot console OAuth first; the IDs are required by +# validation). Optional email backups are manual SNS subscriptions +# to both topics — see SOAK.md. # 2. aws secretsmanager create-secret \ # --name redpanda-connect-bench/license \ # --secret-string file://rpcn.license --region us-east-2 diff --git a/benchmarking/aws/SOAK.md b/benchmarking/aws/SOAK.md index 5c1540ff24..1eaeea09a4 100644 --- a/benchmarking/aws/SOAK.md +++ b/benchmarking/aws/SOAK.md @@ -30,8 +30,9 @@ files named below. `task aws:validate scenario=/` must pass. 3. **Register the dashboard + alarms**: add an entry to `soak_scenarios` in `terraform/persistent/variables.tf` (key → connector + scenario - name), then `TF_VAR_soak_alert_email= task aws:persistent`. - Alarms and the dashboard are generated per entry. + name), then `task aws:persistent`. Alarms and the dashboard are + generated per entry; Slack delivery is on by default via `slack.tf`'s + committed IDs — no extra vars needed. 4. **First runs**: dispatch the nightly workflow manually with the scenario input. The baseline comparator stays advisory until three soak-index entries exist. @@ -53,13 +54,17 @@ files named below. the workflows — no relevant merge, no run. Fails open: a missing entry or unknown SHA runs the soak. - **One-time account setup** (already done in 605419575229, needed again - only for a new account): `TF_VAR_soak_alert_email= task - aws:persistent` (builds the reaper's `bootstrap.zip` itself; the alert - email is deliberately undefaulted — point it at a monitored team - alias, never an individual); create the license secret: `aws secretsmanager + only for a new account): authorize the Slack workspace in the AWS + Chatbot console (OAuth — see `terraform/persistent/slack.tf`) and put + the resulting IDs in that file's defaults, then `task aws:persistent` + (builds the reaper's `bootstrap.zip` itself); optionally add a manual + email backup — `aws sns subscribe --protocol email + --notification-endpoint --topic-arn ` for BOTH the + soak-alerts and orphans topics, then click each confirmation link + (deliberately outside Terraform: no re-apply can unsubscribe it); + create the license secret: `aws secretsmanager create-secret --name redpanda-connect-bench/license --secret-string - file:// --region us-east-2`; confirm the SNS email - subscription. + file:// --region us-east-2`. - **Laptop runs**: always from a git worktree (never a checkout you might branch-switch mid-run), always with credentials that outlive the run — `aws-vault exec` static creds die at ~1h; prefer the scheduled workflow. @@ -72,11 +77,25 @@ files named below. sweeping every 15 min. A bench legitimately running past 4h needs the rule disabled first (and re-enabled after — set a reminder). The persistent stack itself is exempt via its distinct Project tag. -- **Alerts** land at the `redpanda-connect-bench-soak-alerts` SNS topic - (email today; swap the subscription for Chatbot/Slack without touching - alarms). Alarm emails during a run are the acute channel; a red nightly - workflow is the between-builds channel; the `/soak` comment is the - before-merge channel. +- **Alerts** land at the `redpanda-connect-bench-soak-alerts` SNS topic and + deliver to #soak-redpanda-connect via AWS Chatbot + (`terraform/persistent/slack.tf` — the workspace/channel IDs are + committed defaults, so plain `task aws:persistent` keeps Slack wired). + Alarm cards render natively; reaper notices arrive via the + custom-notification envelope in `cleanup-lambda/sweep.go` (plain SNS + text is silently dropped by Chatbot — keep that envelope). Email backup + is a manual SNS subscription to BOTH topics (see one-time setup above) — + deliberately unmanaged, so no re-apply can silently unsubscribe it. + Slack is the only Terraform-managed channel, so blanking its IDs fails + validation instead of leaving the topics unrouted. Alarms during a run + are the acute channel; a red nightly workflow is the between-builds + channel; the `/soak` comment is the before-merge channel. +- **Grafana**: `grafana/soak-dashboard.json` is an importable dashboard + over the same CloudWatch metrics (template dropdowns for + connector/scenario, alarm thresholds drawn in, CloudWatch alarm + annotations). It needs a CloudWatch data source for account + 605419575229 in the Grafana stack — read-only metrics access; the + dashboard binds to it via a data-source variable at import time. ## Known limitations diff --git a/benchmarking/aws/cleanup-lambda/sweep.go b/benchmarking/aws/cleanup-lambda/sweep.go index c86a0c5650..65681a13a6 100644 --- a/benchmarking/aws/cleanup-lambda/sweep.go +++ b/benchmarking/aws/cleanup-lambda/sweep.go @@ -10,6 +10,7 @@ package main import ( "context" + "encoding/json" "errors" "fmt" "log/slog" @@ -30,6 +31,40 @@ import ( "github.com/aws/smithy-go" ) +// perProtocolMessage wraps the human-readable sweep summary in an SNS +// per-protocol envelope: email subscribers get the plain text, while every +// other protocol — which is how AWS Chatbot's Slack delivery subscribes — +// gets Chatbot's custom-notification schema. Chatbot silently DROPS plain +// SNS text (it only forwards formats it recognises), so without this the +// Slack channel would look wired up while never showing a reaper notice. +// +// A non-nil error means the caller must publish msg as plain text WITHOUT +// MessageStructure=json — SNS rejects a structure-json Publish whose body +// isn't a JSON object with a "default" key, so returning raw text here +// would lose the notice entirely instead of degrading to email-only. +func perProtocolMessage(msg string) (string, error) { + chatbot, err := json.Marshal(map[string]any{ + "version": "1.0", + "source": "custom", + "content": map[string]string{ + "textType": "client-markdown", + "title": "bench orphan-cleanup ran", + "description": msg, + }, + }) + if err != nil { + return "", err + } + envelope, err := json.Marshal(map[string]string{ + "default": string(chatbot), + "email": msg, + }) + if err != nil { + return "", err + } + return string(envelope), nil +} + // isGone reports whether err is a not-found-style AWS error: the resource // vanished between the tag index (which lags real deletions by minutes to // hours) and our describe/delete call. Already-gone is a successful no-op, @@ -641,11 +676,20 @@ func Sweep(ctx context.Context, api cleanupAPI, now time.Time, ttl time.Duration if len(report.Failed) > 0 { msg += "\n\nfailed (still accruing cost; retried next sweep):\n" + strings.Join(report.Failed, "\n") } - if _, err := api.Publish(ctx, &sns.PublishInput{ + in := &sns.PublishInput{ TopicArn: aws.String(snsTopicARN), Subject: aws.String("bench orphan-cleanup ran"), - Message: aws.String(msg), - }); err != nil { + } + if envelope, err := perProtocolMessage(msg); err != nil { + // Unreachable for maps of strings, but if it ever fires, + // plain text (email-only delivery) beats a rejected publish. + slog.Error("per-protocol envelope failed; publishing plain text", "err", err) + in.Message = aws.String(msg) + } else { + in.Message = aws.String(envelope) + in.MessageStructure = aws.String("json") + } + if _, err := api.Publish(ctx, in); err != nil { slog.Error("sns publish failed", "err", err) } } diff --git a/benchmarking/aws/cleanup-lambda/sweep_test.go b/benchmarking/aws/cleanup-lambda/sweep_test.go index 0f87ae1d3a..8369043469 100644 --- a/benchmarking/aws/cleanup-lambda/sweep_test.go +++ b/benchmarking/aws/cleanup-lambda/sweep_test.go @@ -10,6 +10,7 @@ package main import ( "context" + "encoding/json" "strconv" "testing" "time" @@ -74,6 +75,7 @@ type FakeAWS struct { DeletedRDSSubnetGroups []string DeletedRDSParamGroups []string SNSMessages []string + SNSStructures []string // CallLog records ":" for every mutating call, in call // order, so tests can assert cross-resource dependency ordering. CallLog []string @@ -387,6 +389,7 @@ func (f *FakeAWS) DeleteRole(_ context.Context, in *iam.DeleteRoleInput) (*iam.D func (f *FakeAWS) Publish(_ context.Context, in *sns.PublishInput) (*sns.PublishOutput, error) { f.SNSMessages = append(f.SNSMessages, aws.ToString(in.Message)) + f.SNSStructures = append(f.SNSStructures, aws.ToString(in.MessageStructure)) return &sns.PublishOutput{}, nil } @@ -809,6 +812,48 @@ func TestSweep_ErroredDeleteNotReportedDestroyed(t *testing.T) { require.NotContains(t, api.SNSMessages[0], "destroyed:", "must not list the errored resource as destroyed") } +// TestSweep_PublishUsesPerProtocolEnvelope pins the SNS message contract: +// MessageStructure=json, with "email" carrying the plain-text summary and +// "default" carrying AWS Chatbot's custom-notification schema. Chatbot +// (the Slack delivery path) silently drops any other shape, so a break +// here means a Slack channel that looks wired up but never shows a reaper +// notice. +func TestSweep_PublishUsesPerProtocolEnvelope(t *testing.T) { + now := time.Date(2026, 5, 21, 12, 0, 0, 0, time.UTC) + old := now.Add(-4 * time.Hour) + api := &FakeAWS{ + TaggedResources: []rgtatypes.ResourceTagMapping{ + {ResourceARN: aws.String("arn:aws:ec2:us-east-2:1:instance/i-old")}, + }, + EC2Instances: map[string]ec2types.Instance{ + "i-old": {InstanceId: aws.String("i-old"), LaunchTime: &old}, + }, + } + _, err := Sweep(t.Context(), api, now, 3*time.Hour, "arn:sns:topic") + require.NoError(t, err) + require.Len(t, api.SNSMessages, 1) + require.Equal(t, []string{"json"}, api.SNSStructures) + + var envelope map[string]string + require.NoError(t, json.Unmarshal([]byte(api.SNSMessages[0]), &envelope)) + require.Contains(t, envelope["email"], "destroyed 1 resources") + require.Contains(t, envelope["email"], "ec2:i-old") + + var chatbot struct { + Version string `json:"version"` + Source string `json:"source"` + Content struct { + TextType string `json:"textType"` + Description string `json:"description"` + } `json:"content"` + } + require.NoError(t, json.Unmarshal([]byte(envelope["default"]), &chatbot)) + require.Equal(t, "1.0", chatbot.Version) + require.Equal(t, "custom", chatbot.Source) + require.Equal(t, "client-markdown", chatbot.Content.TextType) + require.Equal(t, envelope["email"], chatbot.Content.Description) +} + // TestSweep_StaleTagIndexEntryIsSilent: the tag index lags real deletions // by minutes to hours, so a sweep routinely describes resources that no // longer exist and gets a NotFound-style error back. That is a successful diff --git a/benchmarking/aws/grafana/soak-dashboard.json b/benchmarking/aws/grafana/soak-dashboard.json new file mode 100644 index 0000000000..be8fe8949e --- /dev/null +++ b/benchmarking/aws/grafana/soak-dashboard.json @@ -0,0 +1,483 @@ +{ + "__comment": "CDC soak dashboard for Grafana (import via Dashboards -> New -> Import). Reads the RedpandaConnect/Bench CloudWatch namespace published by the soak runner (see runner/cloudwatch.go for the emitter contract and SOAK.md for the pipeline). Data arrives in ~10-minute checkpoint batches with per-minute backfill, so live runs trail real time by up to one checkpoint. The datasource/connector/scenario dropdowns are template variables: new connectors joining the rotation appear automatically.", + "title": "RPCN CDC Soak", + "uid": "rpcn-cdc-soak", + "tags": [ + "redpanda-connect", + "soak", + "cdc" + ], + "timezone": "utc", + "editable": true, + "graphTooltip": 1, + "time": { + "from": "now-7d", + "to": "now" + }, + "refresh": "", + "schemaVersion": 39, + "description": "Sustained-load soak metrics for Redpanda Connect CDC connectors (CON-179 R6). Metrics arrive in ~10-minute checkpoint batches - a flat right edge during a live run is lag, not a stall. Alarm thresholds are drawn as red lines. Defaults to a 7-day window: soaks run nightly at most, so short ranges legitimately show no data.", + "annotations": { + "list": [ + { + "name": "Soak alarms", + "enable": true, + "iconColor": "red", + "datasource": { + "type": "cloudwatch", + "uid": "${ds}" + }, + "queryMode": "Annotations", + "region": "us-east-2", + "namespace": "RedpandaConnect/Bench", + "prefixMatching": true, + "alarmNamePrefix": "rpcn-soak-", + "actionPrefix": "", + "matchExact": false, + "statistic": "Maximum" + } + ] + }, + "templating": { + "list": [ + { + "name": "ds", + "label": "CloudWatch data source", + "type": "datasource", + "query": "cloudwatch", + "current": {}, + "refresh": 1 + }, + { + "name": "connector", + "label": "Connector", + "type": "query", + "datasource": { + "type": "cloudwatch", + "uid": "${ds}" + }, + "query": "dimension_values(us-east-2, RedpandaConnect/Bench, ThroughputMBps, Connector)", + "refresh": 2, + "sort": 1, + "current": {}, + "includeAll": false + }, + { + "name": "scenario", + "label": "Scenario", + "type": "query", + "datasource": { + "type": "cloudwatch", + "uid": "${ds}" + }, + "query": "dimension_values(us-east-2, RedpandaConnect/Bench, ThroughputMBps, Scenario, {\"Connector\":\"$connector\"})", + "refresh": 2, + "sort": 1, + "current": {}, + "includeAll": false + } + ] + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Throughput - broker vs Connect log", + "description": "Broker-side produce rate vs the rate reported in Connect's own log. Divergence means the log is lying or the brokers are seeing traffic Connect isn't reporting. Red line = stall alarm threshold (0.5 MB/s for 10 min pages).", + "gridPos": { + "x": 0, + "y": 0, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "cloudwatch", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "MBs", + "min": 0, + "custom": { + "thresholdsStyle": { + "mode": "line" + }, + "insertNulls": 3600000 + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.5 + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "region": "us-east-2", + "namespace": "RedpandaConnect/Bench", + "metricName": "ThroughputMBps", + "dimensions": { + "Connector": "$connector", + "Scenario": "$scenario" + }, + "statistic": "Average", + "period": "60", + "matchExact": false, + "label": "broker MB/s", + "queryMode": "Metrics", + "metricQueryType": 0, + "metricEditorMode": 0 + }, + { + "refId": "B", + "region": "us-east-2", + "namespace": "RedpandaConnect/Bench", + "metricName": "LogThroughputMBps", + "dimensions": { + "Connector": "$connector", + "Scenario": "$scenario" + }, + "statistic": "Average", + "period": "60", + "matchExact": false, + "label": "Connect log MB/s", + "queryMode": "Metrics", + "metricQueryType": 0, + "metricEditorMode": 0 + } + ] + }, + { + "id": 2, + "type": "timeseries", + "title": "Memory (RSS is what the OOM killer sees)", + "description": "RSS climbing while heap stays flat points at native/off-heap growth. A steady upward RSS slope is the slow-leak class - see the RSS slope panel for the alarmed signal.", + "gridPos": { + "x": 12, + "y": 0, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "cloudwatch", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "min": 0, + "custom": { + "insertNulls": 3600000 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "region": "us-east-2", + "namespace": "RedpandaConnect/Bench", + "metricName": "RSSBytes", + "dimensions": { + "Connector": "$connector", + "Scenario": "$scenario" + }, + "statistic": "Maximum", + "period": "60", + "matchExact": false, + "label": "RSS", + "queryMode": "Metrics", + "metricQueryType": 0, + "metricEditorMode": 0 + }, + { + "refId": "B", + "region": "us-east-2", + "namespace": "RedpandaConnect/Bench", + "metricName": "HeapInUseBytes", + "dimensions": { + "Connector": "$connector", + "Scenario": "$scenario" + }, + "statistic": "Maximum", + "period": "60", + "matchExact": false, + "label": "Go heap in use", + "queryMode": "Metrics", + "metricQueryType": 0, + "metricEditorMode": 0 + } + ] + }, + { + "id": 3, + "type": "timeseries", + "title": "RSS slope (leak detector)", + "description": "Least-squares fit of RSS growth per minute over the trailing window, computed per checkpoint cycle (one point per ~10 min). Red line = leak alarm threshold (2 MB/min sustained for 2 consecutive checkpoints pages; that rate is ~3 GB/day). Negative values = memory settling.", + "gridPos": { + "x": 0, + "y": 8, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "cloudwatch", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "thresholdsStyle": { + "mode": "line" + }, + "axisLabel": "bytes/min", + "insertNulls": 3600000 + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 2000000 + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "region": "us-east-2", + "namespace": "RedpandaConnect/Bench", + "metricName": "RSSSlopeBytesPerMin", + "dimensions": { + "Connector": "$connector", + "Scenario": "$scenario" + }, + "statistic": "Maximum", + "period": "600", + "matchExact": false, + "label": "RSS slope", + "queryMode": "Metrics", + "metricQueryType": 0, + "metricEditorMode": 0 + } + ] + }, + { + "id": 4, + "type": "timeseries", + "title": "End-to-end backlog (seconds behind source)", + "description": "How far the connector is behind the database it replicates. A constant offset (~60s snapshot catch-up) is normal; a climbing line means it is losing the race with the source. Red line = backlog alarm threshold (600s for 10 min pages).", + "gridPos": { + "x": 8, + "y": 8, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "cloudwatch", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0, + "custom": { + "thresholdsStyle": { + "mode": "line" + }, + "insertNulls": 3600000 + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 600 + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "region": "us-east-2", + "namespace": "RedpandaConnect/Bench", + "metricName": "BacklogSeconds", + "dimensions": { + "Connector": "$connector", + "Scenario": "$scenario" + }, + "statistic": "Maximum", + "period": "60", + "matchExact": false, + "label": "backlog", + "queryMode": "Metrics", + "metricQueryType": 0, + "metricEditorMode": 0 + } + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Goroutines", + "description": "Goroutine count in the Connect process. Monotonic growth across the run is a goroutine leak even when memory looks fine.", + "gridPos": { + "x": 16, + "y": 8, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "cloudwatch", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "custom": { + "insertNulls": 3600000 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "region": "us-east-2", + "namespace": "RedpandaConnect/Bench", + "metricName": "Goroutines", + "dimensions": { + "Connector": "$connector", + "Scenario": "$scenario" + }, + "statistic": "Maximum", + "period": "60", + "matchExact": false, + "label": "goroutines", + "queryMode": "Metrics", + "metricQueryType": 0, + "metricEditorMode": 0 + } + ] + }, + { + "id": 6, + "type": "timeseries", + "title": "Records/s + run liveness", + "description": "Delivered records per second against the workload's declared rate, with RunActive (right axis, 0/1) marking when a soak window is actually open - gaps in every other panel outside RunActive=1 are absence of runs, not failures.", + "gridPos": { + "x": 0, + "y": 16, + "w": 24, + "h": 8 + }, + "datasource": { + "type": "cloudwatch", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "custom": { + "insertNulls": 3600000 + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "run active" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "max", + "value": 2 + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.fillOpacity", + "value": 15 + } + ] + } + ] + }, + "targets": [ + { + "refId": "A", + "region": "us-east-2", + "namespace": "RedpandaConnect/Bench", + "metricName": "RecordsPerSec", + "dimensions": { + "Connector": "$connector", + "Scenario": "$scenario" + }, + "statistic": "Average", + "period": "60", + "matchExact": false, + "label": "records/s", + "queryMode": "Metrics", + "metricQueryType": 0, + "metricEditorMode": 0 + }, + { + "refId": "B", + "region": "us-east-2", + "namespace": "RedpandaConnect/Bench", + "metricName": "RunActive", + "dimensions": { + "Connector": "$connector", + "Scenario": "$scenario" + }, + "statistic": "Maximum", + "period": "60", + "matchExact": false, + "label": "run active", + "queryMode": "Metrics", + "metricQueryType": 0, + "metricEditorMode": 0 + } + ] + } + ] +} \ No newline at end of file diff --git a/benchmarking/aws/terraform/persistent/alarms.tf b/benchmarking/aws/terraform/persistent/alarms.tf index d495807e4b..89f59935c4 100644 --- a/benchmarking/aws/terraform/persistent/alarms.tf +++ b/benchmarking/aws/terraform/persistent/alarms.tf @@ -8,29 +8,31 @@ resource "aws_sns_topic" "soak_alerts" { name = "redpanda-connect-bench-soak-alerts" } -variable "soak_alert_email" { - # Endpoint for soak alarm notifications — the pipeline's only acute - # channel, so it must be a monitored team alias, not an individual's - # mailbox (a personal default rots silently when that person moves on). - # No default on purpose: an unconfigured apply fails loudly instead of - # subscribing a stale address. Pass via TF_VAR_soak_alert_email at - # `task aws:persistent` time. SNS sends a confirmation email on first - # apply — the subscription is inactive until the link is clicked. - # Swap for AWS Chatbot / Slack later without touching the alarms. - type = string +# Email backup subscriptions are deliberately NOT managed here. Terraform +# manages exactly one alert channel — Slack (slack.tf, committed IDs) — and +# email backups are manual, out-of-band subscriptions (see SOAK.md): +# +# aws sns subscribe --topic-arn \ +# --protocol email --notification-endpoint +# +# Why not a Terraform resource: an address can't be committed (no team +# alias exists; a personal one doesn't belong in the repo), and a +# subscription count-gated on an ambient TF_VAR_* is silently destroyed by +# any re-apply whose shell didn't re-export it — worse than unmanaged, +# because resurrecting an SNS email subscription needs a human to re-click +# the confirmation link. Manual subscriptions survive every apply. +# +# The pre-existing email subscription in 605419575229 is preserved as +# exactly such an unmanaged subscription by the removed block below (state +# forget, not destroy). +removed { + from = aws_sns_topic_subscription.soak_alerts_email - validation { - condition = can(regex("^[^@\\s]+@[^@\\s]+$", var.soak_alert_email)) - error_message = "soak_alert_email must be a single email address (use a team alias, not a personal mailbox)." + lifecycle { + destroy = false } } -resource "aws_sns_topic_subscription" "soak_alerts_email" { - topic_arn = aws_sns_topic.soak_alerts.arn - protocol = "email" - endpoint = var.soak_alert_email -} - locals { soak_alarm_dims = { for k, v in var.soak_scenarios : k => { @@ -45,7 +47,7 @@ locals { resource "aws_cloudwatch_metric_alarm" "soak_stall" { for_each = local.soak_alarm_dims alarm_name = "rpcn-soak-${each.key}-stall" - alarm_description = "Soak throughput <= 0.5 MB/s for 10 minutes while the run is active — the silent-stall class (#4648/#4655)." + alarm_description = "STALL: the soak connector stopped moving data (throughput ~0 for 10+ min during an active run). It usually still LOOKS healthy — process up, health checks green — so don't trust 'connected'. First: open the rpcn-bench-soak dashboard, confirm a run is actually in progress, then pull the run's log. Runbook: benchmarking/aws/SOAK.md. Past examples: github.com/redpanda-data/connect/issues/4648 (input silently stalls), /4655 (health endpoint lies)." namespace = "RedpandaConnect/Bench" metric_name = "ThroughputMBps" dimensions = each.value @@ -76,7 +78,7 @@ resource "aws_cloudwatch_metric_alarm" "soak_stall" { resource "aws_cloudwatch_metric_alarm" "soak_rss_slope" { for_each = local.soak_alarm_dims alarm_name = "rpcn-soak-${each.key}-rss-slope" - alarm_description = "Soak RSS climbing >= 2 MB/min across consecutive 10-minute checkpoints — the slow-leak class (inc-2861/#4527/#4657)." + alarm_description = "MEMORY LEAK: the soak connector's memory (RSS) is climbing >= 2 MB/min sustained across checkpoints. Sounds small but that's ~3 GB/day — in production this OOM-kills within days. First: open the Memory widget on the rpcn-bench-soak dashboard and look at the RSS slope shape. Runbook: benchmarking/aws/SOAK.md. Past examples: github.com/redpanda-data/connect/issues/4527 (protobuf profile leak), /4657 (growth under back-pressure)." namespace = "RedpandaConnect/Bench" metric_name = "RSSSlopeBytesPerMin" dimensions = each.value @@ -98,7 +100,7 @@ resource "aws_cloudwatch_metric_alarm" "soak_rss_slope" { resource "aws_cloudwatch_metric_alarm" "soak_backlog" { for_each = local.soak_alarm_dims alarm_name = "rpcn-soak-${each.key}-backlog" - alarm_description = "Soak end-to-end backlog >= 600s for 10 minutes — the connector is falling behind its source." + alarm_description = "BACKLOG: the soak connector has fallen 10+ minutes behind the database it replicates and stayed there. Data IS flowing and memory is fine — it's just slower than the source is writing, so the gap grows without bound (in production: a replica going hours stale). First: check BacklogSeconds vs Records/s on the rpcn-bench-soak dashboard — flat-but-high backlog means a one-time hiccup, climbing means it's losing the race. Runbook: benchmarking/aws/SOAK.md." namespace = "RedpandaConnect/Bench" metric_name = "BacklogSeconds" dimensions = each.value diff --git a/benchmarking/aws/terraform/persistent/slack.tf b/benchmarking/aws/terraform/persistent/slack.tf new file mode 100644 index 0000000000..bcd553611c --- /dev/null +++ b/benchmarking/aws/terraform/persistent/slack.tf @@ -0,0 +1,98 @@ +# Slack delivery for soak alarms and orphan-reaper notices via AWS Chatbot +# (console name: "Amazon Q Developer in chat applications"). +# +# One-time manual prerequisite (cannot be Terraform'd): authorize the Slack +# workspace in the AWS Chatbot console of this account (OAuth; may need a +# Slack workspace admin to approve the AWS app). That yields the workspace +# ID committed as the default below, alongside the channel's ID (from the +# channel's "About" tab). Slack is the ONLY Terraform-managed alert +# channel — email backups are manual SNS subscriptions (see alarms.tf and +# SOAK.md) precisely so no ambient variable can silently unsubscribe them — +# so blanking these IDs fails validation rather than leaving both topics +# unrouted. +# +# CloudWatch alarm messages render natively as alarm cards. The reaper's +# notices are published in Chatbot's custom-notification schema via SNS +# per-protocol messages (see cleanup-lambda/sweep.go) — Chatbot silently +# drops plain-text SNS payloads, so without that the reaper channel would +# look connected while delivering nothing. + +# The real IDs are committed as defaults (they are not secrets — both appear +# in every Slack URL), mirroring backend.hcl's account-specific values: +# change them in a private fork if you run this elsewhere. Committed +# defaults, not env vars, because a count-gated resource driven by ambient +# TF_VAR_* would be silently DESTROYED by any re-apply whose shell didn't +# re-export them — the primary alert channel vanishing with no error. + +variable "slack_workspace_id" { + description = "Slack workspace (team) ID from the AWS Chatbot console OAuth." + type = string + default = "TPMVB7YMC" # Redpanda workspace (authorized 2026-08-27) +} + +variable "slack_channel_id" { + description = "Slack channel ID for soak alarms + reaper notices." + type = string + default = "C0BT00TTA11" # #soak-redpanda-connect + + validation { + # Slack is the only Terraform-managed alert channel: blanking these + # would leave BOTH SNS topics with no managed subscriber, silently. + # A fork that truly wants Slack-free operation must set up the manual + # email subscriptions (SOAK.md) and adjust this check consciously. + condition = var.slack_channel_id != "" && var.slack_workspace_id != "" + error_message = "slack_workspace_id and slack_channel_id are required — Slack is the only Terraform-managed alert channel (email backups are manual; see SOAK.md)." + } +} + +# These three were originally created count-gated (state addresses [0]); +# the moved blocks keep the live resources in place now that the gate is +# gone. +moved { + from = aws_iam_role.chatbot_channel[0] + to = aws_iam_role.chatbot_channel +} + +moved { + from = aws_iam_role_policy_attachment.chatbot_channel_cw_read[0] + to = aws_iam_role_policy_attachment.chatbot_channel_cw_read +} + +moved { + from = aws_chatbot_slack_channel_configuration.soak_alerts[0] + to = aws_chatbot_slack_channel_configuration.soak_alerts +} + +# Channel guardrail role: what Chatbot may do ON BEHALF OF channel members. +# Read-only CloudWatch is all the alarm cards need (rendering the metric +# graph); nothing in this channel should ever mutate the account. +resource "aws_iam_role" "chatbot_channel" { + name = "rpcn-bench-chatbot-channel" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "chatbot.amazonaws.com" } + Action = "sts:AssumeRole" + }] + }) +} + +resource "aws_iam_role_policy_attachment" "chatbot_channel_cw_read" { + role = aws_iam_role.chatbot_channel.name + policy_arn = "arn:aws:iam::aws:policy/CloudWatchReadOnlyAccess" +} + +resource "aws_chatbot_slack_channel_configuration" "soak_alerts" { + configuration_name = "rpcn-bench-soak-alerts" + iam_role_arn = aws_iam_role.chatbot_channel.arn + slack_team_id = var.slack_workspace_id + slack_channel_id = var.slack_channel_id + # Guardrail caps every action in the channel regardless of the role above. + guardrail_policy_arns = ["arn:aws:iam::aws:policy/CloudWatchReadOnlyAccess"] + sns_topic_arns = [ + aws_sns_topic.soak_alerts.arn, + aws_sns_topic.orphan_cleanup.arn, + ] + logging_level = "ERROR" +}