From f903238464baf836923e8c5141c82de5cd712abc Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Mon, 6 Jul 2026 21:51:30 +0300 Subject: [PATCH 01/70] postgres_cdc: add multi-schema support --- internal/impl/postgresql/input_pg_stream.go | 56 +++++- internal/impl/postgresql/integration_test.go | 166 +++++++++++++++++- .../impl/postgresql/pglogicalstream/config.go | 6 +- .../pglogicalstream/logical_stream.go | 19 +- .../postgresql/pglogicalstream/pglogrepl.go | 72 +++++--- .../pglogicalstream/schema_resolver.go | 129 ++++++++++++++ .../pglogicalstream/schema_resolver_test.go | 90 ++++++++++ 7 files changed, 496 insertions(+), 42 deletions(-) create mode 100644 internal/impl/postgresql/pglogicalstream/schema_resolver.go create mode 100644 internal/impl/postgresql/pglogicalstream/schema_resolver_test.go diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index b61792a444..f08856dee5 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -14,6 +14,8 @@ import ( "encoding/json" "errors" "fmt" + "strconv" + "strings" "time" "github.com/Jeffail/checkpoint" @@ -81,6 +83,7 @@ Additionally, if ` + "`" + fieldStreamSnapshot + "`" + ` is set to true, then th This input adds the following metadata fields to each message: - table: Name of the table that the message originated from +- pg_schema: The PostgreSQL schema name that the table belongs to (e.g. "public", "tenant_foo"). Useful for per-schema routing when using schema patterns. - operation: Type of operation that generated the message: "read", "insert", "update", or "delete". "read" is from messages that are read in the initial snapshot phase. This will also be "begin" and "commit" if ` + "`" + fieldIncludeTxnMarkers + "`" + ` is enabled - lsn: the log sequence number in postgres - schema: The table schema in benthos common schema format, compatible with processors like parquet_encode @@ -105,8 +108,12 @@ This input adds the following metadata fields to each message: Example(10000). Default(1000)). Field(service.NewStringField(fieldSchema). - Description("The PostgreSQL schema from which to replicate data."). - Examples("public", `"MyCaseSensitiveSchemaNeedingQuotes"`), + Description(`The PostgreSQL schema to replicate data from. Accepts an exact schema name or a glob pattern using ` + "`*`" + ` as a wildcard to match multiple schemas. + +When a pattern is used, all schemas whose names match the pattern are replicated using a single replication slot and publication. This is useful for multi-tenant databases where each tenant has its own schema (e.g. ` + "`tenant_*`" + ` matches ` + "`tenant_foo`" + `, ` + "`tenant_bar`" + `, etc.). + +Double-quoted identifiers are treated as exact names and do not support wildcards.`). + Examples("public", `"MyCaseSensitiveSchemaNeedingQuotes"`, "tenant_*", "*"), ). Field(service.NewStringListField(fieldTables). Description("A list of table names to include in the logical replication. Each table should be specified as a separate item."). @@ -242,6 +249,15 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser if schema, err = conf.FieldString(fieldSchema); err != nil { return nil, err } + if err = validateSchemaPattern(schema); err != nil { + return nil, fmt.Errorf("invalid schema: %w", err) + } + // Normalize unquoted patterns to lower-case: PostgreSQL folds unquoted + // identifiers at creation time, so TENANT_* and tenant_* resolve identically. + // Normalizing early avoids silent case-folding surprises in resolveSchemas. + if !strings.HasPrefix(schema, `"`) { + schema = strings.ToLower(schema) + } if tables, err = conf.FieldStringList(fieldTables); err != nil { return nil, err @@ -321,8 +337,8 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser DBConfig: pgConnConfig, TLSConfig: pgConnConfig.TLSConfig, DBRawDSN: dsn, - DBSchema: schema, - DBTables: tables, + DBSchemaPattern: schema, + DBTables: tables, RefreshAuthToken: iamAuthTokenBuilder, IncludeTxnMarkers: includeTxnMarkers, @@ -361,6 +377,37 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser return conf.WrapBatchInputExtractTracingSpanMapping("postgres_cdc", r) } +// validateSchemaPattern validates a schema name or glob pattern. +// Accepts exact postgres identifiers (letters/digits/underscores) and glob +// patterns that additionally allow '*' as a wildcard character. +// Double-quoted identifiers (e.g. "MySchema") are accepted as exact names; +// wildcards are not allowed inside quotes. +func validateSchemaPattern(s string) error { + if s == "" { + return errors.New("schema cannot be empty") + } + if strings.HasPrefix(s, `"`) { + if !strings.HasSuffix(s, `"`) || len(s) < 2 { + return errors.New("unterminated quoted identifier in schema") + } + if strings.ContainsRune(s, '*') { + return errors.New("wildcard '*' is not allowed inside a quoted schema identifier") + } + return nil + } + for i, ch := range s { + if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '_' || ch == '*' { + continue + } + return fmt.Errorf("invalid character %q at position %d in schema pattern %q", ch, i, s) + } + first := rune(s[0]) + if !(first == '_' || first == '*' || (first >= 'a' && first <= 'z') || (first >= 'A' && first <= 'Z')) { + return fmt.Errorf("schema pattern %q must start with a letter, underscore, or '*'", s) + } + return nil +} + // validateSimpleString ensures we aren't vuln to SQL injection. func validateSimpleString(s string) error { for _, b := range []byte(s) { @@ -475,6 +522,7 @@ func (p *pgStreamInput) processStream(pgStream *pglogicalstream.Stream, batcher } batchMsg := service.NewMessage(mb) batchMsg.MetaSet("table", msg.Table) + batchMsg.MetaSet("pg_schema", msg.Schema) batchMsg.MetaSet("operation", string(msg.Operation)) if msg.LSN != nil { batchMsg.MetaSet("lsn", *msg.LSN) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index c3115986e3..31eb6b8a05 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1052,13 +1052,20 @@ postgres_cdc: _, err = db.Exec(`INSERT INTO "FlightsCompositePK" ("Seq", "Name", "CreatedAt") VALUES ($1, $2, $3);`, 2, "bravo", "2006-01-02T15:04:05Z07:00") require.NoError(t, err) - _, err = db.Exec(`INSERT INTO flights (name, created_at) VALUES ($1, $2);`, "bravo", "2006-01-02T15:04:05Z07:00") + var flightsID int + err = db.QueryRow(`INSERT INTO flights (name, created_at) VALUES ($1, $2) RETURNING id;`, "bravo", "2006-01-02T15:04:05Z07:00").Scan(&flightsID) + require.NoError(t, err) + + _, err = db.Exec(`UPDATE flights SET name = $1 WHERE id = $2;`, "charlie", flightsID) + require.NoError(t, err) + + _, err = db.Exec(`DELETE FROM flights WHERE id = $1;`, flightsID) require.NoError(t, err) assert.EventuallyWithT(t, func(c *assert.CollectT) { outBatchMut.Lock() defer outBatchMut.Unlock() - assert.Len(c, outBatches, 4, "got: %#v", outBatches) + assert.Len(c, outBatches, 6, "got: %#v", outBatches) }, time.Second*25, time.Millisecond*100) require.ElementsMatch( @@ -1068,20 +1075,42 @@ postgres_cdc: map[string]any{ "operation": "read", "table": "FlightsCompositePK", + "pg_schema": "public", }, map[string]any{ "operation": "read", "table": "flights", + "pg_schema": "public", }, map[string]any{ - "operation": "insert", - "table": "flights", - "lsn": "XXX/XXX", + "operation": "insert", + "table": "FlightsCompositePK", + "lsn": "XXX/XXX", + "commit_ts_ms": "SET", + "pg_schema": "public", }, map[string]any{ - "operation": "insert", - "table": "FlightsCompositePK", - "lsn": "XXX/XXX", + "operation": "insert", + "table": "flights", + "lsn": "XXX/XXX", + "commit_ts_ms": "SET", + "pg_schema": "public", + }, + map[string]any{ + "operation": "update", + "table": "flights", + "lsn": "XXX/XXX", + "commit_ts_ms": "SET", + "before": "SET", + "pg_schema": "public", + }, + map[string]any{ + "operation": "delete", + "table": "flights", + "lsn": "XXX/XXX", + "commit_ts_ms": "SET", + "before": "SET", + "pg_schema": "public", }, }, ) @@ -1429,3 +1458,124 @@ postgres_cdc: } assert.Equal(t, "STRING", byName["extra"], "new 'extra' column should have type STRING") } + +func TestIntegrationMultiSchemaSnapshotAndCDC(t *testing.T) { + integration.CheckSkip(t) + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + // Two tenant schemas with the same table name, replicated on a single slot. + for _, schema := range []string{"tenant_a", "tenant_b"} { + _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", schema)) + require.NoError(t, err) + _, err = db.Exec(fmt.Sprintf( + "CREATE TABLE %s.events (id SERIAL PRIMARY KEY, name TEXT)", schema)) + require.NoError(t, err) + } + + // Pre-load snapshot data: 2 rows in tenant_a, 1 in tenant_b. + _, err = db.Exec("INSERT INTO tenant_a.events (name) VALUES ('alice'), ('bob')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_b.events (name) VALUES ('carol')") + require.NoError(t, err) + + type msgMeta struct { + pgSchema string + table string + operation string + lsn string + } + + var ( + mu sync.Mutex + collected []msgMeta + ) + + tmpl := fmt.Sprintf(` +postgres_cdc: + dsn: %s + slot_name: multi_schema_test_slot + stream_snapshot: true + schema: tenant_* + tables: + - events +`, databaseURL) + + sb := service.NewStreamBuilder() + require.NoError(t, sb.SetLoggerYAML(`level: WARN`)) + require.NoError(t, sb.AddInputYAML(tmpl)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + mu.Lock() + defer mu.Unlock() + for _, msg := range batch { + m := msgMeta{} + m.pgSchema, _ = msg.MetaGet("pg_schema") + m.table, _ = msg.MetaGet("table") + m.operation, _ = msg.MetaGet("operation") + m.lsn, _ = msg.MetaGet("lsn") + collected = append(collected, m) + } + return nil + })) + + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + go func() { _ = stream.Run(t.Context()) }() + t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) + + // Wait for all 3 snapshot rows. + assert.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return len(collected) >= 3 + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for snapshot rows") + + // Insert CDC rows. + _, err = db.Exec("INSERT INTO tenant_a.events (name) VALUES ('dave')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_b.events (name) VALUES ('eve')") + require.NoError(t, err) + + // Wait for 2 CDC rows (total 5). + assert.EventuallyWithT(t, func(c *assert.CollectT) { + mu.Lock() + defer mu.Unlock() + assert.Len(c, collected, 5) + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for CDC rows") + + mu.Lock() + defer mu.Unlock() + + var snapshots, cdcMsgs []msgMeta + for _, m := range collected { + if m.operation == "read" { + snapshots = append(snapshots, m) + } else { + cdcMsgs = append(cdcMsgs, m) + } + } + + // Snapshot assertions. + require.Len(t, snapshots, 3) + snapshotSchemas := make(map[string]int) + for _, m := range snapshots { + assert.Equal(t, "events", m.table, "snapshot: table should be bare name without schema prefix") + assert.Empty(t, m.lsn, "snapshot rows have no LSN") + snapshotSchemas[m.pgSchema]++ + } + assert.Equal(t, 2, snapshotSchemas["tenant_a"], "expected 2 snapshot rows from tenant_a") + assert.Equal(t, 1, snapshotSchemas["tenant_b"], "expected 1 snapshot row from tenant_b") + + // CDC assertions. + require.Len(t, cdcMsgs, 2) + cdcSchemas := make(map[string]int) + for _, m := range cdcMsgs { + assert.Equal(t, "insert", m.operation) + assert.Equal(t, "events", m.table) + assert.NotEmpty(t, m.lsn, "CDC rows must have an LSN") + cdcSchemas[m.pgSchema]++ + } + assert.Equal(t, 1, cdcSchemas["tenant_a"], "expected 1 CDC row from tenant_a") + assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") +} diff --git a/internal/impl/postgresql/pglogicalstream/config.go b/internal/impl/postgresql/pglogicalstream/config.go index 73ff4c1c4d..8fed67862e 100644 --- a/internal/impl/postgresql/pglogicalstream/config.go +++ b/internal/impl/postgresql/pglogicalstream/config.go @@ -24,8 +24,10 @@ type Config struct { DBConfig *pgconn.Config DBRawDSN string TLSConfig *tls.Config - DBSchema string - DBTables []string + // DBSchemaPattern is the schema to replicate from. Accepts an exact schema + // name or a glob pattern using '*' as a wildcard (e.g. "tenant_*", "*"). + DBSchemaPattern string + DBTables []string // Refreshes short lived IAM auth token that is treated as a password RefreshAuthToken func(ctx context.Context) error // ReplicationSlotName is the name of the replication slot to use diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 5ebce38064..092669cba8 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -91,18 +91,29 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { return nil, err } - schema, err := sanitize.NormalizePostgresIdentifier(config.DBSchema) + schemas, err := resolveSchemas(ctx, dbConn, config.DBSchemaPattern) if err != nil { - return nil, fmt.Errorf("invalid schema name %q: %w", config.DBSchema, err) + return nil, fmt.Errorf("resolving schema pattern %q: %w", config.DBSchemaPattern, err) } + if len(schemas) == 0 { + return nil, fmt.Errorf("no schemas found matching pattern %q", config.DBSchemaPattern) + } + config.Logger.Infof("Schema pattern %q resolved to %d schema(s): %v", config.DBSchemaPattern, len(schemas), schemas) - tables := []TableFQN{} + normalizedTables := make([]string, 0, len(config.DBTables)) for _, table := range config.DBTables { normalized, err := sanitize.NormalizePostgresIdentifier(table) if err != nil { return nil, fmt.Errorf("invalid table name %q: %w", table, err) } - tables = append(tables, TableFQN{Schema: schema, Table: normalized}) + normalizedTables = append(normalizedTables, normalized) + } + + tables := make([]TableFQN, 0, len(schemas)*len(normalizedTables)) + for _, schema := range schemas { + for _, table := range normalizedTables { + tables = append(tables, TableFQN{Schema: schema, Table: table}) + } } batchSize := 1000 if config.BatchSize > 0 { diff --git a/internal/impl/postgresql/pglogicalstream/pglogrepl.go b/internal/impl/postgresql/pglogicalstream/pglogrepl.go index f92e222d0d..df2f80c933 100644 --- a/internal/impl/postgresql/pglogicalstream/pglogrepl.go +++ b/internal/impl/postgresql/pglogicalstream/pglogrepl.go @@ -23,7 +23,6 @@ import ( "encoding/binary" "errors" "fmt" - "slices" "strconv" "strings" "time" @@ -335,41 +334,66 @@ func CreatePublication(ctx context.Context, conn *pgconn.PgConn, publicationName return nil } - tablesToRemoveFromPublication := []TableFQN{} - tablesToAddToPublication := []TableFQN{} - for _, table := range tables { - if !slices.Contains(pubTables, table) { - tablesToAddToPublication = append(tablesToAddToPublication, table) - } + // Build sets for O(1) lookup — avoids O(n²) slices.Contains when reconciling + // large publication table lists (e.g. 100 schemas × 5 tables = 500 entries). + wantSet := make(map[TableFQN]struct{}, len(tables)) + for _, t := range tables { + wantSet[t] = struct{}{} + } + haveSet := make(map[TableFQN]struct{}, len(pubTables)) + for _, t := range pubTables { + haveSet[t] = struct{}{} } - for _, table := range pubTables { - if !slices.Contains(tables, table) { - tablesToRemoveFromPublication = append(tablesToRemoveFromPublication, table) + var tablesToAdd, tablesToRemove []TableFQN + for _, t := range tables { + if _, ok := haveSet[t]; !ok { + tablesToAdd = append(tablesToAdd, t) + } + } + for _, t := range pubTables { + if _, ok := wantSet[t]; !ok { + tablesToRemove = append(tablesToRemove, t) } } - // remove tables from publication - for _, dropTable := range tablesToRemoveFromPublication { - sq, err := sanitize.SQLQuery(fmt.Sprintf(`ALTER PUBLICATION %s DROP TABLE %s;`, publicationName, dropTable.String())) + // Batch DROP: single ALTER statement for all removed tables. + if len(tablesToRemove) > 0 { + var sb strings.Builder + sb.WriteString(fmt.Sprintf("ALTER PUBLICATION %s DROP TABLE ", publicationName)) + for i, t := range tablesToRemove { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(t.String()) + } + sb.WriteByte(';') + sq, err := sanitize.SQLQuery(sb.String()) if err != nil { - return fmt.Errorf("sanitizing drop table query: %w", err) + return fmt.Errorf("sanitizing drop tables query: %w", err) } - result = conn.Exec(ctx, sq) - if _, err := result.ReadAll(); err != nil { - return fmt.Errorf("removing table from publication: %w", err) + if _, err := conn.Exec(ctx, sq).ReadAll(); err != nil { + return fmt.Errorf("removing tables from publication: %w", err) } } - // add tables to publication - for _, addTable := range tablesToAddToPublication { - sq, err := sanitize.SQLQuery(fmt.Sprintf("ALTER PUBLICATION %s ADD TABLE %s;", publicationName, addTable.String())) + // Batch ADD: single ALTER statement for all new tables. + if len(tablesToAdd) > 0 { + var sb strings.Builder + sb.WriteString(fmt.Sprintf("ALTER PUBLICATION %s ADD TABLE ", publicationName)) + for i, t := range tablesToAdd { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(t.String()) + } + sb.WriteByte(';') + sq, err := sanitize.SQLQuery(sb.String()) if err != nil { - return fmt.Errorf("sanitizing add table query: %w", err) + return fmt.Errorf("sanitizing add tables query: %w", err) } - result = conn.Exec(ctx, sq) - if _, err := result.ReadAll(); err != nil { - return fmt.Errorf("adding table to publication: %w", err) + if _, err := conn.Exec(ctx, sq).ReadAll(); err != nil { + return fmt.Errorf("adding tables to publication: %w", err) } } diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go new file mode 100644 index 0000000000..8af4f7712d --- /dev/null +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -0,0 +1,129 @@ +// Copyright 2024 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md + +package pglogicalstream + +import ( + "context" + "fmt" + "strings" + + "github.com/jackc/pgx/v5/pgconn" + + "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream/sanitize" +) + +// resolveSchemas expands a schema name or glob pattern into the set of +// quoted PostgreSQL identifiers that exist in the database. +// +// For unquoted patterns (e.g. "tenant_*") the pattern is matched +// case-insensitively against information_schema.schemata using LIKE, because +// PostgreSQL folds unquoted identifiers to lower-case at creation time. +// +// For quoted identifiers (e.g. `"MySchema"`) an exact case-sensitive lookup +// is performed. +// +// System schemas (pg_* and information_schema) are always excluded so that +// wildcard patterns like "*" do not attempt to replicate catalog tables. +// +// Returns an error if the query fails or if no schemas match the pattern. +// schemaPatternToLike converts a schema name or glob pattern into the LIKE +// pattern used by resolveSchemas. Extracted for unit testing. +// +// For quoted identifiers the inner name is exact-escaped (no wildcard expansion). +// For unquoted patterns the '*' wildcard is converted to '%' and the input is +// folded to lower-case to match PostgreSQL's identifier folding. +func schemaPatternToLike(pattern string) (string, error) { + if strings.HasPrefix(pattern, `"`) { + unquoted, err := sanitize.UnquotePostgresIdentifier(pattern) + if err != nil { + return "", fmt.Errorf("invalid quoted schema identifier %q: %w", pattern, err) + } + return escapeLike(unquoted), nil + } + return globToLike(strings.ToLower(pattern)), nil +} + +func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) ([]string, error) { + likePattern, err := schemaPatternToLike(pattern) + if err != nil { + return nil, err + } + + q, err := sanitize.SQLQuery( + "SELECT schema_name FROM information_schema.schemata WHERE schema_name LIKE $1 ESCAPE '!' AND schema_name NOT LIKE 'pg!_%' ESCAPE '!' AND schema_name != 'information_schema'", + likePattern, + ) + if err != nil { + return nil, fmt.Errorf("building schema resolution query: %w", err) + } + + results, err := conn.Exec(ctx, q).ReadAll() + if err != nil { + return nil, fmt.Errorf("querying schemas matching %q: %w", pattern, err) + } + + var schemas []string + if len(results) > 0 { + for _, row := range results[0].Rows { + // QuotePostgresIdentifier preserves the exact stored name (including + // case for case-sensitive schemas), unlike NormalizePostgresIdentifier + // which would incorrectly fold to lower-case. + schemas = append(schemas, sanitize.QuotePostgresIdentifier(string(row[0]))) + } + } + return schemas, nil +} + +// globToLike converts an unquoted glob pattern (using '*' as wildcard) into a +// PostgreSQL LIKE pattern that uses '!' as the escape character. +// +// Mapping: +// - '*' → '%' (zero or more characters) +// - '_' → '!_' (literal underscore, not the LIKE single-char wildcard) +// - '%' → '!%' (literal percent, not the LIKE multi-char wildcard) +// - '!' → '!!' (literal escape character) +func globToLike(pattern string) string { + var b strings.Builder + b.Grow(len(pattern) + 4) + for _, ch := range pattern { + switch ch { + case '*': + b.WriteByte('%') + case '_': + b.WriteString("!_") + case '%': + b.WriteString("!%") + case '!': + b.WriteString("!!") + default: + b.WriteRune(ch) + } + } + return b.String() +} + +// escapeLike escapes LIKE metacharacters in s without expanding any wildcards. +// Used for exact quoted-identifier lookups. +func escapeLike(s string) string { + var b strings.Builder + b.Grow(len(s)) + for _, ch := range s { + switch ch { + case '_': + b.WriteString("!_") + case '%': + b.WriteString("!%") + case '!': + b.WriteString("!!") + default: + b.WriteRune(ch) + } + } + return b.String() +} diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go b/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go new file mode 100644 index 0000000000..7ee213f6e4 --- /dev/null +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go @@ -0,0 +1,90 @@ +// Copyright 2024 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md + +package pglogicalstream + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGlobToLike(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"public", "public"}, + {"tenant_*", "tenant!_%"}, + {"*", "%"}, + {"tenant_a", "tenant!_a"}, + {"100%", "100!%"}, + {"a!b", "a!!b"}, + {"multi_*_end", "multi!_%!_end"}, + {"", ""}, + } + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + assert.Equal(t, tt.expected, globToLike(tt.input)) + }) + } +} + +func TestSchemaPatternToLike(t *testing.T) { + tests := []struct { + pattern string + expected string + errContains string + }{ + // Unquoted glob patterns — folded to lower-case, '*' → '%', '_' escaped. + {"public", "public", ""}, + {"tenant_*", "tenant!_%", ""}, + {"*", "%", ""}, + {"schema_1", "schema!_1", ""}, + // Upper-case is folded: TENANT_* matches the same rows as tenant_*. + {"TENANT_*", "tenant!_%", ""}, + // Quoted exact identifier — case preserved, no wildcard expansion. + {`"MySchema"`, "MySchema", ""}, + {`"schema_1"`, "schema!_1", ""}, + {`"has%bang!"`, "has!%bang!!", ""}, + // Unterminated quoted identifier → error. + {`"bad`, "", "invalid quoted schema identifier"}, + } + for _, tt := range tests { + t.Run(tt.pattern, func(t *testing.T) { + got, err := schemaPatternToLike(tt.pattern) + if tt.errContains != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errContains) + return + } + require.NoError(t, err) + assert.Equal(t, tt.expected, got) + }) + } +} + +func TestEscapeLike(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"MySchema", "MySchema"}, + {"schema_1", "schema!_1"}, + {"100%", "100!%"}, + {"bang!bang", "bang!!bang"}, + {"has_a%b!c", "has!_a!%b!!c"}, + {"", ""}, + } + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + assert.Equal(t, tt.expected, escapeLike(tt.input)) + }) + } +} From c38fd7a709654da2c871c1acd994ab8084a71be8 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Mon, 6 Jul 2026 22:27:03 +0300 Subject: [PATCH 02/70] postgres_cdc: reject empty quoted schema identifier and fix misleading godoc --- internal/impl/postgresql/input_pg_stream.go | 2 +- internal/impl/postgresql/pglogicalstream/schema_resolver.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index f08856dee5..733155a281 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -387,7 +387,7 @@ func validateSchemaPattern(s string) error { return errors.New("schema cannot be empty") } if strings.HasPrefix(s, `"`) { - if !strings.HasSuffix(s, `"`) || len(s) < 2 { + if !strings.HasSuffix(s, `"`) || len(s) < 3 { return errors.New("unterminated quoted identifier in schema") } if strings.ContainsRune(s, '*') { diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index 8af4f7712d..8920121e99 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -31,7 +31,9 @@ import ( // System schemas (pg_* and information_schema) are always excluded so that // wildcard patterns like "*" do not attempt to replicate catalog tables. // -// Returns an error if the query fails or if no schemas match the pattern. +// Returns an error if the query fails. Returns (nil, nil) if no schemas match. +// The caller is responsible for treating an empty result as an error. + // schemaPatternToLike converts a schema name or glob pattern into the LIKE // pattern used by resolveSchemas. Extracted for unit testing. // From bda218d4b793dd1a8e91841f3f9172e4521dfd5b Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Tue, 7 Jul 2026 17:54:33 +0300 Subject: [PATCH 03/70] postgres_cdc: add tests/current/ Docker Compose + Taskfile manual test harness Two-schema (tenant_a, tenant_b) Postgres 16 setup that exercises the multi-schema CDC pipeline end-to-end. Also adds schema_validation unit tests that verify invalid patterns are rejected at startup without a DB. --- .../postgresql/tests/current/Taskfile.yaml | 98 +++++++++++++++++++ .../tests/current/docker-compose.yaml | 36 +++++++ .../impl/postgresql/tests/current/setup.sql | 33 +++++++ .../postgresql/tests/current/test_config.yaml | 41 ++++++++ 4 files changed, 208 insertions(+) create mode 100644 internal/impl/postgresql/tests/current/Taskfile.yaml create mode 100644 internal/impl/postgresql/tests/current/docker-compose.yaml create mode 100644 internal/impl/postgresql/tests/current/setup.sql create mode 100644 internal/impl/postgresql/tests/current/test_config.yaml diff --git a/internal/impl/postgresql/tests/current/Taskfile.yaml b/internal/impl/postgresql/tests/current/Taskfile.yaml new file mode 100644 index 0000000000..75551c4d34 --- /dev/null +++ b/internal/impl/postgresql/tests/current/Taskfile.yaml @@ -0,0 +1,98 @@ +version: "3" + +vars: + PG_DSN: '{{.PG_DSN | default "postgres://postgres:postgres@localhost:5433/testdb?sslmode=disable"}}' + +tasks: + # ── Infrastructure ──────────────────────────────────────────────────────────── + + up: + desc: Start PostgreSQL and run schema/data setup + cmds: + - docker compose up -d postgres + - docker compose run --rm setup + + down: + desc: Stop and remove all containers and volumes + cmds: + - docker compose down -v + + reset: + desc: Full teardown + bring back up (also drops and recreates the replication slot) + cmds: + - task: down + - task: up + + # ── Slot management ─────────────────────────────────────────────────────────── + + slot:drop: + desc: Drop the replication slot so the test can be re-run from scratch + cmds: + - | + psql "{{.PG_DSN}}" -c \ + "SELECT pg_drop_replication_slot('multi_schema_test_slot') + FROM pg_replication_slots + WHERE slot_name = 'multi_schema_test_slot';" + + # ── Run the pipeline ────────────────────────────────────────────────────────── + + run: + desc: Run the multi-schema CDC pipeline (streams snapshot then live CDC) + env: + PG_DSN: '{{.PG_DSN}}' + cmds: + - task: slot:drop + - go run ../../../../../cmd/redpanda-connect/main.go run ./test_config.yaml + + # ── Test data ───────────────────────────────────────────────────────────────── + + data:insert: + desc: Insert CDC rows into both tenant schemas (triggers insert events) + cmds: + - psql "{{.PG_DSN}}" -c "INSERT INTO tenant_a.events (name) VALUES ('dave'), ('eve');" + - psql "{{.PG_DSN}}" -c "INSERT INTO tenant_b.events (name) VALUES ('frank');" + + data:update: + desc: Update a row in tenant_a (triggers update event with 'before' field) + cmds: + - psql "{{.PG_DSN}}" -c "UPDATE tenant_a.events SET status = 'updated' WHERE name = 'dave';" + + data:delete: + desc: Delete a row from tenant_b (triggers delete event with 'before' field) + cmds: + - psql "{{.PG_DSN}}" -c "DELETE FROM tenant_b.events WHERE name = 'frank';" + + data:all: + desc: Run all test data mutations in sequence (insert → update → delete) + cmds: + - task: data:insert + - task: data:update + - task: data:delete + + # ── Schema validation smoke test ────────────────────────────────────────────── + + test:invalid-schema: + desc: Confirm that an empty quoted schema identifier is rejected at startup + env: + PG_DSN: '{{.PG_DSN}}' + cmds: + - | + set +e + go run ../../../../../cmd/redpanda-connect/main.go run \ + --set 'input.postgres_cdc.schema=""' \ + ./test_config.yaml 2>&1 | head -5 + echo "exit $?" + # Expects: "invalid schema" error printed, process exits non-zero. + + # ── Quick sanity ────────────────────────────────────────────────────────────── + + psql: + desc: Open a psql shell to the test database + cmds: + - psql "{{.PG_DSN}}" + + status: + desc: Show active replication slots and publications + cmds: + - psql "{{.PG_DSN}}" -c "SELECT slot_name, active FROM pg_replication_slots;" + - psql "{{.PG_DSN}}" -c "SELECT pubname FROM pg_publication;" diff --git a/internal/impl/postgresql/tests/current/docker-compose.yaml b/internal/impl/postgresql/tests/current/docker-compose.yaml new file mode 100644 index 0000000000..896a807027 --- /dev/null +++ b/internal/impl/postgresql/tests/current/docker-compose.yaml @@ -0,0 +1,36 @@ +services: + postgres: + image: postgres:16 + container_name: pgtest-postgres + ports: + - "5433:5432" + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: testdb + # Enable logical replication — required for postgres_cdc + command: postgres -c wal_level=logical -c max_replication_slots=10 -c max_wal_senders=10 + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 3s + timeout: 5s + retries: 10 + + # One-shot setup container: creates schemas, tables, and seed data, then exits. + setup: + image: postgres:16 + container_name: pgtest-setup + depends_on: + postgres: + condition: service_healthy + environment: + PGPASSWORD: postgres + volumes: + - ./setup.sql:/setup.sql:ro + entrypoint: /bin/sh + command: + - -c + - | + psql -h postgres -U postgres -d testdb -f /setup.sql + echo "setup complete" + restart: "no" diff --git a/internal/impl/postgresql/tests/current/setup.sql b/internal/impl/postgresql/tests/current/setup.sql new file mode 100644 index 0000000000..ee40a376cc --- /dev/null +++ b/internal/impl/postgresql/tests/current/setup.sql @@ -0,0 +1,33 @@ +-- Multi-schema CDC test setup +-- Tests: schema glob (tenant_*), pg_schema metadata, commit_ts_ms, before (update/delete) + +-- ── Tenant schemas ──────────────────────────────────────────────────────────── + +CREATE SCHEMA IF NOT EXISTS tenant_a; +CREATE SCHEMA IF NOT EXISTS tenant_b; + +-- ── Events table (same shape in each schema) ────────────────────────────────── + +CREATE TABLE IF NOT EXISTS tenant_a.events ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS tenant_b.events ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- REPLICA IDENTITY FULL so update/delete messages carry the full before-row. +ALTER TABLE tenant_a.events REPLICA IDENTITY FULL; +ALTER TABLE tenant_b.events REPLICA IDENTITY FULL; + +-- ── Seed snapshot rows ──────────────────────────────────────────────────────── +-- These are visible during the initial snapshot (stream_snapshot: true). + +INSERT INTO tenant_a.events (name) VALUES ('alice'), ('bob'); +INSERT INTO tenant_b.events (name) VALUES ('carol'); diff --git a/internal/impl/postgresql/tests/current/test_config.yaml b/internal/impl/postgresql/tests/current/test_config.yaml new file mode 100644 index 0000000000..4a994e30c2 --- /dev/null +++ b/internal/impl/postgresql/tests/current/test_config.yaml @@ -0,0 +1,41 @@ +input: + postgres_cdc: + dsn: ${PG_DSN:postgres://postgres:postgres@localhost:5433/testdb?sslmode=disable} + slot_name: multi_schema_test_slot + stream_snapshot: true + # Glob pattern: replicates both tenant_a and tenant_b via one slot. + schema: tenant_* + tables: + - events + +pipeline: + processors: + # Annotate each message with all new metadata fields so the output clearly + # shows what the feature delivers. + - mapping: | + let op = @operation + let tbl = @table + let schema = @pg_schema + let lsn = @lsn + let ts_ms = @commit_ts_ms + let before = @before + + root = { + "operation": $op, + "pg_schema": $schema, + "table": $tbl, + "payload": this, + "lsn": if $lsn != null { $lsn } else { null }, + "commit_ts_ms": if $ts_ms != null { $ts_ms } else { null }, + "before": if $before != null { $before.string().parse_json() } else { null }, + } + +output: + stdout: + codec: lines + +logger: + level: INFO + +metrics: + none: {} From df2dca6dff49c4f9c50ca795d89815a7d107859a Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Tue, 7 Jul 2026 17:55:08 +0300 Subject: [PATCH 04/70] postgres_cdc: add commit_ts_ms and before metadata fields --- internal/impl/postgresql/input_pg_stream.go | 8 + internal/impl/postgresql/integration_test.go | 6 + .../pglogicalstream/logical_stream.go | 14 ++ .../replication_message_decoders.go | 23 +++ .../pglogicalstream/stream_message.go | 6 + .../tests/schema_validation/validate_test.go | 147 ++++++++++++++++++ 6 files changed, 204 insertions(+) create mode 100644 internal/impl/postgresql/tests/schema_validation/validate_test.go diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 733155a281..7acbe0dd62 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -530,6 +530,14 @@ func (p *pgStreamInput) processStream(pgStream *pglogicalstream.Stream, batcher if msg.ColumnSchema != nil { batchMsg.MetaSetImmut("schema", service.ImmutableAny{V: msg.ColumnSchema}) } + if msg.CommitTs != nil { + batchMsg.MetaSet("commit_ts_ms", strconv.FormatInt(msg.CommitTs.UnixMilli(), 10)) + } + if msg.Before != nil { + if beforeBytes, err := json.Marshal(msg.Before); err == nil { + batchMsg.MetaSet("before", string(beforeBytes)) + } + } if batcher.Add(batchMsg) { flush = true } diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index 31eb6b8a05..6087b62a96 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1029,6 +1029,12 @@ postgres_cdc: if _, ok := d["lsn"]; ok { d["lsn"] = "XXX/XXX" // Consistent LSN for assertions below } + if _, ok := d["commit_ts_ms"]; ok { + d["commit_ts_ms"] = "SET" + } + if _, ok := d["before"]; ok { + d["before"] = "SET" + } delete(d, "schema") // Schema metadata tested separately in TestIntegrationPostgresCDCSchemaMetadata outBatches = append(outBatches, data) } diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 092669cba8..bd231a30f2 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -55,6 +55,7 @@ type Stream struct { heartbeat *heartbeat maxSnapshotWorkers int unchangedToastValue any + currentTxCommitTime *time.Time } // NewPgStream creates a new instance of the Stream struct. @@ -517,6 +518,14 @@ func (s *Stream) processChange(ctx context.Context, msgLSN LSN, xld XLogData, re delete(schemaCache, rel.RelationID) } + // Track commit timestamp: set on BEGIN (available for DML), clear on COMMIT. + if begin, ok := logicalMsg.(*BeginMessage); ok { + t := begin.CommitTime + s.currentTxCommitTime = &t + } else if _, ok := logicalMsg.(*CommitMessage); ok { + s.currentTxCommitTime = nil + } + // parse changes inside the transaction message, err := toStreamMessage(logicalMsg, relations, typeMap, s.unchangedToastValue) if err != nil { @@ -562,6 +571,11 @@ func (s *Stream) processChange(ctx context.Context, msgLSN LSN, xld XLogData, re } } + switch message.Operation { + case InsertOpType, UpdateOpType, DeleteOpType: + message.CommitTs = s.currentTxCommitTime + } + lsn := msgLSN.String() message.LSN = &lsn select { diff --git a/internal/impl/postgresql/pglogicalstream/replication_message_decoders.go b/internal/impl/postgresql/pglogicalstream/replication_message_decoders.go index 02466e85ed..1f93398c15 100644 --- a/internal/impl/postgresql/pglogicalstream/replication_message_decoders.go +++ b/internal/impl/postgresql/pglogicalstream/replication_message_decoders.go @@ -133,6 +133,28 @@ func toStreamMessage(logicalMsg Message, relations map[uint32]*RelationMessage, } } message.Data = values + if logicalMsg.OldTuple != nil { + before := map[string]any{} + for idx, col := range logicalMsg.OldTuple.Columns { + if idx >= len(rel.Columns) { + break + } + colName := rel.Columns[idx].Name + switch col.DataType { + case 'n': + before[colName] = nil + case 'u': + before[colName] = unchangedToastValue + case 't': + val, err := decodeTextColumnData(typeMap, col.Data, rel.Columns[idx].DataType, rel.Columns[idx].TypeModifier) + if err != nil { + return nil, fmt.Errorf("unable to decode before column data: %w", err) + } + before[colName] = val + } + } + message.Before = before + } case *DeleteMessage: rel, ok := relations[logicalMsg.RelationID] if !ok { @@ -159,6 +181,7 @@ func toStreamMessage(logicalMsg Message, relations map[uint32]*RelationMessage, } } message.Data = values + message.Before = values case *TruncateMessage: case *TypeMessage: case *OriginMessage: diff --git a/internal/impl/postgresql/pglogicalstream/stream_message.go b/internal/impl/postgresql/pglogicalstream/stream_message.go index 73e6795fee..0fe927f81f 100644 --- a/internal/impl/postgresql/pglogicalstream/stream_message.go +++ b/internal/impl/postgresql/pglogicalstream/stream_message.go @@ -8,6 +8,8 @@ package pglogicalstream +import "time" + // StreamMode represents the mode of the stream at the time of the message type StreamMode string @@ -47,4 +49,8 @@ type StreamMessage struct { // ColumnSchema contains the table's column schema in benthos common schema format. // It is set as message metadata and excluded from JSON serialization. ColumnSchema any `json:"-"` + // CommitTs is the commit timestamp of the enclosing transaction. Nil for snapshot reads. + CommitTs *time.Time `json:"-"` + // Before holds the pre-change row state for update and delete operations. Nil otherwise. + Before any `json:"-"` } diff --git a/internal/impl/postgresql/tests/schema_validation/validate_test.go b/internal/impl/postgresql/tests/schema_validation/validate_test.go new file mode 100644 index 0000000000..a2acdc2e42 --- /dev/null +++ b/internal/impl/postgresql/tests/schema_validation/validate_test.go @@ -0,0 +1,147 @@ +// Copyright 2024 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md + +// Package schema_validation tests postgres_cdc schema pattern validation +// through the public service config API. No database connection is required: +// invalid schemas are rejected during stream construction (before any network +// I/O), so stream.Run returns synchronously for the invalid-schema cases. +package schema_validation_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" + _ "github.com/redpanda-data/benthos/v4/public/components/pure" // registers none tracer and other built-ins + + "github.com/redpanda-data/connect/v4/internal/license" + _ "github.com/redpanda-data/connect/v4/internal/impl/postgresql" // registers postgres_cdc +) + +// postgresStream builds a postgres_cdc stream with the given YAML schema value +// (the raw fragment that appears after "schema: " in YAML) and injects a test +// enterprise license. Returns the stream ready to run. +func postgresStream(t *testing.T, schemaYAML string) *service.Stream { + t.Helper() + yaml := fmt.Sprintf(` +postgres_cdc: + dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable + schema: %s + slot_name: test_slot + tables: + - events +`, schemaYAML) + + sb := service.NewStreamBuilder() + require.NoError(t, sb.SetLoggerYAML(`level: ERROR`)) + require.NoError(t, sb.AddInputYAML(yaml)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, _ service.MessageBatch) error { + return nil + })) + + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + return stream +} + +// TestInvalidSchemaPatterns verifies that invalid schema values are rejected +// during stream construction — before any database connection is attempted. +// stream.Run returns synchronously (no goroutine needed) when the constructor +// fails. +func TestInvalidSchemaPatterns(t *testing.T) { + tests := []struct { + name string + schemaYAML string + wantErrMsg string + }{ + { + // Regression test: len("") == 2 used to pass the old `len(s) < 2` + // guard. Fixed to `len(s) < 3`. + name: "empty quoted identifier", + schemaYAML: `'""'`, + wantErrMsg: "invalid schema", + }, + { + name: "digit-first unquoted pattern", + schemaYAML: `"1abc"`, + wantErrMsg: "invalid schema", + }, + { + name: "unterminated quoted identifier", + schemaYAML: `'"unclosed'`, + wantErrMsg: "invalid schema", + }, + { + name: "hyphen in unquoted pattern", + schemaYAML: `schema-name`, + wantErrMsg: "invalid schema", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stream := postgresStream(t, tt.schemaYAML) + + // Run returns synchronously when the input constructor fails — + // no timeout context needed, but use one as a safety net. + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + err := stream.Run(ctx) + require.Error(t, err, "expected stream construction to fail") + assert.Contains(t, err.Error(), tt.wantErrMsg, + "error should indicate schema validation failure") + }) + } +} + +// TestValidSchemaPatterns verifies that valid schema values pass construction +// and are only rejected later (at DB-connect time). We run the stream briefly +// and confirm no "invalid schema" error surfaces. +func TestValidSchemaPatterns(t *testing.T) { + tests := []struct { + name string + schemaYAML string + }{ + {name: "exact unquoted schema", schemaYAML: `public`}, + {name: "glob pattern", schemaYAML: `tenant_*`}, + {name: "wildcard", schemaYAML: `"*"`}, + {name: "quoted exact identifier", schemaYAML: `'"MySchema"'`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stream := postgresStream(t, tt.schemaYAML) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + errCh := make(chan error, 1) + go func() { errCh <- stream.Run(ctx) }() + + select { + case err := <-errCh: + // Stream stopped before timeout — must not be a schema error. + if err != nil { + assert.NotContains(t, err.Error(), "invalid schema", + "valid schema %q should not trigger schema validation error", tt.schemaYAML) + } + case <-ctx.Done(): + // Stream is still running after timeout — constructor succeeded, + // stream is attempting DB connection. This is the expected path. + _ = stream.StopWithin(2 * time.Second) + } + }) + } +} From fcb0427d6a1a030a36adf5b963fddf3380565917 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Fri, 10 Jul 2026 12:55:07 +0300 Subject: [PATCH 05/70] postgres_cdc: fix lint and docs --- docs/modules/components/pages/inputs/postgres_cdc.adoc | 10 +++++++++- internal/impl/postgresql/input_pg_stream.go | 8 ++++---- internal/impl/postgresql/pglogicalstream/pglogrepl.go | 4 ++-- .../tests/schema_validation/validate_test.go | 4 ++-- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index a7d64a843b..e1c2257518 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -170,7 +170,11 @@ snapshot_batch_size: 10000 === `schema` -The PostgreSQL schema from which to replicate data. +The PostgreSQL schema to replicate data from. Accepts an exact schema name or a glob pattern using `*` as a wildcard to match multiple schemas. + +When a pattern is used, all schemas whose names match the pattern are replicated using a single replication slot and publication. This is useful for multi-tenant databases where each tenant has its own schema (e.g. `tenant_*` matches `tenant_foo`, `tenant_bar`, etc.). + +Double-quoted identifiers are treated as exact names and do not support wildcards. *Type*: `string` @@ -182,6 +186,10 @@ The PostgreSQL schema from which to replicate data. schema: public schema: '"MyCaseSensitiveSchemaNeedingQuotes"' + +schema: tenant_* + +schema: '*' ``` === `tables` diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index e6e1b509f8..dfec0b4881 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -110,9 +110,9 @@ This input adds the following metadata fields to each message: Example(10000). Default(1000)). Field(service.NewStringField(fieldSchema). - Description(`The PostgreSQL schema to replicate data from. Accepts an exact schema name or a glob pattern using ` + "`*`" + ` as a wildcard to match multiple schemas. + Description(`The PostgreSQL schema to replicate data from. Accepts an exact schema name or a glob pattern using `+"`*`"+` as a wildcard to match multiple schemas. -When a pattern is used, all schemas whose names match the pattern are replicated using a single replication slot and publication. This is useful for multi-tenant databases where each tenant has its own schema (e.g. ` + "`tenant_*`" + ` matches ` + "`tenant_foo`" + `, ` + "`tenant_bar`" + `, etc.). +When a pattern is used, all schemas whose names match the pattern are replicated using a single replication slot and publication. This is useful for multi-tenant databases where each tenant has its own schema (e.g. `+"`tenant_*`"+` matches `+"`tenant_foo`"+`, `+"`tenant_bar`"+`, etc.). Double-quoted identifiers are treated as exact names and do not support wildcards.`). Examples("public", `"MyCaseSensitiveSchemaNeedingQuotes"`, "tenant_*", "*"), @@ -339,8 +339,8 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser DBConfig: pgConnConfig, TLSConfig: pgConnConfig.TLSConfig, DBRawDSN: dsn, - DBSchemaPattern: schema, - DBTables: tables, + DBSchemaPattern: schema, + DBTables: tables, RefreshAuthToken: iamAuthTokenBuilder, IncludeTxnMarkers: includeTxnMarkers, diff --git a/internal/impl/postgresql/pglogicalstream/pglogrepl.go b/internal/impl/postgresql/pglogicalstream/pglogrepl.go index df2f80c933..69385f753f 100644 --- a/internal/impl/postgresql/pglogicalstream/pglogrepl.go +++ b/internal/impl/postgresql/pglogicalstream/pglogrepl.go @@ -360,7 +360,7 @@ func CreatePublication(ctx context.Context, conn *pgconn.PgConn, publicationName // Batch DROP: single ALTER statement for all removed tables. if len(tablesToRemove) > 0 { var sb strings.Builder - sb.WriteString(fmt.Sprintf("ALTER PUBLICATION %s DROP TABLE ", publicationName)) + fmt.Fprintf(&sb, "ALTER PUBLICATION %s DROP TABLE ", publicationName) for i, t := range tablesToRemove { if i > 0 { sb.WriteString(", ") @@ -380,7 +380,7 @@ func CreatePublication(ctx context.Context, conn *pgconn.PgConn, publicationName // Batch ADD: single ALTER statement for all new tables. if len(tablesToAdd) > 0 { var sb strings.Builder - sb.WriteString(fmt.Sprintf("ALTER PUBLICATION %s ADD TABLE ", publicationName)) + fmt.Fprintf(&sb, "ALTER PUBLICATION %s ADD TABLE ", publicationName) for i, t := range tablesToAdd { if i > 0 { sb.WriteString(", ") diff --git a/internal/impl/postgresql/tests/schema_validation/validate_test.go b/internal/impl/postgresql/tests/schema_validation/validate_test.go index a2acdc2e42..e45e4bf2f2 100644 --- a/internal/impl/postgresql/tests/schema_validation/validate_test.go +++ b/internal/impl/postgresql/tests/schema_validation/validate_test.go @@ -21,11 +21,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/redpanda-data/benthos/v4/public/service" _ "github.com/redpanda-data/benthos/v4/public/components/pure" // registers none tracer and other built-ins + "github.com/redpanda-data/benthos/v4/public/service" - "github.com/redpanda-data/connect/v4/internal/license" _ "github.com/redpanda-data/connect/v4/internal/impl/postgresql" // registers postgres_cdc + "github.com/redpanda-data/connect/v4/internal/license" ) // postgresStream builds a postgres_cdc stream with the given YAML schema value From 7bcc3ac5af7cb21e4bbabd97e379443fd477810a Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Fri, 10 Jul 2026 13:21:46 +0300 Subject: [PATCH 06/70] postgres_cdc: review fixes and test coverage --- CHANGELOG.md | 4 +++ .../components/pages/inputs/postgres_cdc.adoc | 2 ++ internal/impl/postgresql/input_pg_stream.go | 9 +++-- internal/impl/postgresql/integration_test.go | 33 +++++++++++++++++++ 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77f8a2e534..93fa77403b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. ## Unreleased +### Added + +- postgres_cdc: Postgres CDC now accepts a glob pattern for the `schema` field (e.g. `tenant_*`), replicating all matching schemas through a single replication slot. Useful for multi-tenant databases where each tenant has its own schema. ([@ness-david-dedu](https://github.com/ness-david-dedu), [#4589](https://github.com/redpanda-data/connect/pull/4589)) + ### Fixed - general: The CGO-enabled distribution binary now embeds the IANA time zone database via the `timetzdata` build tag, matching the other distributions, so `time.LoadLocation` works in minimal runtimes without system tzdata instead of silently falling back to UTC (which shifts JQL date predicates in the `jira` input). ([@squiidz](https://github.com/squiidz), [#4583](https://github.com/redpanda-data/connect/pull/4583)) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index e1c2257518..069c0f1bfe 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -176,6 +176,8 @@ When a pattern is used, all schemas whose names match the pattern are replicated Double-quoted identifiers are treated as exact names and do not support wildcards. +Schema pattern matching runs once at pipeline startup. Schemas created after the pipeline starts will not be picked up until the pipeline is restarted. + *Type*: `string` diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index dfec0b4881..f8d0b0fde3 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -26,6 +26,7 @@ import ( "github.com/redpanda-data/connect/v4/internal/asyncroutine" "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream" + "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream/sanitize" "github.com/redpanda-data/connect/v4/internal/license" ) @@ -114,7 +115,9 @@ This input adds the following metadata fields to each message: When a pattern is used, all schemas whose names match the pattern are replicated using a single replication slot and publication. This is useful for multi-tenant databases where each tenant has its own schema (e.g. `+"`tenant_*`"+` matches `+"`tenant_foo`"+`, `+"`tenant_bar`"+`, etc.). -Double-quoted identifiers are treated as exact names and do not support wildcards.`). +Double-quoted identifiers are treated as exact names and do not support wildcards. + +Schema pattern matching runs once at pipeline startup. Schemas created after the pipeline starts will not be picked up until the pipeline is restarted.`). Examples("public", `"MyCaseSensitiveSchemaNeedingQuotes"`, "tenant_*", "*"), ). Field(service.NewStringListField(fieldTables). @@ -389,8 +392,8 @@ func validateSchemaPattern(s string) error { return errors.New("schema cannot be empty") } if strings.HasPrefix(s, `"`) { - if !strings.HasSuffix(s, `"`) || len(s) < 3 { - return errors.New("unterminated quoted identifier in schema") + if _, err := sanitize.UnquotePostgresIdentifier(s); err != nil { + return fmt.Errorf("invalid quoted schema identifier: %w", err) } if strings.ContainsRune(s, '*') { return errors.New("wildcard '*' is not allowed inside a quoted schema identifier") diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index aeae22b0c7..30068b6d2c 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1600,3 +1600,36 @@ postgres_cdc: assert.Equal(t, 1, cdcSchemas["tenant_a"], "expected 1 CDC row from tenant_a") assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") } + +func TestIntegrationNoSchemasMatchedError(t *testing.T) { + integration.CheckSkip(t) + databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + tmpl := fmt.Sprintf(` +postgres_cdc: + dsn: %s + slot_name: no_schema_match_slot + schema: nonexistent_schema_zzz_* + tables: + - events +`, databaseURL) + + sb := service.NewStreamBuilder() + require.NoError(t, sb.SetLoggerYAML(`level: ERROR`)) + require.NoError(t, sb.AddInputYAML(tmpl)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, _ service.MessageBatch) error { + return nil + })) + + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + err = stream.Run(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "no schemas found matching pattern") +} From d39de9a3e5976a9868ee97674ab54273da689001 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Mon, 13 Jul 2026 10:26:28 +0300 Subject: [PATCH 07/70] postgres_cdc: fix tests --- .../impl/postgresql/pglogicalstream/sanitize/sanitize.go | 2 +- internal/plugins/cdctest/cdc_conformance_test.go | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/internal/impl/postgresql/pglogicalstream/sanitize/sanitize.go b/internal/impl/postgresql/pglogicalstream/sanitize/sanitize.go index febdb3311f..c3826b0037 100644 --- a/internal/impl/postgresql/pglogicalstream/sanitize/sanitize.go +++ b/internal/impl/postgresql/pglogicalstream/sanitize/sanitize.go @@ -384,7 +384,7 @@ func QuotePostgresIdentifier(name string) string { // UnquotePostgresIdentifier returns the valid unescaped identifier. func UnquotePostgresIdentifier(quoted string) (string, error) { var output strings.Builder - if !strings.HasPrefix(quoted, `"`) || !strings.HasSuffix(quoted, `"`) || len(quoted) < 2 { + if !strings.HasPrefix(quoted, `"`) || !strings.HasSuffix(quoted, `"`) || len(quoted) < 3 { return "", errors.New("missing quotes for identifier") } unquoted := quoted[1 : len(quoted)-1] diff --git a/internal/plugins/cdctest/cdc_conformance_test.go b/internal/plugins/cdctest/cdc_conformance_test.go index cd5eac4de5..31141ddc2a 100644 --- a/internal/plugins/cdctest/cdc_conformance_test.go +++ b/internal/plugins/cdctest/cdc_conformance_test.go @@ -92,13 +92,6 @@ var knownNonConformant = map[string]map[string]string{ "salesforce_cdc": { "max_parallel_snapshot_tables": "uses max_parallel_snapshot_objects; migrate", }, - "tigerbeetle_cdc": { - "checkpoint_cache": "non-relational; §5 applicability under triage", - "checkpoint_limit": "non-relational; §5 applicability under triage", - "snapshot_max_batch_size": "non-relational; §5 applicability under triage", - "max_parallel_snapshot_tables": "non-relational; §5 applicability under triage", - "stream_snapshot": "non-relational; §5 applicability under triage", - }, } // componentSchema is the subset of the docs.ComponentSpec JSON emitted by From 0aa387adfd3d9a05ecf4ed8464e6584bd1267d71 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Wed, 15 Jul 2026 15:35:56 +0300 Subject: [PATCH 08/70] test(cdctest): waive tigerbeetle_cdc conformance fields --- .../plugins/cdctest/cdc_conformance_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/internal/plugins/cdctest/cdc_conformance_test.go b/internal/plugins/cdctest/cdc_conformance_test.go index 31141ddc2a..6a3980caba 100644 --- a/internal/plugins/cdctest/cdc_conformance_test.go +++ b/internal/plugins/cdctest/cdc_conformance_test.go @@ -54,6 +54,13 @@ var canonicalFields = []string{ "stream_snapshot", } +// conditionalConnectors lists CDC inputs that are only registered under specific +// build tags (e.g. cgo). They are exempt from the stale-entry guard when not +// registered, but are still subject to conformance checks when they are. +var conditionalConnectors = map[string]bool{ + "tigerbeetle_cdc": true, // requires cgo +} + // knownNonConformant waives specific (connector → field → reason) checks that // have not yet been migrated. New connectors default to strict. Populated from // the actual registry state; shrink it as connectors converge. @@ -92,6 +99,13 @@ var knownNonConformant = map[string]map[string]string{ "salesforce_cdc": { "max_parallel_snapshot_tables": "uses max_parallel_snapshot_objects; migrate", }, + "tigerbeetle_cdc": { + "checkpoint_cache": "uses progress_cache; migrate to checkpoint_cache", + "checkpoint_limit": "no discrete checkpoint limit; TigerBeetle CDC is pure streaming", + "snapshot_max_batch_size": "no snapshot phase; TigerBeetle CDC is pure streaming with no initial snapshot", + "max_parallel_snapshot_tables": "no snapshot phase; TigerBeetle CDC is pure streaming with no initial snapshot", + "stream_snapshot": "no snapshot phase; TigerBeetle CDC is pure streaming with no initial snapshot", + }, } // componentSchema is the subset of the docs.ComponentSpec JSON emitted by @@ -172,6 +186,10 @@ func TestCDCConformance(t *testing.T) { } for name := range knownNonConformant { if _, ok := registered[name]; !ok { + if conditionalConnectors[name] { + t.Logf("SKIPPED stale check for %q: not registered in this build (conditional build tag)", name) + continue + } t.Errorf("stale knownNonConformant entry %q is not a registered CDC input; remove it", name) } } From ba7c3849c22472df2dbd46d92abce94610903f54 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Wed, 15 Jul 2026 17:17:41 +0300 Subject: [PATCH 09/70] postgres_cdc: fix lint --- internal/impl/postgresql/input_pg_stream.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index deb3edc5bb..11ad70d6c4 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -408,7 +408,7 @@ func validateSchemaPattern(s string) error { return fmt.Errorf("invalid character %q at position %d in schema pattern %q", ch, i, s) } first := rune(s[0]) - if !(first == '_' || first == '*' || (first >= 'a' && first <= 'z') || (first >= 'A' && first <= 'Z')) { + if first != '_' && first != '*' && (first < 'a' || first > 'z') && (first < 'A' || first > 'Z') { return fmt.Errorf("schema pattern %q must start with a letter, underscore, or '*'", s) } return nil From caa0b1ace5d20b919def4ef14191a438f7b67abe Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Wed, 22 Jul 2026 23:42:19 +0300 Subject: [PATCH 10/70] postgres_cdc: skip missing tables per-schema instead of failing whole publication --- .../components/pages/inputs/postgres_cdc.adoc | 2 + internal/impl/postgresql/input_pg_stream.go | 4 +- internal/impl/postgresql/integration_test.go | 75 +++++++++++++++++++ .../pglogicalstream/logical_stream.go | 11 +++ .../pglogicalstream/schema_resolver.go | 37 +++++++++ 5 files changed, 128 insertions(+), 1 deletion(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index 069c0f1bfe..70818a8a1d 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -198,6 +198,8 @@ schema: '*' A list of table names to include in the logical replication. Each table should be specified as a separate item. +When `schema` is a glob pattern, this list is resolved against each matched schema independently: a table missing from a given schema is skipped (with a warning logged) rather than failing replication for every matched schema. + *Type*: `array` diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 11ad70d6c4..0ad7e1c887 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -122,7 +122,9 @@ Schema pattern matching runs once at pipeline startup. Schemas created after the Examples("public", `"MyCaseSensitiveSchemaNeedingQuotes"`, "tenant_*", "*"), ). Field(service.NewStringListField(fieldTables). - Description("A list of table names to include in the logical replication. Each table should be specified as a separate item."). + Description(`A list of table names to include in the logical replication. Each table should be specified as a separate item. + +When `+"`schema`"+` is a glob pattern, this list is resolved against each matched schema independently: a table missing from a given schema is skipped (with a warning logged) rather than failing replication for every matched schema.`). Example([]string{"my_table_1", `"MyCaseSensitiveTableNeedingQuotes"`})). Field(service.NewIntField(fieldCheckpointLimit). Description("The maximum number of messages that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level. Any given LSN will not be acknowledged unless all messages under that offset are delivered in order to preserve at least once delivery guarantees."). diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index 55ebafe4f1..344c20f006 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1738,6 +1738,81 @@ postgres_cdc: assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") } +func TestIntegrationMultiSchemaMissingTableDegradesGracefully(t *testing.T) { + integration.CheckSkip(t) + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + // tenant_a is fully provisioned with the "events" table; tenant_b matches + // the schema glob but is missing it (e.g. still being migrated). Before + // this fix, CreatePublication's FOR TABLE clause would reference the + // non-existent tenant_b.events relation and fail publication setup for + // every matched schema, not just the drifted one. + _, err = db.Exec("CREATE SCHEMA tenant_a") + require.NoError(t, err) + _, err = db.Exec("CREATE TABLE tenant_a.events (id SERIAL PRIMARY KEY, name TEXT)") + require.NoError(t, err) + _, err = db.Exec("CREATE SCHEMA tenant_b") + require.NoError(t, err) + + _, err = db.Exec("INSERT INTO tenant_a.events (name) VALUES ('alice')") + require.NoError(t, err) + + type msgMeta struct { + pgSchema string + table string + } + + var ( + mu sync.Mutex + collected []msgMeta + ) + + tmpl := fmt.Sprintf(` +postgres_cdc: + dsn: %s + slot_name: missing_table_degrade_slot + stream_snapshot: true + schema: tenant_* + tables: + - events +`, databaseURL) + + sb := service.NewStreamBuilder() + require.NoError(t, sb.SetLoggerYAML(`level: WARN`)) + require.NoError(t, sb.AddInputYAML(tmpl)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + mu.Lock() + defer mu.Unlock() + for _, msg := range batch { + m := msgMeta{} + m.pgSchema, _ = msg.MetaGet("pg_schema") + m.table, _ = msg.MetaGet("table") + collected = append(collected, m) + } + return nil + })) + + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + go func() { _ = stream.Run(t.Context()) }() + t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) + + // tenant_a should keep streaming even though tenant_b is missing the table. + assert.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return len(collected) >= 1 + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for tenant_a snapshot row; a missing table in tenant_b should not block replication") + + mu.Lock() + defer mu.Unlock() + require.Len(t, collected, 1) + assert.Equal(t, "tenant_a", collected[0].pgSchema) + assert.Equal(t, "events", collected[0].table) +} + func TestIntegrationNoSchemasMatchedError(t *testing.T) { integration.CheckSkip(t) databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 1c0ecf322d..96f4953d46 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -118,10 +118,21 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { tables := make([]TableFQN, 0, len(schemas)*len(normalizedTables)) for _, schema := range schemas { + existingTables, err := resolveExistingTables(ctx, dbConn, schema) + if err != nil { + return nil, fmt.Errorf("resolving tables in schema %q: %w", schema, err) + } for _, table := range normalizedTables { + if _, ok := existingTables[table]; !ok { + config.Logger.Warnf("table %s.%s not found, skipping (schema %s matched pattern %q but does not contain this table)", schema, table, schema, config.DBSchemaPattern) + continue + } tables = append(tables, TableFQN{Schema: schema, Table: table}) } } + if len(tables) == 0 && len(normalizedTables) > 0 { + return nil, fmt.Errorf("none of the configured tables %v were found in any schema matching pattern %q", config.DBTables, config.DBSchemaPattern) + } batchSize := 1000 if config.BatchSize > 0 { batchSize = config.BatchSize diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index 8920121e99..117b822da9 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -82,6 +82,43 @@ func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) ([ return schemas, nil } +// resolveExistingTables returns the set of quoted table identifiers that +// actually exist in the given (already quoted) schema. +// +// This is used to resolve a schema glob × table list combination per-schema +// rather than assuming every matched schema contains every listed table. A +// schema matching the glob but missing one of the configured tables (e.g. a +// tenant schema that's still being provisioned) would otherwise cause +// CreatePublication's FOR TABLE clause to reference a non-existent relation, +// failing publication setup for every schema, not just the drifted one. +func resolveExistingTables(ctx context.Context, conn *pgconn.PgConn, quotedSchema string) (map[string]struct{}, error) { + schema, err := sanitize.UnquotePostgresIdentifier(quotedSchema) + if err != nil { + return nil, fmt.Errorf("unquoting schema identifier %q: %w", quotedSchema, err) + } + + q, err := sanitize.SQLQuery( + "SELECT table_name FROM information_schema.tables WHERE table_schema = $1", + schema, + ) + if err != nil { + return nil, fmt.Errorf("building table resolution query for schema %q: %w", quotedSchema, err) + } + + results, err := conn.Exec(ctx, q).ReadAll() + if err != nil { + return nil, fmt.Errorf("querying tables in schema %q: %w", quotedSchema, err) + } + + existing := map[string]struct{}{} + if len(results) > 0 { + for _, row := range results[0].Rows { + existing[sanitize.QuotePostgresIdentifier(string(row[0]))] = struct{}{} + } + } + return existing, nil +} + // globToLike converts an unquoted glob pattern (using '*' as wildcard) into a // PostgreSQL LIKE pattern that uses '!' as the escape character. // From c161947c20bf8051a5fe14e9020beafb446346d9 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Wed, 22 Jul 2026 23:48:02 +0300 Subject: [PATCH 11/70] postgres_cdc: fix lint --- internal/gateway/authz_endpoint_test.go | 6 +++--- internal/impl/otlp/mock_policy_server_test.go | 6 +++--- internal/impl/postgresql/input_pg_stream.go | 2 +- internal/impl/protobuf/processor_protobuf_test.go | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/gateway/authz_endpoint_test.go b/internal/gateway/authz_endpoint_test.go index b7f240b4f6..899e04b048 100644 --- a/internal/gateway/authz_endpoint_test.go +++ b/internal/gateway/authz_endpoint_test.go @@ -20,7 +20,7 @@ import ( policymaterializerv1 "buf.build/gen/go/redpandadata/common/protocolbuffers/go/redpanda/policymaterializer/v1" "connectrpc.com/connect" "golang.org/x/net/http2" - "golang.org/x/net/http2/h2c" //nolint:staticcheck + "golang.org/x/net/http2/h2c" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -65,8 +65,8 @@ func startPolicyMaterializerServer(t *testing.T, svc policymaterializerv1connect lis, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") require.NoError(t, err) - srv := &http.Server{Handler: h2c.NewHandler(mux, &http2.Server{})} //nolint:staticcheck - go srv.Serve(lis) //nolint:errcheck // test server + srv := &http.Server{Handler: h2c.NewHandler(mux, &http2.Server{})} + go srv.Serve(lis) //nolint:errcheck // test server t.Cleanup(func() { srv.Close() }) return "http://" + lis.Addr().String() diff --git a/internal/impl/otlp/mock_policy_server_test.go b/internal/impl/otlp/mock_policy_server_test.go index 43ae225c7c..64d5781551 100644 --- a/internal/impl/otlp/mock_policy_server_test.go +++ b/internal/impl/otlp/mock_policy_server_test.go @@ -18,7 +18,7 @@ import ( policymaterializerv1 "buf.build/gen/go/redpandadata/common/protocolbuffers/go/redpanda/policymaterializer/v1" "connectrpc.com/connect" "golang.org/x/net/http2" - "golang.org/x/net/http2/h2c" //nolint:staticcheck + "golang.org/x/net/http2/h2c" "github.com/stretchr/testify/require" ) @@ -59,8 +59,8 @@ func startMockPolicyEndpoint(t *testing.T, svc policymaterializerv1connect.Polic lis, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") require.NoError(t, err) - srv := &http.Server{Handler: h2c.NewHandler(mux, &http2.Server{})} //nolint:staticcheck - go srv.Serve(lis) //nolint:errcheck + srv := &http.Server{Handler: h2c.NewHandler(mux, &http2.Server{})} + go srv.Serve(lis) //nolint:errcheck t.Cleanup(func() { srv.Close() }) return "http://" + lis.Addr().String() diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 0ad7e1c887..1bbac06ed1 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -124,7 +124,7 @@ Schema pattern matching runs once at pipeline startup. Schemas created after the Field(service.NewStringListField(fieldTables). Description(`A list of table names to include in the logical replication. Each table should be specified as a separate item. -When `+"`schema`"+` is a glob pattern, this list is resolved against each matched schema independently: a table missing from a given schema is skipped (with a warning logged) rather than failing replication for every matched schema.`). +When ` + "`schema`" + ` is a glob pattern, this list is resolved against each matched schema independently: a table missing from a given schema is skipped (with a warning logged) rather than failing replication for every matched schema.`). Example([]string{"my_table_1", `"MyCaseSensitiveTableNeedingQuotes"`})). Field(service.NewIntField(fieldCheckpointLimit). Description("The maximum number of messages that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level. Any given LSN will not be acknowledged unless all messages under that offset are delivered in order to preserve at least once delivery guarantees."). diff --git a/internal/impl/protobuf/processor_protobuf_test.go b/internal/impl/protobuf/processor_protobuf_test.go index d7f6fd01d9..92d8f6bd90 100644 --- a/internal/impl/protobuf/processor_protobuf_test.go +++ b/internal/impl/protobuf/processor_protobuf_test.go @@ -51,7 +51,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/net/http2" - "golang.org/x/net/http2/h2c" //nolint:staticcheck + "golang.org/x/net/http2/h2c" "google.golang.org/protobuf/types/descriptorpb" "github.com/redpanda-data/benthos/v4/public/service" @@ -463,7 +463,7 @@ func runMockBSRServer(t *testing.T, importPath string) string { fileDescriptorSetServer := &fileDescriptorSetServer{fileDescriptorSet: files} mux.Handle(reflectv1beta1connect.NewFileDescriptorSetServiceHandler(fileDescriptorSetServer)) go func() { - if err := http.Serve(listener, h2c.NewHandler(mux, &http2.Server{})); err != nil && !errors.Is(err, http.ErrServerClosed) { //nolint:staticcheck + if err := http.Serve(listener, h2c.NewHandler(mux, &http2.Server{})); err != nil && !errors.Is(err, http.ErrServerClosed) { require.NoError(t, err) } }() From 0ed96fdf0cd8364c58b62e55963d8cc7cb208fbb Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Thu, 23 Jul 2026 13:03:26 +0300 Subject: [PATCH 12/70] Update internal/impl/postgresql/pglogicalstream/schema_resolver.go Co-authored-by: Joseph Woodward --- internal/impl/postgresql/pglogicalstream/schema_resolver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index 117b822da9..8f547bcff7 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -1,4 +1,4 @@ -// Copyright 2024 Redpanda Data, Inc. +// Copyright 2026 Redpanda Data, Inc. // // Licensed as a Redpanda Enterprise file under the Redpanda Community // License (the "License"); you may not use this file except in compliance with From d893e1ac9bc9efc93464b6e776ffea9d240e3884 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Mon, 27 Jul 2026 15:38:08 +0300 Subject: [PATCH 13/70] postgres_cdc: warn when schema pattern matches privilege-hidden schemas information_schema.schemata only lists schemas the connecting role can see, so a schema hidden by missing USAGE was silently dropped from a matched pattern with no signal to the user. resolveSchemas now cross-checks pg_catalog.pg_namespace, which isn't privilege-filtered, and reports those as inaccessible so logical_stream.go can warn instead of skipping silently. --- .../pglogicalstream/logical_stream.go | 5 +- .../pglogicalstream/schema_resolver.go | 49 +++++++++++-- .../schema_resolver_integration_test.go | 71 +++++++++++++++++++ 3 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 96f4953d46..b7cf77d571 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -98,10 +98,13 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { return nil, err } - schemas, err := resolveSchemas(ctx, dbConn, config.DBSchemaPattern) + schemas, inaccessibleSchemas, err := resolveSchemas(ctx, dbConn, config.DBSchemaPattern) if err != nil { return nil, fmt.Errorf("resolving schema pattern %q: %w", config.DBSchemaPattern, err) } + if len(inaccessibleSchemas) > 0 { + config.Logger.Warnf("schema pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaPattern, inaccessibleSchemas) + } if len(schemas) == 0 { return nil, fmt.Errorf("no schemas found matching pattern %q", config.DBSchemaPattern) } diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index 8f547bcff7..3859ccface 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -51,10 +51,17 @@ func schemaPatternToLike(pattern string) (string, error) { return globToLike(strings.ToLower(pattern)), nil } -func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) ([]string, error) { +// resolveSchemas returns the schemas matching pattern that the connection's +// role has access to (visibleSchemas), plus any schemas that also match +// pattern but are hidden from information_schema.schemata by privileges +// (inaccessibleSchemas). The latter is surfaced separately so callers can +// warn the user instead of silently dropping schemas they expected to be +// included, e.g. `"tenant_*"` matching a schema the configured role can't +// see yet. +func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (visibleSchemas, inaccessibleSchemas []string, err error) { likePattern, err := schemaPatternToLike(pattern) if err != nil { - return nil, err + return nil, nil, err } q, err := sanitize.SQLQuery( @@ -62,24 +69,54 @@ func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) ([ likePattern, ) if err != nil { - return nil, fmt.Errorf("building schema resolution query: %w", err) + return nil, nil, fmt.Errorf("building schema resolution query: %w", err) } results, err := conn.Exec(ctx, q).ReadAll() if err != nil { - return nil, fmt.Errorf("querying schemas matching %q: %w", pattern, err) + return nil, nil, fmt.Errorf("querying schemas matching %q: %w", pattern, err) } + visible := map[string]struct{}{} var schemas []string if len(results) > 0 { for _, row := range results[0].Rows { + name := string(row[0]) + visible[name] = struct{}{} // QuotePostgresIdentifier preserves the exact stored name (including // case for case-sensitive schemas), unlike NormalizePostgresIdentifier // which would incorrectly fold to lower-case. - schemas = append(schemas, sanitize.QuotePostgresIdentifier(string(row[0]))) + schemas = append(schemas, sanitize.QuotePostgresIdentifier(name)) } } - return schemas, nil + + // pg_namespace is not privilege-filtered, so any pattern match here that's + // missing from information_schema.schemata means the role lacks USAGE (or + // similar) on that schema rather than the schema simply not existing. + nsQ, err := sanitize.SQLQuery( + "SELECT nspname FROM pg_catalog.pg_namespace WHERE nspname LIKE $1 ESCAPE '!' AND nspname NOT LIKE 'pg!_%' ESCAPE '!' AND nspname != 'information_schema'", + likePattern, + ) + if err != nil { + return nil, nil, fmt.Errorf("building pg_namespace resolution query: %w", err) + } + + nsResults, err := conn.Exec(ctx, nsQ).ReadAll() + if err != nil { + return nil, nil, fmt.Errorf("querying pg_namespace for schemas matching %q: %w", pattern, err) + } + + var hidden []string + if len(nsResults) > 0 { + for _, row := range nsResults[0].Rows { + name := string(row[0]) + if _, ok := visible[name]; !ok { + hidden = append(hidden, sanitize.QuotePostgresIdentifier(name)) + } + } + } + + return schemas, hidden, nil } // resolveExistingTables returns the set of quoted table identifiers that diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go b/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go new file mode 100644 index 0000000000..6af1d4ad7e --- /dev/null +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go @@ -0,0 +1,71 @@ +// Copyright 2024 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md + +package pglogicalstream + +import ( + "context" + "database/sql" + "testing" + "time" + + _ "github.com/lib/pq" // registers "postgres" driver for sql.Open in tests + + "github.com/jackc/pgx/v5/pgconn" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service/integration" +) + +// TestIntegrationResolveSchemasReportsInaccessibleSchemas verifies that a +// schema pattern matching a schema the connecting role lacks USAGE on is +// reported via inaccessibleSchemas rather than silently dropped, since +// information_schema.schemata alone would make it indistinguishable from a +// schema that simply doesn't exist. +func TestIntegrationResolveSchemasReportsInaccessibleSchemas(t *testing.T) { + integration.CheckSkip(t) + + _, adminURL := createDockerInstance(t) + + adminDB, err := sql.Open("postgres", adminURL) + require.NoError(t, err) + defer adminDB.Close() + + _, err = adminDB.Exec("CREATE SCHEMA visible_schema") + require.NoError(t, err) + _, err = adminDB.Exec("CREATE SCHEMA hidden_schema") + require.NoError(t, err) + + _, err = adminDB.Exec("CREATE ROLE restricted_role LOGIN PASSWORD 'restricted_pw'") + require.NoError(t, err) + _, err = adminDB.Exec("GRANT CONNECT ON DATABASE dbname TO restricted_role") + require.NoError(t, err) + _, err = adminDB.Exec("GRANT USAGE ON SCHEMA visible_schema TO restricted_role") + require.NoError(t, err) + // Deliberately no GRANT on hidden_schema. + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + + restrictedConfig, err := pgconn.ParseConfig(adminURL) + require.NoError(t, err) + restrictedConfig.User = "restricted_role" + restrictedConfig.Password = "restricted_pw" + delete(restrictedConfig.RuntimeParams, "replication") + + restrictedConn, err := pgconn.ConnectConfig(ctx, restrictedConfig) + require.NoError(t, err) + defer closeConn(t, restrictedConn) + + visible, inaccessible, err := resolveSchemas(ctx, restrictedConn, "*_schema") + require.NoError(t, err) + + assert.Equal(t, []string{`"visible_schema"`}, visible) + assert.Equal(t, []string{`"hidden_schema"`}, inaccessible) +} From 0e662de9d78a8903fcb2a77d0a709127442ae83f Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 4 Aug 2026 12:15:05 +0100 Subject: [PATCH 14/70] postgres_cdc: Address minor issues --- docs/modules/components/pages/inputs/postgres_cdc.adoc | 1 + internal/gateway/authz_endpoint_test.go | 6 +++--- internal/impl/otlp/mock_policy_server_test.go | 6 +++--- internal/impl/protobuf/processor_protobuf_test.go | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index 37adc7fe1b..849adbd8ae 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -199,6 +199,7 @@ schema: '*' A list of table names to include in the logical replication. Each table should be specified as a separate item. When `schema` is a glob pattern, this list is resolved against each matched schema independently: a table missing from a given schema is skipped (with a warning logged) rather than failing replication for every matched schema. + If left empty, the underlying PostgreSQL publication is created `FOR ALL TABLES`, which replicates every table in every schema of the database, ignoring `schema`. This also disables `stream_snapshot`, since the initial snapshot is only planned for tables listed here. diff --git a/internal/gateway/authz_endpoint_test.go b/internal/gateway/authz_endpoint_test.go index 899e04b048..b7f240b4f6 100644 --- a/internal/gateway/authz_endpoint_test.go +++ b/internal/gateway/authz_endpoint_test.go @@ -20,7 +20,7 @@ import ( policymaterializerv1 "buf.build/gen/go/redpandadata/common/protocolbuffers/go/redpanda/policymaterializer/v1" "connectrpc.com/connect" "golang.org/x/net/http2" - "golang.org/x/net/http2/h2c" + "golang.org/x/net/http2/h2c" //nolint:staticcheck "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -65,8 +65,8 @@ func startPolicyMaterializerServer(t *testing.T, svc policymaterializerv1connect lis, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") require.NoError(t, err) - srv := &http.Server{Handler: h2c.NewHandler(mux, &http2.Server{})} - go srv.Serve(lis) //nolint:errcheck // test server + srv := &http.Server{Handler: h2c.NewHandler(mux, &http2.Server{})} //nolint:staticcheck + go srv.Serve(lis) //nolint:errcheck // test server t.Cleanup(func() { srv.Close() }) return "http://" + lis.Addr().String() diff --git a/internal/impl/otlp/mock_policy_server_test.go b/internal/impl/otlp/mock_policy_server_test.go index 64d5781551..43ae225c7c 100644 --- a/internal/impl/otlp/mock_policy_server_test.go +++ b/internal/impl/otlp/mock_policy_server_test.go @@ -18,7 +18,7 @@ import ( policymaterializerv1 "buf.build/gen/go/redpandadata/common/protocolbuffers/go/redpanda/policymaterializer/v1" "connectrpc.com/connect" "golang.org/x/net/http2" - "golang.org/x/net/http2/h2c" + "golang.org/x/net/http2/h2c" //nolint:staticcheck "github.com/stretchr/testify/require" ) @@ -59,8 +59,8 @@ func startMockPolicyEndpoint(t *testing.T, svc policymaterializerv1connect.Polic lis, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") require.NoError(t, err) - srv := &http.Server{Handler: h2c.NewHandler(mux, &http2.Server{})} - go srv.Serve(lis) //nolint:errcheck + srv := &http.Server{Handler: h2c.NewHandler(mux, &http2.Server{})} //nolint:staticcheck + go srv.Serve(lis) //nolint:errcheck t.Cleanup(func() { srv.Close() }) return "http://" + lis.Addr().String() diff --git a/internal/impl/protobuf/processor_protobuf_test.go b/internal/impl/protobuf/processor_protobuf_test.go index 92d8f6bd90..d7f6fd01d9 100644 --- a/internal/impl/protobuf/processor_protobuf_test.go +++ b/internal/impl/protobuf/processor_protobuf_test.go @@ -51,7 +51,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/net/http2" - "golang.org/x/net/http2/h2c" + "golang.org/x/net/http2/h2c" //nolint:staticcheck "google.golang.org/protobuf/types/descriptorpb" "github.com/redpanda-data/benthos/v4/public/service" @@ -463,7 +463,7 @@ func runMockBSRServer(t *testing.T, importPath string) string { fileDescriptorSetServer := &fileDescriptorSetServer{fileDescriptorSet: files} mux.Handle(reflectv1beta1connect.NewFileDescriptorSetServiceHandler(fileDescriptorSetServer)) go func() { - if err := http.Serve(listener, h2c.NewHandler(mux, &http2.Server{})); err != nil && !errors.Is(err, http.ErrServerClosed) { + if err := http.Serve(listener, h2c.NewHandler(mux, &http2.Server{})); err != nil && !errors.Is(err, http.ErrServerClosed) { //nolint:staticcheck require.NoError(t, err) } }() From e7dccc60b99d32aaf670db8cfe03a7dc82d7c45c Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 4 Aug 2026 12:53:19 +0100 Subject: [PATCH 15/70] postgres_cdc: fix broken test --- internal/impl/postgresql/integration_test.go | 35 ++++++++++---------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index f675014071..f61148c4a4 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1831,35 +1831,36 @@ postgres_cdc: assert.Equal(t, "events", collected[0].table) } -func TestIntegrationNoSchemasMatchedError(t *testing.T) { +func TestIntegrationNoSchemasMatchedReturnsError(t *testing.T) { integration.CheckSkip(t) databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") require.NoError(t, err) tmpl := fmt.Sprintf(` -postgres_cdc: - dsn: %s - slot_name: no_schema_match_slot - schema: nonexistent_schema_zzz_* - tables: - - events +dsn: %s +slot_name: no_schema_match_slot +schema: nonexistent_schema_zzz_* +tables: + - events `, databaseURL) - sb := service.NewStreamBuilder() - require.NoError(t, sb.SetLoggerYAML(`level: ERROR`)) - require.NoError(t, sb.AddInputYAML(tmpl)) - require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, _ service.MessageBatch) error { - return nil - })) + conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) + require.NoError(t, err) - stream, err := sb.Build() + mgr := service.MockResources() + license.InjectTestService(mgr) + + input, err := newPgStreamInput(conf, mgr) require.NoError(t, err) - license.InjectTestService(stream.Resources()) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - err = stream.Run(ctx) + // Bypass the benthos AsyncReader's infinite connect-retry loop by calling + // Connect directly: a schema-pattern-not-found error is permanent, but + // stream.Run has no path to surface it (it only returns once ctx is done), + // so going through StreamBuilder/Run here would just time out instead. + err = input.Connect(ctx) require.Error(t, err) assert.Contains(t, err.Error(), "no schemas found matching pattern") } From 80e2204e86a555b365b7c5980ec4cf3ab3fdf802 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 4 Aug 2026 13:09:51 +0100 Subject: [PATCH 16/70] postgres_cdc: replace pg_schema with database_schema --- internal/impl/postgresql/input_pg_stream.go | 4 +- internal/impl/postgresql/integration_test.go | 70 +++++++++---------- .../impl/postgresql/tests/current/setup.sql | 2 +- .../postgresql/tests/current/test_config.yaml | 4 +- 4 files changed, 40 insertions(+), 40 deletions(-) diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 13c0a41fd6..5cecc17457 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -85,7 +85,7 @@ Additionally, if ` + "`" + fieldStreamSnapshot + "`" + ` is set to true, then th This input adds the following metadata fields to each message: - table: Name of the table that the message originated from -- pg_schema: The PostgreSQL schema name that the table belongs to (e.g. "public", "tenant_foo"). Useful for per-schema routing when using schema patterns. +- database_schema: The database schema for the table where the message originates from (e.g. "public", "tenant_foo"). Useful for per-schema routing when using schema patterns. - operation: Type of operation that generated the message: "read", "insert", "update", or "delete". "read" is from messages that are read in the initial snapshot phase. This will also be "begin" and "commit" if ` + "`" + fieldIncludeTxnMarkers + "`" + ` is enabled - lsn: the log sequence number in postgres - schema: The table schema in benthos common schema format, compatible with processors like parquet_encode @@ -586,7 +586,7 @@ func (p *pgStreamInput) processStream(pgStream *pglogicalstream.Stream, batcher } batchMsg := service.NewMessage(mb) batchMsg.MetaSet("table", msg.Table) - batchMsg.MetaSet("pg_schema", msg.Schema) + batchMsg.MetaSet("database_schema", msg.Schema) batchMsg.MetaSet("operation", string(msg.Operation)) if msg.LSN != nil { batchMsg.MetaSet("lsn", *msg.LSN) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index f61148c4a4..e183f0f517 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1249,44 +1249,44 @@ postgres_cdc: outBatches, []any{ map[string]any{ - "operation": "read", - "table": "FlightsCompositePK", - "pg_schema": "public", + "operation": "read", + "table": "FlightsCompositePK", + "database_schema": "public", }, map[string]any{ - "operation": "read", - "table": "flights", - "pg_schema": "public", + "operation": "read", + "table": "flights", + "database_schema": "public", }, map[string]any{ - "operation": "insert", - "table": "FlightsCompositePK", - "lsn": "XXX/XXX", - "commit_ts_ms": "SET", - "pg_schema": "public", + "operation": "insert", + "table": "FlightsCompositePK", + "lsn": "XXX/XXX", + "commit_ts_ms": "SET", + "database_schema": "public", }, map[string]any{ - "operation": "insert", - "table": "flights", - "lsn": "XXX/XXX", - "commit_ts_ms": "SET", - "pg_schema": "public", + "operation": "insert", + "table": "flights", + "lsn": "XXX/XXX", + "commit_ts_ms": "SET", + "database_schema": "public", }, map[string]any{ - "operation": "update", - "table": "flights", - "lsn": "XXX/XXX", - "commit_ts_ms": "SET", - "before": "SET", - "pg_schema": "public", + "operation": "update", + "table": "flights", + "lsn": "XXX/XXX", + "commit_ts_ms": "SET", + "before": "SET", + "database_schema": "public", }, map[string]any{ - "operation": "delete", - "table": "flights", - "lsn": "XXX/XXX", - "commit_ts_ms": "SET", - "before": "SET", - "pg_schema": "public", + "operation": "delete", + "table": "flights", + "lsn": "XXX/XXX", + "commit_ts_ms": "SET", + "before": "SET", + "database_schema": "public", }, }, ) @@ -1656,7 +1656,7 @@ func TestIntegrationMultiSchemaSnapshotAndCDC(t *testing.T) { require.NoError(t, err) type msgMeta struct { - pgSchema string + dbSchema string table string operation string lsn string @@ -1685,7 +1685,7 @@ postgres_cdc: defer mu.Unlock() for _, msg := range batch { m := msgMeta{} - m.pgSchema, _ = msg.MetaGet("pg_schema") + m.dbSchema, _ = msg.MetaGet("database_schema") m.table, _ = msg.MetaGet("table") m.operation, _ = msg.MetaGet("operation") m.lsn, _ = msg.MetaGet("lsn") @@ -1738,7 +1738,7 @@ postgres_cdc: for _, m := range snapshots { assert.Equal(t, "events", m.table, "snapshot: table should be bare name without schema prefix") assert.Empty(t, m.lsn, "snapshot rows have no LSN") - snapshotSchemas[m.pgSchema]++ + snapshotSchemas[m.dbSchema]++ } assert.Equal(t, 2, snapshotSchemas["tenant_a"], "expected 2 snapshot rows from tenant_a") assert.Equal(t, 1, snapshotSchemas["tenant_b"], "expected 1 snapshot row from tenant_b") @@ -1750,7 +1750,7 @@ postgres_cdc: assert.Equal(t, "insert", m.operation) assert.Equal(t, "events", m.table) assert.NotEmpty(t, m.lsn, "CDC rows must have an LSN") - cdcSchemas[m.pgSchema]++ + cdcSchemas[m.dbSchema]++ } assert.Equal(t, 1, cdcSchemas["tenant_a"], "expected 1 CDC row from tenant_a") assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") @@ -1777,7 +1777,7 @@ func TestIntegrationMultiSchemaMissingTableDegradesGracefully(t *testing.T) { require.NoError(t, err) type msgMeta struct { - pgSchema string + dbSchema string table string } @@ -1804,7 +1804,7 @@ postgres_cdc: defer mu.Unlock() for _, msg := range batch { m := msgMeta{} - m.pgSchema, _ = msg.MetaGet("pg_schema") + m.dbSchema, _ = msg.MetaGet("database_schema") m.table, _ = msg.MetaGet("table") collected = append(collected, m) } @@ -1827,7 +1827,7 @@ postgres_cdc: mu.Lock() defer mu.Unlock() require.Len(t, collected, 1) - assert.Equal(t, "tenant_a", collected[0].pgSchema) + assert.Equal(t, "tenant_a", collected[0].dbSchema) assert.Equal(t, "events", collected[0].table) } diff --git a/internal/impl/postgresql/tests/current/setup.sql b/internal/impl/postgresql/tests/current/setup.sql index ee40a376cc..512f8373fa 100644 --- a/internal/impl/postgresql/tests/current/setup.sql +++ b/internal/impl/postgresql/tests/current/setup.sql @@ -1,5 +1,5 @@ -- Multi-schema CDC test setup --- Tests: schema glob (tenant_*), pg_schema metadata, commit_ts_ms, before (update/delete) +-- Tests: schema glob (tenant_*), database_schema metadata, commit_ts_ms, before (update/delete) -- ── Tenant schemas ──────────────────────────────────────────────────────────── diff --git a/internal/impl/postgresql/tests/current/test_config.yaml b/internal/impl/postgresql/tests/current/test_config.yaml index 4a994e30c2..9e40b5a1c1 100644 --- a/internal/impl/postgresql/tests/current/test_config.yaml +++ b/internal/impl/postgresql/tests/current/test_config.yaml @@ -15,14 +15,14 @@ pipeline: - mapping: | let op = @operation let tbl = @table - let schema = @pg_schema + let schema = @database_schema let lsn = @lsn let ts_ms = @commit_ts_ms let before = @before root = { "operation": $op, - "pg_schema": $schema, + "database_schema": $schema, "table": $tbl, "payload": this, "lsn": if $lsn != null { $lsn } else { null }, From fc960ceae39e2fbef4457a71fef2311f4dac28d6 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 4 Aug 2026 13:20:58 +0100 Subject: [PATCH 17/70] postgres_cdc: move schema validation to unit test closer to use --- .../impl/postgresql/input_pg_stream_test.go | 72 +++++++++ .../tests/schema_validation/validate_test.go | 147 ------------------ 2 files changed, 72 insertions(+), 147 deletions(-) create mode 100644 internal/impl/postgresql/input_pg_stream_test.go delete mode 100644 internal/impl/postgresql/tests/schema_validation/validate_test.go diff --git a/internal/impl/postgresql/input_pg_stream_test.go b/internal/impl/postgresql/input_pg_stream_test.go new file mode 100644 index 0000000000..19d9723bc2 --- /dev/null +++ b/internal/impl/postgresql/input_pg_stream_test.go @@ -0,0 +1,72 @@ +// Copyright 2024 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md + +package pgstream + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" + + "github.com/redpanda-data/connect/v4/internal/license" +) + +// TestSchemaPatternValidation verifies that the schema field is validated +// during config parsing, before any network I/O is attempted. Success is +// asserted via newPgStreamInput returning no error - the constructor doesn't +// dial the database, so a valid pattern implies validation passed. +func TestSchemaPatternValidation(t *testing.T) { + tests := []struct { + pattern string + errContains string + }{ + {"public", ""}, + {"tenant_*", ""}, + {"*", ""}, + {`"MySchema"`, ""}, + // Regression test: len("") == 2 used to pass the old `len(s) < 2` guard. + // Fixed to `len(s) < 3`. + {`""`, "invalid quoted schema identifier"}, + {"1abc", "must start with a letter"}, + {`"unclosed`, "invalid quoted schema identifier"}, + {"schema-name", "invalid character"}, + {"", "schema cannot be empty"}, + {`"quoted*"`, "wildcard"}, + } + for _, tt := range tests { + t.Run(tt.pattern, func(t *testing.T) { + // Single-quoted so the pattern (which may itself contain double + // quotes, e.g. `"MySchema"`) reaches validateSchemaPattern verbatim. + yaml := fmt.Sprintf(` +dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable +schema: '%s' +slot_name: test_slot +tables: + - events +`, tt.pattern) + + conf, err := newPostgresCDCConfig().ParseYAML(yaml, nil) + require.NoError(t, err) + + mgr := service.MockResources() + license.InjectTestService(mgr) + + _, err = newPgStreamInput(conf, mgr) + if tt.errContains != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errContains) + return + } + require.NoError(t, err) + }) + } +} diff --git a/internal/impl/postgresql/tests/schema_validation/validate_test.go b/internal/impl/postgresql/tests/schema_validation/validate_test.go deleted file mode 100644 index e45e4bf2f2..0000000000 --- a/internal/impl/postgresql/tests/schema_validation/validate_test.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2024 Redpanda Data, Inc. -// -// Licensed as a Redpanda Enterprise file under the Redpanda Community -// License (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md - -// Package schema_validation tests postgres_cdc schema pattern validation -// through the public service config API. No database connection is required: -// invalid schemas are rejected during stream construction (before any network -// I/O), so stream.Run returns synchronously for the invalid-schema cases. -package schema_validation_test - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - _ "github.com/redpanda-data/benthos/v4/public/components/pure" // registers none tracer and other built-ins - "github.com/redpanda-data/benthos/v4/public/service" - - _ "github.com/redpanda-data/connect/v4/internal/impl/postgresql" // registers postgres_cdc - "github.com/redpanda-data/connect/v4/internal/license" -) - -// postgresStream builds a postgres_cdc stream with the given YAML schema value -// (the raw fragment that appears after "schema: " in YAML) and injects a test -// enterprise license. Returns the stream ready to run. -func postgresStream(t *testing.T, schemaYAML string) *service.Stream { - t.Helper() - yaml := fmt.Sprintf(` -postgres_cdc: - dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable - schema: %s - slot_name: test_slot - tables: - - events -`, schemaYAML) - - sb := service.NewStreamBuilder() - require.NoError(t, sb.SetLoggerYAML(`level: ERROR`)) - require.NoError(t, sb.AddInputYAML(yaml)) - require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, _ service.MessageBatch) error { - return nil - })) - - stream, err := sb.Build() - require.NoError(t, err) - license.InjectTestService(stream.Resources()) - return stream -} - -// TestInvalidSchemaPatterns verifies that invalid schema values are rejected -// during stream construction — before any database connection is attempted. -// stream.Run returns synchronously (no goroutine needed) when the constructor -// fails. -func TestInvalidSchemaPatterns(t *testing.T) { - tests := []struct { - name string - schemaYAML string - wantErrMsg string - }{ - { - // Regression test: len("") == 2 used to pass the old `len(s) < 2` - // guard. Fixed to `len(s) < 3`. - name: "empty quoted identifier", - schemaYAML: `'""'`, - wantErrMsg: "invalid schema", - }, - { - name: "digit-first unquoted pattern", - schemaYAML: `"1abc"`, - wantErrMsg: "invalid schema", - }, - { - name: "unterminated quoted identifier", - schemaYAML: `'"unclosed'`, - wantErrMsg: "invalid schema", - }, - { - name: "hyphen in unquoted pattern", - schemaYAML: `schema-name`, - wantErrMsg: "invalid schema", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - stream := postgresStream(t, tt.schemaYAML) - - // Run returns synchronously when the input constructor fails — - // no timeout context needed, but use one as a safety net. - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - err := stream.Run(ctx) - require.Error(t, err, "expected stream construction to fail") - assert.Contains(t, err.Error(), tt.wantErrMsg, - "error should indicate schema validation failure") - }) - } -} - -// TestValidSchemaPatterns verifies that valid schema values pass construction -// and are only rejected later (at DB-connect time). We run the stream briefly -// and confirm no "invalid schema" error surfaces. -func TestValidSchemaPatterns(t *testing.T) { - tests := []struct { - name string - schemaYAML string - }{ - {name: "exact unquoted schema", schemaYAML: `public`}, - {name: "glob pattern", schemaYAML: `tenant_*`}, - {name: "wildcard", schemaYAML: `"*"`}, - {name: "quoted exact identifier", schemaYAML: `'"MySchema"'`}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - stream := postgresStream(t, tt.schemaYAML) - - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - - errCh := make(chan error, 1) - go func() { errCh <- stream.Run(ctx) }() - - select { - case err := <-errCh: - // Stream stopped before timeout — must not be a schema error. - if err != nil { - assert.NotContains(t, err.Error(), "invalid schema", - "valid schema %q should not trigger schema validation error", tt.schemaYAML) - } - case <-ctx.Done(): - // Stream is still running after timeout — constructor succeeded, - // stream is attempting DB connection. This is the expected path. - _ = stream.StopWithin(2 * time.Second) - } - }) - } -} From 3411584a7c1cbe7a1423ed9ef2566431ec63e1fc Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 4 Aug 2026 13:26:13 +0100 Subject: [PATCH 18/70] postgres_cdc: clean up redundant comment --- .../pglogicalstream/schema_resolver.go | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index 3859ccface..4737f7c4cb 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -18,22 +18,6 @@ import ( "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream/sanitize" ) -// resolveSchemas expands a schema name or glob pattern into the set of -// quoted PostgreSQL identifiers that exist in the database. -// -// For unquoted patterns (e.g. "tenant_*") the pattern is matched -// case-insensitively against information_schema.schemata using LIKE, because -// PostgreSQL folds unquoted identifiers to lower-case at creation time. -// -// For quoted identifiers (e.g. `"MySchema"`) an exact case-sensitive lookup -// is performed. -// -// System schemas (pg_* and information_schema) are always excluded so that -// wildcard patterns like "*" do not attempt to replicate catalog tables. -// -// Returns an error if the query fails. Returns (nil, nil) if no schemas match. -// The caller is responsible for treating an empty result as an error. - // schemaPatternToLike converts a schema name or glob pattern into the LIKE // pattern used by resolveSchemas. Extracted for unit testing. // @@ -58,6 +42,17 @@ func schemaPatternToLike(pattern string) (string, error) { // warn the user instead of silently dropping schemas they expected to be // included, e.g. `"tenant_*"` matching a schema the configured role can't // see yet. +// +// For unquoted patterns (e.g. "tenant_*") the pattern is matched +// case-insensitively via LIKE, because PostgreSQL folds unquoted identifiers +// to lower-case at creation time. For quoted identifiers (e.g. `"MySchema"`) +// an exact case-sensitive lookup is performed. System schemas (pg_* and +// information_schema) are always excluded so that wildcard patterns like "*" +// do not attempt to replicate catalog tables. +// +// Returned schema names are quoted PostgreSQL identifiers. Returns an error +// if either query fails; returns a nil visibleSchemas slice (with a nil err) +// if no schemas match — callers should treat that as an error condition. func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (visibleSchemas, inaccessibleSchemas []string, err error) { likePattern, err := schemaPatternToLike(pattern) if err != nil { From bf289596b5a13ba1b210eb5dfdf8ed2261f2c10e Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 4 Aug 2026 13:34:44 +0100 Subject: [PATCH 19/70] postgres_cdc: normalie test structure --- internal/impl/postgresql/integration_test.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index e183f0f517..26217596a5 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1697,7 +1697,11 @@ postgres_cdc: stream, err := sb.Build() require.NoError(t, err) license.InjectTestService(stream.Resources()) - go func() { _ = stream.Run(t.Context()) }() + go func() { + if err := stream.Run(t.Context()); err != nil && !errors.Is(err, context.Canceled) { + t.Error(err) + } + }() t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) // Wait for all 3 snapshot rows. @@ -1814,7 +1818,11 @@ postgres_cdc: stream, err := sb.Build() require.NoError(t, err) license.InjectTestService(stream.Resources()) - go func() { _ = stream.Run(t.Context()) }() + go func() { + if err := stream.Run(t.Context()); err != nil && !errors.Is(err, context.Canceled) { + t.Error(err) + } + }() t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) // tenant_a should keep streaming even though tenant_b is missing the table. From 58d3be8886774b200bea0a7e97b6730cf2a0d97d Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 4 Aug 2026 13:46:46 +0100 Subject: [PATCH 20/70] postgres_cdc: t.Context() --- internal/impl/postgresql/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index 26217596a5..e68882e7bc 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1861,7 +1861,7 @@ tables: input, err := newPgStreamInput(conf, mgr) require.NoError(t, err) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() // Bypass the benthos AsyncReader's infinite connect-retry loop by calling From 6803eca6c66378366969e56d025ddb0b02df2951 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Thu, 6 Aug 2026 12:24:36 +0100 Subject: [PATCH 21/70] postgres_cdc: revert to existing behaviour If no schema is found and tables is empty then we want to ensure we create publication for all tables. --- internal/impl/postgresql/integration_test.go | 67 +++++++++++++++++++ .../pglogicalstream/logical_stream.go | 4 +- 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index e68882e7bc..9bbd8a8882 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1872,3 +1872,70 @@ tables: require.Error(t, err) assert.Contains(t, err.Error(), "no schemas found matching pattern") } + +// TestIntegrationForAllTablesIgnoresNonMatchingSchemaPattern guards the +// documented behaviour of FOR ALL TABLES mode: when `tables` is left empty, +// `schema` has no effect and must not block startup even if it matches no +// schema in the database. +func TestIntegrationForAllTablesIgnoresNonMatchingSchemaPattern(t *testing.T) { + integration.CheckSkip(t) + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + tmpl := fmt.Sprintf(` +postgres_cdc: + dsn: %s + slot_name: for_all_tables_ignores_schema_slot + schema: nonexistent_schema_zzz_* +`, databaseURL) + + // FOR ALL TABLES mode disables the initial snapshot, so replication only + // sees rows written after the slot/publication exist. Insert continuously + // (rather than once upfront) so a row lands after startup completes. + writer := asyncroutine.NewPeriodic(100*time.Millisecond, func() { + _, err := db.Exec("INSERT INTO flights (name, created_at) VALUES ('alice', now());") + require.NoError(t, err) + }) + writer.Start() + t.Cleanup(writer.Stop) + + var ( + mu sync.Mutex + collected []string + ) + + sb := service.NewStreamBuilder() + require.NoError(t, sb.SetLoggerYAML(`level: WARN`)) + require.NoError(t, sb.AddInputYAML(tmpl)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + mu.Lock() + defer mu.Unlock() + for _, msg := range batch { + table, _ := msg.MetaGet("table") + collected = append(collected, table) + } + return nil + })) + + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + go func() { + if err := stream.Run(t.Context()); err != nil && !errors.Is(err, context.Canceled) { + t.Error(err) + } + }() + t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) + + // Non-matching schema pattern must not block FOR ALL TABLES replication: + // the "flights" insert above should still arrive. + assert.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return len(collected) >= 1 + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for FOR ALL TABLES replication; a non-matching schema pattern should be ignored when tables is empty") + + mu.Lock() + defer mu.Unlock() + assert.Contains(t, collected, "flights") +} diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index b7cf77d571..f2d7bd17f8 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -102,10 +102,10 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { if err != nil { return nil, fmt.Errorf("resolving schema pattern %q: %w", config.DBSchemaPattern, err) } - if len(inaccessibleSchemas) > 0 { + if len(inaccessibleSchemas) > 0 && len(config.DBTables) > 0 { config.Logger.Warnf("schema pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaPattern, inaccessibleSchemas) } - if len(schemas) == 0 { + if len(schemas) == 0 && len(config.DBTables) > 0 { return nil, fmt.Errorf("no schemas found matching pattern %q", config.DBSchemaPattern) } config.Logger.Infof("Schema pattern %q resolved to %d schema(s): %v", config.DBSchemaPattern, len(schemas), schemas) From 045cde2e534650affb661c1a6fd1c7acbe04e7ce Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Thu, 6 Aug 2026 13:43:29 +0100 Subject: [PATCH 22/70] postgres_cdc: improve on missing table coverage --- internal/impl/postgresql/integration_test.go | 234 ++++++++++++++++++ .../pglogicalstream/logical_stream.go | 16 +- 2 files changed, 248 insertions(+), 2 deletions(-) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index 9bbd8a8882..35adb4cc28 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1939,3 +1939,237 @@ postgres_cdc: defer mu.Unlock() assert.Contains(t, collected, "flights") } + +func TestIntegrationSchemaAndTableMatchingTest(t *testing.T) { + integration.CheckSkip(t) + + t.Run("exact schema match with missing table fails", func(t *testing.T) { + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + _, err = db.Exec("CREATE TABLE IF NOT EXISTS orders (id SERIAL PRIMARY KEY, name TEXT);") + require.NoError(t, err) + + tmpl := fmt.Sprintf(` +dsn: %s +slot_name: exact_schema_missing_table_slot +schema: public +tables: + - orders + - ordres +`, databaseURL) + + conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) + require.NoError(t, err) + + mgr := service.MockResources() + license.InjectTestService(mgr) + + input, err := newPgStreamInput(conf, mgr) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + + // Bypass the benthos AsyncReader's infinite connect-retry loop, same as + // TestIntegrationNoSchemasMatchedReturnsError. + err = input.Connect(ctx) + require.Error(t, err, "typo'd table %q should fail startup loudly instead of silently streaming only %q", "ordres", "orders") + assert.Contains(t, err.Error(), "ordres") + }) + + t.Run("glob schema matching multiple schemas with one or more missing tables fails", func(t *testing.T) { + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + _, err = db.Exec(` + CREATE SCHEMA tenant_a; + CREATE SCHEMA tenant_b; + + CREATE TABLE tenant_a.orders (id SERIAL PRIMARY KEY, name TEXT); + CREATE TABLE tenant_b.orders (id SERIAL PRIMARY KEY, name TEXT); + `) + require.NoError(t, err) + + tmpl := fmt.Sprintf(` +dsn: %s +slot_name: glob_schema_total_miss_slot +schema: tenant_* +tables: + - orders + - ordres +`, databaseURL) + + conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) + require.NoError(t, err) + + mgr := service.MockResources() + license.InjectTestService(mgr) + + input, err := newPgStreamInput(conf, mgr) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + + err = input.Connect(ctx) + require.Error(t, err, "ordres exists in neither tenant_a nor tenant_b, so it should fail startup instead of silently streaming only tenant_a/b.orders") + assert.Contains(t, err.Error(), "ordres") + }) + + t.Run("glob schema matching multiple schemas with matching tables passes", func(t *testing.T) { + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + _, err = db.Exec(` + CREATE SCHEMA tenant_a; + CREATE SCHEMA tenant_b; + + CREATE TABLE tenant_a.orders (id SERIAL PRIMARY KEY, name TEXT); + CREATE TABLE tenant_b.ordres (id SERIAL PRIMARY KEY, name TEXT); + + INSERT INTO tenant_a.orders (name) VALUES ('alice'); + INSERT INTO tenant_b.ordres (name) VALUES ('bob'); + `) + require.NoError(t, err) + + type msgMeta struct { + dbSchema string + table string + } + + var ( + mu sync.Mutex + collected []msgMeta + ) + + tmpl := fmt.Sprintf(` +postgres_cdc: + dsn: %s + slot_name: glob_schema_partial_match_slot + stream_snapshot: true + schema: tenant_* + tables: + - orders + - ordres +`, databaseURL) + + sb := service.NewStreamBuilder() + require.NoError(t, sb.SetLoggerYAML(`level: WARN`)) + require.NoError(t, sb.AddInputYAML(tmpl)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + mu.Lock() + defer mu.Unlock() + for _, msg := range batch { + m := msgMeta{} + m.dbSchema, _ = msg.MetaGet("database_schema") + m.table, _ = msg.MetaGet("table") + collected = append(collected, m) + } + return nil + })) + + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + go func() { + if err := stream.Run(t.Context()); err != nil && !errors.Is(err, context.Canceled) { + t.Error(err) + } + }() + t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) + + assert.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return len(collected) >= 2 + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for both tenant_a.orders and tenant_b.ordres snapshot rows") + + mu.Lock() + defer mu.Unlock() + require.Len(t, collected, 2) + assert.Contains(t, collected, msgMeta{dbSchema: "tenant_a", table: "orders"}) + assert.Contains(t, collected, msgMeta{dbSchema: "tenant_b", table: "ordres"}) + }) + + t.Run("glob schema matching multiple schemas with all tables present in all schemas passes", func(t *testing.T) { + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + _, err = db.Exec(` + CREATE SCHEMA tenant_a; + CREATE SCHEMA tenant_b; + + CREATE TABLE tenant_a.orders (id SERIAL PRIMARY KEY, name TEXT); + CREATE TABLE tenant_a.ordres (id SERIAL PRIMARY KEY, name TEXT); + CREATE TABLE tenant_b.orders (id SERIAL PRIMARY KEY, name TEXT); + CREATE TABLE tenant_b.ordres (id SERIAL PRIMARY KEY, name TEXT); + + INSERT INTO tenant_a.orders (name) VALUES ('alice'); + INSERT INTO tenant_a.ordres (name) VALUES ('bob'); + INSERT INTO tenant_b.orders (name) VALUES ('carol'); + INSERT INTO tenant_b.ordres (name) VALUES ('dave'); + `) + require.NoError(t, err) + + type msgMeta struct { + dbSchema string + table string + } + + var ( + mu sync.Mutex + collected []msgMeta + ) + + tmpl := fmt.Sprintf(` +postgres_cdc: + dsn: %s + slot_name: glob_schema_full_match_slot + stream_snapshot: true + schema: tenant_* + tables: + - orders + - ordres +`, databaseURL) + + sb := service.NewStreamBuilder() + require.NoError(t, sb.SetLoggerYAML(`level: WARN`)) + require.NoError(t, sb.AddInputYAML(tmpl)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + mu.Lock() + defer mu.Unlock() + for _, msg := range batch { + m := msgMeta{} + m.dbSchema, _ = msg.MetaGet("database_schema") + m.table, _ = msg.MetaGet("table") + collected = append(collected, m) + } + return nil + })) + + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + go func() { + if err := stream.Run(t.Context()); err != nil && !errors.Is(err, context.Canceled) { + t.Error(err) + } + }() + t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) + + assert.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return len(collected) >= 4 + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for all four tenant_{a,b}.{orders,ordres} snapshot rows") + + mu.Lock() + defer mu.Unlock() + require.Len(t, collected, 4) + assert.Contains(t, collected, msgMeta{dbSchema: "tenant_a", table: "orders"}) + assert.Contains(t, collected, msgMeta{dbSchema: "tenant_a", table: "ordres"}) + assert.Contains(t, collected, msgMeta{dbSchema: "tenant_b", table: "orders"}) + assert.Contains(t, collected, msgMeta{dbSchema: "tenant_b", table: "ordres"}) + }) +} diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index f2d7bd17f8..5747803116 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -120,6 +120,7 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { } tables := make([]TableFQN, 0, len(schemas)*len(normalizedTables)) + foundTables := make(map[string]bool, len(normalizedTables)) for _, schema := range schemas { existingTables, err := resolveExistingTables(ctx, dbConn, schema) if err != nil { @@ -131,10 +132,21 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { continue } tables = append(tables, TableFQN{Schema: schema, Table: table}) + foundTables[table] = true } } - if len(tables) == 0 && len(normalizedTables) > 0 { - return nil, fmt.Errorf("none of the configured tables %v were found in any schema matching pattern %q", config.DBTables, config.DBSchemaPattern) + // A table must exist in at least one matched schema. Missing from some + // (but not all) matched schemas is tolerated above as a multi-tenant gap; + // missing from every matched schema is indistinguishable from a typo and + // must fail loudly rather than silently drop the table. + var missingTables []string + for i, table := range normalizedTables { + if !foundTables[table] { + missingTables = append(missingTables, config.DBTables[i]) + } + } + if len(missingTables) > 0 { + return nil, fmt.Errorf("table(s) %v not found in any schema matching pattern %q", missingTables, config.DBSchemaPattern) } batchSize := 1000 if config.BatchSize > 0 { From c9e488b42dc50b324627a702c3a63b64942c6242 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Thu, 6 Aug 2026 14:06:05 +0100 Subject: [PATCH 23/70] update docs --- docs/modules/components/pages/inputs/postgres_cdc.adoc | 4 +++- internal/impl/postgresql/input_pg_stream.go | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index 849adbd8ae..e7f995f242 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -178,6 +178,8 @@ Double-quoted identifiers are treated as exact names and do not support wildcard Schema pattern matching runs once at pipeline startup. Schemas created after the pipeline starts will not be picked up until the pipeline is restarted. +If `tables` is non-empty and this pattern matches no schema in the database, startup fails with an error. This field has no effect when `tables` is left empty - see `tables` below. + *Type*: `string` @@ -198,7 +200,7 @@ schema: '*' A list of table names to include in the logical replication. Each table should be specified as a separate item. -When `schema` is a glob pattern, this list is resolved against each matched schema independently: a table missing from a given schema is skipped (with a warning logged) rather than failing replication for every matched schema. +When `schema` is a glob pattern, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. If left empty, the underlying PostgreSQL publication is created `FOR ALL TABLES`, which replicates every table in every schema of the database, ignoring `schema`. This also disables `stream_snapshot`, since the initial snapshot is only planned for tables listed here. diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 5cecc17457..5598c0b24f 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -122,13 +122,15 @@ When a pattern is used, all schemas whose names match the pattern are replicated Double-quoted identifiers are treated as exact names and do not support wildcards. -Schema pattern matching runs once at pipeline startup. Schemas created after the pipeline starts will not be picked up until the pipeline is restarted.`). +Schema pattern matching runs once at pipeline startup. Schemas created after the pipeline starts will not be picked up until the pipeline is restarted. + +If ` + "`" + fieldTables + "`" + ` is non-empty and this pattern matches no schema in the database, startup fails with an error. This field has no effect when ` + "`" + fieldTables + "`" + ` is left empty - see ` + "`" + fieldTables + "`" + ` below.`). Examples("public", `"MyCaseSensitiveSchemaNeedingQuotes"`, "tenant_*", "*"), ). Field(service.NewStringListField(fieldTables). Description(`A list of table names to include in the logical replication. Each table should be specified as a separate item. -When ` + "`schema`" + ` is a glob pattern, this list is resolved against each matched schema independently: a table missing from a given schema is skipped (with a warning logged) rather than failing replication for every matched schema. +When ` + "`schema`" + ` is a glob pattern, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. If left empty, the underlying PostgreSQL publication is created ` + "`FOR ALL TABLES`" + `, which replicates every table in every schema of the database, ignoring ` + "`" + fieldSchema + "`" + `. This also disables ` + "`" + fieldStreamSnapshot + "`" + `, since the initial snapshot is only planned for tables listed here.`). Example([]string{"my_table_1", `"MyCaseSensitiveTableNeedingQuotes"`})). From 0cd24bdc7ec2a51e6b944a3dd8bcedda8de3f3e0 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Tue, 11 Aug 2026 01:23:56 +0300 Subject: [PATCH 24/70] postgres_cdc: add schema_pattern field for multi-schema replication Introduces schema_pattern as a new optional field for glob-based multi-schema CDC, instead of overloading schema with pattern semantics. This keeps schema backwards compatible: it reverts to exact-name-only behavior (now defaulting to "public"), while schema_pattern opts into the glob resolution path. Setting both is a config error. Co-Authored-By: Claude Sonnet 5 --- .../components/pages/inputs/postgres_cdc.adoc | 38 +++++-- internal/impl/postgresql/input_pg_stream.go | 46 +++++--- .../impl/postgresql/input_pg_stream_test.go | 77 ++++++++++--- internal/impl/postgresql/integration_test.go | 14 +-- .../impl/postgresql/pglogicalstream/config.go | 7 +- .../pglogicalstream/logical_stream.go | 101 ++++++++++-------- 6 files changed, 197 insertions(+), 86 deletions(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index e7f995f242..86702205a5 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -43,7 +43,8 @@ input: include_transaction_markers: false stream_snapshot: false snapshot_batch_size: 1000 - schema: public # No default (required) + schema: public + schema_pattern: "" tables: [] # No default (required) checkpoint_limit: 1024 temporary_slot: false @@ -73,7 +74,8 @@ input: include_transaction_markers: false stream_snapshot: false snapshot_batch_size: 1000 - schema: public # No default (required) + schema: public + schema_pattern: "" tables: [] # No default (required) checkpoint_limit: 1024 temporary_slot: false @@ -170,7 +172,24 @@ snapshot_batch_size: 10000 === `schema` -The PostgreSQL schema to replicate data from. Accepts an exact schema name or a glob pattern using `*` as a wildcard to match multiple schemas. +The PostgreSQL schema from which to replicate data. + + +*Type*: `string` + +*Default*: `"public"` + +```yml +# Examples + +schema: public + +schema: '"MyCaseSensitiveSchemaNeedingQuotes"' +``` + +=== `schema_pattern` + +The PostgreSQL schema pattern to replicate data from. Accepts an exact schema name or a glob pattern using `*` as a wildcard to match multiple schemas. When a pattern is used, all schemas whose names match the pattern are replicated using a single replication slot and publication. This is useful for multi-tenant databases where each tenant has its own schema (e.g. `tenant_*` matches `tenant_foo`, `tenant_bar`, etc.). @@ -180,27 +199,28 @@ Schema pattern matching runs once at pipeline startup. Schemas created after the If `tables` is non-empty and this pattern matches no schema in the database, startup fails with an error. This field has no effect when `tables` is left empty - see `tables` below. +This field is mutually exclusive with `schema`; when set, it takes over schema resolution entirely and `schema` must be left at its default. + *Type*: `string` +*Default*: `""` ```yml # Examples -schema: public - -schema: '"MyCaseSensitiveSchemaNeedingQuotes"' +schema_pattern: tenant_* -schema: tenant_* +schema_pattern: '*' -schema: '*' +schema_pattern: '"MyCaseSensitiveSchemaNeedingQuotes"' ``` === `tables` A list of table names to include in the logical replication. Each table should be specified as a separate item. -When `schema` is a glob pattern, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. +When `schema_pattern` is set, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. If left empty, the underlying PostgreSQL publication is created `FOR ALL TABLES`, which replicates every table in every schema of the database, ignoring `schema`. This also disables `stream_snapshot`, since the initial snapshot is only planned for tables listed here. diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 5598c0b24f..8dbb87f5d0 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -38,6 +38,7 @@ const ( fieldSnapshotMemSafetyFactor = "snapshot_memory_safety_factor" fieldSnapshotBatchSize = "snapshot_batch_size" fieldSchema = "schema" + fieldSchemaPattern = "schema_pattern" fieldTables = "tables" fieldCheckpointLimit = "checkpoint_limit" fieldTemporarySlot = "temporary_slot" @@ -116,7 +117,13 @@ This input adds the following metadata fields to each message: Example(10000). Default(1000)). Field(service.NewStringField(fieldSchema). - Description(`The PostgreSQL schema to replicate data from. Accepts an exact schema name or a glob pattern using `+"`*`"+` as a wildcard to match multiple schemas. + Description("The PostgreSQL schema from which to replicate data."). + Examples("public", `"MyCaseSensitiveSchemaNeedingQuotes"`). + Optional(). + Default("public"), + ). + Field(service.NewStringField(fieldSchemaPattern). + Description(`The PostgreSQL schema pattern to replicate data from. Accepts an exact schema name or a glob pattern using `+"`*`"+` as a wildcard to match multiple schemas. When a pattern is used, all schemas whose names match the pattern are replicated using a single replication slot and publication. This is useful for multi-tenant databases where each tenant has its own schema (e.g. `+"`tenant_*`"+` matches `+"`tenant_foo`"+`, `+"`tenant_bar`"+`, etc.). @@ -124,13 +131,17 @@ Double-quoted identifiers are treated as exact names and do not support wildcard Schema pattern matching runs once at pipeline startup. Schemas created after the pipeline starts will not be picked up until the pipeline is restarted. -If ` + "`" + fieldTables + "`" + ` is non-empty and this pattern matches no schema in the database, startup fails with an error. This field has no effect when ` + "`" + fieldTables + "`" + ` is left empty - see ` + "`" + fieldTables + "`" + ` below.`). - Examples("public", `"MyCaseSensitiveSchemaNeedingQuotes"`, "tenant_*", "*"), +If `+"`"+fieldTables+"`"+` is non-empty and this pattern matches no schema in the database, startup fails with an error. This field has no effect when `+"`"+fieldTables+"`"+` is left empty - see `+"`"+fieldTables+"`"+` below. + +This field is mutually exclusive with `+"`"+fieldSchema+"`"+`; when set, it takes over schema resolution entirely and `+"`"+fieldSchema+"`"+` must be left at its default.`). + Examples("tenant_*", "*", `"MyCaseSensitiveSchemaNeedingQuotes"`). + Optional(). + Default(""), ). Field(service.NewStringListField(fieldTables). Description(`A list of table names to include in the logical replication. Each table should be specified as a separate item. -When ` + "`schema`" + ` is a glob pattern, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. +When ` + "`schema_pattern`" + ` is set, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. If left empty, the underlying PostgreSQL publication is created ` + "`FOR ALL TABLES`" + `, which replicates every table in every schema of the database, ignoring ` + "`" + fieldSchema + "`" + `. This also disables ` + "`" + fieldStreamSnapshot + "`" + `, since the initial snapshot is only planned for tables listed here.`). Example([]string{"my_table_1", `"MyCaseSensitiveTableNeedingQuotes"`})). @@ -231,6 +242,7 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser dbSlotName string temporarySlot bool schema string + schemaPattern string tables []string streamSnapshot bool includeTxnMarkers bool @@ -275,14 +287,23 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser if schema, err = conf.FieldString(fieldSchema); err != nil { return nil, err } - if err = validateSchemaPattern(schema); err != nil { - return nil, fmt.Errorf("invalid schema: %w", err) + + if schemaPattern, err = conf.FieldString(fieldSchemaPattern); err != nil { + return nil, err } - // Normalize unquoted patterns to lower-case: PostgreSQL folds unquoted - // identifiers at creation time, so TENANT_* and tenant_* resolve identically. - // Normalizing early avoids silent case-folding surprises in resolveSchemas. - if !strings.HasPrefix(schema, `"`) { - schema = strings.ToLower(schema) + if schemaPattern != "" { + if schema != "public" { + return nil, errors.New("schema and schema_pattern are mutually exclusive") + } + if err = validateSchemaPattern(schemaPattern); err != nil { + return nil, fmt.Errorf("invalid schema_pattern: %w", err) + } + // Normalize unquoted patterns to lower-case: PostgreSQL folds unquoted + // identifiers at creation time, so TENANT_* and tenant_* resolve identically. + // Normalizing early avoids silent case-folding surprises in resolveSchemas. + if !strings.HasPrefix(schemaPattern, `"`) { + schemaPattern = strings.ToLower(schemaPattern) + } } if tables, err = conf.FieldStringList(fieldTables); err != nil { @@ -363,7 +384,8 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser DBConfig: pgConnConfig, TLSConfig: pgConnConfig.TLSConfig, DBRawDSN: dsn, - DBSchemaPattern: schema, + DBSchema: schema, + DBSchemaPattern: schemaPattern, DBTables: tables, RefreshAuthToken: iamAuthTokenBuilder, diff --git a/internal/impl/postgresql/input_pg_stream_test.go b/internal/impl/postgresql/input_pg_stream_test.go index 19d9723bc2..251a963aa8 100644 --- a/internal/impl/postgresql/input_pg_stream_test.go +++ b/internal/impl/postgresql/input_pg_stream_test.go @@ -20,16 +20,40 @@ import ( "github.com/redpanda-data/connect/v4/internal/license" ) -// TestSchemaPatternValidation verifies that the schema field is validated -// during config parsing, before any network I/O is attempted. Success is -// asserted via newPgStreamInput returning no error - the constructor doesn't -// dial the database, so a valid pattern implies validation passed. +func parsePgStreamInput(t *testing.T, yaml string) (service.BatchInput, error) { + t.Helper() + conf, err := newPostgresCDCConfig().ParseYAML(yaml, nil) + require.NoError(t, err) + + mgr := service.MockResources() + license.InjectTestService(mgr) + + return newPgStreamInput(conf, mgr) +} + +// TestSchemaDefault verifies that the schema field defaults to "public" when +// left unset, matching pre-multi-schema behaviour. +func TestSchemaDefault(t *testing.T) { + yaml := ` +dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable +slot_name: test_slot +tables: + - events +` + _, err := parsePgStreamInput(t, yaml) + require.NoError(t, err) +} + +// TestSchemaPatternValidation verifies that the schema_pattern field is +// validated during config parsing, before any network I/O is attempted. +// Success is asserted via newPgStreamInput returning no error - the +// constructor doesn't dial the database, so a valid pattern implies +// validation passed. func TestSchemaPatternValidation(t *testing.T) { tests := []struct { pattern string errContains string }{ - {"public", ""}, {"tenant_*", ""}, {"*", ""}, {`"MySchema"`, ""}, @@ -39,7 +63,6 @@ func TestSchemaPatternValidation(t *testing.T) { {"1abc", "must start with a letter"}, {`"unclosed`, "invalid quoted schema identifier"}, {"schema-name", "invalid character"}, - {"", "schema cannot be empty"}, {`"quoted*"`, "wildcard"}, } for _, tt := range tests { @@ -48,19 +71,13 @@ func TestSchemaPatternValidation(t *testing.T) { // quotes, e.g. `"MySchema"`) reaches validateSchemaPattern verbatim. yaml := fmt.Sprintf(` dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable -schema: '%s' +schema_pattern: '%s' slot_name: test_slot tables: - events `, tt.pattern) - conf, err := newPostgresCDCConfig().ParseYAML(yaml, nil) - require.NoError(t, err) - - mgr := service.MockResources() - license.InjectTestService(mgr) - - _, err = newPgStreamInput(conf, mgr) + _, err := parsePgStreamInput(t, yaml) if tt.errContains != "" { require.Error(t, err) assert.Contains(t, err.Error(), tt.errContains) @@ -70,3 +87,35 @@ tables: }) } } + +// TestSchemaAndSchemaPatternMutuallyExclusive verifies that setting both +// schema (to a non-default value) and schema_pattern is rejected at config +// construction time. +func TestSchemaAndSchemaPatternMutuallyExclusive(t *testing.T) { + yaml := ` +dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable +schema: tenant_foo +schema_pattern: 'tenant_*' +slot_name: test_slot +tables: + - events +` + _, err := parsePgStreamInput(t, yaml) + require.Error(t, err) + assert.Contains(t, err.Error(), "schema and schema_pattern are mutually exclusive") +} + +// TestSchemaPatternWithDefaultSchemaSucceeds verifies that setting +// schema_pattern while leaving schema untouched (at its "public" default) is +// allowed. +func TestSchemaPatternWithDefaultSchemaSucceeds(t *testing.T) { + yaml := ` +dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable +schema_pattern: 'tenant_*' +slot_name: test_slot +tables: + - events +` + _, err := parsePgStreamInput(t, yaml) + require.NoError(t, err) +} diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index 35adb4cc28..acb5458439 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1672,7 +1672,7 @@ postgres_cdc: dsn: %s slot_name: multi_schema_test_slot stream_snapshot: true - schema: tenant_* + schema_pattern: tenant_* tables: - events `, databaseURL) @@ -1795,7 +1795,7 @@ postgres_cdc: dsn: %s slot_name: missing_table_degrade_slot stream_snapshot: true - schema: tenant_* + schema_pattern: tenant_* tables: - events `, databaseURL) @@ -1847,7 +1847,7 @@ func TestIntegrationNoSchemasMatchedReturnsError(t *testing.T) { tmpl := fmt.Sprintf(` dsn: %s slot_name: no_schema_match_slot -schema: nonexistent_schema_zzz_* +schema_pattern: nonexistent_schema_zzz_* tables: - events `, databaseURL) @@ -1886,7 +1886,7 @@ func TestIntegrationForAllTablesIgnoresNonMatchingSchemaPattern(t *testing.T) { postgres_cdc: dsn: %s slot_name: for_all_tables_ignores_schema_slot - schema: nonexistent_schema_zzz_* + schema_pattern: nonexistent_schema_zzz_* `, databaseURL) // FOR ALL TABLES mode disables the initial snapshot, so replication only @@ -1994,7 +1994,7 @@ tables: tmpl := fmt.Sprintf(` dsn: %s slot_name: glob_schema_total_miss_slot -schema: tenant_* +schema_pattern: tenant_* tables: - orders - ordres @@ -2048,7 +2048,7 @@ postgres_cdc: dsn: %s slot_name: glob_schema_partial_match_slot stream_snapshot: true - schema: tenant_* + schema_pattern: tenant_* tables: - orders - ordres @@ -2127,7 +2127,7 @@ postgres_cdc: dsn: %s slot_name: glob_schema_full_match_slot stream_snapshot: true - schema: tenant_* + schema_pattern: tenant_* tables: - orders - ordres diff --git a/internal/impl/postgresql/pglogicalstream/config.go b/internal/impl/postgresql/pglogicalstream/config.go index 8fed67862e..f6cb5ce99c 100644 --- a/internal/impl/postgresql/pglogicalstream/config.go +++ b/internal/impl/postgresql/pglogicalstream/config.go @@ -24,8 +24,11 @@ type Config struct { DBConfig *pgconn.Config DBRawDSN string TLSConfig *tls.Config - // DBSchemaPattern is the schema to replicate from. Accepts an exact schema - // name or a glob pattern using '*' as a wildcard (e.g. "tenant_*", "*"). + DBSchema string + // DBSchemaPattern is the glob pattern used to replicate from multiple + // schemas at once, using '*' as a wildcard (e.g. "tenant_*", "*"). When + // non-empty, it takes precedence over DBSchema and schemas are resolved + // dynamically at stream creation time. DBSchemaPattern string DBTables []string // Refreshes short lived IAM auth token that is treated as a password diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 5747803116..669acbc66a 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -98,55 +98,72 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { return nil, err } - schemas, inaccessibleSchemas, err := resolveSchemas(ctx, dbConn, config.DBSchemaPattern) - if err != nil { - return nil, fmt.Errorf("resolving schema pattern %q: %w", config.DBSchemaPattern, err) - } - if len(inaccessibleSchemas) > 0 && len(config.DBTables) > 0 { - config.Logger.Warnf("schema pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaPattern, inaccessibleSchemas) - } - if len(schemas) == 0 && len(config.DBTables) > 0 { - return nil, fmt.Errorf("no schemas found matching pattern %q", config.DBSchemaPattern) - } - config.Logger.Infof("Schema pattern %q resolved to %d schema(s): %v", config.DBSchemaPattern, len(schemas), schemas) - - normalizedTables := make([]string, 0, len(config.DBTables)) - for _, table := range config.DBTables { - normalized, err := sanitize.NormalizePostgresIdentifier(table) + var tables []TableFQN + if config.DBSchemaPattern != "" { + schemas, inaccessibleSchemas, err := resolveSchemas(ctx, dbConn, config.DBSchemaPattern) if err != nil { - return nil, fmt.Errorf("invalid table name %q: %w", table, err) + return nil, fmt.Errorf("resolving schema pattern %q: %w", config.DBSchemaPattern, err) } - normalizedTables = append(normalizedTables, normalized) - } + if len(inaccessibleSchemas) > 0 && len(config.DBTables) > 0 { + config.Logger.Warnf("schema pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaPattern, inaccessibleSchemas) + } + if len(schemas) == 0 && len(config.DBTables) > 0 { + return nil, fmt.Errorf("no schemas found matching pattern %q", config.DBSchemaPattern) + } + config.Logger.Infof("Schema pattern %q resolved to %d schema(s): %v", config.DBSchemaPattern, len(schemas), schemas) - tables := make([]TableFQN, 0, len(schemas)*len(normalizedTables)) - foundTables := make(map[string]bool, len(normalizedTables)) - for _, schema := range schemas { - existingTables, err := resolveExistingTables(ctx, dbConn, schema) - if err != nil { - return nil, fmt.Errorf("resolving tables in schema %q: %w", schema, err) + normalizedTables := make([]string, 0, len(config.DBTables)) + for _, table := range config.DBTables { + normalized, err := sanitize.NormalizePostgresIdentifier(table) + if err != nil { + return nil, fmt.Errorf("invalid table name %q: %w", table, err) + } + normalizedTables = append(normalizedTables, normalized) } - for _, table := range normalizedTables { - if _, ok := existingTables[table]; !ok { - config.Logger.Warnf("table %s.%s not found, skipping (schema %s matched pattern %q but does not contain this table)", schema, table, schema, config.DBSchemaPattern) - continue + + tables = make([]TableFQN, 0, len(schemas)*len(normalizedTables)) + foundTables := make(map[string]bool, len(normalizedTables)) + for _, schema := range schemas { + existingTables, err := resolveExistingTables(ctx, dbConn, schema) + if err != nil { + return nil, fmt.Errorf("resolving tables in schema %q: %w", schema, err) + } + for _, table := range normalizedTables { + if _, ok := existingTables[table]; !ok { + config.Logger.Warnf("table %s.%s not found, skipping (schema %s matched pattern %q but does not contain this table)", schema, table, schema, config.DBSchemaPattern) + continue + } + tables = append(tables, TableFQN{Schema: schema, Table: table}) + foundTables[table] = true } - tables = append(tables, TableFQN{Schema: schema, Table: table}) - foundTables[table] = true } - } - // A table must exist in at least one matched schema. Missing from some - // (but not all) matched schemas is tolerated above as a multi-tenant gap; - // missing from every matched schema is indistinguishable from a typo and - // must fail loudly rather than silently drop the table. - var missingTables []string - for i, table := range normalizedTables { - if !foundTables[table] { - missingTables = append(missingTables, config.DBTables[i]) + // A table must exist in at least one matched schema. Missing from some + // (but not all) matched schemas is tolerated above as a multi-tenant gap; + // missing from every matched schema is indistinguishable from a typo and + // must fail loudly rather than silently drop the table. + var missingTables []string + for i, table := range normalizedTables { + if !foundTables[table] { + missingTables = append(missingTables, config.DBTables[i]) + } + } + if len(missingTables) > 0 { + return nil, fmt.Errorf("table(s) %v not found in any schema matching pattern %q", missingTables, config.DBSchemaPattern) + } + } else { + schema, err := sanitize.NormalizePostgresIdentifier(config.DBSchema) + if err != nil { + return nil, fmt.Errorf("invalid schema name %q: %w", config.DBSchema, err) + } + + tables = []TableFQN{} + for _, table := range config.DBTables { + normalized, err := sanitize.NormalizePostgresIdentifier(table) + if err != nil { + return nil, fmt.Errorf("invalid table name %q: %w", table, err) + } + tables = append(tables, TableFQN{Schema: schema, Table: normalized}) } - } - if len(missingTables) > 0 { - return nil, fmt.Errorf("table(s) %v not found in any schema matching pattern %q", missingTables, config.DBSchemaPattern) } batchSize := 1000 if config.BatchSize > 0 { From 33e4049f6c4412252f843138d7bf9c30a8550bd8 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Tue, 11 Aug 2026 10:40:06 +0300 Subject: [PATCH 25/70] postgres_cdc: fix unicode schema_pattern rejection, sync changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit validateSchemaPattern only accepted [a-zA-Z0-9_*], stricter than sanitize.NormalizePostgresIdentifier (unicode.IsLetter/IsDigit) which governs the exact-name schema path. Since schema_pattern is now the sole caller of validateSchemaPattern, an unquoted pattern with non-ASCII letters (e.g. münchen) was rejected at startup even though the equivalent schema value has always been accepted. Widened the character classes to match, using utf8.DecodeRuneInString for the first-character check instead of a raw byte cast. Also corrected the Unreleased changelog entry, which still described the old design where schema itself accepted glob patterns. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 2 +- internal/impl/postgresql/input_pg_stream.go | 8 ++++--- .../impl/postgresql/input_pg_stream_test.go | 23 +++++++++++++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51e5da7716..006280950c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,7 +69,7 @@ All notable changes to this project will be documented in this file. ### Added -- postgres_cdc: Postgres CDC now accepts a glob pattern for the `schema` field (e.g. `tenant_*`), replicating all matching schemas through a single replication slot. Useful for multi-tenant databases where each tenant has its own schema. ([@ness-david-dedu](https://github.com/ness-david-dedu), [#4589](https://github.com/redpanda-data/connect/pull/4589)) +- postgres_cdc: Added a new `schema_pattern` field accepting a glob pattern (e.g. `tenant_*`), replicating all matching schemas through a single replication slot. Useful for multi-tenant databases where each tenant has its own schema. The existing `schema` field is unaffected and continues to take a single exact schema name (defaulting to `public`); `schema` and `schema_pattern` are mutually exclusive. ([@ness-david-dedu](https://github.com/ness-david-dedu), [#4589](https://github.com/redpanda-data/connect/pull/4589)) - aws_dynamodb_cdc: DynamoDB CDC now supports an optional checkpoint_namespace field, allowing multiple independent pipelines to share a single checkpoint table without overwriting each other's checkpoints. ([@squiidz](https://github.com/squiidz), [#4602](https://github.com/redpanda-data/connect/pull/4602)) ### Fixed diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 8dbb87f5d0..d6d7235fd0 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -18,6 +18,8 @@ import ( "strings" "sync" "time" + "unicode" + "unicode/utf8" "github.com/Jeffail/checkpoint" "github.com/Jeffail/shutdown" @@ -444,13 +446,13 @@ func validateSchemaPattern(s string) error { return nil } for i, ch := range s { - if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '_' || ch == '*' { + if unicode.IsLetter(ch) || unicode.IsDigit(ch) || ch == '_' || ch == '*' { continue } return fmt.Errorf("invalid character %q at position %d in schema pattern %q", ch, i, s) } - first := rune(s[0]) - if first != '_' && first != '*' && (first < 'a' || first > 'z') && (first < 'A' || first > 'Z') { + first, _ := utf8.DecodeRuneInString(s) + if first != '_' && first != '*' && !unicode.IsLetter(first) { return fmt.Errorf("schema pattern %q must start with a letter, underscore, or '*'", s) } return nil diff --git a/internal/impl/postgresql/input_pg_stream_test.go b/internal/impl/postgresql/input_pg_stream_test.go index 251a963aa8..2b30df9fa2 100644 --- a/internal/impl/postgresql/input_pg_stream_test.go +++ b/internal/impl/postgresql/input_pg_stream_test.go @@ -57,6 +57,11 @@ func TestSchemaPatternValidation(t *testing.T) { {"tenant_*", ""}, {"*", ""}, {`"MySchema"`, ""}, + // Regression test: validateSchemaPattern must accept the same unicode + // letters/digits that sanitize.NormalizePostgresIdentifier accepts for + // unquoted identifiers (e.g. "münchen"), not just ASCII. + {"münchen", ""}, + {"tenant_ü*", ""}, // Regression test: len("") == 2 used to pass the old `len(s) < 2` guard. // Fixed to `len(s) < 3`. {`""`, "invalid quoted schema identifier"}, @@ -119,3 +124,21 @@ tables: _, err := parsePgStreamInput(t, yaml) require.NoError(t, err) } + +// TestSchemaAcceptsUnicodeIdentifier verifies that the schema field (single +// exact-name path) still accepts unquoted unicode identifiers like +// "münchen", matching sanitize.NormalizePostgresIdentifier which is the sole +// validator on this path (see NewPgStream). schema no longer runs through +// validateSchemaPattern, so this guards against that ASCII-only validator +// regressing this path again in the future. +func TestSchemaAcceptsUnicodeIdentifier(t *testing.T) { + yaml := ` +dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable +schema: münchen +slot_name: test_slot +tables: + - events +` + _, err := parsePgStreamInput(t, yaml) + require.NoError(t, err) +} From b428bd7af2b4cde5023cf4c3ef473e48903865fe Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Tue, 11 Aug 2026 15:52:17 +0300 Subject: [PATCH 26/70] postgres_cdc: fix stale schema_pattern refs in manual test harness internal/impl/postgresql/tests/current/ is a docker-compose-backed Taskfile harness for manually exercising multi-schema CDC against a real Postgres (tenant_a/tenant_b). It predates the schema/schema_pattern split and still used schema: tenant_* directly, which the split turns into a literal (and invalid) exact schema name instead of a glob. - test_config.yaml: schema -> schema_pattern for the tenant_* glob. - Taskfile.yaml test:invalid-schema: schema="" is now a no-op (unset schema_pattern falls back to schema's "public" default), so it no longer exercises a failure path. Repointed at schema_pattern=1abc, which still fails fast in newPgStreamInput before any DB connection, matching the task's original intent. Verified by bringing the compose stack up and running the smoke test; config loads and reaches runtime init (blocked only by the license check in this sandboxed environment, which is unrelated). Co-Authored-By: Claude Sonnet 5 --- internal/impl/postgresql/tests/current/Taskfile.yaml | 11 ++++++++--- .../impl/postgresql/tests/current/test_config.yaml | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/internal/impl/postgresql/tests/current/Taskfile.yaml b/internal/impl/postgresql/tests/current/Taskfile.yaml index 75551c4d34..d81e5413ce 100644 --- a/internal/impl/postgresql/tests/current/Taskfile.yaml +++ b/internal/impl/postgresql/tests/current/Taskfile.yaml @@ -72,17 +72,22 @@ tasks: # ── Schema validation smoke test ────────────────────────────────────────────── test:invalid-schema: - desc: Confirm that an empty quoted schema identifier is rejected at startup + desc: Confirm that a malformed schema_pattern is rejected at startup, before any DB connection is attempted env: PG_DSN: '{{.PG_DSN}}' cmds: - | set +e go run ../../../../../cmd/redpanda-connect/main.go run \ - --set 'input.postgres_cdc.schema=""' \ + --set 'input.postgres_cdc.schema_pattern=1abc' \ ./test_config.yaml 2>&1 | head -5 echo "exit $?" - # Expects: "invalid schema" error printed, process exits non-zero. + # Expects: "invalid schema_pattern" error printed, process exits non-zero. + # schema_pattern is validated eagerly in newPgStreamInput + # (validateSchemaPattern), unlike schema which is only checked later, + # against the live DB connection, inside NewPgStream. + # Note: an empty schema_pattern is NOT an error - it means "unset", and + # the connector falls back to the schema field (default "public"). # ── Quick sanity ────────────────────────────────────────────────────────────── diff --git a/internal/impl/postgresql/tests/current/test_config.yaml b/internal/impl/postgresql/tests/current/test_config.yaml index 9e40b5a1c1..838321a392 100644 --- a/internal/impl/postgresql/tests/current/test_config.yaml +++ b/internal/impl/postgresql/tests/current/test_config.yaml @@ -4,7 +4,7 @@ input: slot_name: multi_schema_test_slot stream_snapshot: true # Glob pattern: replicates both tenant_a and tenant_b via one slot. - schema: tenant_* + schema_pattern: tenant_* tables: - events From b46a4e5af60767b223be62e99c9679d25f85afe8 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Tue, 11 Aug 2026 21:59:54 +0300 Subject: [PATCH 27/70] postgres_cdc: add excluded_schema --- CHANGELOG.md | 1 + .../components/pages/inputs/postgres_cdc.adoc | 23 +++ internal/impl/postgresql/input_pg_stream.go | 33 ++++ .../impl/postgresql/input_pg_stream_test.go | 73 +++++++++ internal/impl/postgresql/integration_test.go | 152 ++++++++++++++++++ .../impl/postgresql/pglogicalstream/config.go | 6 +- .../pglogicalstream/logical_stream.go | 30 ++++ .../pglogicalstream/schema_resolver.go | 51 ++++++ .../pglogicalstream/schema_resolver_test.go | 40 +++++ .../postgresql/tests/current/Taskfile.yaml | 19 ++- .../impl/postgresql/tests/current/setup.sql | 18 ++- .../postgresql/tests/current/test_config.yaml | 4 + .../test_config_exclude_no_pattern.yaml | 22 +++ 13 files changed, 469 insertions(+), 3 deletions(-) create mode 100644 internal/impl/postgresql/tests/current/test_config_exclude_no_pattern.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 006280950c..857a23158d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,7 @@ All notable changes to this project will be documented in this file. ### Added - postgres_cdc: Added a new `schema_pattern` field accepting a glob pattern (e.g. `tenant_*`), replicating all matching schemas through a single replication slot. Useful for multi-tenant databases where each tenant has its own schema. The existing `schema` field is unaffected and continues to take a single exact schema name (defaulting to `public`); `schema` and `schema_pattern` are mutually exclusive. ([@ness-david-dedu](https://github.com/ness-david-dedu), [#4589](https://github.com/redpanda-data/connect/pull/4589)) +- postgres_cdc: Added a new `exclude_schemas` field to carve exceptions out of a broad `schema_pattern` (e.g. `schema_pattern: tenant_*` while skipping `tenant_test`). Accepts the same exact-name/glob/quoted syntax as `schema_pattern`, matches entries against the already-resolved schema list in memory with no extra database round-trips, and requires `schema_pattern` to be set. ([@ness-david-dedu](https://github.com/ness-david-dedu), [#4589](https://github.com/redpanda-data/connect/pull/4589)) - aws_dynamodb_cdc: DynamoDB CDC now supports an optional checkpoint_namespace field, allowing multiple independent pipelines to share a single checkpoint table without overwriting each other's checkpoints. ([@squiidz](https://github.com/squiidz), [#4602](https://github.com/redpanda-data/connect/pull/4602)) ### Fixed diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index 86702205a5..6332e8fd3c 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -45,6 +45,7 @@ input: snapshot_batch_size: 1000 schema: public schema_pattern: "" + exclude_schemas: [] tables: [] # No default (required) checkpoint_limit: 1024 temporary_slot: false @@ -76,6 +77,7 @@ input: snapshot_batch_size: 1000 schema: public schema_pattern: "" + exclude_schemas: [] tables: [] # No default (required) checkpoint_limit: 1024 temporary_slot: false @@ -216,6 +218,27 @@ schema_pattern: '*' schema_pattern: '"MyCaseSensitiveSchemaNeedingQuotes"' ``` +=== `exclude_schemas` + +A list of schema names or glob patterns to exclude from the schemas matched by `schema_pattern`. Only valid when `schema_pattern` is set. + +Each entry uses the same syntax as `schema_pattern`: an exact schema name, a glob pattern using `*` as a wildcard, or a double-quoted exact identifier for an exact, case-sensitive match. + +A schema that matches `schema_pattern` and also matches any entry in this list is excluded from replication. An entry that does not match any schema resolved by `schema_pattern` is silently ignored, so a typo here simply excludes nothing rather than failing startup. + + +*Type*: `array` + +*Default*: `[]` + +```yml +# Examples + +exclude_schemas: + - tenant_internal + - tenant_test_* +``` + === `tables` A list of table names to include in the logical replication. Each table should be specified as a separate item. diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index d6d7235fd0..17959f19e1 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -41,6 +41,7 @@ const ( fieldSnapshotBatchSize = "snapshot_batch_size" fieldSchema = "schema" fieldSchemaPattern = "schema_pattern" + fieldExcludeSchemas = "exclude_schemas" fieldTables = "tables" fieldCheckpointLimit = "checkpoint_limit" fieldTemporarySlot = "temporary_slot" @@ -140,6 +141,16 @@ This field is mutually exclusive with `+"`"+fieldSchema+"`"+`; when set, it take Optional(). Default(""), ). + Field(service.NewStringListField(fieldExcludeSchemas). + Description(`A list of schema names or glob patterns to exclude from the schemas matched by ` + "`" + fieldSchemaPattern + "`" + `. Only valid when ` + "`" + fieldSchemaPattern + "`" + ` is set. + +Each entry uses the same syntax as ` + "`" + fieldSchemaPattern + "`" + `: an exact schema name, a glob pattern using ` + "`*`" + ` as a wildcard, or a double-quoted exact identifier for an exact, case-sensitive match. + +A schema that matches ` + "`" + fieldSchemaPattern + "`" + ` and also matches any entry in this list is excluded from replication. An entry that does not match any schema resolved by ` + "`" + fieldSchemaPattern + "`" + ` is silently ignored, so a typo here simply excludes nothing rather than failing startup.`). + Examples([]string{"tenant_internal", "tenant_test_*"}). + Optional(). + Default([]string{}), + ). Field(service.NewStringListField(fieldTables). Description(`A list of table names to include in the logical replication. Each table should be specified as a separate item. @@ -245,6 +256,7 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser temporarySlot bool schema string schemaPattern string + excludeSchemas []string tables []string streamSnapshot bool includeTxnMarkers bool @@ -308,6 +320,26 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser } } + if excludeSchemas, err = conf.FieldStringList(fieldExcludeSchemas); err != nil { + return nil, err + } + if len(excludeSchemas) > 0 { + if schemaPattern == "" { + return nil, errors.New("exclude_schemas requires schema_pattern to be set") + } + for i, pattern := range excludeSchemas { + if err = validateSchemaPattern(pattern); err != nil { + return nil, fmt.Errorf("invalid exclude_schemas entry %q: %w", pattern, err) + } + // Normalize unquoted patterns to lower-case, mirroring schema_pattern + // above: PostgreSQL folds unquoted identifiers at creation time, so + // TENANT_TEST and tenant_test resolve to the same schema. + if !strings.HasPrefix(pattern, `"`) { + excludeSchemas[i] = strings.ToLower(pattern) + } + } + } + if tables, err = conf.FieldStringList(fieldTables); err != nil { return nil, err } @@ -388,6 +420,7 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser DBRawDSN: dsn, DBSchema: schema, DBSchemaPattern: schemaPattern, + DBExcludeSchemas: excludeSchemas, DBTables: tables, RefreshAuthToken: iamAuthTokenBuilder, diff --git a/internal/impl/postgresql/input_pg_stream_test.go b/internal/impl/postgresql/input_pg_stream_test.go index 2b30df9fa2..d7869eaa4f 100644 --- a/internal/impl/postgresql/input_pg_stream_test.go +++ b/internal/impl/postgresql/input_pg_stream_test.go @@ -125,6 +125,79 @@ tables: require.NoError(t, err) } +// TestExcludeSchemasValidation verifies that each exclude_schemas entry is +// validated with the same rules as schema_pattern - validateSchemaPattern is +// reused rather than re-derived, so this exercises the same error cases +// TestSchemaPatternValidation covers, just reached through a different field. +func TestExcludeSchemasValidation(t *testing.T) { + tests := []struct { + pattern string + errContains string + }{ + {"tenant_test", ""}, + {"tenant_test_*", ""}, + {`"MySchema"`, ""}, + {"1abc", "must start with a letter"}, + {`"unclosed`, "invalid quoted schema identifier"}, + {"schema-name", "invalid character"}, + {`"quoted*"`, "wildcard"}, + } + for _, tt := range tests { + t.Run(tt.pattern, func(t *testing.T) { + // Single-quoted so the pattern (which may itself contain double + // quotes, e.g. `"MySchema"`) reaches validateSchemaPattern verbatim. + yaml := fmt.Sprintf(` +dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable +schema_pattern: 'tenant_*' +exclude_schemas: ['%s'] +slot_name: test_slot +tables: + - events +`, tt.pattern) + + _, err := parsePgStreamInput(t, yaml) + if tt.errContains != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errContains) + return + } + require.NoError(t, err) + }) + } +} + +// TestExcludeSchemasRequiresSchemaPattern verifies that exclude_schemas is +// rejected at config-parse time when schema_pattern is left unset. Both +// single-exact-schema mode and FOR ALL TABLES mode (empty tables) have no +// well-defined candidate set to exclude from, so this is a hard error rather +// than a silent no-op. +func TestExcludeSchemasRequiresSchemaPattern(t *testing.T) { + yaml := ` +dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable +exclude_schemas: [tenant_test] +slot_name: test_slot +tables: + - events +` + _, err := parsePgStreamInput(t, yaml) + require.Error(t, err) + assert.Contains(t, err.Error(), "exclude_schemas requires schema_pattern to be set") +} + +// TestExcludeSchemasEmptyWithoutSchemaPatternSucceeds verifies that leaving +// exclude_schemas at its default empty list does not trip the +// requires-schema_pattern check, since there's nothing to exclude. +func TestExcludeSchemasEmptyWithoutSchemaPatternSucceeds(t *testing.T) { + yaml := ` +dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable +slot_name: test_slot +tables: + - events +` + _, err := parsePgStreamInput(t, yaml) + require.NoError(t, err) +} + // TestSchemaAcceptsUnicodeIdentifier verifies that the schema field (single // exact-name path) still accepts unquoted unicode identifiers like // "münchen", matching sanitize.NormalizePostgresIdentifier which is the sole diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index acb5458439..1b77789324 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1760,6 +1760,158 @@ postgres_cdc: assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") } +// TestIntegrationMultiSchemaExcludeSchemas verifies that exclude_schemas +// carves an exception out of a broad schema_pattern: a schema that matches +// schema_pattern but also matches an exclude_schemas entry contributes no +// rows at all, neither during the initial snapshot nor from subsequent CDC +// changes. +func TestIntegrationMultiSchemaExcludeSchemas(t *testing.T) { + integration.CheckSkip(t) + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + // Three tenant schemas match tenant_*; tenant_c is carved out via exclude_schemas. + for _, schema := range []string{"tenant_a", "tenant_b", "tenant_c"} { + _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", schema)) + require.NoError(t, err) + _, err = db.Exec(fmt.Sprintf( + "CREATE TABLE %s.events (id SERIAL PRIMARY KEY, name TEXT)", schema)) + require.NoError(t, err) + } + + // Pre-load snapshot data, including a row in the excluded schema that must + // never surface. + _, err = db.Exec("INSERT INTO tenant_a.events (name) VALUES ('alice'), ('bob')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_b.events (name) VALUES ('carol')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_c.events (name) VALUES ('mallory')") + require.NoError(t, err) + + type msgMeta struct { + dbSchema string + table string + operation string + lsn string + } + + var ( + mu sync.Mutex + collected []msgMeta + ) + collectedLen := func() int { + mu.Lock() + defer mu.Unlock() + return len(collected) + } + + tmpl := fmt.Sprintf(` +postgres_cdc: + dsn: %s + slot_name: exclude_schemas_test_slot + stream_snapshot: true + schema_pattern: tenant_* + exclude_schemas: + - tenant_c + tables: + - events +`, databaseURL) + + sb := service.NewStreamBuilder() + require.NoError(t, sb.SetLoggerYAML(`level: WARN`)) + require.NoError(t, sb.AddInputYAML(tmpl)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + mu.Lock() + defer mu.Unlock() + for _, msg := range batch { + m := msgMeta{} + m.dbSchema, _ = msg.MetaGet("database_schema") + m.table, _ = msg.MetaGet("table") + m.operation, _ = msg.MetaGet("operation") + m.lsn, _ = msg.MetaGet("lsn") + collected = append(collected, m) + } + return nil + })) + + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + go func() { + if err := stream.Run(t.Context()); err != nil && !errors.Is(err, context.Canceled) { + t.Error(err) + } + }() + t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) + + // Wait for the 3 snapshot rows from the two non-excluded schemas; tenant_c's + // row must never contribute to this count. + assert.Eventually(t, func() bool { + return collectedLen() >= 3 + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for snapshot rows") + + // Insert CDC rows into all three schemas, including the excluded one. + _, err = db.Exec("INSERT INTO tenant_a.events (name) VALUES ('dave')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_b.events (name) VALUES ('eve')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_c.events (name) VALUES ('trudy')") + require.NoError(t, err) + + // Wait for the 2 CDC rows from the non-excluded schemas (total 5). + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.Equal(c, 5, collectedLen()) + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for CDC rows") + + // tenant_c's CDC insert above raced the same replication stream as the + // tenant_a/tenant_b inserts already confirmed above, so if it were going + // to leak through it would have by now; assert the count never climbs + // past 5 to catch a delayed leak instead of just checking once. + assert.Never(t, func() bool { + return collectedLen() > 5 + }, 3*time.Second, 200*time.Millisecond, "received unexpected message(s) from excluded schema tenant_c") + + mu.Lock() + defer mu.Unlock() + + require.Len(t, collected, 5) + for _, m := range collected { + assert.NotEqual(t, "tenant_c", m.dbSchema, "tenant_c is excluded and must never appear, got message: %+v", m) + } + + var snapshots, cdcMsgs []msgMeta + for _, m := range collected { + if m.operation == "read" { + snapshots = append(snapshots, m) + } else { + cdcMsgs = append(cdcMsgs, m) + } + } + + // Snapshot assertions. + require.Len(t, snapshots, 3) + snapshotSchemas := make(map[string]int) + for _, m := range snapshots { + assert.Equal(t, "events", m.table, "snapshot: table should be bare name without schema prefix") + assert.Empty(t, m.lsn, "snapshot rows have no LSN") + snapshotSchemas[m.dbSchema]++ + } + assert.Equal(t, 2, snapshotSchemas["tenant_a"], "expected 2 snapshot rows from tenant_a") + assert.Equal(t, 1, snapshotSchemas["tenant_b"], "expected 1 snapshot row from tenant_b") + + // CDC assertions. + require.Len(t, cdcMsgs, 2) + cdcSchemas := make(map[string]int) + for _, m := range cdcMsgs { + assert.Equal(t, "insert", m.operation) + assert.Equal(t, "events", m.table) + assert.NotEmpty(t, m.lsn, "CDC rows must have an LSN") + cdcSchemas[m.dbSchema]++ + } + assert.Equal(t, 1, cdcSchemas["tenant_a"], "expected 1 CDC row from tenant_a") + assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") +} + func TestIntegrationMultiSchemaMissingTableDegradesGracefully(t *testing.T) { integration.CheckSkip(t) databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") diff --git a/internal/impl/postgresql/pglogicalstream/config.go b/internal/impl/postgresql/pglogicalstream/config.go index f6cb5ce99c..2247421542 100644 --- a/internal/impl/postgresql/pglogicalstream/config.go +++ b/internal/impl/postgresql/pglogicalstream/config.go @@ -30,7 +30,11 @@ type Config struct { // non-empty, it takes precedence over DBSchema and schemas are resolved // dynamically at stream creation time. DBSchemaPattern string - DBTables []string + // DBExcludeSchemas is a list of schema names or glob patterns (same syntax + // as DBSchemaPattern) excluded from the schemas resolved by + // DBSchemaPattern. Only meaningful when DBSchemaPattern is non-empty. + DBExcludeSchemas []string + DBTables []string // Refreshes short lived IAM auth token that is treated as a password RefreshAuthToken func(ctx context.Context) error // ReplicationSlotName is the name of the replication slot to use diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 669acbc66a..a5fe4369b3 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -104,6 +104,36 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { if err != nil { return nil, fmt.Errorf("resolving schema pattern %q: %w", config.DBSchemaPattern, err) } + + if len(config.DBExcludeSchemas) > 0 { + // Filtering happens entirely against the schemas slice we already + // fetched above - no extra DB round-trips per exclude pattern. + var excluded []string + remaining := make([]string, 0, len(schemas)) + for _, schema := range schemas { + var isExcluded bool + for _, pattern := range config.DBExcludeSchemas { + matched, err := schemaMatchesExcludePattern(schema, pattern) + if err != nil { + return nil, fmt.Errorf("evaluating exclude_schemas pattern %q against schema %q: %w", pattern, schema, err) + } + if matched { + isExcluded = true + break + } + } + if isExcluded { + excluded = append(excluded, schema) + continue + } + remaining = append(remaining, schema) + } + if len(excluded) > 0 { + config.Logger.Infof("exclude_schemas %v excluded %d schema(s) %v from schema pattern %q; %d schema(s) remain: %v", config.DBExcludeSchemas, len(excluded), excluded, config.DBSchemaPattern, len(remaining), remaining) + } + schemas = remaining + } + if len(inaccessibleSchemas) > 0 && len(config.DBTables) > 0 { config.Logger.Warnf("schema pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaPattern, inaccessibleSchemas) } diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index 4737f7c4cb..f8d6991007 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -11,6 +11,7 @@ package pglogicalstream import ( "context" "fmt" + "regexp" "strings" "github.com/jackc/pgx/v5/pgconn" @@ -198,3 +199,53 @@ func escapeLike(s string) string { } return b.String() } + +// schemaMatchesExcludePattern reports whether quotedSchemaName - a schema +// identifier in the same quoted form resolveSchemas returns - matches +// excludePattern, an exclude_schemas entry written in the same shape as a +// schema_pattern value (an exact name, a '*' glob, or a double-quoted exact +// identifier). +// +// Matching happens entirely in memory against a schema list we've already +// resolved, unlike resolveSchemas which queries the database: exclude_schemas +// only ever narrows a candidate set that's already been fetched, so there's +// no reason to pay for another round-trip per exclude pattern. Semantics +// mirror schemaPatternToLike without going through SQL: a quoted pattern is +// an exact, case-sensitive match on the unquoted name; an unquoted pattern is +// matched case-insensitively with '*' as a wildcard. +// +// Returns an error only when a quoted operand fails to unquote. This should +// only happen for a malformed excludePattern in practice, since +// quotedSchemaName is always freshly quoted by resolveSchemas. +func schemaMatchesExcludePattern(quotedSchemaName, excludePattern string) (bool, error) { + schemaName, err := sanitize.UnquotePostgresIdentifier(quotedSchemaName) + if err != nil { + return false, fmt.Errorf("unquoting schema identifier %q: %w", quotedSchemaName, err) + } + + if strings.HasPrefix(excludePattern, `"`) { + unquoted, err := sanitize.UnquotePostgresIdentifier(excludePattern) + if err != nil { + return false, fmt.Errorf("invalid quoted schema identifier %q: %w", excludePattern, err) + } + return schemaName == unquoted, nil + } + + re, err := globToRegexp(strings.ToLower(excludePattern)) + if err != nil { + return false, fmt.Errorf("invalid exclude pattern %q: %w", excludePattern, err) + } + return re.MatchString(strings.ToLower(schemaName)), nil +} + +// globToRegexp compiles an unquoted glob pattern (using '*' as a wildcard) +// into an anchored regexp - the in-memory equivalent of globToLike for +// callers matching against values already held in Go rather than via a SQL +// LIKE clause. +func globToRegexp(pattern string) (*regexp.Regexp, error) { + parts := strings.Split(pattern, "*") + for i, part := range parts { + parts[i] = regexp.QuoteMeta(part) + } + return regexp.Compile("^" + strings.Join(parts, ".*") + "$") +} diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go b/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go index 7ee213f6e4..7a352c6949 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go @@ -88,3 +88,43 @@ func TestEscapeLike(t *testing.T) { }) } } + +func TestSchemaMatchesExcludePattern(t *testing.T) { + tests := []struct { + name string + schema string + pattern string + expected bool + errContains string + }{ + // Unquoted patterns - case-insensitive, '*' as wildcard. + {name: "unquoted exact match", schema: `"tenant_a"`, pattern: "tenant_a", expected: true}, + {name: "unquoted exact no match", schema: `"tenant_a"`, pattern: "tenant_b", expected: false}, + {name: "unquoted glob match", schema: `"tenant_test_x"`, pattern: "tenant_test_*", expected: true}, + {name: "unquoted glob no match", schema: `"tenant_prod_x"`, pattern: "tenant_test_*", expected: false}, + {name: "bare wildcard matches everything", schema: `"anything"`, pattern: "*", expected: true}, + // Case-folding: unquoted patterns and unquoted-origin schema names both + // fold to lower-case, mirroring PostgreSQL's identifier folding. + {name: "case-insensitive exact match", schema: `"tenant_a"`, pattern: "TENANT_A", expected: true}, + {name: "case-insensitive glob match", schema: `"Tenant_Test_X"`, pattern: "tenant_test_*", expected: true}, + // Quoted patterns - exact, case-sensitive, no wildcard expansion. + {name: "quoted exact case-sensitive match", schema: `"MySchema"`, pattern: `"MySchema"`, expected: true}, + {name: "quoted exact case mismatch does not match", schema: `"MySchema"`, pattern: `"myschema"`, expected: false}, + {name: "quoted pattern does not expand wildcard", schema: `"tenant_a"`, pattern: `"tenant_*"`, expected: false}, + // Errors - only from a malformed pattern, never from the candidate + // schema name, since that's always freshly quoted by resolveSchemas. + {name: "unterminated quoted pattern errors", schema: `"tenant_a"`, pattern: `"unterminated`, errContains: "invalid quoted schema identifier"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := schemaMatchesExcludePattern(tt.schema, tt.pattern) + if tt.errContains != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errContains) + return + } + require.NoError(t, err) + assert.Equal(t, tt.expected, got) + }) + } +} diff --git a/internal/impl/postgresql/tests/current/Taskfile.yaml b/internal/impl/postgresql/tests/current/Taskfile.yaml index d81e5413ce..0b57a7d25d 100644 --- a/internal/impl/postgresql/tests/current/Taskfile.yaml +++ b/internal/impl/postgresql/tests/current/Taskfile.yaml @@ -47,10 +47,11 @@ tasks: # ── Test data ───────────────────────────────────────────────────────────────── data:insert: - desc: Insert CDC rows into both tenant schemas (triggers insert events) + desc: Insert CDC rows into all three tenant schemas (tenant_c must not appear in the output — it's excluded) cmds: - psql "{{.PG_DSN}}" -c "INSERT INTO tenant_a.events (name) VALUES ('dave'), ('eve');" - psql "{{.PG_DSN}}" -c "INSERT INTO tenant_b.events (name) VALUES ('frank');" + - psql "{{.PG_DSN}}" -c "INSERT INTO tenant_c.events (name) VALUES ('trudy');" data:update: desc: Update a row in tenant_a (triggers update event with 'before' field) @@ -89,6 +90,22 @@ tasks: # Note: an empty schema_pattern is NOT an error - it means "unset", and # the connector falls back to the schema field (default "public"). + test:exclude-schemas-requires-pattern: + desc: Confirm exclude_schemas is rejected at startup when schema_pattern is left unset + env: + PG_DSN: '{{.PG_DSN}}' + cmds: + - | + set +e + go run ../../../../../cmd/redpanda-connect/main.go run \ + ./test_config_exclude_no_pattern.yaml 2>&1 | head -5 + echo "exit $?" + # Expects: "exclude_schemas requires schema_pattern to be set" error + # printed, process exits non-zero. Uses a dedicated fixture file rather + # than --set on test_config.yaml, since --set rejects an empty RHS + # ("foo=" -> "expected foo=bar syntax"), so schema_pattern can't be + # cleared that way. + # ── Quick sanity ────────────────────────────────────────────────────────────── psql: diff --git a/internal/impl/postgresql/tests/current/setup.sql b/internal/impl/postgresql/tests/current/setup.sql index 512f8373fa..3c0c6264fd 100644 --- a/internal/impl/postgresql/tests/current/setup.sql +++ b/internal/impl/postgresql/tests/current/setup.sql @@ -1,10 +1,15 @@ -- Multi-schema CDC test setup --- Tests: schema glob (tenant_*), database_schema metadata, commit_ts_ms, before (update/delete) +-- Tests: schema glob (tenant_*), exclude_schemas (tenant_c), database_schema +-- metadata, commit_ts_ms, before (update/delete) -- ── Tenant schemas ──────────────────────────────────────────────────────────── +-- tenant_a and tenant_b are replicated; tenant_c matches the tenant_* glob in +-- test_config.yaml but is carved out via exclude_schemas — its rows must +-- never appear in the pipeline output. CREATE SCHEMA IF NOT EXISTS tenant_a; CREATE SCHEMA IF NOT EXISTS tenant_b; +CREATE SCHEMA IF NOT EXISTS tenant_c; -- ── Events table (same shape in each schema) ────────────────────────────────── @@ -22,12 +27,23 @@ CREATE TABLE IF NOT EXISTS tenant_b.events ( created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +CREATE TABLE IF NOT EXISTS tenant_c.events ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + -- REPLICA IDENTITY FULL so update/delete messages carry the full before-row. ALTER TABLE tenant_a.events REPLICA IDENTITY FULL; ALTER TABLE tenant_b.events REPLICA IDENTITY FULL; +ALTER TABLE tenant_c.events REPLICA IDENTITY FULL; -- ── Seed snapshot rows ──────────────────────────────────────────────────────── -- These are visible during the initial snapshot (stream_snapshot: true). +-- tenant_c's row (mallory) must NOT appear in the output — see exclude_schemas +-- in test_config.yaml. INSERT INTO tenant_a.events (name) VALUES ('alice'), ('bob'); INSERT INTO tenant_b.events (name) VALUES ('carol'); +INSERT INTO tenant_c.events (name) VALUES ('mallory'); diff --git a/internal/impl/postgresql/tests/current/test_config.yaml b/internal/impl/postgresql/tests/current/test_config.yaml index 838321a392..534b11496e 100644 --- a/internal/impl/postgresql/tests/current/test_config.yaml +++ b/internal/impl/postgresql/tests/current/test_config.yaml @@ -5,6 +5,10 @@ input: stream_snapshot: true # Glob pattern: replicates both tenant_a and tenant_b via one slot. schema_pattern: tenant_* + # tenant_c matches the glob above but is carved out here — its rows must + # never appear in the output below, neither during snapshot nor CDC. + exclude_schemas: + - tenant_c tables: - events diff --git a/internal/impl/postgresql/tests/current/test_config_exclude_no_pattern.yaml b/internal/impl/postgresql/tests/current/test_config_exclude_no_pattern.yaml new file mode 100644 index 0000000000..57c0caf983 --- /dev/null +++ b/internal/impl/postgresql/tests/current/test_config_exclude_no_pattern.yaml @@ -0,0 +1,22 @@ +# Negative-validation fixture for exclude_schemas: schema_pattern is +# deliberately left unset. --set can't assign an empty string (the CLI +# rejects "foo=" as "expected foo=bar syntax"), so this exists as its own +# file instead of overriding test_config.yaml at run time. +input: + postgres_cdc: + dsn: ${PG_DSN:postgres://postgres:postgres@localhost:5433/testdb?sslmode=disable} + slot_name: exclude_schemas_no_pattern_slot + exclude_schemas: + - tenant_c + tables: + - events + +output: + stdout: + codec: lines + +logger: + level: INFO + +metrics: + none: {} From 7a136ba4cdb655f6fc339550801e24323f883e99 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Wed, 12 Aug 2026 13:07:22 +0100 Subject: [PATCH 28/70] postgres_cdc: fix failing signalling tests --- internal/impl/postgresql/signaller_integration_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/impl/postgresql/signaller_integration_test.go b/internal/impl/postgresql/signaller_integration_test.go index fb70c7ff2d..d3ef8a5a2f 100644 --- a/internal/impl/postgresql/signaller_integration_test.go +++ b/internal/impl/postgresql/signaller_integration_test.go @@ -449,6 +449,7 @@ func startSignallingStream(t *testing.T, inputYAML string) (*pgtest.ReceivedMess } delete(m, "schema") delete(m, "commit_ts_ms") + delete(m, "database_schema") received.Add(m) } return nil From 52dbd2bb8984ec4aff9a3a769cdc05026ec7e9c7 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Wed, 12 Aug 2026 13:18:01 +0100 Subject: [PATCH 29/70] postgres_cdc: add further test coverage for uuid schemas --- .../pglogicalstream/schema_resolver.go | 47 ++++++---- .../schema_resolver_integration_test.go | 90 +++++++++++++++++++ .../pglogicalstream/schema_resolver_test.go | 34 +++---- 3 files changed, 140 insertions(+), 31 deletions(-) diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index f8d6991007..da3958a73f 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -20,20 +20,28 @@ import ( ) // schemaPatternToLike converts a schema name or glob pattern into the LIKE -// pattern used by resolveSchemas. Extracted for unit testing. +// pattern used by resolveSchemas, plus whether that pattern must be matched +// case-sensitively. Extracted for unit testing. // -// For quoted identifiers the inner name is exact-escaped (no wildcard expansion). -// For unquoted patterns the '*' wildcard is converted to '%' and the input is -// folded to lower-case to match PostgreSQL's identifier folding. -func schemaPatternToLike(pattern string) (string, error) { +// For quoted identifiers the inner name is exact-escaped (no wildcard +// expansion) and matched case-sensitively, since a quoted identifier's case +// is significant and PostgreSQL does not fold it. For unquoted patterns the +// '*' wildcard is converted to '%' and matching is case-insensitive: this +// mirrors PostgreSQL folding unquoted identifiers to lower-case at creation +// time for the common case, but must hold even when a schema had to be +// created with a quoted identifier for an unrelated reason (e.g. a +// UUID-suffixed tenant schema, which requires quoting because hyphens are +// invalid in unquoted identifiers) and so kept whatever case it was written +// with — an unquoted glob like "tenant_*" is still expected to match it. +func schemaPatternToLike(pattern string) (likePattern string, caseSensitive bool, err error) { if strings.HasPrefix(pattern, `"`) { unquoted, err := sanitize.UnquotePostgresIdentifier(pattern) if err != nil { - return "", fmt.Errorf("invalid quoted schema identifier %q: %w", pattern, err) + return "", false, fmt.Errorf("invalid quoted schema identifier %q: %w", pattern, err) } - return escapeLike(unquoted), nil + return escapeLike(unquoted), true, nil } - return globToLike(strings.ToLower(pattern)), nil + return globToLike(strings.ToLower(pattern)), false, nil } // resolveSchemas returns the schemas matching pattern that the connection's @@ -45,23 +53,30 @@ func schemaPatternToLike(pattern string) (string, error) { // see yet. // // For unquoted patterns (e.g. "tenant_*") the pattern is matched -// case-insensitively via LIKE, because PostgreSQL folds unquoted identifiers -// to lower-case at creation time. For quoted identifiers (e.g. `"MySchema"`) -// an exact case-sensitive lookup is performed. System schemas (pg_* and -// information_schema) are always excluded so that wildcard patterns like "*" -// do not attempt to replicate catalog tables. +// case-insensitively via ILIKE, regardless of whether the matched schema was +// itself created case-insensitively. For quoted identifiers (e.g. +// `"MySchema"`) an exact case-sensitive lookup is performed via LIKE. System +// schemas (pg_* and information_schema) are always excluded case-sensitively +// so that wildcard patterns like "*" do not attempt to replicate catalog +// tables. // // Returned schema names are quoted PostgreSQL identifiers. Returns an error // if either query fails; returns a nil visibleSchemas slice (with a nil err) // if no schemas match — callers should treat that as an error condition. func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (visibleSchemas, inaccessibleSchemas []string, err error) { - likePattern, err := schemaPatternToLike(pattern) + likePattern, caseSensitive, err := schemaPatternToLike(pattern) if err != nil { return nil, nil, err } + // Fixed, code-chosen operator (never derived from user input), so it's + // safe to splice directly into the query text rather than parameterize. + op := "ILIKE" + if caseSensitive { + op = "LIKE" + } q, err := sanitize.SQLQuery( - "SELECT schema_name FROM information_schema.schemata WHERE schema_name LIKE $1 ESCAPE '!' AND schema_name NOT LIKE 'pg!_%' ESCAPE '!' AND schema_name != 'information_schema'", + fmt.Sprintf("SELECT schema_name FROM information_schema.schemata WHERE schema_name %s $1 ESCAPE '!' AND schema_name NOT LIKE 'pg!_%%' ESCAPE '!' AND schema_name != 'information_schema'", op), likePattern, ) if err != nil { @@ -90,7 +105,7 @@ func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (v // missing from information_schema.schemata means the role lacks USAGE (or // similar) on that schema rather than the schema simply not existing. nsQ, err := sanitize.SQLQuery( - "SELECT nspname FROM pg_catalog.pg_namespace WHERE nspname LIKE $1 ESCAPE '!' AND nspname NOT LIKE 'pg!_%' ESCAPE '!' AND nspname != 'information_schema'", + fmt.Sprintf("SELECT nspname FROM pg_catalog.pg_namespace WHERE nspname %s $1 ESCAPE '!' AND nspname NOT LIKE 'pg!_%%' ESCAPE '!' AND nspname != 'information_schema'", op), likePattern, ) if err != nil { diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go b/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go index 6af1d4ad7e..91243a6802 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go @@ -69,3 +69,93 @@ func TestIntegrationResolveSchemasReportsInaccessibleSchemas(t *testing.T) { assert.Equal(t, []string{`"visible_schema"`}, visible) assert.Equal(t, []string{`"hidden_schema"`}, inaccessible) } + +func TestIntegrationResolveSchemasUUIDSuffixedSchemas(t *testing.T) { + integration.CheckSkip(t) + + _, adminURL := createDockerInstance(t) + + adminDB, err := sql.Open("postgres", adminURL) + require.NoError(t, err) + defer adminDB.Close() + + // Quoting is mandatory here because of the UUID's hyphens, regardless of + // case - so both of these preserve their literal casing exactly as written. + const ( + lowerCaseSchema = `"tenant_a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"` + mixedCaseSchema = `"Tenant_9c0b4ef8-bb6d-6bb9-bd38-0a11a0eebc99"` + ) + _, err = adminDB.Exec(`CREATE SCHEMA ` + lowerCaseSchema) + require.NoError(t, err) + _, err = adminDB.Exec(`CREATE SCHEMA ` + mixedCaseSchema) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + + conn, err := pgconn.Connect(ctx, adminURL) + require.NoError(t, err) + defer closeConn(t, conn) + + visible, inaccessible, err := resolveSchemas(ctx, conn, "tenant_*") + require.NoError(t, err) + + // Both schemas match the unquoted glob despite the case difference in + // their literal prefix - matching is case-insensitive independent of how + // the schema itself was created. + assert.ElementsMatch(t, []string{lowerCaseSchema, mixedCaseSchema}, visible) + assert.Empty(t, inaccessible) + + // A quoted pattern is still exact and case-sensitive: it picks out only + // the schema whose case matches the pattern, with no wildcard expansion. + exactVisible, _, err := resolveSchemas(ctx, conn, mixedCaseSchema) + require.NoError(t, err) + assert.Equal(t, []string{mixedCaseSchema}, exactVisible) +} + +func TestIntegrationResolveSchemasBareUUIDSchema(t *testing.T) { + integration.CheckSkip(t) + + _, adminURL := createDockerInstance(t) + + adminDB, err := sql.Open("postgres", adminURL) + require.NoError(t, err) + defer adminDB.Close() + + // Upper-case hex digits, no prefix - quoting is mandatory purely because + // of the hyphens, not because of anything alphabetic. + const bareUUIDSchema = `"A0EEBC99-9C0B-4EF8-BB6D-6BB9BD380A11"` + _, err = adminDB.Exec(`CREATE SCHEMA ` + bareUUIDSchema) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + + conn, err := pgconn.Connect(ctx, adminURL) + require.NoError(t, err) + defer closeConn(t, conn) + + // A bare wildcard has no literal characters to case-compare, so this is + // unaffected by hex-digit casing either way - included as a baseline. + visible, _, err := resolveSchemas(ctx, conn, "*") + require.NoError(t, err) + assert.Contains(t, visible, bareUUIDSchema) + + // The interesting case: an unquoted pattern whose only literal portion is + // a lower-case chunk of the UUID itself (no prefix) must still match the + // upper-case schema case-insensitively. + visible, _, err = resolveSchemas(ctx, conn, "a0eebc99-*") + require.NoError(t, err) + assert.Equal(t, []string{bareUUIDSchema}, visible) + + // Same, but the literal chunk sits in the middle rather than at the start. + visible, _, err = resolveSchemas(ctx, conn, "*-bb6d-*") + require.NoError(t, err) + assert.Equal(t, []string{bareUUIDSchema}, visible) + + // A quoted pattern remains an exact, case-sensitive lookup: the + // differently-cased quoted form matches nothing. + visible, _, err = resolveSchemas(ctx, conn, `"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"`) + require.NoError(t, err) + assert.Empty(t, visible) +} diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go b/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go index 7a352c6949..ae1ee89a92 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go @@ -38,27 +38,30 @@ func TestGlobToLike(t *testing.T) { func TestSchemaPatternToLike(t *testing.T) { tests := []struct { - pattern string - expected string - errContains string + pattern string + expected string + caseSensitive bool + errContains string }{ - // Unquoted glob patterns — folded to lower-case, '*' → '%', '_' escaped. - {"public", "public", ""}, - {"tenant_*", "tenant!_%", ""}, - {"*", "%", ""}, - {"schema_1", "schema!_1", ""}, + // Unquoted glob patterns — folded to lower-case, '*' → '%', '_' escaped, + // matched case-insensitively regardless of how the matched schema was created. + {pattern: "public", expected: "public"}, + {pattern: "tenant_*", expected: "tenant!_%"}, + {pattern: "*", expected: "%"}, + {pattern: "schema_1", expected: "schema!_1"}, // Upper-case is folded: TENANT_* matches the same rows as tenant_*. - {"TENANT_*", "tenant!_%", ""}, - // Quoted exact identifier — case preserved, no wildcard expansion. - {`"MySchema"`, "MySchema", ""}, - {`"schema_1"`, "schema!_1", ""}, - {`"has%bang!"`, "has!%bang!!", ""}, + {pattern: "TENANT_*", expected: "tenant!_%"}, + // Quoted exact identifier — case preserved, no wildcard expansion, + // matched case-sensitively. + {pattern: `"MySchema"`, expected: "MySchema", caseSensitive: true}, + {pattern: `"schema_1"`, expected: "schema!_1", caseSensitive: true}, + {pattern: `"has%bang!"`, expected: "has!%bang!!", caseSensitive: true}, // Unterminated quoted identifier → error. - {`"bad`, "", "invalid quoted schema identifier"}, + {pattern: `"bad`, errContains: "invalid quoted schema identifier"}, } for _, tt := range tests { t.Run(tt.pattern, func(t *testing.T) { - got, err := schemaPatternToLike(tt.pattern) + got, caseSensitive, err := schemaPatternToLike(tt.pattern) if tt.errContains != "" { require.Error(t, err) assert.Contains(t, err.Error(), tt.errContains) @@ -66,6 +69,7 @@ func TestSchemaPatternToLike(t *testing.T) { } require.NoError(t, err) assert.Equal(t, tt.expected, got) + assert.Equal(t, tt.caseSensitive, caseSensitive) }) } } From 55cf17dda5c835ff935ddeb77ee4f5c13c059f7f Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Wed, 12 Aug 2026 16:21:42 +0100 Subject: [PATCH 30/70] postgres_cdc: adress qupted schema --- .../components/pages/inputs/postgres_cdc.adoc | 4 +- internal/impl/postgresql/input_pg_stream.go | 20 +++-- .../impl/postgresql/input_pg_stream_test.go | 10 ++- internal/impl/postgresql/integration_test.go | 85 +++++++++++++++++++ 4 files changed, 109 insertions(+), 10 deletions(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index 7946e3e547..67dd5f4dd0 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -202,6 +202,8 @@ Schema pattern matching runs once at pipeline startup. Schemas created after the If `tables` is non-empty and this pattern matches no schema in the database, startup fails with an error. This field has no effect when `tables` is left empty - see `tables` below. +This pattern is matched against schema names as stored in the database rather than parsed as an identifier, so unquoted patterns may contain characters that are not valid in an unquoted PostgreSQL identifier (e.g. `tenant-*` or `a0eebc99-*` to match a UUID-suffixed schema that had to be created with a quoted identifier). + This field is mutually exclusive with `schema`; when set, it takes over schema resolution entirely and `schema` must be left at its default. @@ -623,7 +625,7 @@ Optional external ID for the role assumption. === `signal_table_name` -The name of the table used to send control signals to the connector, excluding the schema. The table must +The name of the table used to send control signals to the connector, excluding the schema. Not supported when `schema_pattern` is set, since there is no single schema to anchor the signal table to. The table must exist in the schema configured via the `schema` field, and must not also appear in `tables` — the signal table is implicitly added to the publication and excluded from snapshot scans, so listing it in both places is rejected at startup. It must have at least these columns — startup validation checks diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 4eba744b76..d19f659835 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -137,6 +137,8 @@ Schema pattern matching runs once at pipeline startup. Schemas created after the If `+"`"+fieldTables+"`"+` is non-empty and this pattern matches no schema in the database, startup fails with an error. This field has no effect when `+"`"+fieldTables+"`"+` is left empty - see `+"`"+fieldTables+"`"+` below. +This pattern can contain characters that wouldn't be allowed in an unquoted schema name, because it's only ever compared against the real name of each schema in the database - it doesn't have to be a valid name itself. For example, a schema literally named `+"`a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11`"+` (which must have been created using double quotes, since hyphens aren't allowed in an unquoted `+"`CREATE SCHEMA`"+` statement) can still be matched using the unquoted pattern `+"`a0eebc99-*`"+`. + This field is mutually exclusive with `+"`"+fieldSchema+"`"+`; when set, it takes over schema resolution entirely and `+"`"+fieldSchema+"`"+` must be left at its default.`). Examples("tenant_*", "*", `"MyCaseSensitiveSchemaNeedingQuotes"`). Optional(). @@ -535,8 +537,15 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser } // validateSchemaPattern validates a schema name or glob pattern. -// Accepts exact postgres identifiers (letters/digits/underscores) and glob -// patterns that additionally allow '*' as a wildcard character. +// +// Unquoted patterns are matched via ILIKE against stored schema names (see +// resolveSchemas) rather than parsed as an identifier we construct +// ourselves, so the accepted character set here is deliberately as wide as +// what's legal inside a quoted schema name, plus '*' as a wildcard - it is +// not narrowed to unquoted-identifier syntax. This matters because a schema +// that had to be created with a quoted identifier (e.g. a UUID-suffixed +// tenant schema, since hyphens are invalid in unquoted identifiers) must +// still be matchable via an unquoted glob such as "tenant-*" or "a0eebc99-*". // Double-quoted identifiers (e.g. "MySchema") are accepted as exact names; // wildcards are not allowed inside quotes. func validateSchemaPattern(s string) error { @@ -552,11 +561,8 @@ func validateSchemaPattern(s string) error { } return nil } - for i, ch := range s { - if unicode.IsLetter(ch) || unicode.IsDigit(ch) || ch == '_' || ch == '*' { - continue - } - return fmt.Errorf("invalid character %q at position %d in schema pattern %q", ch, i, s) + if strings.ContainsRune(s, '"') { + return fmt.Errorf("unquoted schema pattern %q must not contain '\"'", s) } first, _ := utf8.DecodeRuneInString(s) if first != '_' && first != '*' && !unicode.IsLetter(first) { diff --git a/internal/impl/postgresql/input_pg_stream_test.go b/internal/impl/postgresql/input_pg_stream_test.go index 53aad6f05c..73a3874122 100644 --- a/internal/impl/postgresql/input_pg_stream_test.go +++ b/internal/impl/postgresql/input_pg_stream_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 Redpanda Data, Inc. +// Copyright 2026 Redpanda Data, Inc. // // Licensed as a Redpanda Enterprise file under the Redpanda Community // License (the "License"); you may not use this file except in compliance with @@ -67,8 +67,14 @@ func TestSchemaPatternValidation(t *testing.T) { {`""`, "invalid quoted schema identifier"}, {"1abc", "must start with a letter"}, {`"unclosed`, "invalid quoted schema identifier"}, - {"schema-name", "invalid character"}, + // Regression test: an unquoted pattern is matched against stored + // schema names, not parsed as an identifier, so hyphens (invalid in + // unquoted Postgres identifiers) must still be accepted - e.g. to + // match a UUID-suffixed tenant schema that had to be created quoted. + {"schema-name", ""}, + {"a0eebc99-*", ""}, {`"quoted*"`, "wildcard"}, + {`a"b`, `must not contain '"'`}, } for _, tt := range tests { t.Run(tt.pattern, func(t *testing.T) { diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index 6f7f802359..a795c54b5e 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1898,6 +1898,91 @@ postgres_cdc: assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") } +func TestIntegrationSchemaPatternMatchesHyphenatedUUIDSchema(t *testing.T) { + integration.CheckSkip(t) + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + const uuidSchema = "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11" + _, err = db.Exec(fmt.Sprintf(`CREATE SCHEMA "%s"`, uuidSchema)) + require.NoError(t, err) + _, err = db.Exec(fmt.Sprintf(`CREATE TABLE "%s".events (id SERIAL PRIMARY KEY, name TEXT)`, uuidSchema)) + require.NoError(t, err) + _, err = db.Exec(fmt.Sprintf(`INSERT INTO "%s".events (name) VALUES ('alice')`, uuidSchema)) + require.NoError(t, err) + + type msgMeta struct { + dbSchema string + table string + operation string + } + + var ( + mu sync.Mutex + collected []msgMeta + ) + collectedLen := func() int { + mu.Lock() + defer mu.Unlock() + return len(collected) + } + + tmpl := fmt.Sprintf(` +postgres_cdc: + dsn: %s + slot_name: hyphenated_schema_pattern_slot + stream_snapshot: true + schema_pattern: a0eebc99-* + tables: + - events +`, databaseURL) + + sb := service.NewStreamBuilder() + require.NoError(t, sb.SetLoggerYAML(`level: WARN`)) + require.NoError(t, sb.AddInputYAML(tmpl)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + mu.Lock() + defer mu.Unlock() + for _, msg := range batch { + m := msgMeta{} + m.dbSchema, _ = msg.MetaGet("database_schema") + m.table, _ = msg.MetaGet("table") + m.operation, _ = msg.MetaGet("operation") + collected = append(collected, m) + } + return nil + })) + + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + go func() { + if err := stream.Run(t.Context()); err != nil && !errors.Is(err, context.Canceled) { + t.Error(err) + } + }() + t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) + + assert.Eventually(t, func() bool { + return collectedLen() >= 1 + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for snapshot row from hyphenated UUID schema") + + _, err = db.Exec(fmt.Sprintf(`INSERT INTO "%s".events (name) VALUES ('bob')`, uuidSchema)) + require.NoError(t, err) + + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.Equal(c, 2, collectedLen()) + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for CDC row from hyphenated UUID schema") + + mu.Lock() + defer mu.Unlock() + require.Len(t, collected, 2) + for _, m := range collected { + assert.Equal(t, uuidSchema, m.dbSchema, "database_schema metadata should be the raw, unquoted, case-preserved schema name") + assert.Equal(t, "events", m.table) + } +} + func TestIntegrationMultiSchemaMissingTableDegradesGracefully(t *testing.T) { integration.CheckSkip(t) databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") From f39ff341595d724c78ae5d052eb8ee0ab631801b Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Wed, 12 Aug 2026 16:42:48 +0100 Subject: [PATCH 31/70] postgres_cdc: allow a leading digit for unquoted patterns --- internal/impl/postgresql/input_pg_stream.go | 17 ++++------------- .../impl/postgresql/input_pg_stream_test.go | 14 +++++++++++++- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index d19f659835..05fdf13420 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -18,8 +18,6 @@ import ( "strings" "sync" "time" - "unicode" - "unicode/utf8" "github.com/Jeffail/checkpoint" "github.com/Jeffail/shutdown" @@ -539,13 +537,10 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser // validateSchemaPattern validates a schema name or glob pattern. // // Unquoted patterns are matched via ILIKE against stored schema names (see -// resolveSchemas) rather than parsed as an identifier we construct -// ourselves, so the accepted character set here is deliberately as wide as -// what's legal inside a quoted schema name, plus '*' as a wildcard - it is -// not narrowed to unquoted-identifier syntax. This matters because a schema -// that had to be created with a quoted identifier (e.g. a UUID-suffixed -// tenant schema, since hyphens are invalid in unquoted identifiers) must -// still be matchable via an unquoted glob such as "tenant-*" or "a0eebc99-*". +// resolveSchemas), not parsed as an identifier, so any character or leading +// character is accepted - including hyphens and leading digits - except a +// literal '"'. This lets a glob like "a0eebc99-*" match a UUID-suffixed +// schema that itself had to be created quoted. // Double-quoted identifiers (e.g. "MySchema") are accepted as exact names; // wildcards are not allowed inside quotes. func validateSchemaPattern(s string) error { @@ -564,10 +559,6 @@ func validateSchemaPattern(s string) error { if strings.ContainsRune(s, '"') { return fmt.Errorf("unquoted schema pattern %q must not contain '\"'", s) } - first, _ := utf8.DecodeRuneInString(s) - if first != '_' && first != '*' && !unicode.IsLetter(first) { - return fmt.Errorf("schema pattern %q must start with a letter, underscore, or '*'", s) - } return nil } diff --git a/internal/impl/postgresql/input_pg_stream_test.go b/internal/impl/postgresql/input_pg_stream_test.go index 73a3874122..bce903c4b7 100644 --- a/internal/impl/postgresql/input_pg_stream_test.go +++ b/internal/impl/postgresql/input_pg_stream_test.go @@ -65,7 +65,12 @@ func TestSchemaPatternValidation(t *testing.T) { // Regression test: len("") == 2 used to pass the old `len(s) < 2` guard. // Fixed to `len(s) < 3`. {`""`, "invalid quoted schema identifier"}, - {"1abc", "must start with a letter"}, + // Regression test: a leading digit is not an identifier-syntax + // violation here - the pattern is compared via ILIKE, never spliced + // into an identifier position - so "1abc" must be as valid as any + // other unquoted pattern. See the "9c0b4ef8-*" case below for the + // motivating real-world scenario (a UUID-suffixed tenant schema). + {"1abc", ""}, {`"unclosed`, "invalid quoted schema identifier"}, // Regression test: an unquoted pattern is matched against stored // schema names, not parsed as an identifier, so hyphens (invalid in @@ -73,6 +78,13 @@ func TestSchemaPatternValidation(t *testing.T) { // match a UUID-suffixed tenant schema that had to be created quoted. {"schema-name", ""}, {"a0eebc99-*", ""}, + // Regression test: most UUIDs begin with a hex digit, so a tenant + // schema named e.g. "9c0b4ef8-bb6d-6bb9-bd38-0a11a0eebc99" (created + // quoted, per the a0eebc99-* case above) must be matchable by an + // unquoted glob starting with a digit - the pattern is compared via + // ILIKE, never spliced into an identifier position, so there's no + // syntactic reason to require a letter/underscore/'*' lead-in. + {"9c0b4ef8-*", ""}, {`"quoted*"`, "wildcard"}, {`a"b`, `must not contain '"'`}, } From bbf65cdb27b60c0bbd59e1c529daaa2dfe6de538 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Wed, 12 Aug 2026 16:47:05 +0100 Subject: [PATCH 32/70] postgres_cdc: remove now redundant subtests --- internal/impl/postgresql/input_pg_stream_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/impl/postgresql/input_pg_stream_test.go b/internal/impl/postgresql/input_pg_stream_test.go index bce903c4b7..fe682e4cb9 100644 --- a/internal/impl/postgresql/input_pg_stream_test.go +++ b/internal/impl/postgresql/input_pg_stream_test.go @@ -155,9 +155,9 @@ func TestExcludeSchemasValidation(t *testing.T) { {"tenant_test", ""}, {"tenant_test_*", ""}, {`"MySchema"`, ""}, - {"1abc", "must start with a letter"}, + {"1abc", ""}, {`"unclosed`, "invalid quoted schema identifier"}, - {"schema-name", "invalid character"}, + {"schema-name", ""}, {`"quoted*"`, "wildcard"}, } for _, tt := range tests { From 715e23336d3ba2fe8c2997fb5a678e9c9bfdd2b5 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Wed, 12 Aug 2026 16:58:32 +0100 Subject: [PATCH 33/70] postgres_cdc: ensure we don't discover views --- internal/impl/postgresql/integration_test.go | 86 +++++++++++++++++++ .../pglogicalstream/schema_resolver.go | 21 +++-- 2 files changed, 98 insertions(+), 9 deletions(-) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index a795c54b5e..b91c99205c 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -2062,6 +2062,92 @@ postgres_cdc: assert.Equal(t, "events", collected[0].table) } +// TestIntegrationMultiSchemaViewNamesakeDegradesGracefully guards against a +// relation that exists but isn't publishable: tenant_c matches the schema +// glob and has an "events" view (e.g. a compatibility shim over a renamed +// table), not the "events" table configured. Before restricting +// resolveExistingTables to table_type = 'BASE TABLE', this view counted as +// present, so CreatePublication's FOR TABLE clause referenced it and failed +// setup for every matched schema with "... is not supported for views". +// tenant_c's view should instead be skipped with a warning, the same way a +// genuinely missing table is, leaving tenant_a free to stream. +func TestIntegrationMultiSchemaViewNamesakeDegradesGracefully(t *testing.T) { + integration.CheckSkip(t) + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + _, err = db.Exec("CREATE SCHEMA tenant_a") + require.NoError(t, err) + _, err = db.Exec("CREATE TABLE tenant_a.events (id SERIAL PRIMARY KEY, name TEXT)") + require.NoError(t, err) + _, err = db.Exec("CREATE SCHEMA tenant_c") + require.NoError(t, err) + _, err = db.Exec("CREATE VIEW tenant_c.events AS SELECT 1 AS id, 'namesake'::text AS name") + require.NoError(t, err) + + _, err = db.Exec("INSERT INTO tenant_a.events (name) VALUES ('alice')") + require.NoError(t, err) + + type msgMeta struct { + dbSchema string + table string + } + + var ( + mu sync.Mutex + collected []msgMeta + ) + + tmpl := fmt.Sprintf(` +postgres_cdc: + dsn: %s + slot_name: view_namesake_degrade_slot + stream_snapshot: true + schema_pattern: tenant_* + tables: + - events +`, databaseURL) + + sb := service.NewStreamBuilder() + require.NoError(t, sb.SetLoggerYAML(`level: WARN`)) + require.NoError(t, sb.AddInputYAML(tmpl)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + mu.Lock() + defer mu.Unlock() + for _, msg := range batch { + m := msgMeta{} + m.dbSchema, _ = msg.MetaGet("database_schema") + m.table, _ = msg.MetaGet("table") + collected = append(collected, m) + } + return nil + })) + + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + go func() { + if err := stream.Run(t.Context()); err != nil && !errors.Is(err, context.Canceled) { + t.Error(err) + } + }() + t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) + + // tenant_a should keep streaming even though tenant_c's "events" is a + // view rather than a publishable table. + assert.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return len(collected) >= 1 + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for tenant_a snapshot row; a view namesake in tenant_c should not block replication or fail publication setup") + + mu.Lock() + defer mu.Unlock() + require.Len(t, collected, 1) + assert.Equal(t, "tenant_a", collected[0].dbSchema) + assert.Equal(t, "events", collected[0].table) +} + func TestIntegrationNoSchemasMatchedReturnsError(t *testing.T) { integration.CheckSkip(t) databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index da3958a73f..668314767e 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -130,15 +130,18 @@ func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (v return schemas, hidden, nil } -// resolveExistingTables returns the set of quoted table identifiers that -// actually exist in the given (already quoted) schema. +// resolveExistingTables returns the quoted names of the publishable base +// tables (including partitioned tables) that exist in the given (already +// quoted) schema. // -// This is used to resolve a schema glob × table list combination per-schema -// rather than assuming every matched schema contains every listed table. A -// schema matching the glob but missing one of the configured tables (e.g. a -// tenant schema that's still being provisioned) would otherwise cause -// CreatePublication's FOR TABLE clause to reference a non-existent relation, -// failing publication setup for every schema, not just the drifted one. +// Used to resolve a schema glob × table list combination per-schema, since a +// matched schema may not contain a configured table (e.g. still being +// provisioned) or may have a same-named view/foreign/temporary table +// instead. table_type is restricted to 'BASE TABLE' so either case is +// treated as missing and falls through to the caller's "not found, skipping" +// warning - otherwise CreatePublication's FOR TABLE clause would reference +// an unpublishable relation and fail setup for every matched schema, not +// just the drifted one. func resolveExistingTables(ctx context.Context, conn *pgconn.PgConn, quotedSchema string) (map[string]struct{}, error) { schema, err := sanitize.UnquotePostgresIdentifier(quotedSchema) if err != nil { @@ -146,7 +149,7 @@ func resolveExistingTables(ctx context.Context, conn *pgconn.PgConn, quotedSchem } q, err := sanitize.SQLQuery( - "SELECT table_name FROM information_schema.tables WHERE table_schema = $1", + "SELECT table_name FROM information_schema.tables WHERE table_schema = $1 AND table_type = 'BASE TABLE'", schema, ) if err != nil { From 65b08a9d0b2a349688ac8a1724ab4e5ec8259e88 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Wed, 12 Aug 2026 17:08:13 +0100 Subject: [PATCH 34/70] postgres_cdc: require/assert switch --- internal/impl/postgresql/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index b91c99205c..dab3a68eda 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -2203,7 +2203,7 @@ postgres_cdc: // (rather than once upfront) so a row lands after startup completes. writer := asyncroutine.NewPeriodic(100*time.Millisecond, func() { _, err := db.Exec("INSERT INTO flights (name, created_at) VALUES ('alice', now());") - require.NoError(t, err) + assert.NoError(t, err) }) writer.Start() t.Cleanup(writer.Stop) From 6079ef8fa7fe95ecaf54e80b2f2a781d809f5174 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Wed, 12 Aug 2026 17:14:38 +0100 Subject: [PATCH 35/70] postgres_cdc: update docs --- docs/modules/components/pages/inputs/postgres_cdc.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index 67dd5f4dd0..f8c013b09d 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -202,7 +202,7 @@ Schema pattern matching runs once at pipeline startup. Schemas created after the If `tables` is non-empty and this pattern matches no schema in the database, startup fails with an error. This field has no effect when `tables` is left empty - see `tables` below. -This pattern is matched against schema names as stored in the database rather than parsed as an identifier, so unquoted patterns may contain characters that are not valid in an unquoted PostgreSQL identifier (e.g. `tenant-*` or `a0eebc99-*` to match a UUID-suffixed schema that had to be created with a quoted identifier). +This pattern can contain characters that wouldn't be allowed in an unquoted schema name, because it's only ever compared against the real name of each schema in the database - it doesn't have to be a valid name itself. For example, a schema literally named `a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11` (which must have been created using double quotes, since hyphens aren't allowed in an unquoted `CREATE SCHEMA` statement) can still be matched using the unquoted pattern `a0eebc99-*`. This field is mutually exclusive with `schema`; when set, it takes over schema resolution entirely and `schema` must be left at its default. From a74fdf7853790dab7cdfd26fa0bf25b0534547c8 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Thu, 13 Aug 2026 09:21:29 +0300 Subject: [PATCH 36/70] postgres_cdc: group tables/schema var declarations --- internal/impl/postgresql/pglogicalstream/logical_stream.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 55913cec6f..99439abd3f 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -98,8 +98,10 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { return nil, err } - var tables []TableFQN - var schema string + var ( + tables []TableFQN + schema string + ) if config.DBSchemaPattern != "" { if config.SignalTableName != "" { return nil, errors.New("signal_table_name is not supported when schema_pattern is set") From 24aa2cd59daff86deb00805d134cc8799a860956 Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Thu, 13 Aug 2026 09:43:06 +0300 Subject: [PATCH 37/70] postgres_cdc: auto-discover tables for schema_pattern when tables is empty --- .../components/pages/inputs/postgres_cdc.adoc | 10 +- internal/impl/postgresql/input_pg_stream.go | 12 +- internal/impl/postgresql/integration_test.go | 180 +++++++++++++++--- .../pglogicalstream/logical_stream.go | 47 +++-- 4 files changed, 200 insertions(+), 49 deletions(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index f8c013b09d..1539b17d55 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -145,7 +145,7 @@ When set to true, empty messages with operation types BEGIN and COMMIT are gener === `stream_snapshot` -When set to true, the plugin will first stream a snapshot of all existing data in the database before streaming changes. In order to use this the tables that are being snapshot MUST have a primary key set so that reading from the table can be parallelized. Note that this has no effect if `tables` is left empty, since the snapshot is only planned for tables listed there. +When set to true, the plugin will first stream a snapshot of all existing data in the database before streaming changes. In order to use this the tables that are being snapshot MUST have a primary key set so that reading from the table can be parallelized. Note that this has no effect if `tables` is left empty and `schema_pattern` is NOT set, since in that case the snapshot is only planned for tables listed in `tables`. When `schema_pattern` IS set, leaving `tables` empty auto-discovers tables to snapshot instead - see `tables` below - and every discovered table must have a primary key. *Type*: `bool` @@ -200,7 +200,7 @@ Double-quoted identifiers are treated as exact names and do not support wildcard Schema pattern matching runs once at pipeline startup. Schemas created after the pipeline starts will not be picked up until the pipeline is restarted. -If `tables` is non-empty and this pattern matches no schema in the database, startup fails with an error. This field has no effect when `tables` is left empty - see `tables` below. +If this pattern matches no schema in the database, startup fails with an error - this holds whether or not `tables` is set. See `tables` below for what happens when it's left empty. This pattern can contain characters that wouldn't be allowed in an unquoted schema name, because it's only ever compared against the real name of each schema in the database - it doesn't have to be a valid name itself. For example, a schema literally named `a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11` (which must have been created using double quotes, since hyphens aren't allowed in an unquoted `CREATE SCHEMA` statement) can still be matched using the unquoted pattern `a0eebc99-*`. @@ -229,6 +229,8 @@ Each entry uses the same syntax as `schema_pattern`: an exact schema name, a glo A schema that matches `schema_pattern` and also matches any entry in this list is excluded from replication. An entry that does not match any schema resolved by `schema_pattern` is silently ignored, so a typo here simply excludes nothing rather than failing startup. +This exclusion is applied before `tables` is resolved, so it also takes effect when `tables` is left empty and tables are auto-discovered. + *Type*: `array` @@ -248,7 +250,9 @@ A list of table names to include in the logical replication. Each table should b When `schema_pattern` is set, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. -If left empty, the underlying PostgreSQL publication is created `FOR ALL TABLES`, which replicates every table in every schema of the database, ignoring `schema`. This also disables `stream_snapshot`, since the initial snapshot is only planned for tables listed here. +If left empty while `schema_pattern` is set, every base table in each matched (and un-excluded, see `exclude_schemas`) schema is auto-discovered and published explicitly, instead of listing tables by hand - this is the expected way to replicate "every table" in a multi-tenant, schema-per-tenant setup without also picking up unrelated schemas. Startup fails if no matched schema contains any table. + +If left empty while `schema_pattern` is NOT set, the underlying PostgreSQL publication is instead created `FOR ALL TABLES`, which replicates every table in every schema of the database, ignoring `schema`. This also disables `stream_snapshot`, since the initial snapshot is only planned for tables listed here. *Type*: `array` diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 05fdf13420..9ffda49895 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -104,7 +104,7 @@ This input adds the following metadata fields to each message: ShortDescription("Emit empty BEGIN and COMMIT messages at the start and end of each transaction."). Default(false)). Field(service.NewBoolField(fieldStreamSnapshot). - Description("When set to true, the plugin will first stream a snapshot of all existing data in the database before streaming changes. In order to use this the tables that are being snapshot MUST have a primary key set so that reading from the table can be parallelized. Note that this has no effect if `" + fieldTables + "` is left empty, since the snapshot is only planned for tables listed there."). + Description("When set to true, the plugin will first stream a snapshot of all existing data in the database before streaming changes. In order to use this the tables that are being snapshot MUST have a primary key set so that reading from the table can be parallelized. Note that this has no effect if `" + fieldTables + "` is left empty and `" + fieldSchemaPattern + "` is NOT set, since in that case the snapshot is only planned for tables listed in `" + fieldTables + "`. When `" + fieldSchemaPattern + "` IS set, leaving `" + fieldTables + "` empty auto-discovers tables to snapshot instead - see `" + fieldTables + "` below - and every discovered table must have a primary key."). ShortDescription("Stream a snapshot of all existing data before streaming changes. Snapshot tables must have a primary key."). Example(true). Default(false)). @@ -133,7 +133,7 @@ Double-quoted identifiers are treated as exact names and do not support wildcard Schema pattern matching runs once at pipeline startup. Schemas created after the pipeline starts will not be picked up until the pipeline is restarted. -If `+"`"+fieldTables+"`"+` is non-empty and this pattern matches no schema in the database, startup fails with an error. This field has no effect when `+"`"+fieldTables+"`"+` is left empty - see `+"`"+fieldTables+"`"+` below. +If this pattern matches no schema in the database, startup fails with an error - this holds whether or not `+"`"+fieldTables+"`"+` is set. See `+"`"+fieldTables+"`"+` below for what happens when it's left empty. This pattern can contain characters that wouldn't be allowed in an unquoted schema name, because it's only ever compared against the real name of each schema in the database - it doesn't have to be a valid name itself. For example, a schema literally named `+"`a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11`"+` (which must have been created using double quotes, since hyphens aren't allowed in an unquoted `+"`CREATE SCHEMA`"+` statement) can still be matched using the unquoted pattern `+"`a0eebc99-*`"+`. @@ -147,7 +147,9 @@ This field is mutually exclusive with `+"`"+fieldSchema+"`"+`; when set, it take Each entry uses the same syntax as ` + "`" + fieldSchemaPattern + "`" + `: an exact schema name, a glob pattern using ` + "`*`" + ` as a wildcard, or a double-quoted exact identifier for an exact, case-sensitive match. -A schema that matches ` + "`" + fieldSchemaPattern + "`" + ` and also matches any entry in this list is excluded from replication. An entry that does not match any schema resolved by ` + "`" + fieldSchemaPattern + "`" + ` is silently ignored, so a typo here simply excludes nothing rather than failing startup.`). +A schema that matches ` + "`" + fieldSchemaPattern + "`" + ` and also matches any entry in this list is excluded from replication. An entry that does not match any schema resolved by ` + "`" + fieldSchemaPattern + "`" + ` is silently ignored, so a typo here simply excludes nothing rather than failing startup. + +This exclusion is applied before ` + "`" + fieldTables + "`" + ` is resolved, so it also takes effect when ` + "`" + fieldTables + "`" + ` is left empty and tables are auto-discovered.`). Examples([]string{"tenant_internal", "tenant_test_*"}). Optional(). Default([]string{}), @@ -157,7 +159,9 @@ A schema that matches ` + "`" + fieldSchemaPattern + "`" + ` and also matches an When ` + "`schema_pattern`" + ` is set, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. -If left empty, the underlying PostgreSQL publication is created ` + "`FOR ALL TABLES`" + `, which replicates every table in every schema of the database, ignoring ` + "`" + fieldSchema + "`" + `. This also disables ` + "`" + fieldStreamSnapshot + "`" + `, since the initial snapshot is only planned for tables listed here.`). +If left empty while ` + "`schema_pattern`" + ` is set, every base table in each matched (and un-excluded, see ` + "`" + fieldExcludeSchemas + "`" + `) schema is auto-discovered and published explicitly, instead of listing tables by hand - this is the expected way to replicate "every table" in a multi-tenant, schema-per-tenant setup without also picking up unrelated schemas. Startup fails if no matched schema contains any table. + +If left empty while ` + "`schema_pattern`" + ` is NOT set, the underlying PostgreSQL publication is instead created ` + "`FOR ALL TABLES`" + `, which replicates every table in every schema of the database, ignoring ` + "`" + fieldSchema + "`" + `. This also disables ` + "`" + fieldStreamSnapshot + "`" + `, since the initial snapshot is only planned for tables listed here.`). Example([]string{"my_table_1", `"MyCaseSensitiveTableNeedingQuotes"`})). Field(service.NewIntField(fieldCheckpointLimit). Description("The maximum number of messages that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level. Any given LSN will not be acknowledged unless all messages under that offset are delivered in order to preserve at least once delivery guarantees."). diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index dab3a68eda..07bcf48958 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -2182,36 +2182,99 @@ tables: assert.Contains(t, err.Error(), "no schemas found matching pattern") } -// TestIntegrationForAllTablesIgnoresNonMatchingSchemaPattern guards the -// documented behaviour of FOR ALL TABLES mode: when `tables` is left empty, -// `schema` has no effect and must not block startup even if it matches no -// schema in the database. -func TestIntegrationForAllTablesIgnoresNonMatchingSchemaPattern(t *testing.T) { +// TestIntegrationSchemaPatternNonMatchingFailsEvenWithEmptyTables guards the +// fixed behaviour: schema_pattern always scopes replication, even when +// `tables` is left empty. A schema_pattern matching nothing in the database +// must fail startup rather than silently falling back to a database-wide +// FOR ALL TABLES publication (the old behaviour, which made exclude_schemas +// meaningless whenever tables was left unset). +func TestIntegrationSchemaPatternNonMatchingFailsEvenWithEmptyTables(t *testing.T) { integration.CheckSkip(t) - databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") require.NoError(t, err) tmpl := fmt.Sprintf(` -postgres_cdc: - dsn: %s - slot_name: for_all_tables_ignores_schema_slot - schema_pattern: nonexistent_schema_zzz_* +dsn: %s +slot_name: no_schema_match_empty_tables_slot +schema_pattern: nonexistent_schema_zzz_* `, databaseURL) - // FOR ALL TABLES mode disables the initial snapshot, so replication only - // sees rows written after the slot/publication exist. Insert continuously - // (rather than once upfront) so a row lands after startup completes. - writer := asyncroutine.NewPeriodic(100*time.Millisecond, func() { - _, err := db.Exec("INSERT INTO flights (name, created_at) VALUES ('alice', now());") - assert.NoError(t, err) - }) - writer.Start() - t.Cleanup(writer.Stop) + conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) + require.NoError(t, err) + + mgr := service.MockResources() + license.InjectTestService(mgr) + + input, err := newPgStreamInput(conf, mgr) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + + // Bypass the benthos AsyncReader's infinite connect-retry loop, same as + // TestIntegrationNoSchemasMatchedReturnsError. + err = input.Connect(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "no schemas found matching pattern") +} + +// TestIntegrationMultiSchemaAutoDiscoverExcludeSchemas is +// TestIntegrationMultiSchemaExcludeSchemas with `tables` left unset: it +// verifies that leaving `tables` empty under schema_pattern auto-discovers +// the "events" table in each matched schema instead of falling back to a +// database-wide FOR ALL TABLES publication, so exclude_schemas still carves +// tenant_c out of both the snapshot and CDC. +func TestIntegrationMultiSchemaAutoDiscoverExcludeSchemas(t *testing.T) { + integration.CheckSkip(t) + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + // Three tenant schemas match tenant_*; tenant_c is carved out via exclude_schemas. + for _, schema := range []string{"tenant_a", "tenant_b", "tenant_c"} { + _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", schema)) + require.NoError(t, err) + _, err = db.Exec(fmt.Sprintf( + "CREATE TABLE %s.events (id SERIAL PRIMARY KEY, name TEXT)", schema)) + require.NoError(t, err) + } + + // Pre-load snapshot data, including a row in the excluded schema that must + // never surface. + _, err = db.Exec("INSERT INTO tenant_a.events (name) VALUES ('alice'), ('bob')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_b.events (name) VALUES ('carol')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_c.events (name) VALUES ('mallory')") + require.NoError(t, err) + + type msgMeta struct { + dbSchema string + table string + operation string + lsn string + } var ( mu sync.Mutex - collected []string + collected []msgMeta ) + collectedLen := func() int { + mu.Lock() + defer mu.Unlock() + return len(collected) + } + + // No `tables` field: every base table in tenant_a/tenant_b must be + // auto-discovered without listing "events" by hand. + tmpl := fmt.Sprintf(` +postgres_cdc: + dsn: %s + slot_name: auto_discover_exclude_schemas_test_slot + stream_snapshot: true + schema_pattern: tenant_* + exclude_schemas: + - tenant_c +`, databaseURL) sb := service.NewStreamBuilder() require.NoError(t, sb.SetLoggerYAML(`level: WARN`)) @@ -2220,8 +2283,12 @@ postgres_cdc: mu.Lock() defer mu.Unlock() for _, msg := range batch { - table, _ := msg.MetaGet("table") - collected = append(collected, table) + m := msgMeta{} + m.dbSchema, _ = msg.MetaGet("database_schema") + m.table, _ = msg.MetaGet("table") + m.operation, _ = msg.MetaGet("operation") + m.lsn, _ = msg.MetaGet("lsn") + collected = append(collected, m) } return nil })) @@ -2236,17 +2303,72 @@ postgres_cdc: }() t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) - // Non-matching schema pattern must not block FOR ALL TABLES replication: - // the "flights" insert above should still arrive. + // Wait for the 3 snapshot rows from the two non-excluded schemas; tenant_c's + // row must never contribute to this count. assert.Eventually(t, func() bool { - mu.Lock() - defer mu.Unlock() - return len(collected) >= 1 - }, 30*time.Second, 100*time.Millisecond, "timed out waiting for FOR ALL TABLES replication; a non-matching schema pattern should be ignored when tables is empty") + return collectedLen() >= 3 + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for snapshot rows") + + // Insert CDC rows into all three schemas, including the excluded one. + _, err = db.Exec("INSERT INTO tenant_a.events (name) VALUES ('dave')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_b.events (name) VALUES ('eve')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_c.events (name) VALUES ('trudy')") + require.NoError(t, err) + + // Wait for the 2 CDC rows from the non-excluded schemas (total 5). + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.Equal(c, 5, collectedLen()) + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for CDC rows") + + // tenant_c's CDC insert above raced the same replication stream as the + // tenant_a/tenant_b inserts already confirmed above, so if it were going + // to leak through it would have by now; assert the count never climbs + // past 5 to catch a delayed leak instead of just checking once. + assert.Never(t, func() bool { + return collectedLen() > 5 + }, 3*time.Second, 200*time.Millisecond, "received unexpected message(s) from excluded schema tenant_c") mu.Lock() defer mu.Unlock() - assert.Contains(t, collected, "flights") + + require.Len(t, collected, 5) + for _, m := range collected { + assert.NotEqual(t, "tenant_c", m.dbSchema, "tenant_c is excluded and must never appear, got message: %+v", m) + } + + var snapshots, cdcMsgs []msgMeta + for _, m := range collected { + if m.operation == "read" { + snapshots = append(snapshots, m) + } else { + cdcMsgs = append(cdcMsgs, m) + } + } + + // Snapshot assertions. + require.Len(t, snapshots, 3) + snapshotSchemas := make(map[string]int) + for _, m := range snapshots { + assert.Equal(t, "events", m.table, "snapshot: table should be bare name without schema prefix") + assert.Empty(t, m.lsn, "snapshot rows have no LSN") + snapshotSchemas[m.dbSchema]++ + } + assert.Equal(t, 2, snapshotSchemas["tenant_a"], "expected 2 snapshot rows from tenant_a") + assert.Equal(t, 1, snapshotSchemas["tenant_b"], "expected 1 snapshot row from tenant_b") + + // CDC assertions. + require.Len(t, cdcMsgs, 2) + cdcSchemas := make(map[string]int) + for _, m := range cdcMsgs { + assert.Equal(t, "insert", m.operation) + assert.Equal(t, "events", m.table) + assert.NotEmpty(t, m.lsn, "CDC rows must have an LSN") + cdcSchemas[m.dbSchema]++ + } + assert.Equal(t, 1, cdcSchemas["tenant_a"], "expected 1 CDC row from tenant_a") + assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") } func TestIntegrationSchemaAndTableMatchingTest(t *testing.T) { diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 99439abd3f..29b5c47899 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -140,10 +140,10 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { schemas = remaining } - if len(inaccessibleSchemas) > 0 && len(config.DBTables) > 0 { + if len(inaccessibleSchemas) > 0 { config.Logger.Warnf("schema pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaPattern, inaccessibleSchemas) } - if len(schemas) == 0 && len(config.DBTables) > 0 { + if len(schemas) == 0 { return nil, fmt.Errorf("no schemas found matching pattern %q", config.DBSchemaPattern) } config.Logger.Infof("Schema pattern %q resolved to %d schema(s): %v", config.DBSchemaPattern, len(schemas), schemas) @@ -157,6 +157,14 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { normalizedTables = append(normalizedTables, normalized) } + // With no explicit table list, auto-discover every base table in each + // matched (and exclude_schemas-filtered) schema and publish them + // explicitly. Without this, an empty tables list would fall through to + // CreatePublication's FOR ALL TABLES fallback below, which replicates + // every schema in the database and silently defeats both schema_pattern + // and exclude_schemas. + autoDiscoverTables := len(normalizedTables) == 0 + tables = make([]TableFQN, 0, len(schemas)*len(normalizedTables)) foundTables := make(map[string]bool, len(normalizedTables)) for _, schema := range schemas { @@ -164,6 +172,12 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { if err != nil { return nil, fmt.Errorf("resolving tables in schema %q: %w", schema, err) } + if autoDiscoverTables { + for table := range existingTables { + tables = append(tables, TableFQN{Schema: schema, Table: table}) + } + continue + } for _, table := range normalizedTables { if _, ok := existingTables[table]; !ok { config.Logger.Warnf("table %s.%s not found, skipping (schema %s matched pattern %q but does not contain this table)", schema, table, schema, config.DBSchemaPattern) @@ -173,18 +187,25 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { foundTables[table] = true } } - // A table must exist in at least one matched schema. Missing from some - // (but not all) matched schemas is tolerated above as a multi-tenant gap; - // missing from every matched schema is indistinguishable from a typo and - // must fail loudly rather than silently drop the table. - var missingTables []string - for i, table := range normalizedTables { - if !foundTables[table] { - missingTables = append(missingTables, config.DBTables[i]) + if autoDiscoverTables { + if len(tables) == 0 { + return nil, fmt.Errorf("no tables found in schema(s) %v matching pattern %q", schemas, config.DBSchemaPattern) + } + config.Logger.Infof("%q has no `tables` list configured: auto-discovered %d table(s) across %d schema(s)", config.DBSchemaPattern, len(tables), len(schemas)) + } else { + // A table must exist in at least one matched schema. Missing from some + // (but not all) matched schemas is tolerated above as a multi-tenant gap; + // missing from every matched schema is indistinguishable from a typo and + // must fail loudly rather than silently drop the table. + var missingTables []string + for i, table := range normalizedTables { + if !foundTables[table] { + missingTables = append(missingTables, config.DBTables[i]) + } + } + if len(missingTables) > 0 { + return nil, fmt.Errorf("table(s) %v not found in any schema matching pattern %q", missingTables, config.DBSchemaPattern) } - } - if len(missingTables) > 0 { - return nil, fmt.Errorf("table(s) %v not found in any schema matching pattern %q", missingTables, config.DBSchemaPattern) } } else { var err error From 96b8d78fbb9080358bea7773e0f2e444f4012d8a Mon Sep 17 00:00:00 2001 From: ness-david-dedu Date: Thu, 13 Aug 2026 10:10:04 +0300 Subject: [PATCH 38/70] postgres_cdc: rename schema_pattern/exclude_schemas to schema_include/schema_exclude --- CHANGELOG.md | 4 +- .../components/pages/inputs/postgres_cdc.adoc | 36 +++++----- internal/impl/postgresql/input_pg_stream.go | 66 ++++++++--------- .../impl/postgresql/input_pg_stream_test.go | 54 +++++++------- internal/impl/postgresql/integration_test.go | 72 +++++++++---------- .../impl/postgresql/pglogicalstream/config.go | 14 ++-- .../pglogicalstream/logical_stream.go | 36 +++++----- .../pglogicalstream/schema_resolver.go | 6 +- .../postgresql/tests/current/Taskfile.yaml | 20 +++--- .../impl/postgresql/tests/current/setup.sql | 6 +- .../postgresql/tests/current/test_config.yaml | 4 +- ...est_config_schema_exclude_no_include.yaml} | 6 +- 12 files changed, 162 insertions(+), 162 deletions(-) rename internal/impl/postgresql/tests/current/{test_config_exclude_no_pattern.yaml => test_config_schema_exclude_no_include.yaml} (76%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1624384605..3196a0dc19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,8 +92,8 @@ All notable changes to this project will be documented in this file. ### Added -- postgres_cdc: Added a new `schema_pattern` field accepting a glob pattern (e.g. `tenant_*`), replicating all matching schemas through a single replication slot. Useful for multi-tenant databases where each tenant has its own schema. The existing `schema` field is unaffected and continues to take a single exact schema name (defaulting to `public`); `schema` and `schema_pattern` are mutually exclusive. ([@ness-david-dedu](https://github.com/ness-david-dedu), [#4589](https://github.com/redpanda-data/connect/pull/4589)) -- postgres_cdc: Added a new `exclude_schemas` field to carve exceptions out of a broad `schema_pattern` (e.g. `schema_pattern: tenant_*` while skipping `tenant_test`). Accepts the same exact-name/glob/quoted syntax as `schema_pattern`, matches entries against the already-resolved schema list in memory with no extra database round-trips, and requires `schema_pattern` to be set. ([@ness-david-dedu](https://github.com/ness-david-dedu), [#4589](https://github.com/redpanda-data/connect/pull/4589)) +- postgres_cdc: Added a new `schema_include` field accepting a glob pattern (e.g. `tenant_*`), replicating all matching schemas through a single replication slot. Useful for multi-tenant databases where each tenant has its own schema. Leaving `tables` unset auto-discovers every table in each matched schema instead of listing them by hand. The existing `schema` field is unaffected and continues to take a single exact schema name (defaulting to `public`); `schema` and `schema_include` are mutually exclusive. ([@ness-david-dedu](https://github.com/ness-david-dedu), [#4589](https://github.com/redpanda-data/connect/pull/4589)) +- postgres_cdc: Added a new `schema_exclude` field to carve exceptions out of a broad `schema_include` (e.g. `schema_include: tenant_*` while skipping `tenant_test`). Accepts the same exact-name/glob/quoted syntax as `schema_include`, matches entries against the already-resolved schema list in memory with no extra database round-trips, and requires `schema_include` to be set. ([@ness-david-dedu](https://github.com/ness-david-dedu), [#4589](https://github.com/redpanda-data/connect/pull/4589)) - aws_dynamodb_cdc: DynamoDB CDC now supports an optional checkpoint_namespace field, allowing multiple independent pipelines to share a single checkpoint table without overwriting each other's checkpoints. ([@squiidz](https://github.com/squiidz), [#4602](https://github.com/redpanda-data/connect/pull/4602)) ### Fixed diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index 1539b17d55..3d4b3470f8 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -44,8 +44,8 @@ input: stream_snapshot: false snapshot_batch_size: 1000 schema: public - schema_pattern: "" - exclude_schemas: [] + schema_include: "" + schema_exclude: [] tables: [] # No default (required) checkpoint_limit: 1024 temporary_slot: false @@ -76,8 +76,8 @@ input: stream_snapshot: false snapshot_batch_size: 1000 schema: public - schema_pattern: "" - exclude_schemas: [] + schema_include: "" + schema_exclude: [] tables: [] # No default (required) checkpoint_limit: 1024 temporary_slot: false @@ -145,7 +145,7 @@ When set to true, empty messages with operation types BEGIN and COMMIT are gener === `stream_snapshot` -When set to true, the plugin will first stream a snapshot of all existing data in the database before streaming changes. In order to use this the tables that are being snapshot MUST have a primary key set so that reading from the table can be parallelized. Note that this has no effect if `tables` is left empty and `schema_pattern` is NOT set, since in that case the snapshot is only planned for tables listed in `tables`. When `schema_pattern` IS set, leaving `tables` empty auto-discovers tables to snapshot instead - see `tables` below - and every discovered table must have a primary key. +When set to true, the plugin will first stream a snapshot of all existing data in the database before streaming changes. In order to use this the tables that are being snapshot MUST have a primary key set so that reading from the table can be parallelized. Note that this has no effect if `tables` is left empty and `schema_include` is NOT set, since in that case the snapshot is only planned for tables listed in `tables`. When `schema_include` IS set, leaving `tables` empty auto-discovers tables to snapshot instead - see `tables` below - and every discovered table must have a primary key. *Type*: `bool` @@ -190,7 +190,7 @@ schema: public schema: '"MyCaseSensitiveSchemaNeedingQuotes"' ``` -=== `schema_pattern` +=== `schema_include` The PostgreSQL schema pattern to replicate data from. Accepts an exact schema name or a glob pattern using `*` as a wildcard to match multiple schemas. @@ -214,20 +214,20 @@ This field is mutually exclusive with `schema`; when set, it takes over schema r ```yml # Examples -schema_pattern: tenant_* +schema_include: tenant_* -schema_pattern: '*' +schema_include: '*' -schema_pattern: '"MyCaseSensitiveSchemaNeedingQuotes"' +schema_include: '"MyCaseSensitiveSchemaNeedingQuotes"' ``` -=== `exclude_schemas` +=== `schema_exclude` -A list of schema names or glob patterns to exclude from the schemas matched by `schema_pattern`. Only valid when `schema_pattern` is set. +A list of schema names or glob patterns to exclude from the schemas matched by `schema_include`. Only valid when `schema_include` is set. -Each entry uses the same syntax as `schema_pattern`: an exact schema name, a glob pattern using `*` as a wildcard, or a double-quoted exact identifier for an exact, case-sensitive match. +Each entry uses the same syntax as `schema_include`: an exact schema name, a glob pattern using `*` as a wildcard, or a double-quoted exact identifier for an exact, case-sensitive match. -A schema that matches `schema_pattern` and also matches any entry in this list is excluded from replication. An entry that does not match any schema resolved by `schema_pattern` is silently ignored, so a typo here simply excludes nothing rather than failing startup. +A schema that matches `schema_include` and also matches any entry in this list is excluded from replication. An entry that does not match any schema resolved by `schema_include` is silently ignored, so a typo here simply excludes nothing rather than failing startup. This exclusion is applied before `tables` is resolved, so it also takes effect when `tables` is left empty and tables are auto-discovered. @@ -239,7 +239,7 @@ This exclusion is applied before `tables` is resolved, so it also takes effect w ```yml # Examples -exclude_schemas: +schema_exclude: - tenant_internal - tenant_test_* ``` @@ -248,11 +248,11 @@ exclude_schemas: A list of table names to include in the logical replication. Each table should be specified as a separate item. -When `schema_pattern` is set, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. +When `schema_include` is set, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. -If left empty while `schema_pattern` is set, every base table in each matched (and un-excluded, see `exclude_schemas`) schema is auto-discovered and published explicitly, instead of listing tables by hand - this is the expected way to replicate "every table" in a multi-tenant, schema-per-tenant setup without also picking up unrelated schemas. Startup fails if no matched schema contains any table. +If left empty while `schema_include` is set, every base table in each matched (and un-excluded, see `schema_exclude`) schema is auto-discovered and published explicitly, instead of listing tables by hand - this is the expected way to replicate "every table" in a multi-tenant, schema-per-tenant setup without also picking up unrelated schemas. Startup fails if no matched schema contains any table. -If left empty while `schema_pattern` is NOT set, the underlying PostgreSQL publication is instead created `FOR ALL TABLES`, which replicates every table in every schema of the database, ignoring `schema`. This also disables `stream_snapshot`, since the initial snapshot is only planned for tables listed here. +If left empty while `schema_include` is NOT set, the underlying PostgreSQL publication is instead created `FOR ALL TABLES`, which replicates every table in every schema of the database, ignoring `schema`. This also disables `stream_snapshot`, since the initial snapshot is only planned for tables listed here. *Type*: `array` @@ -629,7 +629,7 @@ Optional external ID for the role assumption. === `signal_table_name` -The name of the table used to send control signals to the connector, excluding the schema. Not supported when `schema_pattern` is set, since there is no single schema to anchor the signal table to. The table must +The name of the table used to send control signals to the connector, excluding the schema. Not supported when `schema_include` is set, since there is no single schema to anchor the signal table to. The table must exist in the schema configured via the `schema` field, and must not also appear in `tables` — the signal table is implicitly added to the publication and excluded from snapshot scans, so listing it in both places is rejected at startup. It must have at least these columns — startup validation checks diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 9ffda49895..9472afb964 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -38,8 +38,8 @@ const ( fieldSnapshotMemSafetyFactor = "snapshot_memory_safety_factor" fieldSnapshotBatchSize = "snapshot_batch_size" fieldSchema = "schema" - fieldSchemaPattern = "schema_pattern" - fieldExcludeSchemas = "exclude_schemas" + fieldSchemaInclude = "schema_include" + fieldSchemaExclude = "schema_exclude" fieldTables = "tables" fieldCheckpointLimit = "checkpoint_limit" fieldTemporarySlot = "temporary_slot" @@ -104,7 +104,7 @@ This input adds the following metadata fields to each message: ShortDescription("Emit empty BEGIN and COMMIT messages at the start and end of each transaction."). Default(false)). Field(service.NewBoolField(fieldStreamSnapshot). - Description("When set to true, the plugin will first stream a snapshot of all existing data in the database before streaming changes. In order to use this the tables that are being snapshot MUST have a primary key set so that reading from the table can be parallelized. Note that this has no effect if `" + fieldTables + "` is left empty and `" + fieldSchemaPattern + "` is NOT set, since in that case the snapshot is only planned for tables listed in `" + fieldTables + "`. When `" + fieldSchemaPattern + "` IS set, leaving `" + fieldTables + "` empty auto-discovers tables to snapshot instead - see `" + fieldTables + "` below - and every discovered table must have a primary key."). + Description("When set to true, the plugin will first stream a snapshot of all existing data in the database before streaming changes. In order to use this the tables that are being snapshot MUST have a primary key set so that reading from the table can be parallelized. Note that this has no effect if `" + fieldTables + "` is left empty and `" + fieldSchemaInclude + "` is NOT set, since in that case the snapshot is only planned for tables listed in `" + fieldTables + "`. When `" + fieldSchemaInclude + "` IS set, leaving `" + fieldTables + "` empty auto-discovers tables to snapshot instead - see `" + fieldTables + "` below - and every discovered table must have a primary key."). ShortDescription("Stream a snapshot of all existing data before streaming changes. Snapshot tables must have a primary key."). Example(true). Default(false)). @@ -124,7 +124,7 @@ This input adds the following metadata fields to each message: Optional(). Default("public"), ). - Field(service.NewStringField(fieldSchemaPattern). + Field(service.NewStringField(fieldSchemaInclude). Description(`The PostgreSQL schema pattern to replicate data from. Accepts an exact schema name or a glob pattern using `+"`*`"+` as a wildcard to match multiple schemas. When a pattern is used, all schemas whose names match the pattern are replicated using a single replication slot and publication. This is useful for multi-tenant databases where each tenant has its own schema (e.g. `+"`tenant_*`"+` matches `+"`tenant_foo`"+`, `+"`tenant_bar`"+`, etc.). @@ -142,12 +142,12 @@ This field is mutually exclusive with `+"`"+fieldSchema+"`"+`; when set, it take Optional(). Default(""), ). - Field(service.NewStringListField(fieldExcludeSchemas). - Description(`A list of schema names or glob patterns to exclude from the schemas matched by ` + "`" + fieldSchemaPattern + "`" + `. Only valid when ` + "`" + fieldSchemaPattern + "`" + ` is set. + Field(service.NewStringListField(fieldSchemaExclude). + Description(`A list of schema names or glob patterns to exclude from the schemas matched by ` + "`" + fieldSchemaInclude + "`" + `. Only valid when ` + "`" + fieldSchemaInclude + "`" + ` is set. -Each entry uses the same syntax as ` + "`" + fieldSchemaPattern + "`" + `: an exact schema name, a glob pattern using ` + "`*`" + ` as a wildcard, or a double-quoted exact identifier for an exact, case-sensitive match. +Each entry uses the same syntax as ` + "`" + fieldSchemaInclude + "`" + `: an exact schema name, a glob pattern using ` + "`*`" + ` as a wildcard, or a double-quoted exact identifier for an exact, case-sensitive match. -A schema that matches ` + "`" + fieldSchemaPattern + "`" + ` and also matches any entry in this list is excluded from replication. An entry that does not match any schema resolved by ` + "`" + fieldSchemaPattern + "`" + ` is silently ignored, so a typo here simply excludes nothing rather than failing startup. +A schema that matches ` + "`" + fieldSchemaInclude + "`" + ` and also matches any entry in this list is excluded from replication. An entry that does not match any schema resolved by ` + "`" + fieldSchemaInclude + "`" + ` is silently ignored, so a typo here simply excludes nothing rather than failing startup. This exclusion is applied before ` + "`" + fieldTables + "`" + ` is resolved, so it also takes effect when ` + "`" + fieldTables + "`" + ` is left empty and tables are auto-discovered.`). Examples([]string{"tenant_internal", "tenant_test_*"}). @@ -157,11 +157,11 @@ This exclusion is applied before ` + "`" + fieldTables + "`" + ` is resolved, so Field(service.NewStringListField(fieldTables). Description(`A list of table names to include in the logical replication. Each table should be specified as a separate item. -When ` + "`schema_pattern`" + ` is set, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. +When ` + "`" + fieldSchemaInclude + "`" + ` is set, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. -If left empty while ` + "`schema_pattern`" + ` is set, every base table in each matched (and un-excluded, see ` + "`" + fieldExcludeSchemas + "`" + `) schema is auto-discovered and published explicitly, instead of listing tables by hand - this is the expected way to replicate "every table" in a multi-tenant, schema-per-tenant setup without also picking up unrelated schemas. Startup fails if no matched schema contains any table. +If left empty while ` + "`" + fieldSchemaInclude + "`" + ` is set, every base table in each matched (and un-excluded, see ` + "`" + fieldSchemaExclude + "`" + `) schema is auto-discovered and published explicitly, instead of listing tables by hand - this is the expected way to replicate "every table" in a multi-tenant, schema-per-tenant setup without also picking up unrelated schemas. Startup fails if no matched schema contains any table. -If left empty while ` + "`schema_pattern`" + ` is NOT set, the underlying PostgreSQL publication is instead created ` + "`FOR ALL TABLES`" + `, which replicates every table in every schema of the database, ignoring ` + "`" + fieldSchema + "`" + `. This also disables ` + "`" + fieldStreamSnapshot + "`" + `, since the initial snapshot is only planned for tables listed here.`). +If left empty while ` + "`" + fieldSchemaInclude + "`" + ` is NOT set, the underlying PostgreSQL publication is instead created ` + "`FOR ALL TABLES`" + `, which replicates every table in every schema of the database, ignoring ` + "`" + fieldSchema + "`" + `. This also disables ` + "`" + fieldStreamSnapshot + "`" + `, since the initial snapshot is only planned for tables listed here.`). Example([]string{"my_table_1", `"MyCaseSensitiveTableNeedingQuotes"`})). Field(service.NewIntField(fieldCheckpointLimit). Description("The maximum number of messages that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level. Any given LSN will not be acknowledged unless all messages under that offset are delivered in order to preserve at least once delivery guarantees."). @@ -251,7 +251,7 @@ This connector uses the naming pattern ` + "`pglog_stream_ 0 { - if schemaPattern == "" { - return nil, errors.New("exclude_schemas requires schema_pattern to be set") + if len(schemaExclude) > 0 { + if schemaInclude == "" { + return nil, errors.New("schema_exclude requires schema_include to be set") } - for i, pattern := range excludeSchemas { + for i, pattern := range schemaExclude { if err = validateSchemaPattern(pattern); err != nil { - return nil, fmt.Errorf("invalid exclude_schemas entry %q: %w", pattern, err) + return nil, fmt.Errorf("invalid schema_exclude entry %q: %w", pattern, err) } - // Normalize unquoted patterns to lower-case, mirroring schema_pattern + // Normalize unquoted patterns to lower-case, mirroring schema_include // above: PostgreSQL folds unquoted identifiers at creation time, so // TENANT_TEST and tenant_test resolve to the same schema. if !strings.HasPrefix(pattern, `"`) { - excludeSchemas[i] = strings.ToLower(pattern) + schemaExclude[i] = strings.ToLower(pattern) } } } @@ -437,8 +437,8 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser } if signalTableName != "" { - if schemaPattern != "" { - return nil, fmt.Errorf("%s is not supported when %s is set", fieldSignalTableName, fieldSchemaPattern) + if schemaInclude != "" { + return nil, fmt.Errorf("%s is not supported when %s is set", fieldSignalTableName, fieldSchemaInclude) } normalizedSignalTable, err := sanitize.NormalizePostgresIdentifier(signalTableName) if err != nil { @@ -492,8 +492,8 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser TLSConfig: pgConnConfig.TLSConfig, DBRawDSN: dsn, DBSchema: schema, - DBSchemaPattern: schemaPattern, - DBExcludeSchemas: excludeSchemas, + DBSchemaInclude: schemaInclude, + DBSchemaExclude: schemaExclude, DBTables: tables, RefreshAuthToken: iamAuthTokenBuilder, diff --git a/internal/impl/postgresql/input_pg_stream_test.go b/internal/impl/postgresql/input_pg_stream_test.go index fe682e4cb9..275b34fee4 100644 --- a/internal/impl/postgresql/input_pg_stream_test.go +++ b/internal/impl/postgresql/input_pg_stream_test.go @@ -44,12 +44,12 @@ tables: require.NoError(t, err) } -// TestSchemaPatternValidation verifies that the schema_pattern field is +// TestSchemaIncludeValidation verifies that the schema_include field is // validated during config parsing, before any network I/O is attempted. // Success is asserted via newPgStreamInput returning no error - the // constructor doesn't dial the database, so a valid pattern implies // validation passed. -func TestSchemaPatternValidation(t *testing.T) { +func TestSchemaIncludeValidation(t *testing.T) { tests := []struct { pattern string errContains string @@ -94,7 +94,7 @@ func TestSchemaPatternValidation(t *testing.T) { // quotes, e.g. `"MySchema"`) reaches validateSchemaPattern verbatim. yaml := fmt.Sprintf(` dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable -schema_pattern: '%s' +schema_include: '%s' slot_name: test_slot tables: - events @@ -111,30 +111,30 @@ tables: } } -// TestSchemaAndSchemaPatternMutuallyExclusive verifies that setting both -// schema (to a non-default value) and schema_pattern is rejected at config +// TestSchemaAndSchemaIncludeMutuallyExclusive verifies that setting both +// schema (to a non-default value) and schema_include is rejected at config // construction time. -func TestSchemaAndSchemaPatternMutuallyExclusive(t *testing.T) { +func TestSchemaAndSchemaIncludeMutuallyExclusive(t *testing.T) { yaml := ` dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable schema: tenant_foo -schema_pattern: 'tenant_*' +schema_include: 'tenant_*' slot_name: test_slot tables: - events ` _, err := parsePgStreamInput(t, yaml) require.Error(t, err) - assert.Contains(t, err.Error(), "schema and schema_pattern are mutually exclusive") + assert.Contains(t, err.Error(), "schema and schema_include are mutually exclusive") } -// TestSchemaPatternWithDefaultSchemaSucceeds verifies that setting -// schema_pattern while leaving schema untouched (at its "public" default) is +// TestSchemaIncludeWithDefaultSchemaSucceeds verifies that setting +// schema_include while leaving schema untouched (at its "public" default) is // allowed. -func TestSchemaPatternWithDefaultSchemaSucceeds(t *testing.T) { +func TestSchemaIncludeWithDefaultSchemaSucceeds(t *testing.T) { yaml := ` dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable -schema_pattern: 'tenant_*' +schema_include: 'tenant_*' slot_name: test_slot tables: - events @@ -143,11 +143,11 @@ tables: require.NoError(t, err) } -// TestExcludeSchemasValidation verifies that each exclude_schemas entry is -// validated with the same rules as schema_pattern - validateSchemaPattern is +// TestSchemaExcludeValidation verifies that each schema_exclude entry is +// validated with the same rules as schema_include - validateSchemaPattern is // reused rather than re-derived, so this exercises the same error cases -// TestSchemaPatternValidation covers, just reached through a different field. -func TestExcludeSchemasValidation(t *testing.T) { +// TestSchemaIncludeValidation covers, just reached through a different field. +func TestSchemaExcludeValidation(t *testing.T) { tests := []struct { pattern string errContains string @@ -166,8 +166,8 @@ func TestExcludeSchemasValidation(t *testing.T) { // quotes, e.g. `"MySchema"`) reaches validateSchemaPattern verbatim. yaml := fmt.Sprintf(` dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable -schema_pattern: 'tenant_*' -exclude_schemas: ['%s'] +schema_include: 'tenant_*' +schema_exclude: ['%s'] slot_name: test_slot tables: - events @@ -184,28 +184,28 @@ tables: } } -// TestExcludeSchemasRequiresSchemaPattern verifies that exclude_schemas is -// rejected at config-parse time when schema_pattern is left unset. Both +// TestSchemaExcludeRequiresSchemaInclude verifies that schema_exclude is +// rejected at config-parse time when schema_include is left unset. Both // single-exact-schema mode and FOR ALL TABLES mode (empty tables) have no // well-defined candidate set to exclude from, so this is a hard error rather // than a silent no-op. -func TestExcludeSchemasRequiresSchemaPattern(t *testing.T) { +func TestSchemaExcludeRequiresSchemaInclude(t *testing.T) { yaml := ` dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable -exclude_schemas: [tenant_test] +schema_exclude: [tenant_test] slot_name: test_slot tables: - events ` _, err := parsePgStreamInput(t, yaml) require.Error(t, err) - assert.Contains(t, err.Error(), "exclude_schemas requires schema_pattern to be set") + assert.Contains(t, err.Error(), "schema_exclude requires schema_include to be set") } -// TestExcludeSchemasEmptyWithoutSchemaPatternSucceeds verifies that leaving -// exclude_schemas at its default empty list does not trip the -// requires-schema_pattern check, since there's nothing to exclude. -func TestExcludeSchemasEmptyWithoutSchemaPatternSucceeds(t *testing.T) { +// TestSchemaExcludeEmptyWithoutSchemaIncludeSucceeds verifies that leaving +// schema_exclude at its default empty list does not trip the +// requires-schema_include check, since there's nothing to exclude. +func TestSchemaExcludeEmptyWithoutSchemaIncludeSucceeds(t *testing.T) { yaml := ` dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable slot_name: test_slot diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index 07bcf48958..b88c2d6e8a 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1658,7 +1658,7 @@ postgres_cdc: dsn: %s slot_name: multi_schema_test_slot stream_snapshot: true - schema_pattern: tenant_* + schema_include: tenant_* tables: - events `, databaseURL) @@ -1746,17 +1746,17 @@ postgres_cdc: assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") } -// TestIntegrationMultiSchemaExcludeSchemas verifies that exclude_schemas -// carves an exception out of a broad schema_pattern: a schema that matches -// schema_pattern but also matches an exclude_schemas entry contributes no +// TestIntegrationSchemaExcludeCarvesOutTenant verifies that schema_exclude +// carves an exception out of a broad schema_include: a schema that matches +// schema_include but also matches a schema_exclude entry contributes no // rows at all, neither during the initial snapshot nor from subsequent CDC // changes. -func TestIntegrationMultiSchemaExcludeSchemas(t *testing.T) { +func TestIntegrationSchemaExcludeCarvesOutTenant(t *testing.T) { integration.CheckSkip(t) databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") require.NoError(t, err) - // Three tenant schemas match tenant_*; tenant_c is carved out via exclude_schemas. + // Three tenant schemas match tenant_*; tenant_c is carved out via schema_exclude. for _, schema := range []string{"tenant_a", "tenant_b", "tenant_c"} { _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", schema)) require.NoError(t, err) @@ -1794,10 +1794,10 @@ func TestIntegrationMultiSchemaExcludeSchemas(t *testing.T) { tmpl := fmt.Sprintf(` postgres_cdc: dsn: %s - slot_name: exclude_schemas_test_slot + slot_name: schema_exclude_test_slot stream_snapshot: true - schema_pattern: tenant_* - exclude_schemas: + schema_include: tenant_* + schema_exclude: - tenant_c tables: - events @@ -1898,7 +1898,7 @@ postgres_cdc: assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") } -func TestIntegrationSchemaPatternMatchesHyphenatedUUIDSchema(t *testing.T) { +func TestIntegrationSchemaIncludeMatchesHyphenatedUUIDSchema(t *testing.T) { integration.CheckSkip(t) databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") require.NoError(t, err) @@ -1930,9 +1930,9 @@ func TestIntegrationSchemaPatternMatchesHyphenatedUUIDSchema(t *testing.T) { tmpl := fmt.Sprintf(` postgres_cdc: dsn: %s - slot_name: hyphenated_schema_pattern_slot + slot_name: hyphenated_schema_include_slot stream_snapshot: true - schema_pattern: a0eebc99-* + schema_include: a0eebc99-* tables: - events `, databaseURL) @@ -2018,7 +2018,7 @@ postgres_cdc: dsn: %s slot_name: missing_table_degrade_slot stream_snapshot: true - schema_pattern: tenant_* + schema_include: tenant_* tables: - events `, databaseURL) @@ -2103,7 +2103,7 @@ postgres_cdc: dsn: %s slot_name: view_namesake_degrade_slot stream_snapshot: true - schema_pattern: tenant_* + schema_include: tenant_* tables: - events `, databaseURL) @@ -2156,7 +2156,7 @@ func TestIntegrationNoSchemasMatchedReturnsError(t *testing.T) { tmpl := fmt.Sprintf(` dsn: %s slot_name: no_schema_match_slot -schema_pattern: nonexistent_schema_zzz_* +schema_include: nonexistent_schema_zzz_* tables: - events `, databaseURL) @@ -2174,21 +2174,21 @@ tables: defer cancel() // Bypass the benthos AsyncReader's infinite connect-retry loop by calling - // Connect directly: a schema-pattern-not-found error is permanent, but + // Connect directly: a schema-include-not-found error is permanent, but // stream.Run has no path to surface it (it only returns once ctx is done), // so going through StreamBuilder/Run here would just time out instead. err = input.Connect(ctx) require.Error(t, err) - assert.Contains(t, err.Error(), "no schemas found matching pattern") + assert.Contains(t, err.Error(), "no schemas found matching schema_include pattern") } -// TestIntegrationSchemaPatternNonMatchingFailsEvenWithEmptyTables guards the -// fixed behaviour: schema_pattern always scopes replication, even when -// `tables` is left empty. A schema_pattern matching nothing in the database +// TestIntegrationSchemaIncludeNonMatchingFailsEvenWithEmptyTables guards the +// fixed behaviour: schema_include always scopes replication, even when +// `tables` is left empty. A schema_include matching nothing in the database // must fail startup rather than silently falling back to a database-wide -// FOR ALL TABLES publication (the old behaviour, which made exclude_schemas +// FOR ALL TABLES publication (the old behaviour, which made schema_exclude // meaningless whenever tables was left unset). -func TestIntegrationSchemaPatternNonMatchingFailsEvenWithEmptyTables(t *testing.T) { +func TestIntegrationSchemaIncludeNonMatchingFailsEvenWithEmptyTables(t *testing.T) { integration.CheckSkip(t) databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") require.NoError(t, err) @@ -2196,7 +2196,7 @@ func TestIntegrationSchemaPatternNonMatchingFailsEvenWithEmptyTables(t *testing. tmpl := fmt.Sprintf(` dsn: %s slot_name: no_schema_match_empty_tables_slot -schema_pattern: nonexistent_schema_zzz_* +schema_include: nonexistent_schema_zzz_* `, databaseURL) conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) @@ -2215,21 +2215,21 @@ schema_pattern: nonexistent_schema_zzz_* // TestIntegrationNoSchemasMatchedReturnsError. err = input.Connect(ctx) require.Error(t, err) - assert.Contains(t, err.Error(), "no schemas found matching pattern") + assert.Contains(t, err.Error(), "no schemas found matching schema_include pattern") } -// TestIntegrationMultiSchemaAutoDiscoverExcludeSchemas is -// TestIntegrationMultiSchemaExcludeSchemas with `tables` left unset: it -// verifies that leaving `tables` empty under schema_pattern auto-discovers +// TestIntegrationSchemaExcludeCarvesOutTenantAutoDiscover is +// TestIntegrationSchemaExcludeCarvesOutTenant with `tables` left unset: it +// verifies that leaving `tables` empty under schema_include auto-discovers // the "events" table in each matched schema instead of falling back to a -// database-wide FOR ALL TABLES publication, so exclude_schemas still carves +// database-wide FOR ALL TABLES publication, so schema_exclude still carves // tenant_c out of both the snapshot and CDC. -func TestIntegrationMultiSchemaAutoDiscoverExcludeSchemas(t *testing.T) { +func TestIntegrationSchemaExcludeCarvesOutTenantAutoDiscover(t *testing.T) { integration.CheckSkip(t) databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") require.NoError(t, err) - // Three tenant schemas match tenant_*; tenant_c is carved out via exclude_schemas. + // Three tenant schemas match tenant_*; tenant_c is carved out via schema_exclude. for _, schema := range []string{"tenant_a", "tenant_b", "tenant_c"} { _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", schema)) require.NoError(t, err) @@ -2269,10 +2269,10 @@ func TestIntegrationMultiSchemaAutoDiscoverExcludeSchemas(t *testing.T) { tmpl := fmt.Sprintf(` postgres_cdc: dsn: %s - slot_name: auto_discover_exclude_schemas_test_slot + slot_name: auto_discover_schema_exclude_test_slot stream_snapshot: true - schema_pattern: tenant_* - exclude_schemas: + schema_include: tenant_* + schema_exclude: - tenant_c `, databaseURL) @@ -2425,7 +2425,7 @@ tables: tmpl := fmt.Sprintf(` dsn: %s slot_name: glob_schema_total_miss_slot -schema_pattern: tenant_* +schema_include: tenant_* tables: - orders - ordres @@ -2479,7 +2479,7 @@ postgres_cdc: dsn: %s slot_name: glob_schema_partial_match_slot stream_snapshot: true - schema_pattern: tenant_* + schema_include: tenant_* tables: - orders - ordres @@ -2558,7 +2558,7 @@ postgres_cdc: dsn: %s slot_name: glob_schema_full_match_slot stream_snapshot: true - schema_pattern: tenant_* + schema_include: tenant_* tables: - orders - ordres diff --git a/internal/impl/postgresql/pglogicalstream/config.go b/internal/impl/postgresql/pglogicalstream/config.go index 42f1e4b52c..0398995081 100644 --- a/internal/impl/postgresql/pglogicalstream/config.go +++ b/internal/impl/postgresql/pglogicalstream/config.go @@ -25,16 +25,16 @@ type Config struct { DBRawDSN string TLSConfig *tls.Config DBSchema string - // DBSchemaPattern is the glob pattern used to replicate from multiple + // DBSchemaInclude is the glob pattern used to replicate from multiple // schemas at once, using '*' as a wildcard (e.g. "tenant_*", "*"). When // non-empty, it takes precedence over DBSchema and schemas are resolved // dynamically at stream creation time. - DBSchemaPattern string - // DBExcludeSchemas is a list of schema names or glob patterns (same syntax - // as DBSchemaPattern) excluded from the schemas resolved by - // DBSchemaPattern. Only meaningful when DBSchemaPattern is non-empty. - DBExcludeSchemas []string - DBTables []string + DBSchemaInclude string + // DBSchemaExclude is a list of schema names or glob patterns (same syntax + // as DBSchemaInclude) excluded from the schemas resolved by + // DBSchemaInclude. Only meaningful when DBSchemaInclude is non-empty. + DBSchemaExclude []string + DBTables []string // Refreshes short lived IAM auth token that is treated as a password RefreshAuthToken func(ctx context.Context) error // ReplicationSlotName is the name of the replication slot to use diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 29b5c47899..a2e9d69201 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -102,26 +102,26 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { tables []TableFQN schema string ) - if config.DBSchemaPattern != "" { + if config.DBSchemaInclude != "" { if config.SignalTableName != "" { - return nil, errors.New("signal_table_name is not supported when schema_pattern is set") + return nil, errors.New("signal_table_name is not supported when schema_include is set") } - schemas, inaccessibleSchemas, err := resolveSchemas(ctx, dbConn, config.DBSchemaPattern) + schemas, inaccessibleSchemas, err := resolveSchemas(ctx, dbConn, config.DBSchemaInclude) if err != nil { - return nil, fmt.Errorf("resolving schema pattern %q: %w", config.DBSchemaPattern, err) + return nil, fmt.Errorf("resolving schema_include pattern %q: %w", config.DBSchemaInclude, err) } - if len(config.DBExcludeSchemas) > 0 { + if len(config.DBSchemaExclude) > 0 { // Filtering happens entirely against the schemas slice we already // fetched above - no extra DB round-trips per exclude pattern. var excluded []string remaining := make([]string, 0, len(schemas)) for _, schema := range schemas { var isExcluded bool - for _, pattern := range config.DBExcludeSchemas { + for _, pattern := range config.DBSchemaExclude { matched, err := schemaMatchesExcludePattern(schema, pattern) if err != nil { - return nil, fmt.Errorf("evaluating exclude_schemas pattern %q against schema %q: %w", pattern, schema, err) + return nil, fmt.Errorf("evaluating schema_exclude pattern %q against schema %q: %w", pattern, schema, err) } if matched { isExcluded = true @@ -135,18 +135,18 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { remaining = append(remaining, schema) } if len(excluded) > 0 { - config.Logger.Infof("exclude_schemas %v excluded %d schema(s) %v from schema pattern %q; %d schema(s) remain: %v", config.DBExcludeSchemas, len(excluded), excluded, config.DBSchemaPattern, len(remaining), remaining) + config.Logger.Infof("schema_exclude %v excluded %d schema(s) %v from schema_include pattern %q; %d schema(s) remain: %v", config.DBSchemaExclude, len(excluded), excluded, config.DBSchemaInclude, len(remaining), remaining) } schemas = remaining } if len(inaccessibleSchemas) > 0 { - config.Logger.Warnf("schema pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaPattern, inaccessibleSchemas) + config.Logger.Warnf("schema_include pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaInclude, inaccessibleSchemas) } if len(schemas) == 0 { - return nil, fmt.Errorf("no schemas found matching pattern %q", config.DBSchemaPattern) + return nil, fmt.Errorf("no schemas found matching schema_include pattern %q", config.DBSchemaInclude) } - config.Logger.Infof("Schema pattern %q resolved to %d schema(s): %v", config.DBSchemaPattern, len(schemas), schemas) + config.Logger.Infof("schema_include pattern %q resolved to %d schema(s): %v", config.DBSchemaInclude, len(schemas), schemas) normalizedTables := make([]string, 0, len(config.DBTables)) for _, table := range config.DBTables { @@ -158,11 +158,11 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { } // With no explicit table list, auto-discover every base table in each - // matched (and exclude_schemas-filtered) schema and publish them + // matched (and schema_exclude-filtered) schema and publish them // explicitly. Without this, an empty tables list would fall through to // CreatePublication's FOR ALL TABLES fallback below, which replicates - // every schema in the database and silently defeats both schema_pattern - // and exclude_schemas. + // every schema in the database and silently defeats both schema_include + // and schema_exclude. autoDiscoverTables := len(normalizedTables) == 0 tables = make([]TableFQN, 0, len(schemas)*len(normalizedTables)) @@ -180,7 +180,7 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { } for _, table := range normalizedTables { if _, ok := existingTables[table]; !ok { - config.Logger.Warnf("table %s.%s not found, skipping (schema %s matched pattern %q but does not contain this table)", schema, table, schema, config.DBSchemaPattern) + config.Logger.Warnf("table %s.%s not found, skipping (schema %s matched schema_include pattern %q but does not contain this table)", schema, table, schema, config.DBSchemaInclude) continue } tables = append(tables, TableFQN{Schema: schema, Table: table}) @@ -189,9 +189,9 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { } if autoDiscoverTables { if len(tables) == 0 { - return nil, fmt.Errorf("no tables found in schema(s) %v matching pattern %q", schemas, config.DBSchemaPattern) + return nil, fmt.Errorf("no tables found in schema(s) %v matching schema_include pattern %q", schemas, config.DBSchemaInclude) } - config.Logger.Infof("%q has no `tables` list configured: auto-discovered %d table(s) across %d schema(s)", config.DBSchemaPattern, len(tables), len(schemas)) + config.Logger.Infof("%q has no `tables` list configured: auto-discovered %d table(s) across %d schema(s)", config.DBSchemaInclude, len(tables), len(schemas)) } else { // A table must exist in at least one matched schema. Missing from some // (but not all) matched schemas is tolerated above as a multi-tenant gap; @@ -204,7 +204,7 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { } } if len(missingTables) > 0 { - return nil, fmt.Errorf("table(s) %v not found in any schema matching pattern %q", missingTables, config.DBSchemaPattern) + return nil, fmt.Errorf("table(s) %v not found in any schema matching schema_include pattern %q", missingTables, config.DBSchemaInclude) } } } else { diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index 668314767e..765404b785 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -220,12 +220,12 @@ func escapeLike(s string) string { // schemaMatchesExcludePattern reports whether quotedSchemaName - a schema // identifier in the same quoted form resolveSchemas returns - matches -// excludePattern, an exclude_schemas entry written in the same shape as a -// schema_pattern value (an exact name, a '*' glob, or a double-quoted exact +// excludePattern, a schema_exclude entry written in the same shape as a +// schema_include value (an exact name, a '*' glob, or a double-quoted exact // identifier). // // Matching happens entirely in memory against a schema list we've already -// resolved, unlike resolveSchemas which queries the database: exclude_schemas +// resolved, unlike resolveSchemas which queries the database: schema_exclude // only ever narrows a candidate set that's already been fetched, so there's // no reason to pay for another round-trip per exclude pattern. Semantics // mirror schemaPatternToLike without going through SQL: a quoted pattern is diff --git a/internal/impl/postgresql/tests/current/Taskfile.yaml b/internal/impl/postgresql/tests/current/Taskfile.yaml index 0b57a7d25d..da2e348d06 100644 --- a/internal/impl/postgresql/tests/current/Taskfile.yaml +++ b/internal/impl/postgresql/tests/current/Taskfile.yaml @@ -73,37 +73,37 @@ tasks: # ── Schema validation smoke test ────────────────────────────────────────────── test:invalid-schema: - desc: Confirm that a malformed schema_pattern is rejected at startup, before any DB connection is attempted + desc: Confirm that a malformed schema_include is rejected at startup, before any DB connection is attempted env: PG_DSN: '{{.PG_DSN}}' cmds: - | set +e go run ../../../../../cmd/redpanda-connect/main.go run \ - --set 'input.postgres_cdc.schema_pattern=1abc' \ + --set 'input.postgres_cdc.schema_include=1abc' \ ./test_config.yaml 2>&1 | head -5 echo "exit $?" - # Expects: "invalid schema_pattern" error printed, process exits non-zero. - # schema_pattern is validated eagerly in newPgStreamInput + # Expects: "invalid schema_include" error printed, process exits non-zero. + # schema_include is validated eagerly in newPgStreamInput # (validateSchemaPattern), unlike schema which is only checked later, # against the live DB connection, inside NewPgStream. - # Note: an empty schema_pattern is NOT an error - it means "unset", and + # Note: an empty schema_include is NOT an error - it means "unset", and # the connector falls back to the schema field (default "public"). - test:exclude-schemas-requires-pattern: - desc: Confirm exclude_schemas is rejected at startup when schema_pattern is left unset + test:schema-exclude-requires-include: + desc: Confirm schema_exclude is rejected at startup when schema_include is left unset env: PG_DSN: '{{.PG_DSN}}' cmds: - | set +e go run ../../../../../cmd/redpanda-connect/main.go run \ - ./test_config_exclude_no_pattern.yaml 2>&1 | head -5 + ./test_config_schema_exclude_no_include.yaml 2>&1 | head -5 echo "exit $?" - # Expects: "exclude_schemas requires schema_pattern to be set" error + # Expects: "schema_exclude requires schema_include to be set" error # printed, process exits non-zero. Uses a dedicated fixture file rather # than --set on test_config.yaml, since --set rejects an empty RHS - # ("foo=" -> "expected foo=bar syntax"), so schema_pattern can't be + # ("foo=" -> "expected foo=bar syntax"), so schema_include can't be # cleared that way. # ── Quick sanity ────────────────────────────────────────────────────────────── diff --git a/internal/impl/postgresql/tests/current/setup.sql b/internal/impl/postgresql/tests/current/setup.sql index 3c0c6264fd..d231c381f7 100644 --- a/internal/impl/postgresql/tests/current/setup.sql +++ b/internal/impl/postgresql/tests/current/setup.sql @@ -1,10 +1,10 @@ -- Multi-schema CDC test setup --- Tests: schema glob (tenant_*), exclude_schemas (tenant_c), database_schema +-- Tests: schema glob (tenant_*), schema_exclude (tenant_c), database_schema -- metadata, commit_ts_ms, before (update/delete) -- ── Tenant schemas ──────────────────────────────────────────────────────────── -- tenant_a and tenant_b are replicated; tenant_c matches the tenant_* glob in --- test_config.yaml but is carved out via exclude_schemas — its rows must +-- test_config.yaml but is carved out via schema_exclude — its rows must -- never appear in the pipeline output. CREATE SCHEMA IF NOT EXISTS tenant_a; @@ -41,7 +41,7 @@ ALTER TABLE tenant_c.events REPLICA IDENTITY FULL; -- ── Seed snapshot rows ──────────────────────────────────────────────────────── -- These are visible during the initial snapshot (stream_snapshot: true). --- tenant_c's row (mallory) must NOT appear in the output — see exclude_schemas +-- tenant_c's row (mallory) must NOT appear in the output — see schema_exclude -- in test_config.yaml. INSERT INTO tenant_a.events (name) VALUES ('alice'), ('bob'); diff --git a/internal/impl/postgresql/tests/current/test_config.yaml b/internal/impl/postgresql/tests/current/test_config.yaml index 534b11496e..b440817dd1 100644 --- a/internal/impl/postgresql/tests/current/test_config.yaml +++ b/internal/impl/postgresql/tests/current/test_config.yaml @@ -4,10 +4,10 @@ input: slot_name: multi_schema_test_slot stream_snapshot: true # Glob pattern: replicates both tenant_a and tenant_b via one slot. - schema_pattern: tenant_* + schema_include: tenant_* # tenant_c matches the glob above but is carved out here — its rows must # never appear in the output below, neither during snapshot nor CDC. - exclude_schemas: + schema_exclude: - tenant_c tables: - events diff --git a/internal/impl/postgresql/tests/current/test_config_exclude_no_pattern.yaml b/internal/impl/postgresql/tests/current/test_config_schema_exclude_no_include.yaml similarity index 76% rename from internal/impl/postgresql/tests/current/test_config_exclude_no_pattern.yaml rename to internal/impl/postgresql/tests/current/test_config_schema_exclude_no_include.yaml index 57c0caf983..014332ed82 100644 --- a/internal/impl/postgresql/tests/current/test_config_exclude_no_pattern.yaml +++ b/internal/impl/postgresql/tests/current/test_config_schema_exclude_no_include.yaml @@ -1,12 +1,12 @@ -# Negative-validation fixture for exclude_schemas: schema_pattern is +# Negative-validation fixture for schema_exclude: schema_include is # deliberately left unset. --set can't assign an empty string (the CLI # rejects "foo=" as "expected foo=bar syntax"), so this exists as its own # file instead of overriding test_config.yaml at run time. input: postgres_cdc: dsn: ${PG_DSN:postgres://postgres:postgres@localhost:5433/testdb?sslmode=disable} - slot_name: exclude_schemas_no_pattern_slot - exclude_schemas: + slot_name: schema_exclude_no_include_slot + schema_exclude: - tenant_c tables: - events From 9897bbdd19eb350f53d3b3aee021f095e317ba23 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Thu, 20 Aug 2026 13:41:58 +0100 Subject: [PATCH 39/70] postgres_cdc: update docs --- docs/modules/components/pages/inputs/postgres_cdc.adoc | 5 ++++- internal/impl/postgresql/input_pg_stream.go | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index 3d4b3470f8..a849989785 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -198,7 +198,10 @@ When a pattern is used, all schemas whose names match the pattern are replicated Double-quoted identifiers are treated as exact names and do not support wildcards. -Schema pattern matching runs once at pipeline startup. Schemas created after the pipeline starts will not be picked up until the pipeline is restarted. +Schema pattern matching is re-evaluated every time the connector connects or reconnects - including the automatic reconnects that follow a transient replication failure - not just once at pipeline startup. This has two consequences that are easy to miss: + +- A schema created after the pipeline started that matches the pattern is picked up on the next reconnect and its tables are added to the publication. However, because the replication slot already exists by then, those tables are treated as already caught up, so with `stream_snapshot` enabled the rows that existed in that schema before it was picked up are never snapshotted and are silently missing from the output - only changes made after the schema is picked up are streamed. +- A schema that is dropped, renamed, or loses its `USAGE` grant between reconnects stops matching and its tables are silently removed from the publication on the next reconnect. A warning is logged when a schema becomes inaccessible due to a lost `USAGE` grant, but that warning is about the schema being inaccessible - it is not logged for a dropped or renamed schema, and nothing is ever logged about the resulting publication drop itself. If this pattern matches no schema in the database, startup fails with an error - this holds whether or not `tables` is set. See `tables` below for what happens when it's left empty. diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 9472afb964..d71ac82a08 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -131,7 +131,10 @@ When a pattern is used, all schemas whose names match the pattern are replicated Double-quoted identifiers are treated as exact names and do not support wildcards. -Schema pattern matching runs once at pipeline startup. Schemas created after the pipeline starts will not be picked up until the pipeline is restarted. +Schema pattern matching is re-evaluated every time the connector connects or reconnects - including the automatic reconnects that follow a transient replication failure - not just once at pipeline startup. This has two consequences that are easy to miss: + +- A schema created after the pipeline started that matches the pattern is picked up on the next reconnect and its tables are added to the publication. However, because the replication slot already exists by then, those tables are treated as already caught up, so with `+"`"+fieldStreamSnapshot+"`"+` enabled the rows that existed in that schema before it was picked up are never snapshotted and are silently missing from the output - only changes made after the schema is picked up are streamed. +- A schema that is dropped, renamed, or loses its `+"`USAGE`"+` grant between reconnects stops matching and its tables are silently removed from the publication on the next reconnect. A warning is logged when a schema becomes inaccessible due to a lost `+"`USAGE`"+` grant, but that warning is about the schema being inaccessible - it is not logged for a dropped or renamed schema, and nothing is ever logged about the resulting publication drop itself. If this pattern matches no schema in the database, startup fails with an error - this holds whether or not `+"`"+fieldTables+"`"+` is set. See `+"`"+fieldTables+"`"+` below for what happens when it's left empty. From 5a78e090690b6f78f38892c79763582a194590f0 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Thu, 20 Aug 2026 16:40:26 +0100 Subject: [PATCH 40/70] postgres_cdc: update tables docs --- docs/modules/components/pages/inputs/postgres_cdc.adoc | 5 +++-- internal/impl/postgresql/input_pg_stream.go | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index a849989785..1f419c285d 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -46,7 +46,7 @@ input: schema: public schema_include: "" schema_exclude: [] - tables: [] # No default (required) + tables: [] checkpoint_limit: 1024 temporary_slot: false slot_name: my_test_slot # No default (required) @@ -78,7 +78,7 @@ input: schema: public schema_include: "" schema_exclude: [] - tables: [] # No default (required) + tables: [] checkpoint_limit: 1024 temporary_slot: false slot_name: my_test_slot # No default (required) @@ -260,6 +260,7 @@ If left empty while `schema_include` is NOT set, the underlying PostgreSQL publi *Type*: `array` +*Default*: `[]` ```yml # Examples diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index d71ac82a08..becbe1f522 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -165,7 +165,9 @@ When ` + "`" + fieldSchemaInclude + "`" + ` is set, this list is resolved agains If left empty while ` + "`" + fieldSchemaInclude + "`" + ` is set, every base table in each matched (and un-excluded, see ` + "`" + fieldSchemaExclude + "`" + `) schema is auto-discovered and published explicitly, instead of listing tables by hand - this is the expected way to replicate "every table" in a multi-tenant, schema-per-tenant setup without also picking up unrelated schemas. Startup fails if no matched schema contains any table. If left empty while ` + "`" + fieldSchemaInclude + "`" + ` is NOT set, the underlying PostgreSQL publication is instead created ` + "`FOR ALL TABLES`" + `, which replicates every table in every schema of the database, ignoring ` + "`" + fieldSchema + "`" + `. This also disables ` + "`" + fieldStreamSnapshot + "`" + `, since the initial snapshot is only planned for tables listed here.`). - Example([]string{"my_table_1", `"MyCaseSensitiveTableNeedingQuotes"`})). + Example([]string{"my_table_1", `"MyCaseSensitiveTableNeedingQuotes"`}). + Optional(). + Default([]string{})). Field(service.NewIntField(fieldCheckpointLimit). Description("The maximum number of messages that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level. Any given LSN will not be acknowledged unless all messages under that offset are delivered in order to preserve at least once delivery guarantees."). ShortDescription("The maximum number of messages that can be processed at a given time."). From bc81de9be3fb2c566b8998176479671610c240ea Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Thu, 20 Aug 2026 22:46:04 +0100 Subject: [PATCH 41/70] postgres_cdc: clean up testing and validation --- internal/impl/postgresql/integration_test.go | 134 +++++++++++------- .../pglogicalstream/logical_stream.go | 5 + 2 files changed, 90 insertions(+), 49 deletions(-) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index b88c2d6e8a..afd4d811fa 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1751,7 +1751,7 @@ postgres_cdc: // schema_include but also matches a schema_exclude entry contributes no // rows at all, neither during the initial snapshot nor from subsequent CDC // changes. -func TestIntegrationSchemaExcludeCarvesOutTenant(t *testing.T) { +func TestIntegrationMultiSchemaExcludeCarvesOutTenant(t *testing.T) { integration.CheckSkip(t) databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") require.NoError(t, err) @@ -1898,7 +1898,7 @@ postgres_cdc: assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") } -func TestIntegrationSchemaIncludeMatchesHyphenatedUUIDSchema(t *testing.T) { +func TestIntegrationMultiSchemaIncludeMatchesHyphenatedUUIDSchema(t *testing.T) { integration.CheckSkip(t) databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") require.NoError(t, err) @@ -2148,12 +2148,14 @@ postgres_cdc: assert.Equal(t, "events", collected[0].table) } -func TestIntegrationNoSchemasMatchedReturnsError(t *testing.T) { +func TestIntegrationMultiSchemaIncludeExcludeConfigValidation(t *testing.T) { integration.CheckSkip(t) - databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") - require.NoError(t, err) - tmpl := fmt.Sprintf(` + t.Run("schema_include matches nothing", func(t *testing.T) { + databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + tmpl := fmt.Sprintf(` dsn: %s slot_name: no_schema_match_slot schema_include: nonexistent_schema_zzz_* @@ -2161,61 +2163,95 @@ tables: - events `, databaseURL) - conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) - require.NoError(t, err) + conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) + require.NoError(t, err) - mgr := service.MockResources() - license.InjectTestService(mgr) + mgr := service.MockResources() + license.InjectTestService(mgr) - input, err := newPgStreamInput(conf, mgr) - require.NoError(t, err) + input, err := newPgStreamInput(conf, mgr) + require.NoError(t, err) - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) - defer cancel() + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() - // Bypass the benthos AsyncReader's infinite connect-retry loop by calling - // Connect directly: a schema-include-not-found error is permanent, but - // stream.Run has no path to surface it (it only returns once ctx is done), - // so going through StreamBuilder/Run here would just time out instead. - err = input.Connect(ctx) - require.Error(t, err) - assert.Contains(t, err.Error(), "no schemas found matching schema_include pattern") -} + err = input.Connect(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "no schemas found matching schema_include pattern") + }) -// TestIntegrationSchemaIncludeNonMatchingFailsEvenWithEmptyTables guards the -// fixed behaviour: schema_include always scopes replication, even when -// `tables` is left empty. A schema_include matching nothing in the database -// must fail startup rather than silently falling back to a database-wide -// FOR ALL TABLES publication (the old behaviour, which made schema_exclude -// meaningless whenever tables was left unset). -func TestIntegrationSchemaIncludeNonMatchingFailsEvenWithEmptyTables(t *testing.T) { - integration.CheckSkip(t) - databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") - require.NoError(t, err) + t.Run("schema_include matches nothing with empty tables", func(t *testing.T) { + databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) - tmpl := fmt.Sprintf(` + tmpl := fmt.Sprintf(` dsn: %s slot_name: no_schema_match_empty_tables_slot schema_include: nonexistent_schema_zzz_* `, databaseURL) - conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) - require.NoError(t, err) + conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) + require.NoError(t, err) - mgr := service.MockResources() - license.InjectTestService(mgr) + mgr := service.MockResources() + license.InjectTestService(mgr) - input, err := newPgStreamInput(conf, mgr) - require.NoError(t, err) + input, err := newPgStreamInput(conf, mgr) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) - defer cancel() + // Bypass the benthos AsyncReader's infinite connect-retry loop, same as + // the "schema_include matches nothing" case above. + err = input.Connect(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "no schemas found matching schema_include pattern") + }) - // Bypass the benthos AsyncReader's infinite connect-retry loop, same as - // TestIntegrationNoSchemasMatchedReturnsError. - err = input.Connect(ctx) - require.Error(t, err) - assert.Contains(t, err.Error(), "no schemas found matching schema_include pattern") + t.Run("schema_exclude excludes every matched schema", func(t *testing.T) { + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + // Both schemas match tenant_*, but schema_exclude below excludes them all. + for _, schema := range []string{"tenant_a", "tenant_b"} { + _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", schema)) + require.NoError(t, err) + _, err = db.Exec(fmt.Sprintf( + "CREATE TABLE %s.events (id SERIAL PRIMARY KEY, name TEXT)", schema)) + require.NoError(t, err) + } + + tmpl := fmt.Sprintf(` +dsn: %s +slot_name: schema_exclude_all_matched_slot +schema_include: tenant_* +schema_exclude: + - tenant_* +tables: + - events +`, databaseURL) + + conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) + require.NoError(t, err) + + mgr := service.MockResources() + license.InjectTestService(mgr) + + input, err := newPgStreamInput(conf, mgr) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + + // Bypass the benthos AsyncReader's infinite connect-retry loop, same as + // the "schema_include matches nothing" case above. + err = input.Connect(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "matched schema(s)") + assert.Contains(t, err.Error(), "excluded all of them") + assert.NotContains(t, err.Error(), "no schemas found matching schema_include pattern") + }) } // TestIntegrationSchemaExcludeCarvesOutTenantAutoDiscover is @@ -2224,7 +2260,7 @@ schema_include: nonexistent_schema_zzz_* // the "events" table in each matched schema instead of falling back to a // database-wide FOR ALL TABLES publication, so schema_exclude still carves // tenant_c out of both the snapshot and CDC. -func TestIntegrationSchemaExcludeCarvesOutTenantAutoDiscover(t *testing.T) { +func TestIntegrationMultiSchemaExcludeCarvesOutTenantAutoDiscover(t *testing.T) { integration.CheckSkip(t) databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") require.NoError(t, err) @@ -2371,7 +2407,7 @@ postgres_cdc: assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") } -func TestIntegrationSchemaAndTableMatchingTest(t *testing.T) { +func TestIntegrationMultiSchemaAndTableMatchingTest(t *testing.T) { integration.CheckSkip(t) t.Run("exact schema match with missing table fails", func(t *testing.T) { @@ -2403,7 +2439,7 @@ tables: defer cancel() // Bypass the benthos AsyncReader's infinite connect-retry loop, same as - // TestIntegrationNoSchemasMatchedReturnsError. + // TestIntegrationSchemaIncludeExcludeConfigValidation. err = input.Connect(ctx) require.Error(t, err, "typo'd table %q should fail startup loudly instead of silently streaming only %q", "ordres", "orders") assert.Contains(t, err.Error(), "ordres") diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index a2e9d69201..d3436dad02 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -110,6 +110,7 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { if err != nil { return nil, fmt.Errorf("resolving schema_include pattern %q: %w", config.DBSchemaInclude, err) } + matchedSchemas := schemas if len(config.DBSchemaExclude) > 0 { // Filtering happens entirely against the schemas slice we already @@ -143,7 +144,11 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { if len(inaccessibleSchemas) > 0 { config.Logger.Warnf("schema_include pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaInclude, inaccessibleSchemas) } + if len(schemas) == 0 { + if len(matchedSchemas) > 0 { + return nil, fmt.Errorf("schema_include pattern %q matched schema(s) %v, but schema_exclude %v excluded all of them", config.DBSchemaInclude, matchedSchemas, config.DBSchemaExclude) + } return nil, fmt.Errorf("no schemas found matching schema_include pattern %q", config.DBSchemaInclude) } config.Logger.Infof("schema_include pattern %q resolved to %d schema(s): %v", config.DBSchemaInclude, len(schemas), schemas) From ad32e8e1be67a12194f4e0a50f77a0df4f9c0b01 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Thu, 20 Aug 2026 23:26:30 +0100 Subject: [PATCH 42/70] postgres_cdc: fix link --- internal/impl/postgresql/input_pg_stream_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/impl/postgresql/input_pg_stream_test.go b/internal/impl/postgresql/input_pg_stream_test.go index 275b34fee4..322de40f24 100644 --- a/internal/impl/postgresql/input_pg_stream_test.go +++ b/internal/impl/postgresql/input_pg_stream_test.go @@ -4,7 +4,7 @@ // License (the "License"); you may not use this file except in compliance with // the License. You may obtain a copy of the License at // -// https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md package pgstream From 6c0ffcb6d719421d03493ad9eba06daee29118cb Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Thu, 20 Aug 2026 23:44:05 +0100 Subject: [PATCH 43/70] fix: --- .../impl/postgresql/pglogicalstream/config.go | 4 +++ .../pglogicalstream/logical_stream.go | 14 ++++++++- .../pglogicalstream/schema_resolver.go | 29 +++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/internal/impl/postgresql/pglogicalstream/config.go b/internal/impl/postgresql/pglogicalstream/config.go index 0398995081..8804efcd5c 100644 --- a/internal/impl/postgresql/pglogicalstream/config.go +++ b/internal/impl/postgresql/pglogicalstream/config.go @@ -63,4 +63,8 @@ type Config struct { UnchangedToastValue any // The interval to send logical messages HeartbeatInterval time.Duration + + // Used to track and report on resolved schemas between reconnects + previouslyResolvedSchemas []string + previouslyInaccessibleSchemas []string } diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index d3436dad02..7ceea54d7b 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -141,9 +141,10 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { schemas = remaining } - if len(inaccessibleSchemas) > 0 { + if len(inaccessibleSchemas) > 0 && !slices.Equal(inaccessibleSchemas, config.previouslyInaccessibleSchemas) { config.Logger.Warnf("schema_include pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaInclude, inaccessibleSchemas) } + config.previouslyInaccessibleSchemas = slices.Clone(inaccessibleSchemas) if len(schemas) == 0 { if len(matchedSchemas) > 0 { @@ -153,6 +154,17 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { } config.Logger.Infof("schema_include pattern %q resolved to %d schema(s): %v", config.DBSchemaInclude, len(schemas), schemas) + if config.previouslyResolvedSchemas != nil { + added, removed := diffSchemaSets(config.previouslyResolvedSchemas, schemas) + if len(added) > 0 { + config.Logger.Warnf("schema_include pattern %q now also matches schema(s) %v that did not match on the previous connect; their tables are being added to the publication, but any rows already in them will NOT be snapshotted even if stream_snapshot is enabled - only changes made from now on will be captured", config.DBSchemaInclude, added) + } + if len(removed) > 0 { + config.Logger.Warnf("schema(s) %v no longer match schema_include pattern %q (dropped, renamed, or the role lost USAGE) since the previous connect; their tables are being removed from the publication and will stop replicating", removed, config.DBSchemaInclude) + } + } + config.previouslyResolvedSchemas = slices.Clone(schemas) + normalizedTables := make([]string, 0, len(config.DBTables)) for _, table := range config.DBTables { normalized, err := sanitize.NormalizePostgresIdentifier(table) diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index 765404b785..bbbbc061a7 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -256,6 +256,35 @@ func schemaMatchesExcludePattern(quotedSchemaName, excludePattern string) (bool, return re.MatchString(strings.ToLower(schemaName)), nil } +// diffSchemaSets reports which schemas are present in current but not +// previous (added) and vice versa (removed), used to detect schema-set drift +// between the schema_include resolution done on this connect and the one +// done on the previous connect/reconnect. A nil previous (the very first +// resolution in the process's lifetime) is not a meaningful "everything was +// just added" drift, so callers should only act on the result when previous +// is non-nil. +func diffSchemaSets(previous, current []string) (added, removed []string) { + previousSet := make(map[string]struct{}, len(previous)) + for _, schema := range previous { + previousSet[schema] = struct{}{} + } + currentSet := make(map[string]struct{}, len(current)) + for _, schema := range current { + currentSet[schema] = struct{}{} + } + for _, schema := range current { + if _, ok := previousSet[schema]; !ok { + added = append(added, schema) + } + } + for _, schema := range previous { + if _, ok := currentSet[schema]; !ok { + removed = append(removed, schema) + } + } + return added, removed +} + // globToRegexp compiles an unquoted glob pattern (using '*' as a wildcard) // into an anchored regexp - the in-memory equivalent of globToLike for // callers matching against values already held in Go rather than via a SQL From 129878a05151c524fbe7cba289915a4512c49efc Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Fri, 21 Aug 2026 02:14:06 +0100 Subject: [PATCH 44/70] postgres_cdc: move schema resolution into dedicated file --- .../pglogicalstream/logical_stream.go | 58 +--------------- .../pglogicalstream/schema_resolver.go | 68 +++++++++++++++++++ 2 files changed, 70 insertions(+), 56 deletions(-) diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 7ceea54d7b..ddeaf31a9c 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -106,64 +106,10 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { if config.SignalTableName != "" { return nil, errors.New("signal_table_name is not supported when schema_include is set") } - schemas, inaccessibleSchemas, err := resolveSchemas(ctx, dbConn, config.DBSchemaInclude) + schemas, err := resolveIncludedSchemas(ctx, dbConn, config) if err != nil { - return nil, fmt.Errorf("resolving schema_include pattern %q: %w", config.DBSchemaInclude, err) - } - matchedSchemas := schemas - - if len(config.DBSchemaExclude) > 0 { - // Filtering happens entirely against the schemas slice we already - // fetched above - no extra DB round-trips per exclude pattern. - var excluded []string - remaining := make([]string, 0, len(schemas)) - for _, schema := range schemas { - var isExcluded bool - for _, pattern := range config.DBSchemaExclude { - matched, err := schemaMatchesExcludePattern(schema, pattern) - if err != nil { - return nil, fmt.Errorf("evaluating schema_exclude pattern %q against schema %q: %w", pattern, schema, err) - } - if matched { - isExcluded = true - break - } - } - if isExcluded { - excluded = append(excluded, schema) - continue - } - remaining = append(remaining, schema) - } - if len(excluded) > 0 { - config.Logger.Infof("schema_exclude %v excluded %d schema(s) %v from schema_include pattern %q; %d schema(s) remain: %v", config.DBSchemaExclude, len(excluded), excluded, config.DBSchemaInclude, len(remaining), remaining) - } - schemas = remaining - } - - if len(inaccessibleSchemas) > 0 && !slices.Equal(inaccessibleSchemas, config.previouslyInaccessibleSchemas) { - config.Logger.Warnf("schema_include pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", config.DBSchemaInclude, inaccessibleSchemas) - } - config.previouslyInaccessibleSchemas = slices.Clone(inaccessibleSchemas) - - if len(schemas) == 0 { - if len(matchedSchemas) > 0 { - return nil, fmt.Errorf("schema_include pattern %q matched schema(s) %v, but schema_exclude %v excluded all of them", config.DBSchemaInclude, matchedSchemas, config.DBSchemaExclude) - } - return nil, fmt.Errorf("no schemas found matching schema_include pattern %q", config.DBSchemaInclude) - } - config.Logger.Infof("schema_include pattern %q resolved to %d schema(s): %v", config.DBSchemaInclude, len(schemas), schemas) - - if config.previouslyResolvedSchemas != nil { - added, removed := diffSchemaSets(config.previouslyResolvedSchemas, schemas) - if len(added) > 0 { - config.Logger.Warnf("schema_include pattern %q now also matches schema(s) %v that did not match on the previous connect; their tables are being added to the publication, but any rows already in them will NOT be snapshotted even if stream_snapshot is enabled - only changes made from now on will be captured", config.DBSchemaInclude, added) - } - if len(removed) > 0 { - config.Logger.Warnf("schema(s) %v no longer match schema_include pattern %q (dropped, renamed, or the role lost USAGE) since the previous connect; their tables are being removed from the publication and will stop replicating", removed, config.DBSchemaInclude) - } + return nil, err } - config.previouslyResolvedSchemas = slices.Clone(schemas) normalizedTables := make([]string, 0, len(config.DBTables)) for _, table := range config.DBTables { diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index bbbbc061a7..dcc8df8bc2 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -12,6 +12,7 @@ import ( "context" "fmt" "regexp" + "slices" "strings" "github.com/jackc/pgx/v5/pgconn" @@ -130,6 +131,73 @@ func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (v return schemas, hidden, nil } +// resolveIncludedSchemas resolves config.DBSchemaInclude against conn, +// applies config.DBSchemaExclude filtering, and logs schema-set drift +// against the previous resolution cached on config (see +// Config.previouslyResolvedSchemas) +func resolveIncludedSchemas(ctx context.Context, conn *pgconn.PgConn, cfg *Config) ([]string, error) { + schemas, inaccessibleSchemas, err := resolveSchemas(ctx, conn, cfg.DBSchemaInclude) + if err != nil { + return nil, fmt.Errorf("resolving schema_include pattern %q: %w", cfg.DBSchemaInclude, err) + } + matchedSchemas := schemas + + if len(cfg.DBSchemaExclude) > 0 { + // Filtering happens entirely against the schemas slice we already + // fetched above - no extra DB round-trips per exclude pattern. + var excluded []string + remaining := make([]string, 0, len(schemas)) + for _, schema := range schemas { + var isExcluded bool + for _, pattern := range cfg.DBSchemaExclude { + matched, err := schemaMatchesExcludePattern(schema, pattern) + if err != nil { + return nil, fmt.Errorf("evaluating schema_exclude pattern %q against schema %q: %w", pattern, schema, err) + } + if matched { + isExcluded = true + break + } + } + if isExcluded { + excluded = append(excluded, schema) + continue + } + remaining = append(remaining, schema) + } + if len(excluded) > 0 { + cfg.Logger.Infof("schema_exclude %v excluded %d schema(s) %v from schema_include pattern %q; %d schema(s) remain: %v", cfg.DBSchemaExclude, len(excluded), excluded, cfg.DBSchemaInclude, len(remaining), remaining) + } + schemas = remaining + } + + if len(inaccessibleSchemas) > 0 && !slices.Equal(inaccessibleSchemas, cfg.previouslyInaccessibleSchemas) { + cfg.Logger.Warnf("schema_include pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", cfg.DBSchemaInclude, inaccessibleSchemas) + } + cfg.previouslyInaccessibleSchemas = slices.Clone(inaccessibleSchemas) + + if len(schemas) == 0 { + if len(matchedSchemas) > 0 { + return nil, fmt.Errorf("schema_include pattern %q matched schema(s) %v, but schema_exclude %v excluded all of them", cfg.DBSchemaInclude, matchedSchemas, cfg.DBSchemaExclude) + } + return nil, fmt.Errorf("no schemas found matching schema_include pattern %q", cfg.DBSchemaInclude) + } + cfg.Logger.Infof("schema_include pattern %q resolved to %d schema(s): %v", cfg.DBSchemaInclude, len(schemas), schemas) + + if cfg.previouslyResolvedSchemas != nil { + added, removed := diffSchemaSets(cfg.previouslyResolvedSchemas, schemas) + if len(added) > 0 { + cfg.Logger.Warnf("schema_include pattern %q now also matches schema(s) %v that did not match on the previous connect; their tables are being added to the publication, but any rows already in them will NOT be snapshotted even if stream_snapshot is enabled - only changes made from now on will be captured", cfg.DBSchemaInclude, added) + } + if len(removed) > 0 { + cfg.Logger.Warnf("schema(s) %v no longer match schema_include pattern %q (dropped, renamed, or the role lost USAGE) since the previous connect; their tables are being removed from the publication and will stop replicating", removed, cfg.DBSchemaInclude) + } + } + cfg.previouslyResolvedSchemas = slices.Clone(schemas) + + return schemas, nil +} + // resolveExistingTables returns the quoted names of the publishable base // tables (including partitioned tables) that exist in the given (already // quoted) schema. From ad1c1325707de76d1586122992e8129a9dbd9ab1 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Fri, 21 Aug 2026 02:28:19 +0100 Subject: [PATCH 45/70] clean up --- .../impl/postgresql/pglogicalstream/config.go | 22 ++-- .../pglogicalstream/schema_resolver.go | 107 +++++------------- .../plugins/cdctest/cdc_conformance_test.go | 21 +--- 3 files changed, 42 insertions(+), 108 deletions(-) diff --git a/internal/impl/postgresql/pglogicalstream/config.go b/internal/impl/postgresql/pglogicalstream/config.go index 8804efcd5c..d8400b7c2f 100644 --- a/internal/impl/postgresql/pglogicalstream/config.go +++ b/internal/impl/postgresql/pglogicalstream/config.go @@ -25,16 +25,14 @@ type Config struct { DBRawDSN string TLSConfig *tls.Config DBSchema string - // DBSchemaInclude is the glob pattern used to replicate from multiple - // schemas at once, using '*' as a wildcard (e.g. "tenant_*", "*"). When - // non-empty, it takes precedence over DBSchema and schemas are resolved - // dynamically at stream creation time. - DBSchemaInclude string - // DBSchemaExclude is a list of schema names or glob patterns (same syntax - // as DBSchemaInclude) excluded from the schemas resolved by - // DBSchemaInclude. Only meaningful when DBSchemaInclude is non-empty. - DBSchemaExclude []string - DBTables []string + DBTables []string + + // Multi schema support + DBSchemaInclude string + DBSchemaExclude []string + previouslyResolvedSchemas []string // track and report on resolved schemas between reconnects + previouslyInaccessibleSchemas []string + // Refreshes short lived IAM auth token that is treated as a password RefreshAuthToken func(ctx context.Context) error // ReplicationSlotName is the name of the replication slot to use @@ -63,8 +61,4 @@ type Config struct { UnchangedToastValue any // The interval to send logical messages HeartbeatInterval time.Duration - - // Used to track and report on resolved schemas between reconnects - previouslyResolvedSchemas []string - previouslyInaccessibleSchemas []string } diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index dcc8df8bc2..eef25e3c89 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -21,19 +21,10 @@ import ( ) // schemaPatternToLike converts a schema name or glob pattern into the LIKE -// pattern used by resolveSchemas, plus whether that pattern must be matched -// case-sensitively. Extracted for unit testing. -// -// For quoted identifiers the inner name is exact-escaped (no wildcard -// expansion) and matched case-sensitively, since a quoted identifier's case -// is significant and PostgreSQL does not fold it. For unquoted patterns the -// '*' wildcard is converted to '%' and matching is case-insensitive: this -// mirrors PostgreSQL folding unquoted identifiers to lower-case at creation -// time for the common case, but must hold even when a schema had to be -// created with a quoted identifier for an unrelated reason (e.g. a -// UUID-suffixed tenant schema, which requires quoting because hyphens are -// invalid in unquoted identifiers) and so kept whatever case it was written -// with — an unquoted glob like "tenant_*" is still expected to match it. +// pattern used by resolveSchemas, plus whether it must be matched +// case-sensitively. Quoted identifiers are matched exactly and +// case-sensitively; unquoted patterns use '*' as a wildcard and match +// case-insensitively, mirroring how PostgreSQL folds unquoted identifiers. func schemaPatternToLike(pattern string) (likePattern string, caseSensitive bool, err error) { if strings.HasPrefix(pattern, `"`) { unquoted, err := sanitize.UnquotePostgresIdentifier(pattern) @@ -45,25 +36,13 @@ func schemaPatternToLike(pattern string) (likePattern string, caseSensitive bool return globToLike(strings.ToLower(pattern)), false, nil } -// resolveSchemas returns the schemas matching pattern that the connection's -// role has access to (visibleSchemas), plus any schemas that also match -// pattern but are hidden from information_schema.schemata by privileges -// (inaccessibleSchemas). The latter is surfaced separately so callers can -// warn the user instead of silently dropping schemas they expected to be -// included, e.g. `"tenant_*"` matching a schema the configured role can't -// see yet. -// -// For unquoted patterns (e.g. "tenant_*") the pattern is matched -// case-insensitively via ILIKE, regardless of whether the matched schema was -// itself created case-insensitively. For quoted identifiers (e.g. -// `"MySchema"`) an exact case-sensitive lookup is performed via LIKE. System -// schemas (pg_* and information_schema) are always excluded case-sensitively -// so that wildcard patterns like "*" do not attempt to replicate catalog -// tables. -// -// Returned schema names are quoted PostgreSQL identifiers. Returns an error -// if either query fails; returns a nil visibleSchemas slice (with a nil err) -// if no schemas match — callers should treat that as an error condition. +// resolveSchemas returns the schemas matching pattern that the role can see +// (visibleSchemas), plus schemas that also match but are hidden by missing +// privileges (inaccessibleSchemas) - surfaced separately so callers can warn +// instead of silently dropping them. System schemas (pg_* and +// information_schema) are always excluded. Returned names are quoted +// PostgreSQL identifiers. A nil visibleSchemas with a nil error means no +// match - callers should treat that as an error. func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (visibleSchemas, inaccessibleSchemas []string, err error) { likePattern, caseSensitive, err := schemaPatternToLike(pattern) if err != nil { @@ -102,9 +81,8 @@ func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (v } } - // pg_namespace is not privilege-filtered, so any pattern match here that's - // missing from information_schema.schemata means the role lacks USAGE (or - // similar) on that schema rather than the schema simply not existing. + // pg_namespace isn't privilege-filtered, so a match here missing from + // information_schema.schemata means the role lacks USAGE on that schema. nsQ, err := sanitize.SQLQuery( fmt.Sprintf("SELECT nspname FROM pg_catalog.pg_namespace WHERE nspname %s $1 ESCAPE '!' AND nspname NOT LIKE 'pg!_%%' ESCAPE '!' AND nspname != 'information_schema'", op), likePattern, @@ -131,10 +109,9 @@ func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (v return schemas, hidden, nil } -// resolveIncludedSchemas resolves config.DBSchemaInclude against conn, -// applies config.DBSchemaExclude filtering, and logs schema-set drift -// against the previous resolution cached on config (see -// Config.previouslyResolvedSchemas) +// resolveIncludedSchemas resolves cfg.DBSchemaInclude against conn, applies +// cfg.DBSchemaExclude filtering, and warns about schema-set drift against +// the previously resolved set cached on cfg. func resolveIncludedSchemas(ctx context.Context, conn *pgconn.PgConn, cfg *Config) ([]string, error) { schemas, inaccessibleSchemas, err := resolveSchemas(ctx, conn, cfg.DBSchemaInclude) if err != nil { @@ -199,17 +176,9 @@ func resolveIncludedSchemas(ctx context.Context, conn *pgconn.PgConn, cfg *Confi } // resolveExistingTables returns the quoted names of the publishable base -// tables (including partitioned tables) that exist in the given (already -// quoted) schema. -// -// Used to resolve a schema glob × table list combination per-schema, since a -// matched schema may not contain a configured table (e.g. still being -// provisioned) or may have a same-named view/foreign/temporary table -// instead. table_type is restricted to 'BASE TABLE' so either case is -// treated as missing and falls through to the caller's "not found, skipping" -// warning - otherwise CreatePublication's FOR TABLE clause would reference -// an unpublishable relation and fail setup for every matched schema, not -// just the drifted one. +// tables in the given (already quoted) schema. Restricted to table_type = +// 'BASE TABLE' so a same-named view or foreign table is treated as missing +// rather than breaking CreatePublication. func resolveExistingTables(ctx context.Context, conn *pgconn.PgConn, quotedSchema string) (map[string]struct{}, error) { schema, err := sanitize.UnquotePostgresIdentifier(quotedSchema) if err != nil { @@ -286,23 +255,11 @@ func escapeLike(s string) string { return b.String() } -// schemaMatchesExcludePattern reports whether quotedSchemaName - a schema -// identifier in the same quoted form resolveSchemas returns - matches -// excludePattern, a schema_exclude entry written in the same shape as a -// schema_include value (an exact name, a '*' glob, or a double-quoted exact -// identifier). -// -// Matching happens entirely in memory against a schema list we've already -// resolved, unlike resolveSchemas which queries the database: schema_exclude -// only ever narrows a candidate set that's already been fetched, so there's -// no reason to pay for another round-trip per exclude pattern. Semantics -// mirror schemaPatternToLike without going through SQL: a quoted pattern is -// an exact, case-sensitive match on the unquoted name; an unquoted pattern is -// matched case-insensitively with '*' as a wildcard. -// -// Returns an error only when a quoted operand fails to unquote. This should -// only happen for a malformed excludePattern in practice, since -// quotedSchemaName is always freshly quoted by resolveSchemas. +// schemaMatchesExcludePattern reports whether quotedSchemaName matches +// excludePattern, using the same pattern syntax as schema_include (exact +// name, '*' glob, or quoted exact identifier). Matches in memory against an +// already-resolved schema list, so no extra DB round-trip is needed. Returns +// an error only if a quoted operand fails to unquote. func schemaMatchesExcludePattern(quotedSchemaName, excludePattern string) (bool, error) { schemaName, err := sanitize.UnquotePostgresIdentifier(quotedSchemaName) if err != nil { @@ -324,13 +281,9 @@ func schemaMatchesExcludePattern(quotedSchemaName, excludePattern string) (bool, return re.MatchString(strings.ToLower(schemaName)), nil } -// diffSchemaSets reports which schemas are present in current but not -// previous (added) and vice versa (removed), used to detect schema-set drift -// between the schema_include resolution done on this connect and the one -// done on the previous connect/reconnect. A nil previous (the very first -// resolution in the process's lifetime) is not a meaningful "everything was -// just added" drift, so callers should only act on the result when previous -// is non-nil. +// diffSchemaSets reports schemas present in current but not previous +// (added) and vice versa (removed). Callers should ignore the result when +// previous is nil - that's the first resolution, not real drift. func diffSchemaSets(previous, current []string) (added, removed []string) { previousSet := make(map[string]struct{}, len(previous)) for _, schema := range previous { @@ -353,10 +306,8 @@ func diffSchemaSets(previous, current []string) (added, removed []string) { return added, removed } -// globToRegexp compiles an unquoted glob pattern (using '*' as a wildcard) -// into an anchored regexp - the in-memory equivalent of globToLike for -// callers matching against values already held in Go rather than via a SQL -// LIKE clause. +// globToRegexp compiles an unquoted glob pattern ('*' as wildcard) into an +// anchored regexp - the in-memory equivalent of globToLike. func globToRegexp(pattern string) (*regexp.Regexp, error) { parts := strings.Split(pattern, "*") for i, part := range parts { diff --git a/internal/plugins/cdctest/cdc_conformance_test.go b/internal/plugins/cdctest/cdc_conformance_test.go index 6a3980caba..cd5eac4de5 100644 --- a/internal/plugins/cdctest/cdc_conformance_test.go +++ b/internal/plugins/cdctest/cdc_conformance_test.go @@ -54,13 +54,6 @@ var canonicalFields = []string{ "stream_snapshot", } -// conditionalConnectors lists CDC inputs that are only registered under specific -// build tags (e.g. cgo). They are exempt from the stale-entry guard when not -// registered, but are still subject to conformance checks when they are. -var conditionalConnectors = map[string]bool{ - "tigerbeetle_cdc": true, // requires cgo -} - // knownNonConformant waives specific (connector → field → reason) checks that // have not yet been migrated. New connectors default to strict. Populated from // the actual registry state; shrink it as connectors converge. @@ -100,11 +93,11 @@ var knownNonConformant = map[string]map[string]string{ "max_parallel_snapshot_tables": "uses max_parallel_snapshot_objects; migrate", }, "tigerbeetle_cdc": { - "checkpoint_cache": "uses progress_cache; migrate to checkpoint_cache", - "checkpoint_limit": "no discrete checkpoint limit; TigerBeetle CDC is pure streaming", - "snapshot_max_batch_size": "no snapshot phase; TigerBeetle CDC is pure streaming with no initial snapshot", - "max_parallel_snapshot_tables": "no snapshot phase; TigerBeetle CDC is pure streaming with no initial snapshot", - "stream_snapshot": "no snapshot phase; TigerBeetle CDC is pure streaming with no initial snapshot", + "checkpoint_cache": "non-relational; §5 applicability under triage", + "checkpoint_limit": "non-relational; §5 applicability under triage", + "snapshot_max_batch_size": "non-relational; §5 applicability under triage", + "max_parallel_snapshot_tables": "non-relational; §5 applicability under triage", + "stream_snapshot": "non-relational; §5 applicability under triage", }, } @@ -186,10 +179,6 @@ func TestCDCConformance(t *testing.T) { } for name := range knownNonConformant { if _, ok := registered[name]; !ok { - if conditionalConnectors[name] { - t.Logf("SKIPPED stale check for %q: not registered in this build (conditional build tag)", name) - continue - } t.Errorf("stale knownNonConformant entry %q is not a registered CDC input; remove it", name) } } From 591d750493701b4b6ac9246459afcd3dc534e3ff Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Fri, 21 Aug 2026 10:41:32 +0100 Subject: [PATCH 46/70] postgres_cdc: improve schema discovery on reconnect loops --- .../pglogicalstream/logical_stream.go | 19 ++++----- .../pglogicalstream/schema_resolver.go | 42 +++++++++++++------ .../schema_resolver_integration_test.go | 2 +- .../pglogicalstream/schema_resolver_test.go | 2 +- 4 files changed, 40 insertions(+), 25 deletions(-) diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index ddeaf31a9c..e557d57afa 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -120,21 +120,20 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { normalizedTables = append(normalizedTables, normalized) } - // With no explicit table list, auto-discover every base table in each - // matched (and schema_exclude-filtered) schema and publish them - // explicitly. Without this, an empty tables list would fall through to - // CreatePublication's FOR ALL TABLES fallback below, which replicates - // every schema in the database and silently defeats both schema_include - // and schema_exclude. + // tables empty here would otherwise fall through to CreatePublication's + // FOR ALL TABLES fallback, replicating the whole database and defeating + // schema_include/schema_exclude - auto-discover per matched schema instead. autoDiscoverTables := len(normalizedTables) == 0 + existingTablesBySchema, err := resolveExistingTables(ctx, dbConn, schemas) + if err != nil { + return nil, fmt.Errorf("resolving tables in schema(s) %v: %w", schemas, err) + } + tables = make([]TableFQN, 0, len(schemas)*len(normalizedTables)) foundTables := make(map[string]bool, len(normalizedTables)) for _, schema := range schemas { - existingTables, err := resolveExistingTables(ctx, dbConn, schema) - if err != nil { - return nil, fmt.Errorf("resolving tables in schema %q: %w", schema, err) - } + existingTables := existingTablesBySchema[schema] if autoDiscoverTables { for table := range existingTables { tables = append(tables, TableFQN{Schema: schema, Table: table}) diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index eef25e3c89..0c2f3a637a 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -176,32 +176,48 @@ func resolveIncludedSchemas(ctx context.Context, conn *pgconn.PgConn, cfg *Confi } // resolveExistingTables returns the quoted names of the publishable base -// tables in the given (already quoted) schema. Restricted to table_type = -// 'BASE TABLE' so a same-named view or foreign table is treated as missing -// rather than breaking CreatePublication. -func resolveExistingTables(ctx context.Context, conn *pgconn.PgConn, quotedSchema string) (map[string]struct{}, error) { - schema, err := sanitize.UnquotePostgresIdentifier(quotedSchema) - if err != nil { - return nil, fmt.Errorf("unquoting schema identifier %q: %w", quotedSchema, err) +// tables in each of the given (already quoted) schemas, keyed by quoted +// schema name. A single query covering every schema, rather than one per +// schema, keeps this to one round-trip regardless of tenant count - the +// difference between one query and, say, one hundred on a multi-tenant, +// schema-per-tenant database on every connect and reconnect. Restricted to +// table_type = 'BASE TABLE' so a same-named view or foreign table is treated +// as missing rather than breaking CreatePublication. +func resolveExistingTables(ctx context.Context, conn *pgconn.PgConn, quotedSchemas []string) (map[string]map[string]struct{}, error) { + rawToQuoted := make(map[string]string, len(quotedSchemas)) + args := make([]any, len(quotedSchemas)) + placeholders := make([]string, len(quotedSchemas)) + for i, quotedSchema := range quotedSchemas { + schema, err := sanitize.UnquotePostgresIdentifier(quotedSchema) + if err != nil { + return nil, fmt.Errorf("unquoting schema identifier %q: %w", quotedSchema, err) + } + rawToQuoted[schema] = quotedSchema + args[i] = schema + placeholders[i] = fmt.Sprintf("$%d", i+1) } q, err := sanitize.SQLQuery( - "SELECT table_name FROM information_schema.tables WHERE table_schema = $1 AND table_type = 'BASE TABLE'", - schema, + fmt.Sprintf("SELECT table_schema, table_name FROM information_schema.tables WHERE table_schema IN (%s) AND table_type = 'BASE TABLE'", strings.Join(placeholders, ", ")), + args..., ) if err != nil { - return nil, fmt.Errorf("building table resolution query for schema %q: %w", quotedSchema, err) + return nil, fmt.Errorf("building table resolution query for schema(s) %v: %w", quotedSchemas, err) } results, err := conn.Exec(ctx, q).ReadAll() if err != nil { - return nil, fmt.Errorf("querying tables in schema %q: %w", quotedSchema, err) + return nil, fmt.Errorf("querying tables in schema(s) %v: %w", quotedSchemas, err) } - existing := map[string]struct{}{} + existing := make(map[string]map[string]struct{}, len(quotedSchemas)) + for _, quotedSchema := range quotedSchemas { + existing[quotedSchema] = map[string]struct{}{} + } if len(results) > 0 { for _, row := range results[0].Rows { - existing[sanitize.QuotePostgresIdentifier(string(row[0]))] = struct{}{} + quotedSchema := rawToQuoted[string(row[0])] + existing[quotedSchema][sanitize.QuotePostgresIdentifier(string(row[1]))] = struct{}{} } } return existing, nil diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go b/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go index 91243a6802..a7afbfa48f 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 Redpanda Data, Inc. +// Copyright 2026 Redpanda Data, Inc. // // Licensed as a Redpanda Enterprise file under the Redpanda Community // License (the "License"); you may not use this file except in compliance with diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go b/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go index ae1ee89a92..7931fddb81 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 Redpanda Data, Inc. +// Copyright 2026 Redpanda Data, Inc. // // Licensed as a Redpanda Enterprise file under the Redpanda Community // License (the "License"); you may not use this file except in compliance with From 0a86a33da4cfa1f320e6d5544391b7a3959aad03 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Fri, 21 Aug 2026 11:10:52 +0100 Subject: [PATCH 47/70] clean up logging --- internal/impl/postgresql/pglogicalstream/logical_stream.go | 2 +- internal/impl/postgresql/pglogicalstream/schema_resolver.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index e557d57afa..cb48c712bf 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -153,7 +153,7 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { if len(tables) == 0 { return nil, fmt.Errorf("no tables found in schema(s) %v matching schema_include pattern %q", schemas, config.DBSchemaInclude) } - config.Logger.Infof("%q has no `tables` list configured: auto-discovered %d table(s) across %d schema(s)", config.DBSchemaInclude, len(tables), len(schemas)) + config.Logger.Debugf("%q has no `tables` list configured: auto-discovered %d table(s) across %d schema(s)", config.DBSchemaInclude, len(tables), len(schemas)) } else { // A table must exist in at least one matched schema. Missing from some // (but not all) matched schemas is tolerated above as a multi-tenant gap; diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/schema_resolver.go index 0c2f3a637a..2304e0477f 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/schema_resolver.go @@ -143,7 +143,7 @@ func resolveIncludedSchemas(ctx context.Context, conn *pgconn.PgConn, cfg *Confi remaining = append(remaining, schema) } if len(excluded) > 0 { - cfg.Logger.Infof("schema_exclude %v excluded %d schema(s) %v from schema_include pattern %q; %d schema(s) remain: %v", cfg.DBSchemaExclude, len(excluded), excluded, cfg.DBSchemaInclude, len(remaining), remaining) + cfg.Logger.Debugf("schema_exclude %v excluded %d schema(s) %v from schema_include pattern %q; %d schema(s) remain: %v", cfg.DBSchemaExclude, len(excluded), excluded, cfg.DBSchemaInclude, len(remaining), remaining) } schemas = remaining } @@ -159,7 +159,7 @@ func resolveIncludedSchemas(ctx context.Context, conn *pgconn.PgConn, cfg *Confi } return nil, fmt.Errorf("no schemas found matching schema_include pattern %q", cfg.DBSchemaInclude) } - cfg.Logger.Infof("schema_include pattern %q resolved to %d schema(s): %v", cfg.DBSchemaInclude, len(schemas), schemas) + cfg.Logger.Debugf("schema_include pattern %q resolved to %d schema(s): %v", cfg.DBSchemaInclude, len(schemas), schemas) if cfg.previouslyResolvedSchemas != nil { added, removed := diffSchemaSets(cfg.previouslyResolvedSchemas, schemas) From 8f3c09f29e1a02d6ceca6998c909f6e1f38a5273 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Sat, 22 Aug 2026 00:09:34 +0100 Subject: [PATCH 48/70] move to package --- cmd/tools/integration/packages.json | 1 + internal/impl/postgresql/input_pg_stream.go | 9 +- .../impl/postgresql/pglogicalstream/config.go | 10 +- .../pglogicalstream/logical_stream.go | 15 +- .../resolver.go} | 173 ++++++++++-------- .../resolver_integration_test.go} | 69 ++++--- .../resolver_test.go} | 2 +- 7 files changed, 162 insertions(+), 117 deletions(-) rename internal/impl/postgresql/pglogicalstream/{schema_resolver.go => multischema/resolver.go} (76%) rename internal/impl/postgresql/pglogicalstream/{schema_resolver_integration_test.go => multischema/resolver_integration_test.go} (74%) rename internal/impl/postgresql/pglogicalstream/{schema_resolver_test.go => multischema/resolver_test.go} (99%) diff --git a/cmd/tools/integration/packages.json b/cmd/tools/integration/packages.json index 6bcdf7deb0..bcc6dd0812 100644 --- a/cmd/tools/integration/packages.json +++ b/cmd/tools/integration/packages.json @@ -47,6 +47,7 @@ {"path":"./internal/impl/otlp"}, {"path":"./internal/impl/postgresql"}, {"path":"./internal/impl/postgresql/pglogicalstream"}, + {"path":"./internal/impl/postgresql/pglogicalstream/multischema"}, {"path":"./internal/impl/pulsar","timeout":"10m"}, {"path":"./internal/impl/qdrant"}, {"path":"./internal/impl/questdb"}, diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index becbe1f522..84b8d2044c 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -27,6 +27,7 @@ import ( "github.com/redpanda-data/connect/v4/internal/asyncroutine" "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream" + "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream/multischema" "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream/sanitize" "github.com/redpanda-data/connect/v4/internal/license" ) @@ -491,14 +492,18 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser snapshotMetrics := mgr.Metrics().NewGauge("postgres_snapshot_progress", "table") replicationLag := mgr.Metrics().NewGauge("postgres_replication_lag_bytes") + var schemaResolver *multischema.Resolver + if schemaInclude != "" { + schemaResolver = multischema.NewResolver(schemaInclude, schemaExclude) + } + i := &pgStreamInput{ streamConfig: &pglogicalstream.Config{ DBConfig: pgConnConfig, TLSConfig: pgConnConfig.TLSConfig, DBRawDSN: dsn, DBSchema: schema, - DBSchemaInclude: schemaInclude, - DBSchemaExclude: schemaExclude, + SchemaResolver: schemaResolver, DBTables: tables, RefreshAuthToken: iamAuthTokenBuilder, diff --git a/internal/impl/postgresql/pglogicalstream/config.go b/internal/impl/postgresql/pglogicalstream/config.go index d8400b7c2f..3c87d0bcaa 100644 --- a/internal/impl/postgresql/pglogicalstream/config.go +++ b/internal/impl/postgresql/pglogicalstream/config.go @@ -16,6 +16,8 @@ import ( "github.com/jackc/pgx/v5/pgconn" "github.com/redpanda-data/benthos/v4/public/service" + + "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream/multischema" ) // Config is the configuration for the pglogicalstream plugin @@ -27,11 +29,9 @@ type Config struct { DBSchema string DBTables []string - // Multi schema support - DBSchemaInclude string - DBSchemaExclude []string - previouslyResolvedSchemas []string // track and report on resolved schemas between reconnects - previouslyInaccessibleSchemas []string + // SchemaResolver resolves schema_include/schema_exclude into the schemas + // to replicate. Non-nil only when schema_include is set. + SchemaResolver *multischema.Resolver // Refreshes short lived IAM auth token that is treated as a password RefreshAuthToken func(ctx context.Context) error diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index cb48c712bf..9b29cfdb95 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -26,6 +26,7 @@ import ( "github.com/redpanda-data/benthos/v4/public/service" "github.com/redpanda-data/connect/v4/internal/asyncroutine" + "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream/multischema" "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream/sanitize" ) @@ -102,11 +103,11 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { tables []TableFQN schema string ) - if config.DBSchemaInclude != "" { + if config.SchemaResolver != nil { if config.SignalTableName != "" { return nil, errors.New("signal_table_name is not supported when schema_include is set") } - schemas, err := resolveIncludedSchemas(ctx, dbConn, config) + schemas, err := config.SchemaResolver.Resolve(ctx, dbConn, config.Logger) if err != nil { return nil, err } @@ -125,7 +126,7 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { // schema_include/schema_exclude - auto-discover per matched schema instead. autoDiscoverTables := len(normalizedTables) == 0 - existingTablesBySchema, err := resolveExistingTables(ctx, dbConn, schemas) + existingTablesBySchema, err := multischema.ResolveExistingTables(ctx, dbConn, schemas) if err != nil { return nil, fmt.Errorf("resolving tables in schema(s) %v: %w", schemas, err) } @@ -142,7 +143,7 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { } for _, table := range normalizedTables { if _, ok := existingTables[table]; !ok { - config.Logger.Warnf("table %s.%s not found, skipping (schema %s matched schema_include pattern %q but does not contain this table)", schema, table, schema, config.DBSchemaInclude) + config.Logger.Warnf("table %s.%s not found, skipping (schema %s matched schema_include pattern %q but does not contain this table)", schema, table, schema, config.SchemaResolver.Include) continue } tables = append(tables, TableFQN{Schema: schema, Table: table}) @@ -151,9 +152,9 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { } if autoDiscoverTables { if len(tables) == 0 { - return nil, fmt.Errorf("no tables found in schema(s) %v matching schema_include pattern %q", schemas, config.DBSchemaInclude) + return nil, fmt.Errorf("no tables found in schema(s) %v matching schema_include pattern %q", schemas, config.SchemaResolver.Include) } - config.Logger.Debugf("%q has no `tables` list configured: auto-discovered %d table(s) across %d schema(s)", config.DBSchemaInclude, len(tables), len(schemas)) + config.Logger.Debugf("%q has no `tables` list configured: auto-discovered %d table(s) across %d schema(s)", config.SchemaResolver.Include, len(tables), len(schemas)) } else { // A table must exist in at least one matched schema. Missing from some // (but not all) matched schemas is tolerated above as a multi-tenant gap; @@ -166,7 +167,7 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { } } if len(missingTables) > 0 { - return nil, fmt.Errorf("table(s) %v not found in any schema matching schema_include pattern %q", missingTables, config.DBSchemaInclude) + return nil, fmt.Errorf("table(s) %v not found in any schema matching schema_include pattern %q", missingTables, config.SchemaResolver.Include) } } } else { diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver.go b/internal/impl/postgresql/pglogicalstream/multischema/resolver.go similarity index 76% rename from internal/impl/postgresql/pglogicalstream/schema_resolver.go rename to internal/impl/postgresql/pglogicalstream/multischema/resolver.go index 2304e0477f..4579d90eb2 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver.go +++ b/internal/impl/postgresql/pglogicalstream/multischema/resolver.go @@ -6,7 +6,10 @@ // // https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md -package pglogicalstream +// Package multischema resolves a schema_include/schema_exclude +// configuration (replicating from multiple PostgreSQL schemas matched by a +// glob pattern) into the concrete set of schemas and tables to replicate. +package multischema import ( "context" @@ -17,14 +20,95 @@ import ( "github.com/jackc/pgx/v5/pgconn" + "github.com/redpanda-data/benthos/v4/public/service" + "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream/sanitize" ) -// schemaPatternToLike converts a schema name or glob pattern into the LIKE -// pattern used by resolveSchemas, plus whether it must be matched -// case-sensitively. Quoted identifiers are matched exactly and -// case-sensitively; unquoted patterns use '*' as a wildcard and match -// case-insensitively, mirroring how PostgreSQL folds unquoted identifiers. +type Resolver struct { + // Include is the schema_include glob pattern. + Include string + // Exclude is the schema_exclude list, evaluated against the schemas + // matched by Include. + Exclude []string + + previouslyResolved []string + previouslyInaccessible []string +} + +// NewResolver returns a Resolver for the given schema_include/schema_exclude +// configuration. Callers should only construct one when schema_include is +// set. +func NewResolver(include string, exclude []string) *Resolver { + return &Resolver{Include: include, Exclude: exclude} +} + +// Resolve resolves r.Include against conn, applies r.Exclude filtering, and +// warns about schema-set drift against the previously resolved set from an +// earlier call to Resolve on this same Resolver. +func (r *Resolver) Resolve(ctx context.Context, conn *pgconn.PgConn, logger *service.Logger) ([]string, error) { + schemas, inaccessibleSchemas, err := resolveSchemas(ctx, conn, r.Include) + if err != nil { + return nil, fmt.Errorf("resolving schema_include pattern %q: %w", r.Include, err) + } + matchedSchemas := schemas + + if len(r.Exclude) > 0 { + // Filtering happens entirely against the schemas slice we already + // fetched above - no extra DB round-trips per exclude pattern. + var excluded []string + remaining := make([]string, 0, len(schemas)) + for _, schema := range schemas { + var isExcluded bool + for _, pattern := range r.Exclude { + matched, err := schemaMatchesExcludePattern(schema, pattern) + if err != nil { + return nil, fmt.Errorf("evaluating schema_exclude pattern %q against schema %q: %w", pattern, schema, err) + } + if matched { + isExcluded = true + break + } + } + if isExcluded { + excluded = append(excluded, schema) + continue + } + remaining = append(remaining, schema) + } + if len(excluded) > 0 { + logger.Debugf("schema_exclude %v excluded %d schema(s) %v from schema_include pattern %q; %d schema(s) remain: %v", r.Exclude, len(excluded), excluded, r.Include, len(remaining), remaining) + } + schemas = remaining + } + + if len(inaccessibleSchemas) > 0 && !slices.Equal(inaccessibleSchemas, r.previouslyInaccessible) { + logger.Warnf("schema_include pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", r.Include, inaccessibleSchemas) + } + r.previouslyInaccessible = slices.Clone(inaccessibleSchemas) + + if len(schemas) == 0 { + if len(matchedSchemas) > 0 { + return nil, fmt.Errorf("schema_include pattern %q matched schema(s) %v, but schema_exclude %v excluded all of them", r.Include, matchedSchemas, r.Exclude) + } + return nil, fmt.Errorf("no schemas found matching schema_include pattern %q", r.Include) + } + logger.Debugf("schema_include pattern %q resolved to %d schema(s): %v", r.Include, len(schemas), schemas) + + if r.previouslyResolved != nil { + added, removed := diffSchemaSets(r.previouslyResolved, schemas) + if len(added) > 0 { + logger.Warnf("schema_include pattern %q now also matches schema(s) %v that did not match on the previous connect; their tables are being added to the publication, but any rows already in them will NOT be snapshotted even if stream_snapshot is enabled - only changes made from now on will be captured", r.Include, added) + } + if len(removed) > 0 { + logger.Warnf("schema(s) %v no longer match schema_include pattern %q (dropped, renamed, or the role lost USAGE) since the previous connect; their tables are being removed from the publication and will stop replicating", removed, r.Include) + } + } + r.previouslyResolved = slices.Clone(schemas) + + return schemas, nil +} + func schemaPatternToLike(pattern string) (likePattern string, caseSensitive bool, err error) { if strings.HasPrefix(pattern, `"`) { unquoted, err := sanitize.UnquotePostgresIdentifier(pattern) @@ -36,13 +120,6 @@ func schemaPatternToLike(pattern string) (likePattern string, caseSensitive bool return globToLike(strings.ToLower(pattern)), false, nil } -// resolveSchemas returns the schemas matching pattern that the role can see -// (visibleSchemas), plus schemas that also match but are hidden by missing -// privileges (inaccessibleSchemas) - surfaced separately so callers can warn -// instead of silently dropping them. System schemas (pg_* and -// information_schema) are always excluded. Returned names are quoted -// PostgreSQL identifiers. A nil visibleSchemas with a nil error means no -// match - callers should treat that as an error. func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (visibleSchemas, inaccessibleSchemas []string, err error) { likePattern, caseSensitive, err := schemaPatternToLike(pattern) if err != nil { @@ -109,73 +186,7 @@ func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (v return schemas, hidden, nil } -// resolveIncludedSchemas resolves cfg.DBSchemaInclude against conn, applies -// cfg.DBSchemaExclude filtering, and warns about schema-set drift against -// the previously resolved set cached on cfg. -func resolveIncludedSchemas(ctx context.Context, conn *pgconn.PgConn, cfg *Config) ([]string, error) { - schemas, inaccessibleSchemas, err := resolveSchemas(ctx, conn, cfg.DBSchemaInclude) - if err != nil { - return nil, fmt.Errorf("resolving schema_include pattern %q: %w", cfg.DBSchemaInclude, err) - } - matchedSchemas := schemas - - if len(cfg.DBSchemaExclude) > 0 { - // Filtering happens entirely against the schemas slice we already - // fetched above - no extra DB round-trips per exclude pattern. - var excluded []string - remaining := make([]string, 0, len(schemas)) - for _, schema := range schemas { - var isExcluded bool - for _, pattern := range cfg.DBSchemaExclude { - matched, err := schemaMatchesExcludePattern(schema, pattern) - if err != nil { - return nil, fmt.Errorf("evaluating schema_exclude pattern %q against schema %q: %w", pattern, schema, err) - } - if matched { - isExcluded = true - break - } - } - if isExcluded { - excluded = append(excluded, schema) - continue - } - remaining = append(remaining, schema) - } - if len(excluded) > 0 { - cfg.Logger.Debugf("schema_exclude %v excluded %d schema(s) %v from schema_include pattern %q; %d schema(s) remain: %v", cfg.DBSchemaExclude, len(excluded), excluded, cfg.DBSchemaInclude, len(remaining), remaining) - } - schemas = remaining - } - - if len(inaccessibleSchemas) > 0 && !slices.Equal(inaccessibleSchemas, cfg.previouslyInaccessibleSchemas) { - cfg.Logger.Warnf("schema_include pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", cfg.DBSchemaInclude, inaccessibleSchemas) - } - cfg.previouslyInaccessibleSchemas = slices.Clone(inaccessibleSchemas) - - if len(schemas) == 0 { - if len(matchedSchemas) > 0 { - return nil, fmt.Errorf("schema_include pattern %q matched schema(s) %v, but schema_exclude %v excluded all of them", cfg.DBSchemaInclude, matchedSchemas, cfg.DBSchemaExclude) - } - return nil, fmt.Errorf("no schemas found matching schema_include pattern %q", cfg.DBSchemaInclude) - } - cfg.Logger.Debugf("schema_include pattern %q resolved to %d schema(s): %v", cfg.DBSchemaInclude, len(schemas), schemas) - - if cfg.previouslyResolvedSchemas != nil { - added, removed := diffSchemaSets(cfg.previouslyResolvedSchemas, schemas) - if len(added) > 0 { - cfg.Logger.Warnf("schema_include pattern %q now also matches schema(s) %v that did not match on the previous connect; their tables are being added to the publication, but any rows already in them will NOT be snapshotted even if stream_snapshot is enabled - only changes made from now on will be captured", cfg.DBSchemaInclude, added) - } - if len(removed) > 0 { - cfg.Logger.Warnf("schema(s) %v no longer match schema_include pattern %q (dropped, renamed, or the role lost USAGE) since the previous connect; their tables are being removed from the publication and will stop replicating", removed, cfg.DBSchemaInclude) - } - } - cfg.previouslyResolvedSchemas = slices.Clone(schemas) - - return schemas, nil -} - -// resolveExistingTables returns the quoted names of the publishable base +// ResolveExistingTables returns the quoted names of the publishable base // tables in each of the given (already quoted) schemas, keyed by quoted // schema name. A single query covering every schema, rather than one per // schema, keeps this to one round-trip regardless of tenant count - the @@ -183,7 +194,7 @@ func resolveIncludedSchemas(ctx context.Context, conn *pgconn.PgConn, cfg *Confi // schema-per-tenant database on every connect and reconnect. Restricted to // table_type = 'BASE TABLE' so a same-named view or foreign table is treated // as missing rather than breaking CreatePublication. -func resolveExistingTables(ctx context.Context, conn *pgconn.PgConn, quotedSchemas []string) (map[string]map[string]struct{}, error) { +func ResolveExistingTables(ctx context.Context, conn *pgconn.PgConn, quotedSchemas []string) (map[string]map[string]struct{}, error) { rawToQuoted := make(map[string]string, len(quotedSchemas)) args := make([]any, len(quotedSchemas)) placeholders := make([]string, len(quotedSchemas)) diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go b/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go similarity index 74% rename from internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go rename to internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go index a7afbfa48f..a330e5130b 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver_integration_test.go +++ b/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go @@ -6,11 +6,12 @@ // // https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md -package pglogicalstream +package multischema import ( "context" "database/sql" + "fmt" "testing" "time" @@ -19,15 +20,12 @@ import ( "github.com/jackc/pgx/v5/pgconn" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" "github.com/redpanda-data/benthos/v4/public/service/integration" ) -// TestIntegrationResolveSchemasReportsInaccessibleSchemas verifies that a -// schema pattern matching a schema the connecting role lacks USAGE on is -// reported via inaccessibleSchemas rather than silently dropped, since -// information_schema.schemata alone would make it indistinguishable from a -// schema that simply doesn't exist. func TestIntegrationResolveSchemasReportsInaccessibleSchemas(t *testing.T) { integration.CheckSkip(t) @@ -79,8 +77,6 @@ func TestIntegrationResolveSchemasUUIDSuffixedSchemas(t *testing.T) { require.NoError(t, err) defer adminDB.Close() - // Quoting is mandatory here because of the UUID's hyphens, regardless of - // case - so both of these preserve their literal casing exactly as written. const ( lowerCaseSchema = `"tenant_a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"` mixedCaseSchema = `"Tenant_9c0b4ef8-bb6d-6bb9-bd38-0a11a0eebc99"` @@ -100,14 +96,9 @@ func TestIntegrationResolveSchemasUUIDSuffixedSchemas(t *testing.T) { visible, inaccessible, err := resolveSchemas(ctx, conn, "tenant_*") require.NoError(t, err) - // Both schemas match the unquoted glob despite the case difference in - // their literal prefix - matching is case-insensitive independent of how - // the schema itself was created. assert.ElementsMatch(t, []string{lowerCaseSchema, mixedCaseSchema}, visible) assert.Empty(t, inaccessible) - // A quoted pattern is still exact and case-sensitive: it picks out only - // the schema whose case matches the pattern, with no wildcard expansion. exactVisible, _, err := resolveSchemas(ctx, conn, mixedCaseSchema) require.NoError(t, err) assert.Equal(t, []string{mixedCaseSchema}, exactVisible) @@ -135,27 +126,63 @@ func TestIntegrationResolveSchemasBareUUIDSchema(t *testing.T) { require.NoError(t, err) defer closeConn(t, conn) - // A bare wildcard has no literal characters to case-compare, so this is - // unaffected by hex-digit casing either way - included as a baseline. visible, _, err := resolveSchemas(ctx, conn, "*") require.NoError(t, err) assert.Contains(t, visible, bareUUIDSchema) - // The interesting case: an unquoted pattern whose only literal portion is - // a lower-case chunk of the UUID itself (no prefix) must still match the - // upper-case schema case-insensitively. visible, _, err = resolveSchemas(ctx, conn, "a0eebc99-*") require.NoError(t, err) assert.Equal(t, []string{bareUUIDSchema}, visible) - // Same, but the literal chunk sits in the middle rather than at the start. visible, _, err = resolveSchemas(ctx, conn, "*-bb6d-*") require.NoError(t, err) assert.Equal(t, []string{bareUUIDSchema}, visible) - // A quoted pattern remains an exact, case-sensitive lookup: the - // differently-cased quoted form matches nothing. visible, _, err = resolveSchemas(ctx, conn, `"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"`) require.NoError(t, err) assert.Empty(t, visible) } + +func closeConn(t testing.TB, conn *pgconn.PgConn) { + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + require.NoError(t, conn.Close(ctx)) +} + +func createDockerInstance(t *testing.T) (cleanup func(), dbURL string) { + ctr, err := testcontainers.Run(t.Context(), "postgres:16", + testcontainers.WithExposedPorts("5432/tcp"), + testcontainers.WithEnv(map[string]string{ + "POSTGRES_PASSWORD": "secret", + "POSTGRES_USER": "user_name", + "POSTGRES_DB": "dbname", + }), + testcontainers.WithCmd("postgres", "-c", "wal_level=logical"), + testcontainers.WithWaitStrategy( + wait.ForListeningPort("5432/tcp").WithStartupTimeout(2*time.Minute), + ), + ) + testcontainers.CleanupContainer(t, ctr) + require.NoError(t, err) + + host, err := ctr.Host(t.Context()) + require.NoError(t, err) + mp, err := ctr.MappedPort(t.Context(), "5432/tcp") + require.NoError(t, err) + + databaseURL := fmt.Sprintf("user=user_name password=secret dbname=dbname sslmode=disable host=%s port=%s replication=database", host, mp.Port()) + + var db *sql.DB + require.Eventually(t, func() bool { + if db, err = sql.Open("postgres", databaseURL); err != nil { + return false + } + return db.Ping() == nil + }, 2*time.Minute, time.Second) + + cleanup = func() { + // Container cleanup is handled by testcontainers.CleanupContainer + } + + return cleanup, databaseURL +} diff --git a/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go b/internal/impl/postgresql/pglogicalstream/multischema/resolver_test.go similarity index 99% rename from internal/impl/postgresql/pglogicalstream/schema_resolver_test.go rename to internal/impl/postgresql/pglogicalstream/multischema/resolver_test.go index 7931fddb81..d06b5362dd 100644 --- a/internal/impl/postgresql/pglogicalstream/schema_resolver_test.go +++ b/internal/impl/postgresql/pglogicalstream/multischema/resolver_test.go @@ -6,7 +6,7 @@ // // https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md -package pglogicalstream +package multischema import ( "testing" From c4a5a17ec0c4d326f00df137208e276268d0f175 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Mon, 24 Aug 2026 14:58:44 +0100 Subject: [PATCH 49/70] clean up --- .../postgresql/tests/current/Taskfile.yaml | 120 ------------------ .../tests/current/docker-compose.yaml | 36 ------ .../impl/postgresql/tests/current/setup.sql | 49 ------- .../postgresql/tests/current/test_config.yaml | 45 ------- ...test_config_schema_exclude_no_include.yaml | 22 ---- 5 files changed, 272 deletions(-) delete mode 100644 internal/impl/postgresql/tests/current/Taskfile.yaml delete mode 100644 internal/impl/postgresql/tests/current/docker-compose.yaml delete mode 100644 internal/impl/postgresql/tests/current/setup.sql delete mode 100644 internal/impl/postgresql/tests/current/test_config.yaml delete mode 100644 internal/impl/postgresql/tests/current/test_config_schema_exclude_no_include.yaml diff --git a/internal/impl/postgresql/tests/current/Taskfile.yaml b/internal/impl/postgresql/tests/current/Taskfile.yaml deleted file mode 100644 index da2e348d06..0000000000 --- a/internal/impl/postgresql/tests/current/Taskfile.yaml +++ /dev/null @@ -1,120 +0,0 @@ -version: "3" - -vars: - PG_DSN: '{{.PG_DSN | default "postgres://postgres:postgres@localhost:5433/testdb?sslmode=disable"}}' - -tasks: - # ── Infrastructure ──────────────────────────────────────────────────────────── - - up: - desc: Start PostgreSQL and run schema/data setup - cmds: - - docker compose up -d postgres - - docker compose run --rm setup - - down: - desc: Stop and remove all containers and volumes - cmds: - - docker compose down -v - - reset: - desc: Full teardown + bring back up (also drops and recreates the replication slot) - cmds: - - task: down - - task: up - - # ── Slot management ─────────────────────────────────────────────────────────── - - slot:drop: - desc: Drop the replication slot so the test can be re-run from scratch - cmds: - - | - psql "{{.PG_DSN}}" -c \ - "SELECT pg_drop_replication_slot('multi_schema_test_slot') - FROM pg_replication_slots - WHERE slot_name = 'multi_schema_test_slot';" - - # ── Run the pipeline ────────────────────────────────────────────────────────── - - run: - desc: Run the multi-schema CDC pipeline (streams snapshot then live CDC) - env: - PG_DSN: '{{.PG_DSN}}' - cmds: - - task: slot:drop - - go run ../../../../../cmd/redpanda-connect/main.go run ./test_config.yaml - - # ── Test data ───────────────────────────────────────────────────────────────── - - data:insert: - desc: Insert CDC rows into all three tenant schemas (tenant_c must not appear in the output — it's excluded) - cmds: - - psql "{{.PG_DSN}}" -c "INSERT INTO tenant_a.events (name) VALUES ('dave'), ('eve');" - - psql "{{.PG_DSN}}" -c "INSERT INTO tenant_b.events (name) VALUES ('frank');" - - psql "{{.PG_DSN}}" -c "INSERT INTO tenant_c.events (name) VALUES ('trudy');" - - data:update: - desc: Update a row in tenant_a (triggers update event with 'before' field) - cmds: - - psql "{{.PG_DSN}}" -c "UPDATE tenant_a.events SET status = 'updated' WHERE name = 'dave';" - - data:delete: - desc: Delete a row from tenant_b (triggers delete event with 'before' field) - cmds: - - psql "{{.PG_DSN}}" -c "DELETE FROM tenant_b.events WHERE name = 'frank';" - - data:all: - desc: Run all test data mutations in sequence (insert → update → delete) - cmds: - - task: data:insert - - task: data:update - - task: data:delete - - # ── Schema validation smoke test ────────────────────────────────────────────── - - test:invalid-schema: - desc: Confirm that a malformed schema_include is rejected at startup, before any DB connection is attempted - env: - PG_DSN: '{{.PG_DSN}}' - cmds: - - | - set +e - go run ../../../../../cmd/redpanda-connect/main.go run \ - --set 'input.postgres_cdc.schema_include=1abc' \ - ./test_config.yaml 2>&1 | head -5 - echo "exit $?" - # Expects: "invalid schema_include" error printed, process exits non-zero. - # schema_include is validated eagerly in newPgStreamInput - # (validateSchemaPattern), unlike schema which is only checked later, - # against the live DB connection, inside NewPgStream. - # Note: an empty schema_include is NOT an error - it means "unset", and - # the connector falls back to the schema field (default "public"). - - test:schema-exclude-requires-include: - desc: Confirm schema_exclude is rejected at startup when schema_include is left unset - env: - PG_DSN: '{{.PG_DSN}}' - cmds: - - | - set +e - go run ../../../../../cmd/redpanda-connect/main.go run \ - ./test_config_schema_exclude_no_include.yaml 2>&1 | head -5 - echo "exit $?" - # Expects: "schema_exclude requires schema_include to be set" error - # printed, process exits non-zero. Uses a dedicated fixture file rather - # than --set on test_config.yaml, since --set rejects an empty RHS - # ("foo=" -> "expected foo=bar syntax"), so schema_include can't be - # cleared that way. - - # ── Quick sanity ────────────────────────────────────────────────────────────── - - psql: - desc: Open a psql shell to the test database - cmds: - - psql "{{.PG_DSN}}" - - status: - desc: Show active replication slots and publications - cmds: - - psql "{{.PG_DSN}}" -c "SELECT slot_name, active FROM pg_replication_slots;" - - psql "{{.PG_DSN}}" -c "SELECT pubname FROM pg_publication;" diff --git a/internal/impl/postgresql/tests/current/docker-compose.yaml b/internal/impl/postgresql/tests/current/docker-compose.yaml deleted file mode 100644 index 896a807027..0000000000 --- a/internal/impl/postgresql/tests/current/docker-compose.yaml +++ /dev/null @@ -1,36 +0,0 @@ -services: - postgres: - image: postgres:16 - container_name: pgtest-postgres - ports: - - "5433:5432" - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: testdb - # Enable logical replication — required for postgres_cdc - command: postgres -c wal_level=logical -c max_replication_slots=10 -c max_wal_senders=10 - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] - interval: 3s - timeout: 5s - retries: 10 - - # One-shot setup container: creates schemas, tables, and seed data, then exits. - setup: - image: postgres:16 - container_name: pgtest-setup - depends_on: - postgres: - condition: service_healthy - environment: - PGPASSWORD: postgres - volumes: - - ./setup.sql:/setup.sql:ro - entrypoint: /bin/sh - command: - - -c - - | - psql -h postgres -U postgres -d testdb -f /setup.sql - echo "setup complete" - restart: "no" diff --git a/internal/impl/postgresql/tests/current/setup.sql b/internal/impl/postgresql/tests/current/setup.sql deleted file mode 100644 index d231c381f7..0000000000 --- a/internal/impl/postgresql/tests/current/setup.sql +++ /dev/null @@ -1,49 +0,0 @@ --- Multi-schema CDC test setup --- Tests: schema glob (tenant_*), schema_exclude (tenant_c), database_schema --- metadata, commit_ts_ms, before (update/delete) - --- ── Tenant schemas ──────────────────────────────────────────────────────────── --- tenant_a and tenant_b are replicated; tenant_c matches the tenant_* glob in --- test_config.yaml but is carved out via schema_exclude — its rows must --- never appear in the pipeline output. - -CREATE SCHEMA IF NOT EXISTS tenant_a; -CREATE SCHEMA IF NOT EXISTS tenant_b; -CREATE SCHEMA IF NOT EXISTS tenant_c; - --- ── Events table (same shape in each schema) ────────────────────────────────── - -CREATE TABLE IF NOT EXISTS tenant_a.events ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'active', - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE TABLE IF NOT EXISTS tenant_b.events ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'active', - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE TABLE IF NOT EXISTS tenant_c.events ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'active', - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - --- REPLICA IDENTITY FULL so update/delete messages carry the full before-row. -ALTER TABLE tenant_a.events REPLICA IDENTITY FULL; -ALTER TABLE tenant_b.events REPLICA IDENTITY FULL; -ALTER TABLE tenant_c.events REPLICA IDENTITY FULL; - --- ── Seed snapshot rows ──────────────────────────────────────────────────────── --- These are visible during the initial snapshot (stream_snapshot: true). --- tenant_c's row (mallory) must NOT appear in the output — see schema_exclude --- in test_config.yaml. - -INSERT INTO tenant_a.events (name) VALUES ('alice'), ('bob'); -INSERT INTO tenant_b.events (name) VALUES ('carol'); -INSERT INTO tenant_c.events (name) VALUES ('mallory'); diff --git a/internal/impl/postgresql/tests/current/test_config.yaml b/internal/impl/postgresql/tests/current/test_config.yaml deleted file mode 100644 index b440817dd1..0000000000 --- a/internal/impl/postgresql/tests/current/test_config.yaml +++ /dev/null @@ -1,45 +0,0 @@ -input: - postgres_cdc: - dsn: ${PG_DSN:postgres://postgres:postgres@localhost:5433/testdb?sslmode=disable} - slot_name: multi_schema_test_slot - stream_snapshot: true - # Glob pattern: replicates both tenant_a and tenant_b via one slot. - schema_include: tenant_* - # tenant_c matches the glob above but is carved out here — its rows must - # never appear in the output below, neither during snapshot nor CDC. - schema_exclude: - - tenant_c - tables: - - events - -pipeline: - processors: - # Annotate each message with all new metadata fields so the output clearly - # shows what the feature delivers. - - mapping: | - let op = @operation - let tbl = @table - let schema = @database_schema - let lsn = @lsn - let ts_ms = @commit_ts_ms - let before = @before - - root = { - "operation": $op, - "database_schema": $schema, - "table": $tbl, - "payload": this, - "lsn": if $lsn != null { $lsn } else { null }, - "commit_ts_ms": if $ts_ms != null { $ts_ms } else { null }, - "before": if $before != null { $before.string().parse_json() } else { null }, - } - -output: - stdout: - codec: lines - -logger: - level: INFO - -metrics: - none: {} diff --git a/internal/impl/postgresql/tests/current/test_config_schema_exclude_no_include.yaml b/internal/impl/postgresql/tests/current/test_config_schema_exclude_no_include.yaml deleted file mode 100644 index 014332ed82..0000000000 --- a/internal/impl/postgresql/tests/current/test_config_schema_exclude_no_include.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Negative-validation fixture for schema_exclude: schema_include is -# deliberately left unset. --set can't assign an empty string (the CLI -# rejects "foo=" as "expected foo=bar syntax"), so this exists as its own -# file instead of overriding test_config.yaml at run time. -input: - postgres_cdc: - dsn: ${PG_DSN:postgres://postgres:postgres@localhost:5433/testdb?sslmode=disable} - slot_name: schema_exclude_no_include_slot - schema_exclude: - - tenant_c - tables: - - events - -output: - stdout: - codec: lines - -logger: - level: INFO - -metrics: - none: {} From 306358078e660879717f664a26b5a0ab30d0a8ad Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Mon, 24 Aug 2026 15:02:45 +0100 Subject: [PATCH 50/70] postgres_cdc: add docs to exported function --- internal/impl/postgresql/pglogicalstream/multischema/resolver.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/impl/postgresql/pglogicalstream/multischema/resolver.go b/internal/impl/postgresql/pglogicalstream/multischema/resolver.go index 4579d90eb2..57b2e835b6 100644 --- a/internal/impl/postgresql/pglogicalstream/multischema/resolver.go +++ b/internal/impl/postgresql/pglogicalstream/multischema/resolver.go @@ -25,6 +25,7 @@ import ( "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pglogicalstream/sanitize" ) +// Resolver is responsible for helping resolve DB schemas for multi-schema support. type Resolver struct { // Include is the schema_include glob pattern. Include string From b70f5e30694e737fc86fa030aff138fd2c0d279a Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Mon, 24 Aug 2026 17:53:40 +0100 Subject: [PATCH 51/70] exclude partition tables --- internal/impl/postgresql/integration_test.go | 119 ++++++++++++++++++ .../pglogicalstream/multischema/resolver.go | 15 ++- 2 files changed, 128 insertions(+), 6 deletions(-) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index afd4d811fa..51fb40973a 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -2407,6 +2407,125 @@ postgres_cdc: assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") } +func TestIntegrationMultiSchemaIncludeAutoDiscoverExcludesPartitionedParent(t *testing.T) { + integration.CheckSkip(t) + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + _, err = db.Exec("CREATE SCHEMA tenant_a") + require.NoError(t, err) + _, err = db.Exec(` +CREATE TABLE tenant_a.orders ( + id INT NOT NULL, + created_at DATE NOT NULL, + PRIMARY KEY (id, created_at) +) PARTITION BY RANGE (created_at)`) + require.NoError(t, err) + _, err = db.Exec(` +CREATE TABLE tenant_a.orders_2025 PARTITION OF tenant_a.orders + FOR VALUES FROM ('2025-01-01') TO ('2026-01-01')`) + require.NoError(t, err) + _, err = db.Exec(` +CREATE TABLE tenant_a.orders_2026 PARTITION OF tenant_a.orders + FOR VALUES FROM ('2026-01-01') TO ('2027-01-01')`) + require.NoError(t, err) + + // Insert through the parent, as real usage would, letting Postgres route + // each row to the correct leaf partition. + _, err = db.Exec("INSERT INTO tenant_a.orders (id, created_at) VALUES (1, '2025-06-01')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_a.orders (id, created_at) VALUES (2, '2026-06-01')") + require.NoError(t, err) + + type msgMeta struct { + dbSchema string + table string + id int64 + } + + var ( + mu sync.Mutex + collected []msgMeta + ) + collectedLen := func() int { + mu.Lock() + defer mu.Unlock() + return len(collected) + } + + // No `tables` field: the partitioned "orders" table's leaf partitions must + // be auto-discovered without listing them by hand, and the partitioned + // parent itself must not be discovered. + tmpl := fmt.Sprintf(` +postgres_cdc: + dsn: %s + slot_name: partitioned_parent_excluded_slot + stream_snapshot: true + schema_include: tenant_* +`, databaseURL) + + sb := service.NewStreamBuilder() + require.NoError(t, sb.SetLoggerYAML(`level: WARN`)) + require.NoError(t, sb.AddInputYAML(tmpl)) + require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { + mu.Lock() + defer mu.Unlock() + for _, msg := range batch { + structured, err := msg.AsStructured() + if err != nil { + return err + } + id, err := structured.(map[string]any)["id"].(json.Number).Int64() + if err != nil { + return err + } + m := msgMeta{id: id} + m.dbSchema, _ = msg.MetaGet("database_schema") + m.table, _ = msg.MetaGet("table") + collected = append(collected, m) + } + return nil + })) + + stream, err := sb.Build() + require.NoError(t, err) + license.InjectTestService(stream.Resources()) + go func() { + if err := stream.Run(t.Context()); err != nil && !errors.Is(err, context.Canceled) { + t.Error(err) + } + }() + t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) + + // Exactly the 2 rows inserted, not 4: if the partitioned parent were + // wrongly auto-discovered alongside its leaf partitions, every row would + // be emitted twice (once from scanning the parent, once from scanning the + // leaf it belongs to). + assert.Eventually(t, func() bool { + return collectedLen() >= 2 + }, 30*time.Second, 100*time.Millisecond, "timed out waiting for snapshot rows; the partitioned parent may have swallowed the leaf partitions' rows, or TABLESAMPLE against the parent may have failed outright") + + // Give any erroneous duplicate emissions from the parent a chance to + // surface before asserting the final count. + assert.Never(t, func() bool { + return collectedLen() > 2 + }, 3*time.Second, 200*time.Millisecond, "received more snapshot rows than were inserted; the partitioned parent was likely auto-discovered alongside its leaf partitions") + + mu.Lock() + defer mu.Unlock() + + require.Len(t, collected, 2) + + seenIDs := make(map[int64]int) + for _, m := range collected { + assert.Equal(t, "tenant_a", m.dbSchema) + assert.NotEqual(t, "orders", m.table, "the partitioned parent must not be auto-discovered as a table in its own right") + assert.Contains(t, []string{"orders_2025", "orders_2026"}, m.table, "expected only leaf partitions to be auto-discovered") + seenIDs[m.id]++ + } + assert.Equal(t, map[int64]int{1: 1, 2: 1}, seenIDs, "each inserted row should be emitted exactly once") +} + func TestIntegrationMultiSchemaAndTableMatchingTest(t *testing.T) { integration.CheckSkip(t) diff --git a/internal/impl/postgresql/pglogicalstream/multischema/resolver.go b/internal/impl/postgresql/pglogicalstream/multischema/resolver.go index 57b2e835b6..9b8cf4af37 100644 --- a/internal/impl/postgresql/pglogicalstream/multischema/resolver.go +++ b/internal/impl/postgresql/pglogicalstream/multischema/resolver.go @@ -190,11 +190,11 @@ func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (v // ResolveExistingTables returns the quoted names of the publishable base // tables in each of the given (already quoted) schemas, keyed by quoted // schema name. A single query covering every schema, rather than one per -// schema, keeps this to one round-trip regardless of tenant count - the -// difference between one query and, say, one hundred on a multi-tenant, -// schema-per-tenant database on every connect and reconnect. Restricted to -// table_type = 'BASE TABLE' so a same-named view or foreign table is treated -// as missing rather than breaking CreatePublication. +// schema, keeps this to one round-trip regardless of tenant count. +// +// Queries pg_class/pg_namespace directly, filtered to relkind = 'r', so +// discovery only returns ordinary tables (including leaf partitions) and +// excludes partitioned parents, views, and foreign tables. func ResolveExistingTables(ctx context.Context, conn *pgconn.PgConn, quotedSchemas []string) (map[string]map[string]struct{}, error) { rawToQuoted := make(map[string]string, len(quotedSchemas)) args := make([]any, len(quotedSchemas)) @@ -210,7 +210,10 @@ func ResolveExistingTables(ctx context.Context, conn *pgconn.PgConn, quotedSchem } q, err := sanitize.SQLQuery( - fmt.Sprintf("SELECT table_schema, table_name FROM information_schema.tables WHERE table_schema IN (%s) AND table_type = 'BASE TABLE'", strings.Join(placeholders, ", ")), + fmt.Sprintf(`SELECT n.nspname AS table_schema, c.relname AS table_name +FROM pg_catalog.pg_class c +JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace +WHERE n.nspname IN (%s) AND c.relkind = 'r'`, strings.Join(placeholders, ", ")), args..., ) if err != nil { From 8ff138270ac96bdf1b8552d824e00e57404e92b3 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Mon, 24 Aug 2026 18:21:35 +0100 Subject: [PATCH 52/70] clean up tests --- internal/impl/postgresql/input_pg_stream.go | 8 - internal/impl/postgresql/integration_test.go | 255 +----------------- .../multischema/resolver_integration_test.go | 85 ++++++ 3 files changed, 93 insertions(+), 255 deletions(-) diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 84b8d2044c..ca59e5abce 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -549,14 +549,6 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser } // validateSchemaPattern validates a schema name or glob pattern. -// -// Unquoted patterns are matched via ILIKE against stored schema names (see -// resolveSchemas), not parsed as an identifier, so any character or leading -// character is accepted - including hyphens and leading digits - except a -// literal '"'. This lets a glob like "a0eebc99-*" match a UUID-suffixed -// schema that itself had to be created quoted. -// Double-quoted identifiers (e.g. "MySchema") are accepted as exact names; -// wildcards are not allowed inside quotes. func validateSchemaPattern(s string) error { if s == "" { return errors.New("schema cannot be empty") diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index 51fb40973a..48959fe401 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -1912,20 +1912,14 @@ func TestIntegrationMultiSchemaIncludeMatchesHyphenatedUUIDSchema(t *testing.T) require.NoError(t, err) type msgMeta struct { - dbSchema string - table string - operation string + dbSchema string + table string } var ( mu sync.Mutex collected []msgMeta ) - collectedLen := func() int { - mu.Lock() - defer mu.Unlock() - return len(collected) - } tmpl := fmt.Sprintf(` postgres_cdc: @@ -1947,7 +1941,6 @@ postgres_cdc: m := msgMeta{} m.dbSchema, _ = msg.MetaGet("database_schema") m.table, _ = msg.MetaGet("table") - m.operation, _ = msg.MetaGet("operation") collected = append(collected, m) } return nil @@ -1964,23 +1957,16 @@ postgres_cdc: t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) assert.Eventually(t, func() bool { - return collectedLen() >= 1 + mu.Lock() + defer mu.Unlock() + return len(collected) >= 1 }, 30*time.Second, 100*time.Millisecond, "timed out waiting for snapshot row from hyphenated UUID schema") - _, err = db.Exec(fmt.Sprintf(`INSERT INTO "%s".events (name) VALUES ('bob')`, uuidSchema)) - require.NoError(t, err) - - assert.EventuallyWithT(t, func(c *assert.CollectT) { - assert.Equal(c, 2, collectedLen()) - }, 30*time.Second, 100*time.Millisecond, "timed out waiting for CDC row from hyphenated UUID schema") - mu.Lock() defer mu.Unlock() - require.Len(t, collected, 2) - for _, m := range collected { - assert.Equal(t, uuidSchema, m.dbSchema, "database_schema metadata should be the raw, unquoted, case-preserved schema name") - assert.Equal(t, "events", m.table) - } + require.Len(t, collected, 1) + assert.Equal(t, uuidSchema, collected[0].dbSchema, "database_schema metadata should be the raw, unquoted, case-preserved schema name") + assert.Equal(t, "events", collected[0].table) } func TestIntegrationMultiSchemaMissingTableDegradesGracefully(t *testing.T) { @@ -2148,112 +2134,6 @@ postgres_cdc: assert.Equal(t, "events", collected[0].table) } -func TestIntegrationMultiSchemaIncludeExcludeConfigValidation(t *testing.T) { - integration.CheckSkip(t) - - t.Run("schema_include matches nothing", func(t *testing.T) { - databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") - require.NoError(t, err) - - tmpl := fmt.Sprintf(` -dsn: %s -slot_name: no_schema_match_slot -schema_include: nonexistent_schema_zzz_* -tables: - - events -`, databaseURL) - - conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) - require.NoError(t, err) - - mgr := service.MockResources() - license.InjectTestService(mgr) - - input, err := newPgStreamInput(conf, mgr) - require.NoError(t, err) - - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) - defer cancel() - - err = input.Connect(ctx) - require.Error(t, err) - assert.Contains(t, err.Error(), "no schemas found matching schema_include pattern") - }) - - t.Run("schema_include matches nothing with empty tables", func(t *testing.T) { - databaseURL, _, err := ResourceWithPostgreSQLVersion(t, "16") - require.NoError(t, err) - - tmpl := fmt.Sprintf(` -dsn: %s -slot_name: no_schema_match_empty_tables_slot -schema_include: nonexistent_schema_zzz_* -`, databaseURL) - - conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) - require.NoError(t, err) - - mgr := service.MockResources() - license.InjectTestService(mgr) - - input, err := newPgStreamInput(conf, mgr) - require.NoError(t, err) - - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) - defer cancel() - - // Bypass the benthos AsyncReader's infinite connect-retry loop, same as - // the "schema_include matches nothing" case above. - err = input.Connect(ctx) - require.Error(t, err) - assert.Contains(t, err.Error(), "no schemas found matching schema_include pattern") - }) - - t.Run("schema_exclude excludes every matched schema", func(t *testing.T) { - databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") - require.NoError(t, err) - - // Both schemas match tenant_*, but schema_exclude below excludes them all. - for _, schema := range []string{"tenant_a", "tenant_b"} { - _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", schema)) - require.NoError(t, err) - _, err = db.Exec(fmt.Sprintf( - "CREATE TABLE %s.events (id SERIAL PRIMARY KEY, name TEXT)", schema)) - require.NoError(t, err) - } - - tmpl := fmt.Sprintf(` -dsn: %s -slot_name: schema_exclude_all_matched_slot -schema_include: tenant_* -schema_exclude: - - tenant_* -tables: - - events -`, databaseURL) - - conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) - require.NoError(t, err) - - mgr := service.MockResources() - license.InjectTestService(mgr) - - input, err := newPgStreamInput(conf, mgr) - require.NoError(t, err) - - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) - defer cancel() - - // Bypass the benthos AsyncReader's infinite connect-retry loop, same as - // the "schema_include matches nothing" case above. - err = input.Connect(ctx) - require.Error(t, err) - assert.Contains(t, err.Error(), "matched schema(s)") - assert.Contains(t, err.Error(), "excluded all of them") - assert.NotContains(t, err.Error(), "no schemas found matching schema_include pattern") - }) -} - // TestIntegrationSchemaExcludeCarvesOutTenantAutoDiscover is // TestIntegrationSchemaExcludeCarvesOutTenant with `tables` left unset: it // verifies that leaving `tables` empty under schema_include auto-discovers @@ -2407,125 +2287,6 @@ postgres_cdc: assert.Equal(t, 1, cdcSchemas["tenant_b"], "expected 1 CDC row from tenant_b") } -func TestIntegrationMultiSchemaIncludeAutoDiscoverExcludesPartitionedParent(t *testing.T) { - integration.CheckSkip(t) - databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") - require.NoError(t, err) - - _, err = db.Exec("CREATE SCHEMA tenant_a") - require.NoError(t, err) - _, err = db.Exec(` -CREATE TABLE tenant_a.orders ( - id INT NOT NULL, - created_at DATE NOT NULL, - PRIMARY KEY (id, created_at) -) PARTITION BY RANGE (created_at)`) - require.NoError(t, err) - _, err = db.Exec(` -CREATE TABLE tenant_a.orders_2025 PARTITION OF tenant_a.orders - FOR VALUES FROM ('2025-01-01') TO ('2026-01-01')`) - require.NoError(t, err) - _, err = db.Exec(` -CREATE TABLE tenant_a.orders_2026 PARTITION OF tenant_a.orders - FOR VALUES FROM ('2026-01-01') TO ('2027-01-01')`) - require.NoError(t, err) - - // Insert through the parent, as real usage would, letting Postgres route - // each row to the correct leaf partition. - _, err = db.Exec("INSERT INTO tenant_a.orders (id, created_at) VALUES (1, '2025-06-01')") - require.NoError(t, err) - _, err = db.Exec("INSERT INTO tenant_a.orders (id, created_at) VALUES (2, '2026-06-01')") - require.NoError(t, err) - - type msgMeta struct { - dbSchema string - table string - id int64 - } - - var ( - mu sync.Mutex - collected []msgMeta - ) - collectedLen := func() int { - mu.Lock() - defer mu.Unlock() - return len(collected) - } - - // No `tables` field: the partitioned "orders" table's leaf partitions must - // be auto-discovered without listing them by hand, and the partitioned - // parent itself must not be discovered. - tmpl := fmt.Sprintf(` -postgres_cdc: - dsn: %s - slot_name: partitioned_parent_excluded_slot - stream_snapshot: true - schema_include: tenant_* -`, databaseURL) - - sb := service.NewStreamBuilder() - require.NoError(t, sb.SetLoggerYAML(`level: WARN`)) - require.NoError(t, sb.AddInputYAML(tmpl)) - require.NoError(t, sb.AddBatchConsumerFunc(func(_ context.Context, batch service.MessageBatch) error { - mu.Lock() - defer mu.Unlock() - for _, msg := range batch { - structured, err := msg.AsStructured() - if err != nil { - return err - } - id, err := structured.(map[string]any)["id"].(json.Number).Int64() - if err != nil { - return err - } - m := msgMeta{id: id} - m.dbSchema, _ = msg.MetaGet("database_schema") - m.table, _ = msg.MetaGet("table") - collected = append(collected, m) - } - return nil - })) - - stream, err := sb.Build() - require.NoError(t, err) - license.InjectTestService(stream.Resources()) - go func() { - if err := stream.Run(t.Context()); err != nil && !errors.Is(err, context.Canceled) { - t.Error(err) - } - }() - t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) - - // Exactly the 2 rows inserted, not 4: if the partitioned parent were - // wrongly auto-discovered alongside its leaf partitions, every row would - // be emitted twice (once from scanning the parent, once from scanning the - // leaf it belongs to). - assert.Eventually(t, func() bool { - return collectedLen() >= 2 - }, 30*time.Second, 100*time.Millisecond, "timed out waiting for snapshot rows; the partitioned parent may have swallowed the leaf partitions' rows, or TABLESAMPLE against the parent may have failed outright") - - // Give any erroneous duplicate emissions from the parent a chance to - // surface before asserting the final count. - assert.Never(t, func() bool { - return collectedLen() > 2 - }, 3*time.Second, 200*time.Millisecond, "received more snapshot rows than were inserted; the partitioned parent was likely auto-discovered alongside its leaf partitions") - - mu.Lock() - defer mu.Unlock() - - require.Len(t, collected, 2) - - seenIDs := make(map[int64]int) - for _, m := range collected { - assert.Equal(t, "tenant_a", m.dbSchema) - assert.NotEqual(t, "orders", m.table, "the partitioned parent must not be auto-discovered as a table in its own right") - assert.Contains(t, []string{"orders_2025", "orders_2026"}, m.table, "expected only leaf partitions to be auto-discovered") - seenIDs[m.id]++ - } - assert.Equal(t, map[int64]int{1: 1, 2: 1}, seenIDs, "each inserted row should be emitted exactly once") -} - func TestIntegrationMultiSchemaAndTableMatchingTest(t *testing.T) { integration.CheckSkip(t) diff --git a/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go b/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go index a330e5130b..ef222d9a57 100644 --- a/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go +++ b/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go @@ -23,6 +23,7 @@ import ( "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" + "github.com/redpanda-data/benthos/v4/public/service" "github.com/redpanda-data/benthos/v4/public/service/integration" ) @@ -143,6 +144,90 @@ func TestIntegrationResolveSchemasBareUUIDSchema(t *testing.T) { assert.Empty(t, visible) } +func TestIntegrationResolveExistingTablesExcludesPartitionedParent(t *testing.T) { + integration.CheckSkip(t) + + _, adminURL := createDockerInstance(t) + + adminDB, err := sql.Open("postgres", adminURL) + require.NoError(t, err) + defer adminDB.Close() + + _, err = adminDB.Exec("CREATE SCHEMA tenant_a") + require.NoError(t, err) + _, err = adminDB.Exec(` +CREATE TABLE tenant_a.orders ( + id INT NOT NULL, + created_at DATE NOT NULL, + PRIMARY KEY (id, created_at) +) PARTITION BY RANGE (created_at)`) + require.NoError(t, err) + _, err = adminDB.Exec(` +CREATE TABLE tenant_a.orders_2025 PARTITION OF tenant_a.orders + FOR VALUES FROM ('2025-01-01') TO ('2026-01-01')`) + require.NoError(t, err) + _, err = adminDB.Exec(` +CREATE TABLE tenant_a.orders_2026 PARTITION OF tenant_a.orders + FOR VALUES FROM ('2026-01-01') TO ('2027-01-01')`) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + + conn, err := pgconn.Connect(ctx, adminURL) + require.NoError(t, err) + defer closeConn(t, conn) + + existing, err := ResolveExistingTables(ctx, conn, []string{`"tenant_a"`}) + require.NoError(t, err) + + tables := existing[`"tenant_a"`] + assert.NotContains(t, tables, `"orders"`, "the partitioned parent must not be discovered as a table in its own right") + assert.Contains(t, tables, `"orders_2025"`) + assert.Contains(t, tables, `"orders_2026"`) + assert.Len(t, tables, 2, "only the leaf partitions should be discovered") +} + +func TestIntegrationResolveSchemasConfigValidation(t *testing.T) { + integration.CheckSkip(t) + + _, adminURL := createDockerInstance(t) + + adminDB, err := sql.Open("postgres", adminURL) + require.NoError(t, err) + defer adminDB.Close() + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + + conn, err := pgconn.Connect(ctx, adminURL) + require.NoError(t, err) + defer closeConn(t, conn) + + logger := service.MockResources().Logger() + + t.Run("schema_include matches nothing", func(t *testing.T) { + resolver := NewResolver("nonexistent_schema_zzz_*", nil) + _, err := resolver.Resolve(ctx, conn, logger) + require.Error(t, err) + assert.Contains(t, err.Error(), "no schemas found matching schema_include pattern") + }) + + t.Run("schema_exclude excludes every matched schema", func(t *testing.T) { + _, err = adminDB.Exec("CREATE SCHEMA tenant_a") + require.NoError(t, err) + _, err = adminDB.Exec("CREATE SCHEMA tenant_b") + require.NoError(t, err) + + resolver := NewResolver("tenant_*", []string{"tenant_*"}) + _, err := resolver.Resolve(ctx, conn, logger) + require.Error(t, err) + assert.Contains(t, err.Error(), "matched schema(s)") + assert.Contains(t, err.Error(), "excluded all of them") + assert.NotContains(t, err.Error(), "no schemas found matching schema_include pattern") + }) +} + func closeConn(t testing.TB, conn *pgconn.PgConn) { ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() From 292160acae8cd4e5cec2893fb53550080af42ab0 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Mon, 24 Aug 2026 21:07:36 +0100 Subject: [PATCH 53/70] remove ordering sensitivity --- .../postgresql/pglogicalstream/multischema/resolver.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/impl/postgresql/pglogicalstream/multischema/resolver.go b/internal/impl/postgresql/pglogicalstream/multischema/resolver.go index 9b8cf4af37..28dd7fc857 100644 --- a/internal/impl/postgresql/pglogicalstream/multischema/resolver.go +++ b/internal/impl/postgresql/pglogicalstream/multischema/resolver.go @@ -83,8 +83,11 @@ func (r *Resolver) Resolve(ctx context.Context, conn *pgconn.PgConn, logger *ser schemas = remaining } - if len(inaccessibleSchemas) > 0 && !slices.Equal(inaccessibleSchemas, r.previouslyInaccessible) { - logger.Warnf("schema_include pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", r.Include, inaccessibleSchemas) + // Compared set-wise, not with slices.Equal: resolveSchemas' pg_namespace + // query has no ORDER BY, so an unchanged set of inaccessible schemas can + // still come back in a different order between reconnects. + if newlyInaccessible, _ := diffSchemaSets(r.previouslyInaccessible, inaccessibleSchemas); len(newlyInaccessible) > 0 { + logger.Warnf("schema_include pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", r.Include, newlyInaccessible) } r.previouslyInaccessible = slices.Clone(inaccessibleSchemas) From b6a6a73265bebd8e39d9cdf55f6898a3199f031b Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 25 Aug 2026 10:06:31 +0100 Subject: [PATCH 54/70] postgres_cdc: improve filtering of tables --- internal/impl/postgresql/integration_test.go | 125 +++++++++++++++--- .../pglogicalstream/logical_stream.go | 8 +- .../pglogicalstream/multischema/resolver.go | 36 +++-- .../multischema/resolver_integration_test.go | 18 ++- 4 files changed, 149 insertions(+), 38 deletions(-) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index 48959fe401..a5f1fa8b3f 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -2134,12 +2134,6 @@ postgres_cdc: assert.Equal(t, "events", collected[0].table) } -// TestIntegrationSchemaExcludeCarvesOutTenantAutoDiscover is -// TestIntegrationSchemaExcludeCarvesOutTenant with `tables` left unset: it -// verifies that leaving `tables` empty under schema_include auto-discovers -// the "events" table in each matched schema instead of falling back to a -// database-wide FOR ALL TABLES publication, so schema_exclude still carves -// tenant_c out of both the snapshot and CDC. func TestIntegrationMultiSchemaExcludeCarvesOutTenantAutoDiscover(t *testing.T) { integration.CheckSkip(t) databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") @@ -2154,6 +2148,24 @@ func TestIntegrationMultiSchemaExcludeCarvesOutTenantAutoDiscover(t *testing.T) require.NoError(t, err) } + // tenant_a additionally has a partitioned "orders" table, to verify + // auto-discovery excludes the parent while still picking up its leaves. + _, err = db.Exec(` +CREATE TABLE tenant_a.orders ( + id INT NOT NULL, + created_at DATE NOT NULL, + PRIMARY KEY (id, created_at) +) PARTITION BY RANGE (created_at)`) + require.NoError(t, err) + _, err = db.Exec(` +CREATE TABLE tenant_a.orders_2025 PARTITION OF tenant_a.orders + FOR VALUES FROM ('2025-01-01') TO ('2026-01-01')`) + require.NoError(t, err) + _, err = db.Exec(` +CREATE TABLE tenant_a.orders_2026 PARTITION OF tenant_a.orders + FOR VALUES FROM ('2026-01-01') TO ('2027-01-01')`) + require.NoError(t, err) + // Pre-load snapshot data, including a row in the excluded schema that must // never surface. _, err = db.Exec("INSERT INTO tenant_a.events (name) VALUES ('alice'), ('bob')") @@ -2163,6 +2175,13 @@ func TestIntegrationMultiSchemaExcludeCarvesOutTenantAutoDiscover(t *testing.T) _, err = db.Exec("INSERT INTO tenant_c.events (name) VALUES ('mallory')") require.NoError(t, err) + // Insert through the parent, as real usage would, letting Postgres route + // each row to the correct leaf partition. + _, err = db.Exec("INSERT INTO tenant_a.orders (id, created_at) VALUES (1, '2025-06-01')") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO tenant_a.orders (id, created_at) VALUES (2, '2026-06-01')") + require.NoError(t, err) + type msgMeta struct { dbSchema string table string @@ -2219,12 +2238,19 @@ postgres_cdc: }() t.Cleanup(func() { require.NoError(t, stream.StopWithin(10*time.Second)) }) - // Wait for the 3 snapshot rows from the two non-excluded schemas; tenant_c's - // row must never contribute to this count. + // Wait for the 5 snapshot rows from the two non-excluded schemas (3 + // "events" rows plus 2 "orders" rows split across its leaf partitions); + // tenant_c's row must never contribute to this count. assert.Eventually(t, func() bool { - return collectedLen() >= 3 + return collectedLen() >= 5 }, 30*time.Second, 100*time.Millisecond, "timed out waiting for snapshot rows") + // Give any erroneous duplicate emissions from the partitioned parent a + // chance to surface before moving on to the CDC phase. + assert.Never(t, func() bool { + return collectedLen() > 5 + }, 3*time.Second, 200*time.Millisecond, "received more snapshot rows than were inserted; the partitioned parent was likely auto-discovered alongside its leaf partitions") + // Insert CDC rows into all three schemas, including the excluded one. _, err = db.Exec("INSERT INTO tenant_a.events (name) VALUES ('dave')") require.NoError(t, err) @@ -2233,25 +2259,26 @@ postgres_cdc: _, err = db.Exec("INSERT INTO tenant_c.events (name) VALUES ('trudy')") require.NoError(t, err) - // Wait for the 2 CDC rows from the non-excluded schemas (total 5). + // Wait for the 2 CDC rows from the non-excluded schemas (total 7). assert.EventuallyWithT(t, func(c *assert.CollectT) { - assert.Equal(c, 5, collectedLen()) + assert.Equal(c, 7, collectedLen()) }, 30*time.Second, 100*time.Millisecond, "timed out waiting for CDC rows") // tenant_c's CDC insert above raced the same replication stream as the // tenant_a/tenant_b inserts already confirmed above, so if it were going // to leak through it would have by now; assert the count never climbs - // past 5 to catch a delayed leak instead of just checking once. + // past 7 to catch a delayed leak instead of just checking once. assert.Never(t, func() bool { - return collectedLen() > 5 + return collectedLen() > 7 }, 3*time.Second, 200*time.Millisecond, "received unexpected message(s) from excluded schema tenant_c") mu.Lock() defer mu.Unlock() - require.Len(t, collected, 5) + require.Len(t, collected, 7) for _, m := range collected { assert.NotEqual(t, "tenant_c", m.dbSchema, "tenant_c is excluded and must never appear, got message: %+v", m) + assert.NotEqual(t, "orders", m.table, "the partitioned parent must not be auto-discovered as a table in its own right") } var snapshots, cdcMsgs []msgMeta @@ -2264,15 +2291,24 @@ postgres_cdc: } // Snapshot assertions. - require.Len(t, snapshots, 3) + require.Len(t, snapshots, 5) snapshotSchemas := make(map[string]int) + ordersLeaves := make(map[string]int) for _, m := range snapshots { - assert.Equal(t, "events", m.table, "snapshot: table should be bare name without schema prefix") assert.Empty(t, m.lsn, "snapshot rows have no LSN") - snapshotSchemas[m.dbSchema]++ + switch m.table { + case "events": + snapshotSchemas[m.dbSchema]++ + case "orders_2025", "orders_2026": + assert.Equal(t, "tenant_a", m.dbSchema, "orders and its partitions only exist in tenant_a") + ordersLeaves[m.table]++ + default: + t.Errorf("unexpected table in snapshot: %+v", m) + } } - assert.Equal(t, 2, snapshotSchemas["tenant_a"], "expected 2 snapshot rows from tenant_a") - assert.Equal(t, 1, snapshotSchemas["tenant_b"], "expected 1 snapshot row from tenant_b") + assert.Equal(t, 2, snapshotSchemas["tenant_a"], "expected 2 \"events\" snapshot rows from tenant_a") + assert.Equal(t, 1, snapshotSchemas["tenant_b"], "expected 1 \"events\" snapshot row from tenant_b") + assert.Equal(t, map[string]int{"orders_2025": 1, "orders_2026": 1}, ordersLeaves, "expected exactly one snapshot row from each leaf partition, and none from the partitioned parent") // CDC assertions. require.Len(t, cdcMsgs, 2) @@ -2520,3 +2556,54 @@ postgres_cdc: assert.Contains(t, collected, msgMeta{dbSchema: "tenant_b", table: "ordres"}) }) } + +func TestIntegrationMultiSchemaExplicitTablesAcceptsPartitionedTable(t *testing.T) { + integration.CheckSkip(t) + databaseURL, db, err := ResourceWithPostgreSQLVersion(t, "16") + require.NoError(t, err) + + _, err = db.Exec(` + CREATE SCHEMA tenant_a; + CREATE SCHEMA tenant_b; + + CREATE TABLE tenant_a.orders ( + id INT NOT NULL, + created_at DATE NOT NULL, + PRIMARY KEY (id, created_at) + ) PARTITION BY RANGE (created_at); + CREATE TABLE tenant_a.orders_2025 PARTITION OF tenant_a.orders + FOR VALUES FROM ('2025-01-01') TO ('2026-01-01'); + + CREATE TABLE tenant_b.orders ( + id INT NOT NULL, + created_at DATE NOT NULL, + PRIMARY KEY (id, created_at) + ) PARTITION BY RANGE (created_at); + CREATE TABLE tenant_b.orders_2025 PARTITION OF tenant_b.orders + FOR VALUES FROM ('2025-01-01') TO ('2026-01-01'); + `) + require.NoError(t, err) + + tmpl := fmt.Sprintf(` +dsn: %s +slot_name: partitioned_table_explicit_slot +schema_include: tenant_* +tables: + - orders +`, databaseURL) + + conf, err := newPostgresCDCConfig().ParseYAML(tmpl, nil) + require.NoError(t, err) + + mgr := service.MockResources() + license.InjectTestService(mgr) + + input, err := newPgStreamInput(conf, mgr) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + + err = input.Connect(ctx) + require.NoError(t, err, "orders is a partitioned table that exists in every matched schema and should be accepted, not reported as missing") +} diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 9b29cfdb95..6410b87f4c 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -136,7 +136,13 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { for _, schema := range schemas { existingTables := existingTablesBySchema[schema] if autoDiscoverTables { - for table := range existingTables { + // Only ordinary tables: a partitioned parent must not be + // auto-discovered alongside its leaf partitions (which are + // themselves ordinary tables) - see ResolveExistingTables. + for table, relkind := range existingTables { + if relkind != multischema.RelKindOrdinaryTable { + continue + } tables = append(tables, TableFQN{Schema: schema, Table: table}) } continue diff --git a/internal/impl/postgresql/pglogicalstream/multischema/resolver.go b/internal/impl/postgresql/pglogicalstream/multischema/resolver.go index 28dd7fc857..f909080cc4 100644 --- a/internal/impl/postgresql/pglogicalstream/multischema/resolver.go +++ b/internal/impl/postgresql/pglogicalstream/multischema/resolver.go @@ -190,15 +190,25 @@ func resolveSchemas(ctx context.Context, conn *pgconn.PgConn, pattern string) (v return schemas, hidden, nil } -// ResolveExistingTables returns the quoted names of the publishable base -// tables in each of the given (already quoted) schemas, keyed by quoted -// schema name. A single query covering every schema, rather than one per -// schema, keeps this to one round-trip regardless of tenant count. +// RelKindOrdinaryTable and RelKindPartitionedTable are the pg_class.relkind +// values ResolveExistingTables reports: an ordinary table or leaf partition, +// and a partitioned table's parent. Both are valid ALTER PUBLICATION ... ADD +// TABLE targets from PostgreSQL 13 onward. +const ( + RelKindOrdinaryTable = byte('r') + RelKindPartitionedTable = byte('p') +) + +// ResolveExistingTables returns the relkind of every publishable relation in +// each of the given (already quoted) schemas, keyed by quoted schema name +// and then quoted table name. One query covers every schema, keeping this to +// a single round-trip regardless of tenant count. // -// Queries pg_class/pg_namespace directly, filtered to relkind = 'r', so -// discovery only returns ordinary tables (including leaf partitions) and -// excludes partitioned parents, views, and foreign tables. -func ResolveExistingTables(ctx context.Context, conn *pgconn.PgConn, quotedSchemas []string) (map[string]map[string]struct{}, error) { +// Existence checks for explicitly-listed tables should accept either +// relkind. Auto-discovery should keep only RelKindOrdinaryTable - including +// a partitioned parent would double-count rows already covered by its leaf +// partitions, and TABLESAMPLE rejects partitioned parents outright. +func ResolveExistingTables(ctx context.Context, conn *pgconn.PgConn, quotedSchemas []string) (map[string]map[string]byte, error) { rawToQuoted := make(map[string]string, len(quotedSchemas)) args := make([]any, len(quotedSchemas)) placeholders := make([]string, len(quotedSchemas)) @@ -213,10 +223,10 @@ func ResolveExistingTables(ctx context.Context, conn *pgconn.PgConn, quotedSchem } q, err := sanitize.SQLQuery( - fmt.Sprintf(`SELECT n.nspname AS table_schema, c.relname AS table_name + fmt.Sprintf(`SELECT n.nspname AS table_schema, c.relname AS table_name, c.relkind FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace -WHERE n.nspname IN (%s) AND c.relkind = 'r'`, strings.Join(placeholders, ", ")), +WHERE n.nspname IN (%s) AND c.relkind IN ('r', 'p')`, strings.Join(placeholders, ", ")), args..., ) if err != nil { @@ -228,14 +238,14 @@ WHERE n.nspname IN (%s) AND c.relkind = 'r'`, strings.Join(placeholders, ", ")), return nil, fmt.Errorf("querying tables in schema(s) %v: %w", quotedSchemas, err) } - existing := make(map[string]map[string]struct{}, len(quotedSchemas)) + existing := make(map[string]map[string]byte, len(quotedSchemas)) for _, quotedSchema := range quotedSchemas { - existing[quotedSchema] = map[string]struct{}{} + existing[quotedSchema] = map[string]byte{} } if len(results) > 0 { for _, row := range results[0].Rows { quotedSchema := rawToQuoted[string(row[0])] - existing[quotedSchema][sanitize.QuotePostgresIdentifier(string(row[1]))] = struct{}{} + existing[quotedSchema][sanitize.QuotePostgresIdentifier(string(row[1]))] = row[2][0] } } return existing, nil diff --git a/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go b/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go index ef222d9a57..3373fe217c 100644 --- a/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go +++ b/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go @@ -144,7 +144,15 @@ func TestIntegrationResolveSchemasBareUUIDSchema(t *testing.T) { assert.Empty(t, visible) } -func TestIntegrationResolveExistingTablesExcludesPartitionedParent(t *testing.T) { +// TestIntegrationResolveExistingTablesReportsRelKind verifies that +// ResolveExistingTables reports a partitioned table's parent and its leaf +// partitions with distinct relkinds, rather than excluding the parent +// outright: callers need both, since a partitioned parent is a valid +// publication member if explicitly listed by the user, but must not be +// auto-discovered alongside its own leaf partitions (that filtering is the +// auto-discovery caller's responsibility, not this function's - see +// logical_stream.go's use of RelKindOrdinaryTable). +func TestIntegrationResolveExistingTablesReportsRelKind(t *testing.T) { integration.CheckSkip(t) _, adminURL := createDockerInstance(t) @@ -182,10 +190,10 @@ CREATE TABLE tenant_a.orders_2026 PARTITION OF tenant_a.orders require.NoError(t, err) tables := existing[`"tenant_a"`] - assert.NotContains(t, tables, `"orders"`, "the partitioned parent must not be discovered as a table in its own right") - assert.Contains(t, tables, `"orders_2025"`) - assert.Contains(t, tables, `"orders_2026"`) - assert.Len(t, tables, 2, "only the leaf partitions should be discovered") + require.Len(t, tables, 3, "the parent and both leaf partitions should all be reported") + assert.Equal(t, RelKindPartitionedTable, tables[`"orders"`], "the partitioned parent should be reported, not excluded") + assert.Equal(t, RelKindOrdinaryTable, tables[`"orders_2025"`]) + assert.Equal(t, RelKindOrdinaryTable, tables[`"orders_2026"`]) } func TestIntegrationResolveSchemasConfigValidation(t *testing.T) { From cf9e72bc7abfc2f4c343c8774097b8ce875ff873 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 25 Aug 2026 10:12:21 +0100 Subject: [PATCH 55/70] clean up --- .../impl/postgresql/input_pg_stream_test.go | 18 ------------------ .../pglogicalstream/sanitize/sanitize_test.go | 1 + 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/internal/impl/postgresql/input_pg_stream_test.go b/internal/impl/postgresql/input_pg_stream_test.go index 322de40f24..6e43213b81 100644 --- a/internal/impl/postgresql/input_pg_stream_test.go +++ b/internal/impl/postgresql/input_pg_stream_test.go @@ -216,24 +216,6 @@ tables: require.NoError(t, err) } -// TestSchemaAcceptsUnicodeIdentifier verifies that the schema field (single -// exact-name path) still accepts unquoted unicode identifiers like -// "münchen", matching sanitize.NormalizePostgresIdentifier which is the sole -// validator on this path (see NewPgStream). schema no longer runs through -// validateSchemaPattern, so this guards against that ASCII-only validator -// regressing this path again in the future. -func TestSchemaAcceptsUnicodeIdentifier(t *testing.T) { - yaml := ` -dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable -schema: münchen -slot_name: test_slot -tables: - - events -` - _, err := parsePgStreamInput(t, yaml) - require.NoError(t, err) -} - func TestNewPgStreamInputSignalTableName(t *testing.T) { env := service.NewEnvironment() spec := newPostgresCDCConfig() diff --git a/internal/impl/postgresql/pglogicalstream/sanitize/sanitize_test.go b/internal/impl/postgresql/pglogicalstream/sanitize/sanitize_test.go index e79b233fb7..4e013ee634 100644 --- a/internal/impl/postgresql/pglogicalstream/sanitize/sanitize_test.go +++ b/internal/impl/postgresql/pglogicalstream/sanitize/sanitize_test.go @@ -280,6 +280,7 @@ func TestIdentifierValidation(t *testing.T) { `_Foobar`, strings.Repeat("a", 63), strings.Repeat("A", 63), + `münchen`, } for _, i := range unquoted { From 3d8ce20bab497af545144137557af613671a500c Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 25 Aug 2026 11:10:00 +0100 Subject: [PATCH 56/70] address test clean up --- internal/impl/postgresql/integration_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index a5f1fa8b3f..941c8b4290 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -2600,6 +2600,7 @@ tables: input, err := newPgStreamInput(conf, mgr) require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, input.Close(context.Background())) }) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() From ad33babd3f9dd3263383b5db6c7c6c9166f85ce8 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 25 Aug 2026 11:10:12 +0100 Subject: [PATCH 57/70] address filtering log --- .../pglogicalstream/multischema/resolver.go | 56 +++++++++-------- .../multischema/resolver_integration_test.go | 63 +++++++++++++++++++ 2 files changed, 94 insertions(+), 25 deletions(-) diff --git a/internal/impl/postgresql/pglogicalstream/multischema/resolver.go b/internal/impl/postgresql/pglogicalstream/multischema/resolver.go index f909080cc4..ad5a94f5cb 100644 --- a/internal/impl/postgresql/pglogicalstream/multischema/resolver.go +++ b/internal/impl/postgresql/pglogicalstream/multischema/resolver.go @@ -55,37 +55,20 @@ func (r *Resolver) Resolve(ctx context.Context, conn *pgconn.PgConn, logger *ser matchedSchemas := schemas if len(r.Exclude) > 0 { - // Filtering happens entirely against the schemas slice we already - // fetched above - no extra DB round-trips per exclude pattern. var excluded []string - remaining := make([]string, 0, len(schemas)) - for _, schema := range schemas { - var isExcluded bool - for _, pattern := range r.Exclude { - matched, err := schemaMatchesExcludePattern(schema, pattern) - if err != nil { - return nil, fmt.Errorf("evaluating schema_exclude pattern %q against schema %q: %w", pattern, schema, err) - } - if matched { - isExcluded = true - break - } - } - if isExcluded { - excluded = append(excluded, schema) - continue - } - remaining = append(remaining, schema) + schemas, excluded, err = r.filterExcluded(schemas) + if err != nil { + return nil, err } if len(excluded) > 0 { - logger.Debugf("schema_exclude %v excluded %d schema(s) %v from schema_include pattern %q; %d schema(s) remain: %v", r.Exclude, len(excluded), excluded, r.Include, len(remaining), remaining) + logger.Debugf("schema_exclude %v excluded %d schema(s) %v from schema_include pattern %q; %d schema(s) remain: %v", r.Exclude, len(excluded), excluded, r.Include, len(schemas), schemas) + } + + if inaccessibleSchemas, _, err = r.filterExcluded(inaccessibleSchemas); err != nil { + return nil, err } - schemas = remaining } - // Compared set-wise, not with slices.Equal: resolveSchemas' pg_namespace - // query has no ORDER BY, so an unchanged set of inaccessible schemas can - // still come back in a different order between reconnects. if newlyInaccessible, _ := diffSchemaSets(r.previouslyInaccessible, inaccessibleSchemas); len(newlyInaccessible) > 0 { logger.Warnf("schema_include pattern %q matches schema(s) %v that the configured role cannot see (missing USAGE privilege); they will be skipped", r.Include, newlyInaccessible) } @@ -113,6 +96,29 @@ func (r *Resolver) Resolve(ctx context.Context, conn *pgconn.PgConn, logger *ser return schemas, nil } +func (r *Resolver) filterExcluded(schemas []string) (remaining, excluded []string, err error) { + remaining = make([]string, 0, len(schemas)) + for _, schema := range schemas { + var isExcluded bool + for _, pattern := range r.Exclude { + matched, err := schemaMatchesExcludePattern(schema, pattern) + if err != nil { + return nil, nil, fmt.Errorf("evaluating schema_exclude pattern %q against schema %q: %w", pattern, schema, err) + } + if matched { + isExcluded = true + break + } + } + if isExcluded { + excluded = append(excluded, schema) + continue + } + remaining = append(remaining, schema) + } + return remaining, excluded, nil +} + func schemaPatternToLike(pattern string) (likePattern string, caseSensitive bool, err error) { if strings.HasPrefix(pattern, `"`) { unquoted, err := sanitize.UnquotePostgresIdentifier(pattern) diff --git a/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go b/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go index 3373fe217c..cd119b0ceb 100644 --- a/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go +++ b/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go @@ -12,6 +12,8 @@ import ( "context" "database/sql" "fmt" + "log/slog" + "strings" "testing" "time" @@ -25,6 +27,8 @@ import ( "github.com/redpanda-data/benthos/v4/public/service" "github.com/redpanda-data/benthos/v4/public/service/integration" + + "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pgtest" ) func TestIntegrationResolveSchemasReportsInaccessibleSchemas(t *testing.T) { @@ -69,6 +73,65 @@ func TestIntegrationResolveSchemasReportsInaccessibleSchemas(t *testing.T) { assert.Equal(t, []string{`"hidden_schema"`}, inaccessible) } +func TestIntegrationResolverExcludedInaccessibleSchemaDoesNotWarn(t *testing.T) { + integration.CheckSkip(t) + + _, adminURL := createDockerInstance(t) + + adminDB, err := sql.Open("postgres", adminURL) + require.NoError(t, err) + defer adminDB.Close() + + _, err = adminDB.Exec("CREATE SCHEMA tenant_visible") + require.NoError(t, err) + _, err = adminDB.Exec("CREATE SCHEMA tenant_hidden") + require.NoError(t, err) + _, err = adminDB.Exec("CREATE SCHEMA tenant_internal") + require.NoError(t, err) + + _, err = adminDB.Exec("CREATE ROLE restricted_role2 LOGIN PASSWORD 'restricted_pw'") + require.NoError(t, err) + _, err = adminDB.Exec("GRANT CONNECT ON DATABASE dbname TO restricted_role2") + require.NoError(t, err) + _, err = adminDB.Exec("GRANT USAGE ON SCHEMA tenant_visible TO restricted_role2") + require.NoError(t, err) + // Deliberately no GRANT on tenant_hidden or tenant_internal. + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + + restrictedConfig, err := pgconn.ParseConfig(adminURL) + require.NoError(t, err) + restrictedConfig.User = "restricted_role2" + restrictedConfig.Password = "restricted_pw" + delete(restrictedConfig.RuntimeParams, "replication") + + restrictedConn, err := pgconn.ConnectConfig(ctx, restrictedConfig) + require.NoError(t, err) + defer closeConn(t, restrictedConn) + + logs := pgtest.NewTestLogCapture() + logger := service.NewLoggerFromSlog(slog.New(logs)) + + resolver := NewResolver("tenant_*", []string{"tenant_internal"}) + schemas, err := resolver.Resolve(ctx, restrictedConn, logger) + require.NoError(t, err) + + assert.Equal(t, []string{`"tenant_visible"`}, schemas) + + var sawHidden, sawInternal bool + for _, m := range logs.Messages() { + if strings.Contains(m, "tenant_hidden") { + sawHidden = true + } + if strings.Contains(m, "tenant_internal") { + sawInternal = true + } + } + assert.True(t, sawHidden, "expected a warning naming the non-excluded inaccessible schema tenant_hidden, got: %v", logs.Messages()) + assert.False(t, sawInternal, "must not warn about excluded schema tenant_internal, got: %v", logs.Messages()) +} + func TestIntegrationResolveSchemasUUIDSuffixedSchemas(t *testing.T) { integration.CheckSkip(t) From e252b5ca3bf11de3e7dcd46201e09ff181b1bbe4 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 25 Aug 2026 11:39:00 +0100 Subject: [PATCH 58/70] clean up --- internal/impl/postgresql/input_pg_stream.go | 1 - internal/impl/postgresql/integration_test.go | 5 +++++ internal/impl/postgresql/pglogicalstream/pglogrepl.go | 11 +++++++---- .../impl/postgresql/pglogicalstream/pglogrepl_test.go | 2 +- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index ca59e5abce..77d3644067 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -155,7 +155,6 @@ A schema that matches ` + "`" + fieldSchemaInclude + "`" + ` and also matches an This exclusion is applied before ` + "`" + fieldTables + "`" + ` is resolved, so it also takes effect when ` + "`" + fieldTables + "`" + ` is left empty and tables are auto-discovered.`). Examples([]string{"tenant_internal", "tenant_test_*"}). - Optional(). Default([]string{}), ). Field(service.NewStringListField(fieldTables). diff --git a/internal/impl/postgresql/integration_test.go b/internal/impl/postgresql/integration_test.go index 941c8b4290..66d8694d45 100644 --- a/internal/impl/postgresql/integration_test.go +++ b/internal/impl/postgresql/integration_test.go @@ -2607,4 +2607,9 @@ tables: err = input.Connect(ctx) require.NoError(t, err, "orders is a partitioned table that exists in every matched schema and should be accepted, not reported as missing") + + require.NoError(t, input.Close(context.Background())) + + err = input.Connect(ctx) + require.NoError(t, err, "reconnecting with the same explicitly-listed partitioned table must reconcile cleanly, not attempt to drop its leaf partitions from the publication") } diff --git a/internal/impl/postgresql/pglogicalstream/pglogrepl.go b/internal/impl/postgresql/pglogicalstream/pglogrepl.go index 69385f753f..ee5d40baf6 100644 --- a/internal/impl/postgresql/pglogicalstream/pglogrepl.go +++ b/internal/impl/postgresql/pglogicalstream/pglogrepl.go @@ -405,10 +405,13 @@ func CreatePublication(ctx context.Context, conn *pgconn.PgConn, publicationName func GetPublicationTables(ctx context.Context, conn *pgconn.PgConn, publicationName string) ([]TableFQN, bool, error) { query, err := sanitize.SQLQuery(` SELECT DISTINCT - tablename as table_name, - schemaname as schema_name - FROM pg_publication_tables - WHERE pubname = $1 + c.relname AS table_name, + n.nspname AS schema_name + FROM pg_publication_rel pr + JOIN pg_publication p ON p.oid = pr.prpubid + JOIN pg_class c ON c.oid = pr.prrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE p.pubname = $1 ORDER BY schema_name, table_name; `, publicationName) if err != nil { diff --git a/internal/impl/postgresql/pglogicalstream/pglogrepl_test.go b/internal/impl/postgresql/pglogicalstream/pglogrepl_test.go index 4cc6b81547..c4ebaf0db5 100644 --- a/internal/impl/postgresql/pglogicalstream/pglogrepl_test.go +++ b/internal/impl/postgresql/pglogicalstream/pglogrepl_test.go @@ -255,7 +255,7 @@ func TestIntegrationCreatePublication(t *testing.T) { err = CreatePublication(t.Context(), conn, publicationWithTables, []TableFQN{{schema, `"test_table"`}}) require.NoError(t, err) - tables, forAllTables, err = GetPublicationTables(t.Context(), conn, publicationName) + tables, forAllTables, err = GetPublicationTables(t.Context(), conn, publicationWithTables) require.NoError(t, err) assert.NotEmpty(t, tables) assert.Len(t, tables, 1) From f9429f705a103fe51137c2438f3ed93d7d2d285c Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 25 Aug 2026 12:16:35 +0100 Subject: [PATCH 59/70] test coverage --- .../multischema/resolver_integration_test.go | 87 +++++++++++++++++++ .../multischema/resolver_test.go | 65 ++++++++++++++ 2 files changed, 152 insertions(+) diff --git a/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go b/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go index cd119b0ceb..5faacac776 100644 --- a/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go +++ b/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go @@ -132,6 +132,93 @@ func TestIntegrationResolverExcludedInaccessibleSchemaDoesNotWarn(t *testing.T) assert.False(t, sawInternal, "must not warn about excluded schema tenant_internal, got: %v", logs.Messages()) } +func TestIntegrationResolverWarnsOnSchemaSetDriftBetweenReconnects(t *testing.T) { + integration.CheckSkip(t) + + _, adminURL := createDockerInstance(t) + + adminDB, err := sql.Open("postgres", adminURL) + require.NoError(t, err) + defer adminDB.Close() + + _, err = adminDB.Exec("CREATE SCHEMA tenant_a") + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + + conn, err := pgconn.Connect(ctx, adminURL) + require.NoError(t, err) + defer closeConn(t, conn) + + logs := pgtest.NewTestLogCapture() + logger := service.NewLoggerFromSlog(slog.New(logs)) + resolver := NewResolver("tenant_*", nil) + + // First resolve: only tenant_a matches. previouslyResolved is nil going + // in, so this must not be treated as drift - no added/removed warning. + schemas, err := resolver.Resolve(ctx, conn, logger) + require.NoError(t, err) + assert.Equal(t, []string{`"tenant_a"`}, schemas) + assertNoMessageContains(t, logs.Messages(), "now also matches", "no longer match") + + // Second resolve (simulating a reconnect): tenant_b now also matches. + // Must warn that it was added, and must not also claim anything was + // removed. + before := len(logs.Messages()) + _, err = adminDB.Exec("CREATE SCHEMA tenant_b") + require.NoError(t, err) + + schemas, err = resolver.Resolve(ctx, conn, logger) + require.NoError(t, err) + assert.ElementsMatch(t, []string{`"tenant_a"`, `"tenant_b"`}, schemas) + + added := logs.Messages()[before:] + assertAnyMessageContains(t, added, "now also matches", "tenant_b") + assertNoMessageContains(t, added, "no longer match") + + // Third resolve (another reconnect): tenant_a is dropped. Must warn that + // it was removed, and must not re-warn about tenant_b (it's no longer + // new - it was already part of the previously resolved set). + before = len(logs.Messages()) + _, err = adminDB.Exec("DROP SCHEMA tenant_a") + require.NoError(t, err) + + schemas, err = resolver.Resolve(ctx, conn, logger) + require.NoError(t, err) + assert.Equal(t, []string{`"tenant_b"`}, schemas) + + removed := logs.Messages()[before:] + assertAnyMessageContains(t, removed, "no longer match", "tenant_a") + assertNoMessageContains(t, removed, "now also matches") +} + +func assertAnyMessageContains(t *testing.T, messages []string, substrs ...string) { + t.Helper() + for _, m := range messages { + matchesAll := true + for _, s := range substrs { + if !strings.Contains(m, s) { + matchesAll = false + break + } + } + if matchesAll { + return + } + } + assert.Fail(t, "expected a log message containing all of the given substrings", "substrings: %v, got: %v", substrs, messages) +} + +func assertNoMessageContains(t *testing.T, messages []string, substrs ...string) { + t.Helper() + for _, m := range messages { + for _, s := range substrs { + assert.NotContains(t, m, s, "unexpected log message: %v", m) + } + } +} + func TestIntegrationResolveSchemasUUIDSuffixedSchemas(t *testing.T) { integration.CheckSkip(t) diff --git a/internal/impl/postgresql/pglogicalstream/multischema/resolver_test.go b/internal/impl/postgresql/pglogicalstream/multischema/resolver_test.go index d06b5362dd..14c5532d89 100644 --- a/internal/impl/postgresql/pglogicalstream/multischema/resolver_test.go +++ b/internal/impl/postgresql/pglogicalstream/multischema/resolver_test.go @@ -132,3 +132,68 @@ func TestSchemaMatchesExcludePattern(t *testing.T) { }) } } + +func TestDiffSchemaSets(t *testing.T) { + tests := []struct { + name string + previous []string + current []string + expectedAdded []string + expectedRemoved []string + }{ + // nil previous is the first resolution, not drift - Resolve's caller + // is responsible for ignoring this result in that case (see the + // doc comment on diffSchemaSets), but the function itself still + // reports every current schema as "added" since it has nothing to + // compare against. + { + name: "nil previous, empty current", + previous: nil, + current: nil, + expectedAdded: nil, expectedRemoved: nil, + }, + { + name: "nil previous, non-empty current", + previous: nil, + current: []string{`"a"`, `"b"`}, + expectedAdded: []string{`"a"`, `"b"`}, expectedRemoved: nil, + }, + { + name: "no change", + previous: []string{`"a"`, `"b"`}, + current: []string{`"a"`, `"b"`}, + expectedAdded: nil, expectedRemoved: nil, + }, + { + name: "added only", + previous: []string{`"a"`}, + current: []string{`"a"`, `"b"`}, + expectedAdded: []string{`"b"`}, expectedRemoved: nil, + }, + { + name: "removed only", + previous: []string{`"a"`, `"b"`}, + current: []string{`"a"`}, + expectedAdded: nil, expectedRemoved: []string{`"b"`}, + }, + { + name: "added and removed", + previous: []string{`"a"`, `"b"`}, + current: []string{`"b"`, `"c"`}, + expectedAdded: []string{`"c"`}, expectedRemoved: []string{`"a"`}, + }, + { + name: "everything replaced", + previous: []string{`"a"`, `"b"`}, + current: []string{`"c"`, `"d"`}, + expectedAdded: []string{`"c"`, `"d"`}, expectedRemoved: []string{`"a"`, `"b"`}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + added, removed := diffSchemaSets(tt.previous, tt.current) + assert.Equal(t, tt.expectedAdded, added, "added") + assert.Equal(t, tt.expectedRemoved, removed, "removed") + }) + } +} From 7f30013facd4f8d95e0fe0b3d0412aa5018a670c Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 25 Aug 2026 14:30:53 +0100 Subject: [PATCH 60/70] Fix logging issue --- internal/impl/postgresql/bench/Taskfile.yaml | 2 +- internal/impl/postgresql/bench/create.sql | 61 +++++++++++++++++-- .../pglogicalstream/logical_stream.go | 6 +- 3 files changed, 60 insertions(+), 9 deletions(-) diff --git a/internal/impl/postgresql/bench/Taskfile.yaml b/internal/impl/postgresql/bench/Taskfile.yaml index d8990ea492..438704e272 100644 --- a/internal/impl/postgresql/bench/Taskfile.yaml +++ b/internal/impl/postgresql/bench/Taskfile.yaml @@ -57,7 +57,7 @@ tasks: psql:truncate: desc: Truncate all benchmark tables cmds: - - psql {{.PG_DSN}} -c "TRUNCATE public.users, public.products, public.cart;" + - psql {{.PG_DSN}} -c "TRUNCATE public.users, public.products, public.cart, tenant_a.users, tenant_b.users, tenant_c.users;" psql:drop-slot: desc: Drop the bench_slot replication slot so benchmarks can be re-run diff --git a/internal/impl/postgresql/bench/create.sql b/internal/impl/postgresql/bench/create.sql index d4b8d14d77..d44b9938d7 100644 --- a/internal/impl/postgresql/bench/create.sql +++ b/internal/impl/postgresql/bench/create.sql @@ -1,9 +1,9 @@ -- PostgreSQL Benchmark Setup Script -CREATE TABLE IF NOT EXISTS public.rpcn_signal_table ( - id SERIAL PRIMARY KEY, - type VARCHAR(32), - data TEXT -); +-- CREATE TABLE IF NOT EXISTS public.rpcn_signal_table ( +-- id SERIAL PRIMARY KEY, +-- type VARCHAR(32), +-- data TEXT +-- ); CREATE TABLE IF NOT EXISTS public.users ( id SERIAL PRIMARY KEY, @@ -42,3 +42,54 @@ CREATE TABLE IF NOT EXISTS public.cart ( info TEXT NOT NULL ); ALTER TABLE public.cart REPLICA IDENTITY FULL; + +-- schema 1 +CREATE SCHEMA IF NOT EXISTS tenant_a; +CREATE TABLE IF NOT EXISTS tenant_a.users ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) NOT NULL, + surname VARCHAR(100) NOT NULL, + about TEXT NOT NULL, + email VARCHAR(255) NOT NULL, + date_of_birth DATE, + join_date DATE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + is_active BOOLEAN NOT NULL DEFAULT TRUE, + login_count INT NOT NULL DEFAULT 0, + balance DECIMAL(10,2) NOT NULL DEFAULT 0.00 +); +ALTER TABLE tenant_a.users REPLICA IDENTITY FULL; + +-- schema 2 +CREATE SCHEMA IF NOT EXISTS tenant_b; +CREATE TABLE IF NOT EXISTS tenant_b.users ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) NOT NULL, + surname VARCHAR(100) NOT NULL, + about TEXT NOT NULL, + email VARCHAR(255) NOT NULL, + date_of_birth DATE, + join_date DATE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + is_active BOOLEAN NOT NULL DEFAULT TRUE, + login_count INT NOT NULL DEFAULT 0, + balance DECIMAL(10,2) NOT NULL DEFAULT 0.00 +); +ALTER TABLE tenant_b.users REPLICA IDENTITY FULL; + +-- schema 3 +CREATE SCHEMA IF NOT EXISTS tenant_c; +CREATE TABLE IF NOT EXISTS tenant_c.users ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) NOT NULL, + surname VARCHAR(100) NOT NULL, + about TEXT NOT NULL, + email VARCHAR(255) NOT NULL, + date_of_birth DATE, + join_date DATE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + is_active BOOLEAN NOT NULL DEFAULT TRUE, + login_count INT NOT NULL DEFAULT 0, + balance DECIMAL(10,2) NOT NULL DEFAULT 0.00 +); +ALTER TABLE tenant_c.users REPLICA IDENTITY FULL; diff --git a/internal/impl/postgresql/pglogicalstream/logical_stream.go b/internal/impl/postgresql/pglogicalstream/logical_stream.go index 6410b87f4c..05ea8de474 100644 --- a/internal/impl/postgresql/pglogicalstream/logical_stream.go +++ b/internal/impl/postgresql/pglogicalstream/logical_stream.go @@ -149,7 +149,7 @@ func NewPgStream(ctx context.Context, config *Config) (*Stream, error) { } for _, table := range normalizedTables { if _, ok := existingTables[table]; !ok { - config.Logger.Warnf("table %s.%s not found, skipping (schema %s matched schema_include pattern %q but does not contain this table)", schema, table, schema, config.SchemaResolver.Include) + config.Logger.Warnf("Table %s.%s not found, skipping (schema %s matched schema_include pattern %q but does not contain this table)", schema, table, schema, config.SchemaResolver.Include) continue } tables = append(tables, TableFQN{Schema: schema, Table: table}) @@ -753,7 +753,7 @@ func (s *Stream) processSnapshot(ctx context.Context, snapshotter *snapshotter) if len(ranges) > 1 { s.logger.Infof( - "created plan in %v to split %s into %d chunks of %d and process in parallel", + "Created plan in %v to split %s into %d chunks of %d and process in parallel", time.Since(planStartTime), table, len(ranges), @@ -761,7 +761,7 @@ func (s *Stream) processSnapshot(ctx context.Context, snapshotter *snapshotter) ) } else { s.logger.Infof( - "created plan in %v to scan %s sequentially", + "Created plan in %v to scan %s sequentially", time.Since(planStartTime), table, ) From a03b4fdecdd9bb2d4f3d2699bac041c29d347e05 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 25 Aug 2026 14:36:18 +0100 Subject: [PATCH 61/70] update benchmark --- internal/impl/postgresql/bench/benchmark_config.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/impl/postgresql/bench/benchmark_config.yaml b/internal/impl/postgresql/bench/benchmark_config.yaml index a673fc8503..eeb8f34c3f 100644 --- a/internal/impl/postgresql/bench/benchmark_config.yaml +++ b/internal/impl/postgresql/bench/benchmark_config.yaml @@ -6,6 +6,9 @@ input: dsn: ${PG_DSN:postgres://postgres:postgres@localhost:5432/testdb?sslmode=disable} stream_snapshot: true schema: public + schema_include: "*" + schema_exclude: + - tenant_c tables: - users - products @@ -26,10 +29,10 @@ output: - benchmark: interval: 1s count_bytes: true - # file: - # path: "./benchmark_results.json" - # codec: lines - drop: {} + file: + path: "./benchmark_results.json" + codec: lines + # drop: {} logger: level: INFO From dbf1a815206bb6b77335b3461549b80e7ffd4a00 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 25 Aug 2026 14:42:47 +0100 Subject: [PATCH 62/70] update sql script --- internal/impl/postgresql/bench/users.sql | 37 +++++++++++++++--------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/internal/impl/postgresql/bench/users.sql b/internal/impl/postgresql/bench/users.sql index 70898f7cae..c9eff6845c 100644 --- a/internal/impl/postgresql/bench/users.sql +++ b/internal/impl/postgresql/bench/users.sql @@ -1,16 +1,27 @@ -- PostgreSQL Benchmark - Users Data (150K rows, ~500KB per row) -- Prerequisites: Run create.sql first -INSERT INTO public.users (name, surname, about, email, date_of_birth, join_date, created_at, is_active, login_count, balance) -SELECT - 'user-' || n, - 'surname-' || n, - repeat('This is about user ' || n || '. ', 25000), - 'user' || n || '@example.com', - NOW() - (n % 10000 || ' days')::interval, - NOW(), - NOW(), - (n % 2 = 0), - n % 100, - ((n % 1000) + (n % 100) / 100.0)::decimal(10,2) -FROM generate_series(1, 150000) AS n; +DO $$ +DECLARE + tbl text; + num_rows int := 5000; +BEGIN + FOREACH tbl IN ARRAY ARRAY['public.users', 'tenant_a.users', 'tenant_b.users', 'tenant_c.users'] + LOOP + EXECUTE format($fmt$ + INSERT INTO %s (name, surname, about, email, date_of_birth, join_date, created_at, is_active, login_count, balance) + SELECT + 'user-' || n, + 'surname-' || n, + repeat('This is about user ' || n || '. ', 25000), + 'user' || n || '@example.com', + NOW() - (n %% 10000 || ' days')::interval, + NOW(), + NOW(), + (n %% 2 = 0), + n %% 100, + ((n %% 1000) + (n %% 100) / 100.0)::decimal(10,2) + FROM generate_series(1, %s) AS n + $fmt$, tbl, num_rows); + END LOOP; +END $$; From 2129b1b1d783eff0817196b42fbf136072b91150 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 25 Aug 2026 15:11:07 +0100 Subject: [PATCH 63/70] docs --- docs/modules/components/pages/inputs/postgres_cdc.adoc | 4 ++-- internal/impl/postgresql/input_pg_stream.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index 1f419c285d..0b507caa9e 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -200,8 +200,8 @@ Double-quoted identifiers are treated as exact names and do not support wildcard Schema pattern matching is re-evaluated every time the connector connects or reconnects - including the automatic reconnects that follow a transient replication failure - not just once at pipeline startup. This has two consequences that are easy to miss: -- A schema created after the pipeline started that matches the pattern is picked up on the next reconnect and its tables are added to the publication. However, because the replication slot already exists by then, those tables are treated as already caught up, so with `stream_snapshot` enabled the rows that existed in that schema before it was picked up are never snapshotted and are silently missing from the output - only changes made after the schema is picked up are streamed. -- A schema that is dropped, renamed, or loses its `USAGE` grant between reconnects stops matching and its tables are silently removed from the publication on the next reconnect. A warning is logged when a schema becomes inaccessible due to a lost `USAGE` grant, but that warning is about the schema being inaccessible - it is not logged for a dropped or renamed schema, and nothing is ever logged about the resulting publication drop itself. +- A schema created after the pipeline started that matches the pattern is picked up on the next reconnect and its tables are added to the publication - a warning naming the schema is logged when this happens. However, because the replication slot already exists by then, those tables are treated as already caught up, so with `stream_snapshot` enabled the rows that existed in that schema before it was picked up are never snapshotted - only changes made after the schema is picked up are streamed, and the missed rows cannot be recovered short of resetting the replication slot. +- A schema that is dropped, renamed, or loses its `USAGE` grant between reconnects stops matching and its tables are removed from the publication on the next reconnect - a warning naming the schema is logged for all three cases, whether it dropped out via `USAGE` loss, a rename, or being dropped outright. If this pattern matches no schema in the database, startup fails with an error - this holds whether or not `tables` is set. See `tables` below for what happens when it's left empty. diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 77d3644067..4b328959a8 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -134,8 +134,8 @@ Double-quoted identifiers are treated as exact names and do not support wildcard Schema pattern matching is re-evaluated every time the connector connects or reconnects - including the automatic reconnects that follow a transient replication failure - not just once at pipeline startup. This has two consequences that are easy to miss: -- A schema created after the pipeline started that matches the pattern is picked up on the next reconnect and its tables are added to the publication. However, because the replication slot already exists by then, those tables are treated as already caught up, so with `+"`"+fieldStreamSnapshot+"`"+` enabled the rows that existed in that schema before it was picked up are never snapshotted and are silently missing from the output - only changes made after the schema is picked up are streamed. -- A schema that is dropped, renamed, or loses its `+"`USAGE`"+` grant between reconnects stops matching and its tables are silently removed from the publication on the next reconnect. A warning is logged when a schema becomes inaccessible due to a lost `+"`USAGE`"+` grant, but that warning is about the schema being inaccessible - it is not logged for a dropped or renamed schema, and nothing is ever logged about the resulting publication drop itself. +- A schema created after the pipeline started that matches the pattern is picked up on the next reconnect and its tables are added to the publication - a warning naming the schema is logged when this happens. However, because the replication slot already exists by then, those tables are treated as already caught up, so with `+"`"+fieldStreamSnapshot+"`"+` enabled the rows that existed in that schema before it was picked up are never snapshotted - only changes made after the schema is picked up are streamed, and the missed rows cannot be recovered short of resetting the replication slot. +- A schema that is dropped, renamed, or loses its `+"`USAGE`"+` grant between reconnects stops matching and its tables are removed from the publication on the next reconnect - a warning naming the schema is logged for all three cases, whether it dropped out via `+"`USAGE`"+` loss, a rename, or being dropped outright. If this pattern matches no schema in the database, startup fails with an error - this holds whether or not `+"`"+fieldTables+"`"+` is set. See `+"`"+fieldTables+"`"+` below for what happens when it's left empty. From 20d552d3d94453393cc005a98541109d700607aa Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 25 Aug 2026 15:24:14 +0100 Subject: [PATCH 64/70] clean up test --- .../multischema/resolver_integration_test.go | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go b/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go index 5faacac776..f9605425d9 100644 --- a/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go +++ b/internal/impl/postgresql/pglogicalstream/multischema/resolver_integration_test.go @@ -34,7 +34,7 @@ import ( func TestIntegrationResolveSchemasReportsInaccessibleSchemas(t *testing.T) { integration.CheckSkip(t) - _, adminURL := createDockerInstance(t) + adminURL := createDockerInstance(t) adminDB, err := sql.Open("postgres", adminURL) require.NoError(t, err) @@ -76,7 +76,7 @@ func TestIntegrationResolveSchemasReportsInaccessibleSchemas(t *testing.T) { func TestIntegrationResolverExcludedInaccessibleSchemaDoesNotWarn(t *testing.T) { integration.CheckSkip(t) - _, adminURL := createDockerInstance(t) + adminURL := createDockerInstance(t) adminDB, err := sql.Open("postgres", adminURL) require.NoError(t, err) @@ -135,7 +135,7 @@ func TestIntegrationResolverExcludedInaccessibleSchemaDoesNotWarn(t *testing.T) func TestIntegrationResolverWarnsOnSchemaSetDriftBetweenReconnects(t *testing.T) { integration.CheckSkip(t) - _, adminURL := createDockerInstance(t) + adminURL := createDockerInstance(t) adminDB, err := sql.Open("postgres", adminURL) require.NoError(t, err) @@ -222,7 +222,7 @@ func assertNoMessageContains(t *testing.T, messages []string, substrs ...string) func TestIntegrationResolveSchemasUUIDSuffixedSchemas(t *testing.T) { integration.CheckSkip(t) - _, adminURL := createDockerInstance(t) + adminURL := createDockerInstance(t) adminDB, err := sql.Open("postgres", adminURL) require.NoError(t, err) @@ -258,7 +258,7 @@ func TestIntegrationResolveSchemasUUIDSuffixedSchemas(t *testing.T) { func TestIntegrationResolveSchemasBareUUIDSchema(t *testing.T) { integration.CheckSkip(t) - _, adminURL := createDockerInstance(t) + adminURL := createDockerInstance(t) adminDB, err := sql.Open("postgres", adminURL) require.NoError(t, err) @@ -305,7 +305,7 @@ func TestIntegrationResolveSchemasBareUUIDSchema(t *testing.T) { func TestIntegrationResolveExistingTablesReportsRelKind(t *testing.T) { integration.CheckSkip(t) - _, adminURL := createDockerInstance(t) + adminURL := createDockerInstance(t) adminDB, err := sql.Open("postgres", adminURL) require.NoError(t, err) @@ -349,7 +349,7 @@ CREATE TABLE tenant_a.orders_2026 PARTITION OF tenant_a.orders func TestIntegrationResolveSchemasConfigValidation(t *testing.T) { integration.CheckSkip(t) - _, adminURL := createDockerInstance(t) + adminURL := createDockerInstance(t) adminDB, err := sql.Open("postgres", adminURL) require.NoError(t, err) @@ -392,7 +392,7 @@ func closeConn(t testing.TB, conn *pgconn.PgConn) { require.NoError(t, conn.Close(ctx)) } -func createDockerInstance(t *testing.T) (cleanup func(), dbURL string) { +func createDockerInstance(t *testing.T) (dbURL string) { ctr, err := testcontainers.Run(t.Context(), "postgres:16", testcontainers.WithExposedPorts("5432/tcp"), testcontainers.WithEnv(map[string]string{ @@ -417,15 +417,15 @@ func createDockerInstance(t *testing.T) (cleanup func(), dbURL string) { var db *sql.DB require.Eventually(t, func() bool { + if db != nil { + db.Close() + } if db, err = sql.Open("postgres", databaseURL); err != nil { return false } return db.Ping() == nil }, 2*time.Minute, time.Second) + t.Cleanup(func() { require.NoError(t, db.Close()) }) - cleanup = func() { - // Container cleanup is handled by testcontainers.CleanupContainer - } - - return cleanup, databaseURL + return databaseURL } From 3451cecf7b91cbb3d0ba06692940d95092577cfb Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 25 Aug 2026 16:20:32 +0100 Subject: [PATCH 65/70] address all tables --- .../postgresql/pglogicalstream/pglogrepl.go | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/internal/impl/postgresql/pglogicalstream/pglogrepl.go b/internal/impl/postgresql/pglogicalstream/pglogrepl.go index ee5d40baf6..a87d087435 100644 --- a/internal/impl/postgresql/pglogicalstream/pglogrepl.go +++ b/internal/impl/postgresql/pglogicalstream/pglogrepl.go @@ -403,6 +403,24 @@ func CreatePublication(ctx context.Context, conn *pgconn.PgConn, publicationName // GetPublicationTables returns a list of tables currently in the publication // Arguments, in order: list of the tables, exist for all tables, error. func GetPublicationTables(ctx context.Context, conn *pgconn.PgConn, publicationName string) ([]TableFQN, bool, error) { + pubQuery, err := sanitize.SQLQuery(` + SELECT puballtables + FROM pg_publication + WHERE pubname = $1; + `, publicationName) + if err != nil { + return nil, false, fmt.Errorf("getting publication tables: %w", err) + } + + pubRows, err := conn.Exec(ctx, pubQuery).ReadAll() + if err != nil { + return nil, false, fmt.Errorf("getting publication tables: %w", err) + } + if len(pubRows) == 0 || len(pubRows[0].Rows) == 0 { + return nil, false, fmt.Errorf("publication %q does not exist", publicationName) + } + forAllTables := string(pubRows[0].Rows[0][0]) == "t" + query, err := sanitize.SQLQuery(` SELECT DISTINCT c.relname AS table_name, @@ -427,7 +445,7 @@ func GetPublicationTables(ctx context.Context, conn *pgconn.PgConn, publicationN } if len(rows) == 0 || len(rows[0].Rows) == 0 { - return nil, true, nil // Publication exists and is for all tables + return nil, forAllTables, nil } tables := make([]TableFQN, 0, len(rows)) @@ -439,7 +457,7 @@ func GetPublicationTables(ctx context.Context, conn *pgconn.PgConn, publicationN tables = append(tables, TableFQN{Table: table, Schema: schema}) } - return tables, false, nil + return tables, forAllTables, nil } // StartReplicationOptions are the options for the START_REPLICATION command. From d39e0811dc5e57d216761018378d02b70b17025f Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 25 Aug 2026 17:00:01 +0100 Subject: [PATCH 66/70] for all tables --- .../postgresql/pglogicalstream/pglogrepl.go | 8 +-- .../pglogicalstream/pglogrepl_test.go | 50 +++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/internal/impl/postgresql/pglogicalstream/pglogrepl.go b/internal/impl/postgresql/pglogicalstream/pglogrepl.go index a87d087435..67f0a2419a 100644 --- a/internal/impl/postgresql/pglogicalstream/pglogrepl.go +++ b/internal/impl/postgresql/pglogicalstream/pglogrepl.go @@ -328,9 +328,11 @@ func CreatePublication(ctx context.Context, conn *pgconn.PgConn, publicationName return fmt.Errorf("getting publication tables: %w", err) } - // list of tables to publish is empty and publication is for all tables - // no update is needed - if forAllTables && len(pubTables) == 0 { + // No-op only if the caller still wants FOR ALL TABLES too. Checking + // pubTables instead of tables would always be true here (a FOR ALL + // TABLES publication has no pg_publication_rel rows), silently ignoring + // a caller that has since narrowed to an explicit table list. + if forAllTables && len(tables) == 0 { return nil } diff --git a/internal/impl/postgresql/pglogicalstream/pglogrepl_test.go b/internal/impl/postgresql/pglogicalstream/pglogrepl_test.go index c4ebaf0db5..b001aa6178 100644 --- a/internal/impl/postgresql/pglogicalstream/pglogrepl_test.go +++ b/internal/impl/postgresql/pglogicalstream/pglogrepl_test.go @@ -344,6 +344,56 @@ func TestIntegrationCreatePublication(t *testing.T) { assert.False(t, forAllTables) } +// TestIntegrationCreatePublicationNarrowingFromForAllTablesFailsLoudly +// guards against a silent no-op when a pipeline that previously ran with an +// empty table list (FOR ALL TABLES) reconnects with a narrowed, explicit +// table list on the same slot/publication: CreatePublication must attempt +// the reconcile - which Postgres rejects, since tables can't be added to or +// dropped from a FOR ALL TABLES publication - rather than silently leaving +// the publication as FOR ALL TABLES with no error and no warning. +func TestIntegrationCreatePublicationNarrowingFromForAllTablesFailsLoudly(t *testing.T) { + integration.CheckSkip(t) + + cleanup, dbURL := createDockerInstance(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(t.Context(), time.Second*5) + defer cancel() + + conn, err := pgconn.Connect(ctx, dbURL) + require.NoError(t, err) + defer closeConn(t, conn) + + multiReader := conn.Exec(t.Context(), "CREATE TABLE orders (id serial PRIMARY KEY, name text);") + _, err = multiReader.ReadAll() + require.NoError(t, err) + + publicationName := "narrowing_test_publication" + schema := `"public"` + + // First connect: tables left empty, so the publication is created FOR + // ALL TABLES. + err = CreatePublication(t.Context(), conn, publicationName, []TableFQN{}) + require.NoError(t, err) + + tables, forAllTables, err := GetPublicationTables(t.Context(), conn, publicationName) + require.NoError(t, err) + assert.Empty(t, tables) + assert.True(t, forAllTables) + + // Reconnect: the config has since been narrowed to an explicit table + // list. This must fail loudly, not silently leave the publication + // unchanged. + err = CreatePublication(t.Context(), conn, publicationName, []TableFQN{{schema, `"orders"`}}) + require.Error(t, err, "narrowing an existing FOR ALL TABLES publication to an explicit table list should fail, not silently no-op") + + // The publication must still be untouched - still FOR ALL TABLES. + tables, forAllTables, err = GetPublicationTables(t.Context(), conn, publicationName) + require.NoError(t, err) + assert.Empty(t, tables) + assert.True(t, forAllTables) +} + func TestIntegrationStartReplication(t *testing.T) { integration.CheckSkip(t) From 5ab78ee8b2d95ed3cec0089841ba0d6a2fe98e30 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 25 Aug 2026 17:05:42 +0100 Subject: [PATCH 67/70] docs --- docs/modules/components/pages/inputs/postgres_cdc.adoc | 2 +- internal/impl/postgresql/input_pg_stream.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index 0b507caa9e..bac78ed9c4 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -253,7 +253,7 @@ A list of table names to include in the logical replication. Each table should b When `schema_include` is set, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. -If left empty while `schema_include` is set, every base table in each matched (and un-excluded, see `schema_exclude`) schema is auto-discovered and published explicitly, instead of listing tables by hand - this is the expected way to replicate "every table" in a multi-tenant, schema-per-tenant setup without also picking up unrelated schemas. Startup fails if no matched schema contains any table. +If left empty while `schema_include` is set, every base table in each matched (and un-excluded, see `schema_exclude`) schema is auto-discovered and published explicitly, instead of listing tables by hand - this is the expected way to replicate "every table" in a multi-tenant, schema-per-tenant setup without also picking up unrelated schemas. Startup fails if no matched schema contains any table. A partitioned table is auto-discovered as its individual leaf partitions rather than the table itself, so the `table` metadata on emitted messages will be the partition's name (e.g. `orders_2025`), not the parent's - list the parent explicitly in this field instead if you want it published as a single relation. If left empty while `schema_include` is NOT set, the underlying PostgreSQL publication is instead created `FOR ALL TABLES`, which replicates every table in every schema of the database, ignoring `schema`. This also disables `stream_snapshot`, since the initial snapshot is only planned for tables listed here. diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 4b328959a8..958eeb717f 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -162,7 +162,7 @@ This exclusion is applied before ` + "`" + fieldTables + "`" + ` is resolved, so When ` + "`" + fieldSchemaInclude + "`" + ` is set, this list is resolved against each matched schema independently: a table missing from some (but not all) of the matched schemas is skipped for those schemas only (with a warning logged), tolerating multi-tenant setups where a table hasn't been provisioned in every schema yet. A table that's missing from every matched schema, however, is treated as a configuration error (most likely a typo) and startup fails, naming the missing table. -If left empty while ` + "`" + fieldSchemaInclude + "`" + ` is set, every base table in each matched (and un-excluded, see ` + "`" + fieldSchemaExclude + "`" + `) schema is auto-discovered and published explicitly, instead of listing tables by hand - this is the expected way to replicate "every table" in a multi-tenant, schema-per-tenant setup without also picking up unrelated schemas. Startup fails if no matched schema contains any table. +If left empty while ` + "`" + fieldSchemaInclude + "`" + ` is set, every base table in each matched (and un-excluded, see ` + "`" + fieldSchemaExclude + "`" + `) schema is auto-discovered and published explicitly, instead of listing tables by hand - this is the expected way to replicate "every table" in a multi-tenant, schema-per-tenant setup without also picking up unrelated schemas. Startup fails if no matched schema contains any table. A partitioned table is auto-discovered as its individual leaf partitions rather than the table itself, so the ` + "`table`" + ` metadata on emitted messages will be the partition's name (e.g. ` + "`orders_2025`" + `), not the parent's - list the parent explicitly in this field instead if you want it published as a single relation. If left empty while ` + "`" + fieldSchemaInclude + "`" + ` is NOT set, the underlying PostgreSQL publication is instead created ` + "`FOR ALL TABLES`" + `, which replicates every table in every schema of the database, ignoring ` + "`" + fieldSchema + "`" + `. This also disables ` + "`" + fieldStreamSnapshot + "`" + `, since the initial snapshot is only planned for tables listed here.`). Example([]string{"my_table_1", `"MyCaseSensitiveTableNeedingQuotes"`}). From 2739c370ada1037c204b0b1a1b9687af989b35bf Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 25 Aug 2026 17:10:59 +0100 Subject: [PATCH 68/70] reset bench files --- .../impl/postgresql/bench/benchmark_config.yaml | 14 +++++++------- internal/impl/postgresql/bench/create.sql | 5 ----- internal/impl/postgresql/bench/users.sql | 4 ++-- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/internal/impl/postgresql/bench/benchmark_config.yaml b/internal/impl/postgresql/bench/benchmark_config.yaml index eeb8f34c3f..964b0dbd57 100644 --- a/internal/impl/postgresql/bench/benchmark_config.yaml +++ b/internal/impl/postgresql/bench/benchmark_config.yaml @@ -6,9 +6,9 @@ input: dsn: ${PG_DSN:postgres://postgres:postgres@localhost:5432/testdb?sslmode=disable} stream_snapshot: true schema: public - schema_include: "*" - schema_exclude: - - tenant_c + # schema_include: "*" + # schema_exclude: + # - tenant_c tables: - users - products @@ -29,10 +29,10 @@ output: - benchmark: interval: 1s count_bytes: true - file: - path: "./benchmark_results.json" - codec: lines - # drop: {} + # file: + # path: "./benchmark_results.json" + # codec: lines + drop: {} logger: level: INFO diff --git a/internal/impl/postgresql/bench/create.sql b/internal/impl/postgresql/bench/create.sql index d44b9938d7..2be3bc4842 100644 --- a/internal/impl/postgresql/bench/create.sql +++ b/internal/impl/postgresql/bench/create.sql @@ -1,9 +1,4 @@ -- PostgreSQL Benchmark Setup Script --- CREATE TABLE IF NOT EXISTS public.rpcn_signal_table ( --- id SERIAL PRIMARY KEY, --- type VARCHAR(32), --- data TEXT --- ); CREATE TABLE IF NOT EXISTS public.users ( id SERIAL PRIMARY KEY, diff --git a/internal/impl/postgresql/bench/users.sql b/internal/impl/postgresql/bench/users.sql index c9eff6845c..607da4aa2e 100644 --- a/internal/impl/postgresql/bench/users.sql +++ b/internal/impl/postgresql/bench/users.sql @@ -4,9 +4,9 @@ DO $$ DECLARE tbl text; - num_rows int := 5000; + num_rows int := 150000; BEGIN - FOREACH tbl IN ARRAY ARRAY['public.users', 'tenant_a.users', 'tenant_b.users', 'tenant_c.users'] + FOREACH tbl IN ARRAY ARRAY['public.users', 'tenant_a.users', 'tenant_b.users'] LOOP EXECUTE format($fmt$ INSERT INTO %s (name, surname, about, email, date_of_birth, join_date, created_at, is_active, login_count, balance) From 7e38870be719391337495c83295d58a6de21b1f8 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 25 Aug 2026 18:05:58 +0100 Subject: [PATCH 69/70] clean up schema vs schema_include config --- .../components/pages/inputs/postgres_cdc.adoc | 4 +- internal/impl/postgresql/input_pg_stream.go | 13 ++- .../impl/postgresql/input_pg_stream_test.go | 97 +++++++++++++++---- 3 files changed, 87 insertions(+), 27 deletions(-) diff --git a/docs/modules/components/pages/inputs/postgres_cdc.adoc b/docs/modules/components/pages/inputs/postgres_cdc.adoc index bac78ed9c4..2ca4d50bac 100644 --- a/docs/modules/components/pages/inputs/postgres_cdc.adoc +++ b/docs/modules/components/pages/inputs/postgres_cdc.adoc @@ -175,7 +175,7 @@ snapshot_batch_size: 10000 === `schema` -The PostgreSQL schema from which to replicate data. +The PostgreSQL schema from which to replicate data. Ignored when `schema_include` is used instead. *Type*: `string` @@ -207,7 +207,7 @@ If this pattern matches no schema in the database, startup fails with an error - This pattern can contain characters that wouldn't be allowed in an unquoted schema name, because it's only ever compared against the real name of each schema in the database - it doesn't have to be a valid name itself. For example, a schema literally named `a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11` (which must have been created using double quotes, since hyphens aren't allowed in an unquoted `CREATE SCHEMA` statement) can still be matched using the unquoted pattern `a0eebc99-*`. -This field is mutually exclusive with `schema`; when set, it takes over schema resolution entirely and `schema` must be left at its default. +When set, this field takes over schema resolution entirely; `schema` is ignored (with a warning logged if it was explicitly set) rather than combined with it. *Type*: `string` diff --git a/internal/impl/postgresql/input_pg_stream.go b/internal/impl/postgresql/input_pg_stream.go index 958eeb717f..6089fe4be9 100644 --- a/internal/impl/postgresql/input_pg_stream.go +++ b/internal/impl/postgresql/input_pg_stream.go @@ -56,6 +56,8 @@ const ( // FieldAWSIAMAuthEnabled enabled field. FieldAWSIAMAuthEnabled = "enabled" shutdownTimeout = 5 * time.Second + + defaultSchema = "public" ) func notImportedAWSOptFn(_ context.Context, awsConf *service.ParsedConfig, _ *pgconn.Config, _ *service.Logger) (TokenBuilder, error) { @@ -120,10 +122,10 @@ This input adds the following metadata fields to each message: Example(10000). Default(1000)). Field(service.NewStringField(fieldSchema). - Description("The PostgreSQL schema from which to replicate data."). + Description("The PostgreSQL schema from which to replicate data. Ignored when `"+fieldSchemaInclude+"` is used instead."). Examples("public", `"MyCaseSensitiveSchemaNeedingQuotes"`). Optional(). - Default("public"), + Default(defaultSchema), ). Field(service.NewStringField(fieldSchemaInclude). Description(`The PostgreSQL schema pattern to replicate data from. Accepts an exact schema name or a glob pattern using `+"`*`"+` as a wildcard to match multiple schemas. @@ -141,7 +143,7 @@ If this pattern matches no schema in the database, startup fails with an error - This pattern can contain characters that wouldn't be allowed in an unquoted schema name, because it's only ever compared against the real name of each schema in the database - it doesn't have to be a valid name itself. For example, a schema literally named `+"`a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11`"+` (which must have been created using double quotes, since hyphens aren't allowed in an unquoted `+"`CREATE SCHEMA`"+` statement) can still be matched using the unquoted pattern `+"`a0eebc99-*`"+`. -This field is mutually exclusive with `+"`"+fieldSchema+"`"+`; when set, it takes over schema resolution entirely and `+"`"+fieldSchema+"`"+` must be left at its default.`). +When set, this field takes over schema resolution entirely; `+"`"+fieldSchema+"`"+` is ignored (with a warning logged if it was explicitly set) rather than combined with it.`). Examples("tenant_*", "*", `"MyCaseSensitiveSchemaNeedingQuotes"`). Optional(). Default(""), @@ -361,8 +363,9 @@ func newPgStreamInput(conf *service.ParsedConfig, mgr *service.Resources) (s ser return nil, err } if schemaInclude != "" { - if schema != "public" { - return nil, errors.New("schema and schema_include are mutually exclusive") + if schema != defaultSchema { + // log that we're ignoring schema if schema_include is set. + mgr.Logger().Warnf("Field '%s' configured, ignoring field '%s' configuration", fieldSchemaInclude, fieldSchema) } if err = validateSchemaPattern(schemaInclude); err != nil { return nil, fmt.Errorf("invalid schema_include: %w", err) diff --git a/internal/impl/postgresql/input_pg_stream_test.go b/internal/impl/postgresql/input_pg_stream_test.go index 6e43213b81..56d75118f9 100644 --- a/internal/impl/postgresql/input_pg_stream_test.go +++ b/internal/impl/postgresql/input_pg_stream_test.go @@ -10,6 +10,8 @@ package pgstream import ( "fmt" + "log/slog" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -17,20 +19,10 @@ import ( "github.com/redpanda-data/benthos/v4/public/service" + "github.com/redpanda-data/connect/v4/internal/impl/postgresql/pgtest" "github.com/redpanda-data/connect/v4/internal/license" ) -func parsePgStreamInput(t *testing.T, yaml string) (service.BatchInput, error) { - t.Helper() - conf, err := newPostgresCDCConfig().ParseYAML(yaml, nil) - require.NoError(t, err) - - mgr := service.MockResources() - license.InjectTestService(mgr) - - return newPgStreamInput(conf, mgr) -} - // TestSchemaDefault verifies that the schema field defaults to "public" when // left unset, matching pre-multi-schema behaviour. func TestSchemaDefault(t *testing.T) { @@ -111,10 +103,7 @@ tables: } } -// TestSchemaAndSchemaIncludeMutuallyExclusive verifies that setting both -// schema (to a non-default value) and schema_include is rejected at config -// construction time. -func TestSchemaAndSchemaIncludeMutuallyExclusive(t *testing.T) { +func TestSchemaIgnoredWhenSchemaIncludeSet(t *testing.T) { yaml := ` dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable schema: tenant_foo @@ -123,14 +112,28 @@ slot_name: test_slot tables: - events ` - _, err := parsePgStreamInput(t, yaml) - require.Error(t, err) - assert.Contains(t, err.Error(), "schema and schema_include are mutually exclusive") + conf, err := newPostgresCDCConfig().ParseYAML(yaml, nil) + require.NoError(t, err) + + logs := pgtest.NewTestLogCapture() + mgr := service.MockResources(service.MockResourcesOptUseLogger(service.NewLoggerFromSlog(slog.New(logs)))) + license.InjectTestService(mgr) + + _, err = newPgStreamInput(conf, mgr) + require.NoError(t, err) + + var sawWarning bool + for _, m := range logs.Messages() { + if strings.Contains(m, fieldSchema) && strings.Contains(m, fieldSchemaInclude) { + sawWarning = true + } + } + assert.True(t, sawWarning, "expected a warning that %s is ignored in favor of %s, got: %v", fieldSchema, fieldSchemaInclude, logs.Messages()) } // TestSchemaIncludeWithDefaultSchemaSucceeds verifies that setting // schema_include while leaving schema untouched (at its "public" default) is -// allowed. +// allowed and logs no warning about schema being ignored. func TestSchemaIncludeWithDefaultSchemaSucceeds(t *testing.T) { yaml := ` dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable @@ -139,8 +142,51 @@ slot_name: test_slot tables: - events ` - _, err := parsePgStreamInput(t, yaml) + conf, err := newPostgresCDCConfig().ParseYAML(yaml, nil) require.NoError(t, err) + + logs := pgtest.NewTestLogCapture() + mgr := service.MockResources(service.MockResourcesOptUseLogger(service.NewLoggerFromSlog(slog.New(logs)))) + license.InjectTestService(mgr) + + _, err = newPgStreamInput(conf, mgr) + require.NoError(t, err) + + for _, m := range logs.Messages() { + assert.NotContains(t, m, fieldSchema+" is set", "schema was left at its default, so no warning about it should be logged, got: %v", m) + } +} + +// TestSchemaExplicitlySetToDefaultAlongsideSchemaIncludeDoesNotWarn documents +// a known, accepted gap: since schema's value is compared against its own +// default ("public") to decide whether to warn, a user who explicitly writes +// schema: public alongside schema_include is indistinguishable from one who +// left schema unset, so no warning is logged either way. This is considered +// acceptable because "public" is inert here regardless of whether it came +// from the user or the default - unlike any other value, which does warn +// (see TestSchemaIgnoredWhenSchemaIncludeSet). +func TestSchemaExplicitlySetToDefaultAlongsideSchemaIncludeDoesNotWarn(t *testing.T) { + yaml := ` +dsn: postgres://testuser:testpass@localhost:5432/testdb?sslmode=disable +schema: public +schema_include: 'tenant_*' +slot_name: test_slot +tables: + - events +` + conf, err := newPostgresCDCConfig().ParseYAML(yaml, nil) + require.NoError(t, err) + + logs := pgtest.NewTestLogCapture() + mgr := service.MockResources(service.MockResourcesOptUseLogger(service.NewLoggerFromSlog(slog.New(logs)))) + license.InjectTestService(mgr) + + _, err = newPgStreamInput(conf, mgr) + require.NoError(t, err) + + for _, m := range logs.Messages() { + assert.NotContains(t, m, fieldSchema+" is set", "explicit schema: public is indistinguishable from the default, so no warning is logged, got: %v", m) + } } // TestSchemaExcludeValidation verifies that each schema_exclude entry is @@ -291,3 +337,14 @@ signal_table_name: rpcn_signal_table }) } } + +func parsePgStreamInput(t *testing.T, yaml string) (service.BatchInput, error) { + t.Helper() + conf, err := newPostgresCDCConfig().ParseYAML(yaml, nil) + require.NoError(t, err) + + mgr := service.MockResources() + license.InjectTestService(mgr) + + return newPgStreamInput(conf, mgr) +} From 95373a486bb0dc73fec12b1c84ea3e37fdb53db6 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Tue, 25 Aug 2026 19:14:02 +0100 Subject: [PATCH 70/70] reset cl --- CHANGELOG.md | 44 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3196a0dc19..6bb7e3a8e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,48 @@ Changelog All notable changes to this project will be documented in this file. +## 4.106.0 - 2026-08-20 + +### Added + +- salesforce_cdc: Added decode-failure bounds and classification of schema-fetch errors (deterministic vs transient) to prevent infinite retry loops and livelocks, with terminal failures surfaced clearly to the health check. ([@squiidz](https://github.com/squiidz), [#4689](https://github.com/redpanda-data/connect/pull/4689)) +- mongodb, mongodb_cdc: Added AWS IAM authentication (`MONGODB-AWS`) for MongoDB Atlas to the `mongodb` input, output, processor and cache, and to the `mongodb_cdc` input, via a new `aws` configuration block supporting the ambient credential chain, static keys, and assume-role chaining. ([@squiidz](https://github.com/squiidz), [#4690](https://github.com/redpanda-data/connect/pull/4690)) +- mongodb_cdc: The input now checkpoints as soon as the initial snapshot completes and is fully acknowledged, so restarts resume the stream instead of re-running the snapshot; a stream position that can no longer be resumed from (for example one that has aged out of the oplog) is recovered by re-running the snapshot, bounded by a breaker that fails loudly instead of churning, with a new `on_unresumable_position` field controlling the lossy no-snapshot case (default `fail`) and a new `checkpoint_write_timeout` field bounding the detached checkpoint writes (the post-snapshot store and the recovery clear). ([@squiidz](https://github.com/squiidz), [#4690](https://github.com/redpanda-data/connect/pull/4690)) + +### Fixed + +- aws_dynamodb_cdc: Fixed silent data loss in snapshot handling by gating checkpoint persistence on downstream acknowledgments, ensuring rejected batches are redelivered instead of skipped. ([@squiidz](https://github.com/squiidz), [#4687](https://github.com/redpanda-data/connect/pull/4687)) +- aws_dynamodb_cdc: Fixed stream rotation and restart scenarios where start_from: latest was incorrectly applied to child shards and checkpoint-less shards discovered after initial setup, causing silent loss of backlog. ([@squiidz](https://github.com/squiidz), [#4687](https://github.com/redpanda-data/connect/pull/4687)) +- cockroachdb_changefeed: Fixed unbounded silent data loss where transaction rows and backfill batches sharing timestamps could skip data on restart; now checkpoints only persist resolved timestamps to guarantee no loss. ([@squiidz](https://github.com/squiidz), [#4688](https://github.com/redpanda-data/connect/pull/4688)) +- salesforce_cdc: Fixed multiple silent-loss paths in Pub/Sub gRPC handling and ack functions: full buffer now applies backpressure instead of dropping events, schema/decode failures reconnect without losing batches, and nacks now pin checkpoints. ([@squiidz](https://github.com/squiidz), [#4689](https://github.com/redpanda-data/connect/pull/4689)) +- salesforce_cdc: Fixed off-by-one error in schema-retry budgeting and credential refresh in unanchored schema retries to prevent indefinite stalls under the default unlimited reconnect policy. ([@squiidz](https://github.com/squiidz), [#4689](https://github.com/redpanda-data/connect/pull/4689)) + +### Changed + +- aws_dynamodb_cdc: Added auto_replay_nacks support to automatically retry transient downstream failures in-process, with nacks now advancing checkpoints when auto_replay_nacks is disabled. ([@squiidz](https://github.com/squiidz), [#4687](https://github.com/redpanda-data/connect/pull/4687)) +- cockroachdb_changefeed: Changed nack handling to advance cursors when auto_replay_nacks is disabled, treating it as an opt-in to drop rejected messages per the framework contract. ([@squiidz](https://github.com/squiidz), [#4688](https://github.com/redpanda-data/connect/pull/4688)) +- general: Updated CDC connector documentation across Microsoft SQL Server, MongoDB, and OracleDB with measured performance characteristics, scaling limitations, and configuration guidance based on real-world benchmarking. ([@prakhargarg105](https://github.com/prakhargarg105), [#4691](https://github.com/redpanda-data/connect/pull/4691)) + +## 4.105.0 - 2026-08-13 + +### Added + +- postgres_cdc: Added support for control signals in PostgreSQL CDC by detecting and forwarding rows inserted into a configurable signal table downstream like regular messages. ([@josephwoodward](https://github.com/josephwoodward), [#4637](https://github.com/redpanda-data/connect/pull/4637)) +- iceberg: Added an opt-in `merge_strategy: copy-on-write` for row-level `upsert`/`delete`, which materialises mutations by rewriting whole data files so the table only ever contains plain data files. This makes mutations readable by engine-backed catalogs that cannot handle merge-on-read equality deletes, such as the Databricks Unity Catalog and Snowflake. The default remains `merge-on-read`. ([@Jeffail](https://github.com/Jeffail), [#4666](https://github.com/redpanda-data/connect/pull/4666)) +- iceberg: Added a `commit.cleanup_on_failure` field (default `true`) to disable connector-side cleanup of files written by failed commits, as an escape hatch for incident recovery. Disabling it can only leak orphan files, which regular orphan-file maintenance reclaims. ([@Jeffail](https://github.com/Jeffail), [#4666](https://github.com/redpanda-data/connect/pull/4666)) + +### Fixed + +- iceberg: Fixed a regression introduced in 4.99.0 where a commit that landed server-side but was reported as failed (ambiguous 5xx, timeout, lost acknowledgement, or an unclassified error) had its just-written parquet files deleted by the failure-path cleanup, leaving the table unreadable. Failure cleanup is now gated on a provable catalog rejection, and commits detected as landed are reported as success, which also prevents the duplicate rows that redelivery produced. ([@Jeffail](https://github.com/Jeffail), [#4666](https://github.com/redpanda-data/connect/pull/4666)) +- iceberg: Fixed no-timezone `timestamp` columns being written to parquet with `isAdjustedToUTC=true`, which is spec-incorrect and made them read back as `timestamptz`. New tables are written correctly; the encoding is pinned per table via a `redpanda-connect.timestamp-encoding` property so an existing table never changes or mixes encodings. ([@Jeffail](https://github.com/Jeffail), [#4666](https://github.com/redpanda-data/connect/pull/4666)) +- iceberg: Fixed commits failing against catalogs that prohibit clients setting particular table properties (for example the Databricks Unity Catalog and `schema.name-mapping.default`) by learning the prohibited keys from the catalog's rejection and stripping them from subsequent commits. ([@Jeffail](https://github.com/Jeffail), [#4666](https://github.com/redpanda-data/connect/pull/4666)) +- iceberg: Fixed several `identifier_fields` value shapes that silently matched no rows on `upsert`/`delete` — non-UTC `time` values, decimal floating-point ties, and `[]byte` values for string key columns — and fixed base64 mangling of binary and fixed column values during copy-on-write rewrites. All write paths now share a single value canonicaliser with the insert path. ([@Jeffail](https://github.com/Jeffail), [#4666](https://github.com/redpanda-data/connect/pull/4666)) + +### Change + +- oracledb_cdc: Snapshot performance improvements by reusing seeded schema metadata [@josephwoodward](https://github.com/josephwoodward), [#4695](https://github.com/redpanda-data/connect/pull/4695)) +- iceberg: Merge-key input strictness now matches the insert path: string-typed values for integer and boolean key columns (for example `{"id": "42"}` against a `BIGINT` key) previously matched by accident and are now rejected with an actionable error, and nanosecond-precision timestamp `identifier_fields` are now rejected under `merge-on-read` as they already were under copy-on-write. A table whose `write.delete.mode` property is explicitly `merge-on-read` also now rejects `copy-on-write` mutations rather than silently overriding the property. ([@Jeffail](https://github.com/Jeffail), [#4666](https://github.com/redpanda-data/connect/pull/4666)) + ## 4.104.0 - 2026-08-06 ### Fixed @@ -92,8 +134,6 @@ All notable changes to this project will be documented in this file. ### Added -- postgres_cdc: Added a new `schema_include` field accepting a glob pattern (e.g. `tenant_*`), replicating all matching schemas through a single replication slot. Useful for multi-tenant databases where each tenant has its own schema. Leaving `tables` unset auto-discovers every table in each matched schema instead of listing them by hand. The existing `schema` field is unaffected and continues to take a single exact schema name (defaulting to `public`); `schema` and `schema_include` are mutually exclusive. ([@ness-david-dedu](https://github.com/ness-david-dedu), [#4589](https://github.com/redpanda-data/connect/pull/4589)) -- postgres_cdc: Added a new `schema_exclude` field to carve exceptions out of a broad `schema_include` (e.g. `schema_include: tenant_*` while skipping `tenant_test`). Accepts the same exact-name/glob/quoted syntax as `schema_include`, matches entries against the already-resolved schema list in memory with no extra database round-trips, and requires `schema_include` to be set. ([@ness-david-dedu](https://github.com/ness-david-dedu), [#4589](https://github.com/redpanda-data/connect/pull/4589)) - aws_dynamodb_cdc: DynamoDB CDC now supports an optional checkpoint_namespace field, allowing multiple independent pipelines to share a single checkpoint table without overwriting each other's checkpoints. ([@squiidz](https://github.com/squiidz), [#4602](https://github.com/redpanda-data/connect/pull/4602)) ### Fixed