From 1cdee64365f5a3d038198499b0dcce2bd41db5ea Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Mon, 15 Jun 2026 19:13:59 +0000 Subject: [PATCH 01/81] feat: Add AWS RDS hostgroups config and runtime tables - Add `mysql_aws_rds_hostgroups` and `runtime_mysql_aws_rds_hostgroups`; register in the admin and config DBs and in the disk<->memory sync table set. - Wire config<->runtime in `MySQL_HostGroups_Manager`: in-memory table, incoming staging, table generation, commit, and save/dump handling, with nullable `green_writer_hostgroup` / `green_reader_hostgroup` support. - Implement `LOAD/SAVE MYSQL SERVERS` paths; save-to-memory excludes `auto_generated` rows so only user configuration is persisted. - Add `proxysql.cnf` read/write and the `CHECKSUM MYSQL RDS HOSTGROUPS` command. Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 15 ++++ include/ProxySQL_Admin_Tables_Definitions.h | 23 +++++ lib/Admin_Bootstrap.cpp | 3 + lib/Admin_Handler.cpp | 9 ++ lib/MySQL_HostGroups_Manager.cpp | 82 +++++++++++++++++ lib/ProxySQL_Admin.cpp | 90 +++++++++++++++++++ lib/ProxySQL_Config.cpp | 98 +++++++++++++++++++++ 7 files changed, 320 insertions(+) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index ef77d67429..99918e96f0 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -64,6 +64,18 @@ "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , " \ "comment VARCHAR , UNIQUE (reader_hostgroup))" +#define MYHGM_MYSQL_AWS_RDS_HOSTGROUPS "CREATE TABLE mysql_aws_rds_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0), " \ + "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0), " \ + "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0), " \ + "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , " \ + "domain_name VARCHAR NOT NULL DEFAULT '', " \ + "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000, " \ + "check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800, " \ + "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0, " \ + "comment VARCHAR NOT NULL DEFAULT '', " \ + "auto_generated INT CHECK (auto_generated IN (0,1)) NOT NULL DEFAULT 0," \ + "UNIQUE (reader_hostgroup))" + #define MYHGM_GEN_ADMIN_RUNTIME_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers ORDER BY hostgroup_id, hostname, port" #define MYHGM_MYSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , hostgroup_settings VARCHAR CHECK (JSON_VALID(hostgroup_settings) OR hostgroup_settings = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" @@ -697,6 +709,9 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { pthread_mutex_t AWS_Aurora_Info_mutex; std::map AWS_Aurora_Info_Map; + void generate_mysql_aws_rds_hostgroups_table(); + SQLite3_result *incoming_aws_rds_hostgroups; + void generate_mysql_hostgroup_attributes_table(); SQLite3_result *incoming_hostgroup_attributes; diff --git a/include/ProxySQL_Admin_Tables_Definitions.h b/include/ProxySQL_Admin_Tables_Definitions.h index 46f21dfbed..a50e1967f8 100644 --- a/include/ProxySQL_Admin_Tables_Definitions.h +++ b/include/ProxySQL_Admin_Tables_Definitions.h @@ -235,6 +235,29 @@ #define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS "CREATE TABLE runtime_mysql_aws_aurora_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , aurora_port INT NOT NUlL DEFAULT 3306 , domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , add_lag_ms INT NOT NULL CHECK (add_lag_ms >= 0 AND add_lag_ms <= 600000) DEFAULT 30 , min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup))" +// AWS RDS hostgroups; adds blue/green (green_*_hostgroup) over aurora. +// The runtime table carries one extra runtime-only column: auto_generated. +#define ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_HOSTGROUPS "CREATE TABLE mysql_aws_rds_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , " \ + "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0) , " \ + "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0) , " \ + "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , " \ + "domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , " \ + "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , " \ + "check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , " \ + "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , " \ + "comment VARCHAR NOT NULL DEFAULT '' , UNIQUE (reader_hostgroup))" + +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_RDS_HOSTGROUPS "CREATE TABLE runtime_mysql_aws_rds_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , " \ + "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0) , " \ + "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0) , " \ + "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , " \ + "domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , " \ + "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , " \ + "check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , " \ + "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , " \ + "comment VARCHAR NOT NULL DEFAULT '' , " \ + "auto_generated INT CHECK (auto_generated IN (0,1)) NOT NULL DEFAULT 0 , UNIQUE (reader_hostgroup))" + #define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_0 "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" #define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_2 "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" diff --git a/lib/Admin_Bootstrap.cpp b/lib/Admin_Bootstrap.cpp index ca6623b8b0..2e5f0ce38f 100644 --- a/lib/Admin_Bootstrap.cpp +++ b/lib/Admin_Bootstrap.cpp @@ -753,6 +753,8 @@ bool ProxySQL_Admin::init(const bootstrap_info_t& bootstrap_info) { insert_into_tables_defs(tables_defs_admin,"runtime_mysql_galera_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GALERA_HOSTGROUPS); insert_into_tables_defs(tables_defs_admin,"mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); insert_into_tables_defs(tables_defs_admin,"runtime_mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS); + insert_into_tables_defs(tables_defs_admin,"mysql_aws_rds_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_HOSTGROUPS); + insert_into_tables_defs(tables_defs_admin,"runtime_mysql_aws_rds_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_RDS_HOSTGROUPS); insert_into_tables_defs(tables_defs_admin,"mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); insert_into_tables_defs(tables_defs_admin,"runtime_mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_HOSTGROUP_ATTRIBUTES); insert_into_tables_defs(tables_defs_admin,"mysql_servers_ssl_params", ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS); @@ -837,6 +839,7 @@ bool ProxySQL_Admin::init(const bootstrap_info_t& bootstrap_info) { insert_into_tables_defs(tables_defs_config,"mysql_group_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS); insert_into_tables_defs(tables_defs_config,"mysql_galera_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS); insert_into_tables_defs(tables_defs_config,"mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + insert_into_tables_defs(tables_defs_config,"mysql_aws_rds_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_HOSTGROUPS); insert_into_tables_defs(tables_defs_config,"mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); insert_into_tables_defs(tables_defs_config,"mysql_servers_ssl_params", ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS); insert_into_tables_defs(tables_defs_config,"mysql_query_rules", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES); diff --git a/lib/Admin_Handler.cpp b/lib/Admin_Handler.cpp index 9cfedf44ce..275a8174ca 100644 --- a/lib/Admin_Handler.cpp +++ b/lib/Admin_Handler.cpp @@ -4273,6 +4273,15 @@ void admin_session_handler(S* sess, void *_pa, PtrSize_t *pkt) { tablename=(char *)"MYSQL AURORA HOSTGROUPS"; SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); } + if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL RDS HOSTGROUPS") && !strncasecmp("CHECKSUM MEMORY MYSQL RDS HOSTGROUPS", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL RDS HOSTGROUPS") && !strncasecmp("CHECKSUM MEM MYSQL RDS HOSTGROUPS", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MYSQL RDS HOSTGROUPS") && !strncasecmp("CHECKSUM MYSQL RDS HOSTGROUPS", query_no_space, strlen(query_no_space)))){ + char *q=(char *)"SELECT * FROM mysql_aws_rds_hostgroups ORDER BY writer_hostgroup"; + tablename=(char *)"MYSQL RDS HOSTGROUPS"; + SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); + } if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL HOSTGROUP ATTRIBUTES") && !strncasecmp("CHECKSUM MEMORY MYSQL HOSTGROUP ATTRIBUTES", query_no_space, strlen(query_no_space))) || (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL HOSTGROUP ATTRIBUTES") && !strncasecmp("CHECKSUM MEM MYSQL HOSTGROUP ATTRIBUTES", query_no_space, strlen(query_no_space))) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 2afdc09076..63df00425d 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -727,6 +727,7 @@ MySQL_HostGroups_Manager::MySQL_HostGroups_Manager() { mydb->execute(MYHGM_MYSQL_GROUP_REPLICATION_HOSTGROUPS); mydb->execute(MYHGM_MYSQL_GALERA_HOSTGROUPS); mydb->execute(MYHGM_MYSQL_AWS_AURORA_HOSTGROUPS); + mydb->execute(MYHGM_MYSQL_AWS_RDS_HOSTGROUPS); mydb->execute(MYHGM_MYSQL_HOSTGROUP_ATTRIBUTES); mydb->execute(MYHGM_MYSQL_SERVERS_SSL_PARAMS); mydb->execute("CREATE INDEX IF NOT EXISTS idx_mysql_servers_hostname_port ON mysql_servers (hostname,port)"); @@ -736,6 +737,7 @@ MySQL_HostGroups_Manager::MySQL_HostGroups_Manager() { incoming_group_replication_hostgroups=NULL; incoming_galera_hostgroups=NULL; incoming_aws_aurora_hostgroups = NULL; + incoming_aws_rds_hostgroups = NULL; incoming_hostgroup_attributes = NULL; incoming_mysql_servers_ssl_params = NULL; incoming_mysql_servers_v2 = NULL; @@ -1546,6 +1548,13 @@ bool MySQL_HostGroups_Manager::commit( generate_mysql_aws_aurora_hostgroups_table(); } + // AWS RDS + if (incoming_aws_rds_hostgroups) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_aws_rds_hostgroups\n"); + mydb->execute("DELETE FROM mysql_aws_rds_hostgroups"); + generate_mysql_aws_rds_hostgroups_table(); + } + // hostgroup attributes if (incoming_hostgroup_attributes) { proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_hostgroup_attributes\n"); @@ -2247,6 +2256,9 @@ SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql(const string& name) if (name == "mysql_aws_aurora_hostgroups") { query=(char *)"SELECT writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms," "check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment FROM mysql_aws_aurora_hostgroups"; + } else if (name == "mysql_aws_rds_hostgroups") { + query=(char *)"SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader," + "domain_name,check_interval_ms,check_timeout_ms,autopurge_missing_checks,comment,auto_generated FROM mysql_aws_rds_hostgroups"; } else if (name == "mysql_galera_hostgroups") { query=(char *)"SELECT writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers,writer_is_also_reader,max_transactions_behind,comment FROM mysql_galera_hostgroups"; } else if (name == "mysql_group_replication_hostgroups") { @@ -3077,6 +3089,8 @@ void MySQL_HostGroups_Manager::save_incoming_mysql_table(SQLite3_result *s, cons SQLite3_result ** inc = NULL; if (name == "mysql_aws_aurora_hostgroups") { inc = &incoming_aws_aurora_hostgroups; + } else if (name == "mysql_aws_rds_hostgroups") { + inc = &incoming_aws_rds_hostgroups; } else if (name == "mysql_galera_hostgroups") { inc = &incoming_galera_hostgroups; } else if (name == "mysql_group_replication_hostgroups") { @@ -6265,6 +6279,74 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { pthread_mutex_unlock(&AWS_Aurora_Info_mutex); } +/** + * @brief Regenerates the runtime in-memory `mysql_aws_rds_hostgroups` table from `incoming_aws_rds_hostgroups`. + * + * The incoming resultset comes from the admin config table (11 columns, no `auto_generated`); config-loaded + * entries are user-defined, so `auto_generated` is stored as 0. `green_writer_hostgroup` and + * `green_reader_hostgroup` are optional and bound as SQL NULL when absent. + */ +void MySQL_HostGroups_Manager::generate_mysql_aws_rds_hostgroups_table() { + if (incoming_aws_rds_hostgroups==NULL) { + return; + } + int rc; + char *query=(char *)"INSERT INTO mysql_aws_rds_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "writer_is_also_reader,domain_name,check_interval_ms,check_timeout_ms,autopurge_missing_checks,comment,auto_generated) VALUES " + "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)"; + auto [rc1, statement_unique] = mydb->prepare_v2(query); + ASSERT_SQLITE_OK(rc1, mydb); + sqlite3_stmt *statement = statement_unique.get(); + proxy_info("New mysql_aws_rds_hostgroups table\n"); + for (std::vector::iterator it = incoming_aws_rds_hostgroups->rows.begin() ; it != incoming_aws_rds_hostgroups->rows.end(); ++it) { + SQLite3_row *r=*it; + int writer_hostgroup=atoi(r->fields[0]); + int reader_hostgroup=atoi(r->fields[1]); + const char *gw_str = r->fields[2]; + const char *gr_str = r->fields[3]; + int green_writer_hostgroup = (gw_str && gw_str[0]) ? atoi(gw_str) : -1; + int green_reader_hostgroup = (gr_str && gr_str[0]) ? atoi(gr_str) : -1; + int active=atoi(r->fields[4]); + int writer_is_also_reader = atoi(r->fields[5]); + int check_interval_ms = atoi(r->fields[7]); + int check_timeout_ms = atoi(r->fields[8]); + int autopurge_missing_checks = atoi(r->fields[9]); + // entries loaded from the admin config table are always user-defined + int auto_generated = 0; + proxy_info("Loading AWS RDS info for (%d,%d,%d,%d,%s,%d,\"%s\",%d,%d,%d,%d,\"%s\")\n", writer_hostgroup,reader_hostgroup, + green_writer_hostgroup,green_reader_hostgroup,(active ? "on" : "off"),writer_is_also_reader,r->fields[6], + check_interval_ms,check_timeout_ms,autopurge_missing_checks,auto_generated,r->fields[10]); + rc=(*proxy_sqlite3_bind_int64)(statement, 1, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 2, reader_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + if (green_writer_hostgroup >= 0) { + rc=(*proxy_sqlite3_bind_int64)(statement, 3, green_writer_hostgroup); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 3); + } + ASSERT_SQLITE_OK(rc, mydb); + if (green_reader_hostgroup >= 0) { + rc=(*proxy_sqlite3_bind_int64)(statement, 4, green_reader_hostgroup); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 4); + } + ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 5, active); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 6, writer_is_also_reader); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 7, r->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 8, check_interval_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 9, check_timeout_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 10, autopurge_missing_checks); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 11, r->fields[10], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 12, auto_generated); ASSERT_SQLITE_OK(rc, mydb); + + SAFE_SQLITE3_STEP2(statement); + rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, mydb); + } + delete incoming_aws_rds_hostgroups; + incoming_aws_rds_hostgroups=NULL; +} + //void MySQL_HostGroups_Manager::aws_aurora_replication_lag_action(int _whid, int _rhid, char *address, unsigned int port, float current_replication_lag, bool enable, bool verbose) { diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index ddbc57d248..a61c3fb6e7 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -147,6 +147,7 @@ static const vector mysql_servers_tablenames = { "mysql_group_replication_hostgroups", "mysql_galera_hostgroups", "mysql_aws_aurora_hostgroups", + "mysql_aws_rds_hostgroups", "mysql_hostgroup_attributes", "mysql_servers_ssl_params", }; @@ -1509,6 +1510,8 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign || strstr(query_no_space,"runtime_mysql_aws_aurora_hostgroups") || + strstr(query_no_space,"runtime_mysql_aws_rds_hostgroups") + || strstr(query_no_space,"runtime_mysql_hostgroup_attributes") || strstr(query_no_space,"runtime_mysql_servers_ssl_params") @@ -7547,6 +7550,77 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { if(resultset) delete resultset; resultset=NULL; + // dump mysql_aws_rds_hostgroups + // The runtime table carries the extra runtime-only 'auto_generated' column; the config table + // does not. 'dump_table_mysql' always returns 12 columns (last is 'auto_generated'); we bind + // 12 for the runtime table and only the first 11 for the config table. 'green_writer_hostgroup' + // and 'green_reader_hostgroup' (fields 2,3) are nullable and bound as NULL when absent. + + if (_runtime) { + query=(char *)"DELETE FROM main.runtime_mysql_aws_rds_hostgroups"; + } else { + query=(char *)"DELETE FROM main.mysql_aws_rds_hostgroups"; + } + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); + admindb->execute(query); + resultset=MyHGM->dump_table_mysql("mysql_aws_rds_hostgroups"); + if (resultset) { + int rc; + sqlite3_stmt *statement=NULL; + + char *query=NULL; + if (_runtime) { + query=(char *)"INSERT INTO runtime_mysql_aws_rds_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader,domain_name,check_interval_ms,check_timeout_ms,autopurge_missing_checks,comment,auto_generated) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)"; + } else { + query=(char *)"INSERT INTO mysql_aws_rds_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader,domain_name,check_interval_ms,check_timeout_ms,autopurge_missing_checks,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"; + } + + auto [rc1, statement_unique] = admindb->prepare_v2(query); + rc = rc1; + statement = statement_unique.get(); + ASSERT_SQLITE_OK(rc, admindb); + + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + // auto_generated (field 11) entries are created at runtime by the monitor; they are NOT + // user configuration, so they must not be persisted to the memory config table. They are + // still written to the runtime table. + if (!_runtime && r->fields[11] && atoi(r->fields[11]) != 0) { + continue; + } + rc=(*proxy_sqlite3_bind_int64)(statement, 1, atoi(r->fields[0])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 2, atoi(r->fields[1])); ASSERT_SQLITE_OK(rc, admindb); + if (r->fields[2] && r->fields[2][0]) { + rc=(*proxy_sqlite3_bind_int64)(statement, 3, atoi(r->fields[2])); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 3); + } + ASSERT_SQLITE_OK(rc, admindb); + if (r->fields[3] && r->fields[3][0]) { + rc=(*proxy_sqlite3_bind_int64)(statement, 4, atoi(r->fields[3])); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 4); + } + ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 5, atoi(r->fields[4])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 6, atoi(r->fields[5])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement, 7, r->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 8, atoi(r->fields[7])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 9, atoi(r->fields[8])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 10, atoi(r->fields[9])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement, 11, r->fields[10], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + if (_runtime) { + rc=(*proxy_sqlite3_bind_int64)(statement, 12, atoi(r->fields[11])); ASSERT_SQLITE_OK(rc, admindb); + } + + SAFE_SQLITE3_STEP2(statement); + rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, admindb); + } + } + if(resultset) delete resultset; + resultset=NULL; + // dump mysql_hostgroup_attributes StrQuery = "DELETE FROM main."; @@ -7878,6 +7952,7 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc SQLite3_result *resultset_group_replication=NULL; SQLite3_result *resultset_galera=NULL; SQLite3_result *resultset_aws_aurora=NULL; + SQLite3_result *resultset_aws_rds=NULL; SQLite3_result *resultset_hostgroup_attributes=NULL; SQLite3_result *resultset_mysql_servers_ssl_params=NULL; @@ -8041,6 +8116,17 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc MyHGM->save_incoming_mysql_table(resultset_aws_aurora,"mysql_aws_aurora_hostgroups"); } + // support for AWS RDS, table mysql_aws_rds_hostgroups + query=(char *)"SELECT a.* FROM mysql_aws_rds_hostgroups a LEFT JOIN mysql_aws_rds_hostgroups b ON (a.writer_hostgroup=b.reader_hostgroup) WHERE b.reader_hostgroup IS NULL ORDER BY writer_hostgroup"; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); + admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_aws_rds); + if (error) { + proxy_error("Error on %s : %s\n", query, error); + } else { + // Pass the resultset to MyHGM + MyHGM->save_incoming_mysql_table(resultset_aws_rds,"mysql_aws_rds_hostgroups"); + } + // support for hostgroup attributes, table mysql_hostgroup_attributes query = (char *)"SELECT * FROM mysql_hostgroup_attributes ORDER BY hostgroup_id"; proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); @@ -8100,6 +8186,10 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc //delete resultset_aws_aurora; // do not delete, resultset is stored in MyHGM resultset_aws_aurora=NULL; } + if (resultset_aws_rds) { + //delete resultset_aws_rds; // do not delete, resultset is stored in MyHGM + resultset_aws_rds=NULL; + } if (resultset_hostgroup_attributes) { resultset_hostgroup_attributes = NULL; } diff --git a/lib/ProxySQL_Config.cpp b/lib/ProxySQL_Config.cpp index 0f5c7cc6f0..31e106a977 100644 --- a/lib/ProxySQL_Config.cpp +++ b/lib/ProxySQL_Config.cpp @@ -1094,6 +1094,42 @@ int ProxySQL_Config::Write_MySQL_Servers_to_configfile(std::string& data) { } } + if (sqlite_resultset) + delete sqlite_resultset; + + query=(char *)"SELECT * FROM mysql_aws_rds_hostgroups"; + admindb->execute_statement(query, &error, &cols, &affected_rows, &sqlite_resultset); + if (error) { + proxy_error("Error on read from mysql_aws_rds_hostgroups: %s\n", error); + return -1; + } else { + if (sqlite_resultset) { + data += "mysql_aws_rds_hostgroups:\n(\n"; + bool isNext = false; + for (auto r : sqlite_resultset->rows) { + if (isNext) + data += ",\n"; + data += "\t{\n"; + addField(data, "writer_hostgroup", r->fields[0], ""); + addField(data, "reader_hostgroup", r->fields[1], ""); + // green_writer_hostgroup / green_reader_hostgroup are nullable; addField skips NULLs + addField(data, "green_writer_hostgroup", r->fields[2], ""); + addField(data, "green_reader_hostgroup", r->fields[3], ""); + addField(data, "active", r->fields[4], ""); + addField(data, "writer_is_also_reader", r->fields[5], ""); + addField(data, "domain_name", r->fields[6]); + addField(data, "check_interval_ms", r->fields[7], ""); + addField(data, "check_timeout_ms", r->fields[8], ""); + addField(data, "autopurge_missing_checks", r->fields[9], ""); + addField(data, "comment", r->fields[10]); + + data += "\t}"; + isNext = true; + } + data += "\n)\n"; + } + } + if (sqlite_resultset) delete sqlite_resultset; @@ -1451,6 +1487,68 @@ int ProxySQL_Config::Read_MySQL_Servers_from_configfile(std::string& error) { rows++; } } + + if (root.exists("mysql_aws_rds_hostgroups")==true) { + const Setting &mysql_aws_rds_hostgroups = root["mysql_aws_rds_hostgroups"]; + int count = mysql_aws_rds_hostgroups.getLength(); + // green_writer_hostgroup / green_reader_hostgroup are nullable -> passed as %s ("NULL" or an integer) + char *q=(char *)"INSERT OR REPLACE INTO mysql_aws_rds_hostgroups (writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active, writer_is_also_reader, domain_name, check_interval_ms, check_timeout_ms, autopurge_missing_checks, comment ) VALUES (%d, %d, %s, %s, %d, %d, '%s', %d, %d, %d, '%s')"; + for (i=0; i< count; i++) { + const Setting &line = mysql_aws_rds_hostgroups[i]; + int writer_hostgroup; + int reader_hostgroup; + int green_writer_hostgroup; + int green_reader_hostgroup; + int active=1; // default + int writer_is_also_reader; + int check_interval_ms; + int check_timeout_ms; + int autopurge_missing_checks; + std::string comment=""; + std::string domain_name=""; + if (line.lookupValue("writer_hostgroup", writer_hostgroup)==false) { + proxy_error("Admin: detected a mysql_aws_rds_hostgroups in config file without a mandatory writer_hostgroup\n"); + continue; + } + if (line.lookupValue("reader_hostgroup", reader_hostgroup)==false) { + proxy_error("Admin: detected a mysql_aws_rds_hostgroups in config file without a mandatory reader_hostgroup\n"); + continue; + } + char green_writer_str[24]; + char green_reader_str[24]; + if (line.lookupValue("green_writer_hostgroup", green_writer_hostgroup)==false) { + strcpy(green_writer_str, "NULL"); + } else { + snprintf(green_writer_str, sizeof(green_writer_str), "%d", green_writer_hostgroup); + } + if (line.lookupValue("green_reader_hostgroup", green_reader_hostgroup)==false) { + strcpy(green_reader_str, "NULL"); + } else { + snprintf(green_reader_str, sizeof(green_reader_str), "%d", green_reader_hostgroup); + } + if (line.lookupValue("active", active)==false) active=1; + if (line.lookupValue("writer_is_also_reader", writer_is_also_reader)==false) writer_is_also_reader=0; + if (line.lookupValue("check_interval_ms", check_interval_ms)==false) check_interval_ms=1000; + if (line.lookupValue("check_timeout_ms", check_timeout_ms)==false) check_timeout_ms=800; + if (line.lookupValue("autopurge_missing_checks", autopurge_missing_checks)==false) autopurge_missing_checks=0; + line.lookupValue("comment", comment); + line.lookupValue("domain_name", domain_name); + char *o1=strdup(comment.c_str()); + char *o=escape_string_single_quotes(o1, false); + char *p1=strdup(domain_name.c_str()); + char *p=escape_string_single_quotes(p1, false); + char *query=(char *)malloc(strlen(q)+strlen(o)+strlen(p)+256); // 128 vs sizeof(int)*8 + sprintf(query,q, writer_hostgroup, reader_hostgroup, green_writer_str, green_reader_str, active, writer_is_also_reader, p, check_interval_ms, check_timeout_ms, autopurge_missing_checks, o); + admindb->execute(query); + if (o!=o1) free(o); + free(o1); + if (p!=p1) free(p); + free(p1); + free(query); + rows++; + } + } + if (root.exists("mysql_hostgroup_attributes") == true) { const Setting &mysql_hostgroup_attributes = root["mysql_hostgroup_attributes"]; int count = mysql_hostgroup_attributes.getLength(); From 83b5901dd28f0d05759b0e8c542e40d862b405bc Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Mon, 15 Jun 2026 21:09:30 +0000 Subject: [PATCH 02/81] feat: Add AWS RDS topology monitor thread - Add an RDS monitor thread (dispatcher + one worker per writer hostgroup), mirroring the Aurora monitor and driven by `AWS_RDS_Hosts_resultset`. - Each worker pings a host and runs a persistent two-state probe of `mysql.rds_topology`: check existence, then fetch metadata. - Detect role/status columns by name and branch on shape (Single/Multi-AZ Instance vs Multi-AZ Cluster); handlers currently only log. - Handle a missing topology table gracefully; add `MON_AWS_RDS` task type. Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 6 + include/MySQL_Monitor.hpp | 10 + lib/MySQL_HostGroups_Manager.cpp | 41 +++ lib/MySQL_Monitor.cpp | 432 +++++++++++++++++++++++++++++ 4 files changed, 489 insertions(+) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 99918e96f0..dd3ef96570 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -1122,6 +1122,12 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { * be taken or not. */ void update_aws_aurora_hosts_monitor_resultset(bool lock=false); + /** + * @brief Rebuilds `GloMyMon->AWS_RDS_Hosts_resultset` (and its checksum) from the + * `mysql_servers` x `mysql_aws_rds_hostgroups` join used by the RDS monitor thread. + * @param lock when true, the monitor's `aws_rds_mutex` is taken internally. + */ + void update_aws_rds_hosts_monitor_resultset(bool lock=false); SQLite3_result * get_stats_mysql_gtid_executed(); void generate_mysql_gtid_executed_tables(); diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 9b0f814a44..b77e49b0f8 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -204,6 +204,7 @@ enum MySQL_Monitor_State_Data_Task_Type { MON_REPLICATION_LAG, MON_GALERA, MON_AWS_AURORA, + MON_AWS_RDS, MON_READ_ONLY__AND__AWS_RDS_TOPOLOGY_DISCOVERY }; @@ -413,6 +414,7 @@ class MySQL_Monitor { pthread_mutex_t group_replication_mutex; // for simplicity, a mutex instead of a rwlock pthread_mutex_t galera_mutex; // for simplicity, a mutex instead of a rwlock pthread_mutex_t aws_aurora_mutex; // for simplicity, a mutex instead of a rwlock + pthread_mutex_t aws_rds_mutex; // for simplicity, a mutex instead of a rwlock pthread_mutex_t mysql_servers_mutex; // for simplicity, a mutex instead of a rwlock pthread_mutex_t proxysql_servers_mutex; //std::map Group_Replication_Hosts_Map; @@ -423,6 +425,9 @@ class MySQL_Monitor { std::map AWS_Aurora_Hosts_Map; SQLite3_result *AWS_Aurora_Hosts_resultset; uint64_t AWS_Aurora_Hosts_resultset_checksum; + // host list consumed by the AWS RDS monitor thread (join of mysql_servers x mysql_aws_rds_hostgroups) + SQLite3_result *AWS_RDS_Hosts_resultset; + uint64_t AWS_RDS_Hosts_resultset_checksum; unsigned int num_threads; unsigned int aux_threads; unsigned int started_threads; @@ -470,6 +475,11 @@ class MySQL_Monitor { void * monitor_group_replication_2(); void * monitor_galera(); void * monitor_aws_aurora(); + void * monitor_aws_rds(); + // Invoked once the topology shape is detected on mysql.rds_topology. + // 'result' holds the fetched rows. + void rds_monitor_handle_instance_topology(unsigned int writer_hostgroup, unsigned int reader_hostgroup, int green_writer_hostgroup, int green_reader_hostgroup, MYSQL_RES* result); + void rds_monitor_handle_cluster_topology(unsigned int writer_hostgroup, unsigned int reader_hostgroup, MYSQL_RES* result); void * monitor_replication_lag(); void * monitor_dns_cache(); void * run(); diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 63df00425d..a5e27d2b67 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -6345,6 +6345,13 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_rds_hostgroups_table() { } delete incoming_aws_rds_hostgroups; incoming_aws_rds_hostgroups=NULL; + + // publish the refreshed host list to the RDS monitor thread + if (GloMyMon) { + pthread_mutex_lock(&GloMyMon->aws_rds_mutex); + update_aws_rds_hosts_monitor_resultset(false); + pthread_mutex_unlock(&GloMyMon->aws_rds_mutex); + } } @@ -6923,6 +6930,40 @@ void MySQL_HostGroups_Manager::update_aws_aurora_hosts_monitor_resultset(bool lo } } +const char SELECT_AWS_RDS_SERVERS_FOR_MONITOR[] { + "SELECT writer_hostgroup, reader_hostgroup, hostname, port, MAX(use_ssl) use_ssl, green_writer_hostgroup," + " green_reader_hostgroup, check_interval_ms, check_timeout_ms, autopurge_missing_checks, domain_name FROM mysql_servers" + " JOIN mysql_aws_rds_hostgroups ON" + " hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE active=1 AND status NOT IN (2,3)" + " GROUP BY writer_hostgroup, hostname, port" +}; + +void MySQL_HostGroups_Manager::update_aws_rds_hosts_monitor_resultset(bool lock) { + if (lock) { + pthread_mutex_lock(&GloMyMon->aws_rds_mutex); + } + + SQLite3_result* resultset = nullptr; + { + char* error = nullptr; + int cols = 0; + int affected_rows = 0; + mydb->execute_statement(SELECT_AWS_RDS_SERVERS_FOR_MONITOR, &error, &cols, &affected_rows, &resultset); + } + + if (resultset) { + if (GloMyMon->AWS_RDS_Hosts_resultset) { + delete GloMyMon->AWS_RDS_Hosts_resultset; + } + GloMyMon->AWS_RDS_Hosts_resultset=resultset; + GloMyMon->AWS_RDS_Hosts_resultset_checksum=resultset->raw_checksum(); + } + + if (lock) { + pthread_mutex_unlock(&GloMyMon->aws_rds_mutex); + } +} + MySrvC* MySQL_HostGroups_Manager::find_server_in_hg(unsigned int _hid, const std::string& addr, int port) { MySrvC* f_server = nullptr; diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 1ea5077175..8137b526c6 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -788,6 +788,8 @@ void MySQL_Monitor_State_Data::init_async() { break; case MON_AWS_AURORA: break; + case MON_AWS_RDS: + break; } } @@ -881,6 +883,14 @@ void * monitor_aws_aurora_pthread(void *arg) { return NULL; } +void * monitor_aws_rds_pthread(void *arg) { + set_thread_name("MonitorRDS", GloVars.set_thread_name); + // Wait for GloMTH to be initialized + if (!wait_for_glo_mth()) return NULL; // quick exit during shutdown/restart + GloMyMon->monitor_aws_rds(); + return NULL; +} + void * monitor_replication_lag_pthread(void *arg) { #ifndef NOJEM bool cache=false; @@ -1077,10 +1087,13 @@ MySQL_Monitor::MySQL_Monitor() { Galera_Hosts_resultset=NULL; pthread_mutex_init(&aws_aurora_mutex,NULL); + pthread_mutex_init(&aws_rds_mutex,NULL); pthread_mutex_init(&mysql_servers_mutex,NULL); pthread_mutex_init(&proxysql_servers_mutex, NULL); AWS_Aurora_Hosts_resultset=NULL; AWS_Aurora_Hosts_resultset_checksum = 0; + AWS_RDS_Hosts_resultset=NULL; + AWS_RDS_Hosts_resultset_checksum = 0; shutdown=false; monitor_enabled=true; // default // create new SQLite datatabase @@ -1181,6 +1194,10 @@ MySQL_Monitor::~MySQL_Monitor() { delete AWS_Aurora_Hosts_resultset; AWS_Aurora_Hosts_resultset=NULL; } + if (AWS_RDS_Hosts_resultset) { + delete AWS_RDS_Hosts_resultset; + AWS_RDS_Hosts_resultset=NULL; + } std::map::iterator it2; AWS_Aurora_monitor_node *node=NULL; for (it2 = AWS_Aurora_Hosts_Map.begin(); it2 != AWS_Aurora_Hosts_Map.end(); ++it2) { @@ -5034,6 +5051,13 @@ void * MySQL_Monitor::run() { assert(0); // LCOV_EXCL_STOP } + pthread_t monitor_aws_rds_thread; + if (pthread_create(&monitor_aws_rds_thread, &attr, &monitor_aws_rds_pthread,NULL) != 0) { + // LCOV_EXCL_START + proxy_error("Thread creation\n"); + assert(0); + // LCOV_EXCL_STOP + } pthread_t monitor_replication_lag_thread; if (pthread_create(&monitor_replication_lag_thread, &attr, &monitor_replication_lag_pthread,NULL) != 0) { // LCOV_EXCL_START @@ -5133,6 +5157,7 @@ void * MySQL_Monitor::run() { pthread_join(monitor_group_replication_thread,NULL); pthread_join(monitor_galera_thread,NULL); pthread_join(monitor_aws_aurora_thread,NULL); + pthread_join(monitor_aws_rds_thread,NULL); pthread_join(monitor_replication_lag_thread,NULL); My_Conn_Pool->purge_all_connections(); @@ -6406,6 +6431,413 @@ void * MySQL_Monitor::monitor_aws_aurora() { return NULL; } +// Runs an async query + store_result on the monitor connection, honoring the +// per-check timeout and the global shutdown flag. +// Returns: 0 success, 1 timeout/query-error, 2 shutdown requested. +static int aws_rds_async_query(MySQL_Monitor_State_Data *mmsd, const char *query) { + mmsd->t1 = monotonic_time(); + mmsd->interr = 0; + mmsd->async_exit_status = mysql_query_start(&mmsd->interr, mmsd->mysql, query); + while (mmsd->async_exit_status) { + mmsd->async_exit_status = wait_for_mysql(mmsd->mysql, mmsd->async_exit_status); + const unsigned long long now = monotonic_time(); + if (now > mmsd->t1 + mmsd->aws_aurora_check_timeout_ms * 1000) { + mmsd->mysql_error_msg = strdup("timeout check"); + return 1; + } + if (GloMyMon->shutdown == true) return 2; + if ((mmsd->async_exit_status & MYSQL_WAIT_TIMEOUT) == 0) { + mmsd->async_exit_status = mysql_query_cont(&mmsd->interr, mmsd->mysql, mmsd->async_exit_status); + } + } + mmsd->async_exit_status = mysql_store_result_start(&mmsd->result, mmsd->mysql); + while (mmsd->async_exit_status) { + mmsd->async_exit_status = wait_for_mysql(mmsd->mysql, mmsd->async_exit_status); + const unsigned long long now = monotonic_time(); + if (now > mmsd->t1 + mmsd->aws_aurora_check_timeout_ms * 1000) { + mmsd->mysql_error_msg = strdup("timeout check"); + return 1; + } + if (GloMyMon->shutdown == true) return 2; + if ((mmsd->async_exit_status & MYSQL_WAIT_TIMEOUT) == 0) { + mmsd->async_exit_status = mysql_store_result_cont(&mmsd->result, mmsd->mysql, mmsd->async_exit_status); + } + } + if (mmsd->interr) { // query failed (may be ER_NO_SUCH_TABLE 1146) + mmsd->mysql_error_msg = strdup(mysql_error(mmsd->mysql)); + return 1; + } + return 0; +} + +// State of the per-host RDS topology probe. +enum RDS_Topology_Monitor_State { + TOPOLOGY_TABLE_CHECK, // verify mysql.rds_topology exists + TOPOLOGY_METADATA_FETCH // table confirmed present; fetch and branch on its metadata +}; + +void * monitor_RDS_thread_HG(void *arg) { + unsigned int wHG = *(unsigned int *)arg; + unsigned int rHG = 0; + unsigned int num_hosts = 0; + unsigned int cur_host_idx = 0; + unsigned int check_interval_ms = 0; + unsigned int check_timeout_ms = 0; + int green_writer_hostgroup = -1; + int green_reader_hostgroup = -1; + set_thread_name("MonitorRDSHG", GloVars.set_thread_name); + proxy_info("Started Monitor thread for AWS RDS writer HG %u\n", wHG); + + // Wait for GloMTH to be initialized + if (!wait_for_glo_mth()) return NULL; // quick exit during shutdown/restart + unsigned int MySQL_Monitor__thread_MySQL_Thread_Variables_version; + MySQL_Thread * mysql_thr = new MySQL_Thread(); + mysql_thr->curtime = monotonic_time(); + MySQL_Monitor__thread_MySQL_Thread_Variables_version = GloMTH->get_global_version(); + mysql_thr->refresh_variables(); + + uint64_t initial_raw_checksum = 0; + + // initial data load from the monitor resultset (columns: 0 writer_hostgroup, + // 1 reader_hostgroup, 2 hostname, 3 port, 4 use_ssl, 5 green_writer_hostgroup, + // 6 green_reader_hostgroup, 7 check_interval_ms, 8 check_timeout_ms, + // 9 autopurge_missing_checks, 10 domain_name) + pthread_mutex_lock(&GloMyMon->aws_rds_mutex); + initial_raw_checksum = GloMyMon->AWS_RDS_Hosts_resultset_checksum; + for (std::vector::iterator it = GloMyMon->AWS_RDS_Hosts_resultset->rows.begin() ; it != GloMyMon->AWS_RDS_Hosts_resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + if (atoi(r->fields[0]) == (int)wHG) { + num_hosts++; + if (rHG == 0) rHG = atoi(r->fields[1]); + if (green_writer_hostgroup < 0 && r->fields[5] && r->fields[5][0]) green_writer_hostgroup = atoi(r->fields[5]); + if (green_reader_hostgroup < 0 && r->fields[6] && r->fields[6][0]) green_reader_hostgroup = atoi(r->fields[6]); + if (check_interval_ms == 0) check_interval_ms = atoi(r->fields[7]); + if (check_timeout_ms == 0) check_timeout_ms = atoi(r->fields[8]); + } + } + host_def_t *hpa = (host_def_t *)malloc(sizeof(host_def_t)*(num_hosts ? num_hosts : 1)); + for (std::vector::iterator it = GloMyMon->AWS_RDS_Hosts_resultset->rows.begin() ; it != GloMyMon->AWS_RDS_Hosts_resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + if (atoi(r->fields[0]) == (int)wHG) { + hpa[cur_host_idx].host = strdup(r->fields[2]); + hpa[cur_host_idx].port = atoi(r->fields[3]); + hpa[cur_host_idx].use_ssl = atoi(r->fields[4]); + cur_host_idx++; + } + } + if (num_hosts && cur_host_idx >= num_hosts) cur_host_idx = num_hosts - 1; + pthread_mutex_unlock(&GloMyMon->aws_rds_mutex); + + bool exit_now = false; + unsigned long long t1 = 0; + unsigned long long next_loop_at = 0; + bool crc = false; + uint64_t current_raw_checksum = 0; + size_t rnd; + bool found_pingable_host = false; + bool rc_ping = false; + MySQL_Monitor_State_Data *mmsd = NULL; + RDS_Topology_Monitor_State topology_state = TOPOLOGY_TABLE_CHECK; + + t1 = monotonic_time(); + + while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true && exit_now==false) { + unsigned int glover; + t1 = monotonic_time(); + + if (!GloMTH) goto __exit_monitor_RDS_thread_HG_now; // quick exit during shutdown/restart + + // if variables changed, refresh and force a new check + glover = GloMTH->get_global_version(); + if (MySQL_Monitor__thread_MySQL_Thread_Variables_version < glover) { + MySQL_Monitor__thread_MySQL_Thread_Variables_version = glover; + mysql_thr->refresh_variables(); + next_loop_at = 0; + } + + // if the host list/definition changed, terminate so the dispatcher respawns + pthread_mutex_lock(&GloMyMon->aws_rds_mutex); + current_raw_checksum = GloMyMon->AWS_RDS_Hosts_resultset_checksum; + pthread_mutex_unlock(&GloMyMon->aws_rds_mutex); + if (current_raw_checksum != initial_raw_checksum) { + exit_now = true; + break; + } + + if (num_hosts == 0) { + next_loop_at = t1 + (check_interval_ms ? check_interval_ms : 1000) * 1000; + usleep(50000); + continue; + } + + if (t1 < next_loop_at) { + unsigned long long st = next_loop_at - t1; + if (st > 50000) st = 50000; + usleep(st); + continue; + } + + // pick a pingable host: random first, then shuffle and scan + found_pingable_host = false; + rnd = (size_t) rand(); + rnd %= num_hosts; + rc_ping = GloMyMon->server_responds_to_ping(hpa[rnd].host, hpa[rnd].port); + if (rc_ping) { + found_pingable_host = true; + cur_host_idx = rnd; + } else { + MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, wHG, hpa[rnd].host, hpa[rnd].port, ER_PROXYSQL_AWS_NO_PINGABLE_SRV); + shuffle_hosts(hpa, num_hosts); + for (unsigned int i=0; (found_pingable_host == false && iserver_responds_to_ping(hpa[i].host, hpa[i].port); + if (rc_ping) { + found_pingable_host = true; + cur_host_idx = i; + } else { + MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, wHG, hpa[i].host, hpa[i].port, ER_PROXYSQL_AWS_NO_PINGABLE_SRV); + } + } + } + if (found_pingable_host == false) { + proxy_error("No node is pingable for AWS RDS cluster with writer HG %u\n", wHG); + next_loop_at = t1 + check_interval_ms * 1000; + continue; + } + + mmsd = new MySQL_Monitor_State_Data(MON_AWS_RDS, hpa[cur_host_idx].host, hpa[cur_host_idx].port, hpa[cur_host_idx].use_ssl); + mmsd->writer_hostgroup = wHG; + mmsd->aws_aurora_check_timeout_ms = check_timeout_ms; // reuse the generic per-check timeout field + mmsd->mysql = GloMyMon->My_Conn_Pool->get_connection(mmsd->hostname, mmsd->port, mmsd); + mmsd->t1 = t1; + + crc = false; + if (mmsd->mysql == NULL) { // need a new connection + bool rc = mmsd->create_new_connection(); + if (mmsd->mysql) GloMyMon->My_Conn_Pool->conn_register(mmsd); + crc = true; + if (rc == false) { + proxy_error("Error on AWS RDS check for %s:%d. Unable to create a connection.\n", mmsd->hostname, mmsd->port); + MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_AWS_HEALTH_CHECK_CONN_TIMEOUT); + goto __end_of_loop; + } + } + + if (topology_state == TOPOLOGY_TABLE_CHECK) { + // State TOPOLOGY_TABLE_CHECK: confirm mysql.rds_topology exists. Once seen + // we advance to TOPOLOGY_METADATA_FETCH and skip this check on subsequent + // iterations, until a fetch reports the table is gone. + + int qrc = aws_rds_async_query(mmsd, "SELECT 1 FROM information_schema.TABLES WHERE TABLE_SCHEMA='mysql' AND TABLE_NAME='rds_topology'"); + if (qrc == 2) goto __exit_monitor_RDS_thread_HG_now; + if (qrc != 0) { + proxy_error("AWS RDS topology availability check failed for %s:%d : %s\n", mmsd->hostname, mmsd->port, mmsd->mysql_error_msg ? mmsd->mysql_error_msg : "unknown"); + goto __end_of_loop; + } + bool table_available = (mmsd->result && mysql_num_rows(mmsd->result) > 0); + if (mmsd->result) { mysql_free_result(mmsd->result); mmsd->result = NULL; } + if (!table_available) { + // no blue/green deployment or multi-az cluster discovery in progress; nothing to do + proxy_debug(PROXY_DEBUG_MONITOR, 5, "mysql.rds_topology not present on %s:%d (RDS writer HG %u); skipping\n", mmsd->hostname, mmsd->port, wHG); + goto __end_of_loop; + } + topology_state = TOPOLOGY_METADATA_FETCH; + } else if (topology_state == TOPOLOGY_METADATA_FETCH) { + // State TOPOLOGY_METADATA_FETCH: fetch topology metadata. The column set + // differs by RDS type (the Multi-AZ Cluster topology table may not expose + // 'role'/'status' at all), so dump all columns and detect what is present. + + int qrc = aws_rds_async_query(mmsd, "SELECT * FROM mysql.rds_topology"); + if (qrc == 2) goto __exit_monitor_RDS_thread_HG_now; + if (qrc != 0) { + unsigned int err = mmsd->mysql ? mysql_errno(mmsd->mysql) : 0; + if (err == 1146) { + // the table vanished (ER_NO_SUCH_TABLE), e.g. a blue/green deployment + // was cancelled: re-check its existence on the next iteration. + topology_state = TOPOLOGY_TABLE_CHECK; + proxy_debug(PROXY_DEBUG_MONITOR, 5, "mysql.rds_topology vanished on %s:%d (RDS writer HG %u); rechecking availability\n", mmsd->hostname, mmsd->port, wHG); + } else { + proxy_error("AWS RDS topology fetch failed for %s:%d : %s\n", mmsd->hostname, mmsd->port, mmsd->mysql_error_msg ? mmsd->mysql_error_msg : "unknown"); + } + goto __end_of_loop; + } + + // locate the 'role' and 'status' columns by name; they may be absent + unsigned int num_rows = mmsd->result ? (unsigned int)mysql_num_rows(mmsd->result) : 0; + int role_idx = -1, status_idx = -1; + if (mmsd->result) { + unsigned int num_fields = mysql_num_fields(mmsd->result); + MYSQL_FIELD *fields = mysql_fetch_fields(mmsd->result); + for (unsigned int i=0; i= 0 && status_idx >= 0) { + MYSQL_ROW row = mysql_fetch_row(mmsd->result); + if (row && row[role_idx] != NULL && row[status_idx] != NULL) instance_topology = true; + mysql_data_seek(mmsd->result, 0); // rewind for the handlers + } + if (num_rows > 0 && instance_topology) { + GloMyMon->rds_monitor_handle_instance_topology(wHG, rHG, green_writer_hostgroup, green_reader_hostgroup, mmsd->result); + } else if (num_rows > 0) { + GloMyMon->rds_monitor_handle_cluster_topology(wHG, rHG, mmsd->result); + } + if (mmsd->result) { mysql_free_result(mmsd->result); mmsd->result = NULL; } + } + +__end_of_loop: + mmsd->t2 = monotonic_time(); + next_loop_at = t1 + (check_interval_ms * 1000); + if (mmsd->t2 > t1) next_loop_at -= (mmsd->t2 - t1); + if (mmsd->mysql) { + if (mmsd->mysql_error_msg) { + GloMyMon->My_Conn_Pool->destroy_mysql_connection(mmsd); + } else if (crc) { + if (mmsd->set_wait_timeout()) GloMyMon->My_Conn_Pool->put_connection(mmsd->hostname, mmsd); + else GloMyMon->My_Conn_Pool->destroy_mysql_connection(mmsd); + } else { + GloMyMon->My_Conn_Pool->put_connection(mmsd->hostname, mmsd); + } + } + delete mmsd; + mmsd = NULL; + } + +__exit_monitor_RDS_thread_HG_now: + if (mmsd) { delete mmsd; mmsd = NULL; } + for (unsigned int i=0; icurtime = monotonic_time(); + MySQL_Monitor__thread_MySQL_Thread_Variables_version = GloMTH->get_global_version(); + mysql_thr->refresh_variables(); + + uint64_t last_raw_checksum = 0; + unsigned int *hgs_array = NULL; + pthread_t *pthreads_array = NULL; + unsigned int hgs_num = 0; + + while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true) { + unsigned int glover; + if (!GloMTH) return NULL; // quick exit during shutdown/restart + + glover = GloMTH->get_global_version(); + if (MySQL_Monitor__thread_MySQL_Thread_Variables_version < glover) { + MySQL_Monitor__thread_MySQL_Thread_Variables_version = glover; + mysql_thr->refresh_variables(); + } + + // respawn the per-writer-HG workers when the host list/definition changes + pthread_mutex_lock(&aws_rds_mutex); + uint64_t new_raw_checksum = AWS_RDS_Hosts_resultset->raw_checksum(); + pthread_mutex_unlock(&aws_rds_mutex); + if (new_raw_checksum != last_raw_checksum) { + proxy_info("Detected new/changed definition for AWS RDS monitoring\n"); + last_raw_checksum = new_raw_checksum; + if (pthreads_array) { + for (unsigned int i=0; i < hgs_num; i++) { + pthread_join(pthreads_array[i], NULL); + proxy_info("Stopped Monitor thread for AWS RDS writer HG %u\n", hgs_array[i]); + } + free(pthreads_array); + free(hgs_array); + pthreads_array = NULL; + hgs_array = NULL; + } + hgs_num = 0; + pthread_mutex_lock(&aws_rds_mutex); + unsigned int num_rows = AWS_RDS_Hosts_resultset->rows_count; + if (num_rows) { + unsigned int *tmp_hgs_array = (unsigned int *)malloc(sizeof(unsigned int)*num_rows); + for (std::vector::iterator it = AWS_RDS_Hosts_resultset->rows.begin() ; it != AWS_RDS_Hosts_resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + int wHG = atoi(r->fields[0]); + bool found = false; + for (unsigned int i=0; i < hgs_num; i++) { + if (tmp_hgs_array[i] == (unsigned int)wHG) found = true; + } + if (found == false) { + tmp_hgs_array[hgs_num] = wHG; + hgs_num++; + } + } + proxy_info("Activating Monitoring of %u AWS RDS clusters\n", hgs_num); + hgs_array = (unsigned int *)malloc(sizeof(unsigned int)*hgs_num); + pthreads_array = (pthread_t *)malloc(sizeof(pthread_t)*hgs_num); + for (unsigned int i=0; i < hgs_num; i++) { + hgs_array[i] = tmp_hgs_array[i]; + proxy_info("Starting Monitor thread for AWS RDS writer HG %u\n", hgs_array[i]); + if (pthread_create(&pthreads_array[i], NULL, monitor_RDS_thread_HG, &hgs_array[i]) != 0) { + // LCOV_EXCL_START + proxy_error("Thread creation\n"); + assert(0); + // LCOV_EXCL_STOP + } + } + free(tmp_hgs_array); + } + pthread_mutex_unlock(&aws_rds_mutex); + } + + usleep(10000); + } + // on shutdown, join any running per-HG workers + if (pthreads_array) { + for (unsigned int i=0; i < hgs_num; i++) { + pthread_join(pthreads_array[i], NULL); + } + free(pthreads_array); + free(hgs_array); + } + if (mysql_thr) { + delete mysql_thr; + mysql_thr = NULL; + } + return NULL; +} + unsigned int MySQL_Monitor::estimate_lag(char* server_id, AWS_Aurora_status_entry** aase, unsigned int idx, unsigned int add_lag_ms, unsigned int min_lag_ms, unsigned int lag_num_checks) { assert(aase); assert(server_id); From a5314739fb7163040c823b43c114cbcab537de12 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Wed, 17 Jun 2026 08:25:47 +0000 Subject: [PATCH 03/81] feat: Wire read_only monitor to AWS RDS blue/green monitoring - The read_only monitor flags a detected blue/green deployment and the RDS monitor picks it up to drive the switchover, gated by a new opt-in config `mysql-aws_blue_green_deployment_auto_discovery`. - Scope the dedicated RDS monitor to blue/green deployments only, renaming AWS_RDS -> AWS_RDS_BGD throughout. - Keep Multi-AZ Cluster auto-discovery unchanged in the read_only monitor. - Split the hostgroup schema to distinguish user-defined from auto-generated entries. Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 39 +- include/MySQL_Monitor.hpp | 91 +++- include/MySQL_Thread.h | 2 + include/ProxySQL_Admin_Tables_Definitions.h | 8 +- include/proxysql_structs.h | 2 + lib/Admin_Bootstrap.cpp | 6 +- lib/Admin_Handler.cpp | 10 +- lib/MySQL_HostGroups_Manager.cpp | 153 ++++-- lib/MySQL_Monitor.cpp | 558 ++++++++++++-------- lib/MySQL_Thread.cpp | 4 + lib/ProxySQL_Admin.cpp | 32 +- lib/ProxySQL_Config.cpp | 20 +- 12 files changed, 619 insertions(+), 306 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index dd3ef96570..0491c7ca36 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -64,7 +64,7 @@ "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , " \ "comment VARCHAR , UNIQUE (reader_hostgroup))" -#define MYHGM_MYSQL_AWS_RDS_HOSTGROUPS "CREATE TABLE mysql_aws_rds_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0), " \ +#define MYHGM_MYSQL_AWS_RDS_BGD_HOSTGROUPS "CREATE TABLE mysql_aws_rds_bgd_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0), " \ "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0), " \ "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0), " \ "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , " \ @@ -709,8 +709,17 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { pthread_mutex_t AWS_Aurora_Info_mutex; std::map AWS_Aurora_Info_Map; - void generate_mysql_aws_rds_hostgroups_table(); - SQLite3_result *incoming_aws_rds_hostgroups; + /** + * @brief Materializes the runtime `mysql_aws_rds_bgd_hostgroups` table from the staged + * `incoming_aws_rds_bgd_hostgroups` resultset. + * + * @details Inserts each staged row with `auto_generated=0` (config-loaded entries are + * user-defined) and NULL green hostgroups preserved, clears the staging resultset, then + * republishes the host list to the RDS BGD monitor thread via + * `update_aws_rds_bgd_hosts_monitor_resultset()`. No-op when nothing is staged. + */ + void generate_mysql_aws_rds_bgd_hostgroups_table(); + SQLite3_result *incoming_aws_rds_bgd_hostgroups; void generate_mysql_hostgroup_attributes_table(); SQLite3_result *incoming_hostgroup_attributes; @@ -1123,11 +1132,27 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { */ void update_aws_aurora_hosts_monitor_resultset(bool lock=false); /** - * @brief Rebuilds `GloMyMon->AWS_RDS_Hosts_resultset` (and its checksum) from the - * `mysql_servers` x `mysql_aws_rds_hostgroups` join used by the RDS monitor thread. - * @param lock when true, the monitor's `aws_rds_mutex` is taken internally. + * @brief Rebuilds the AWS RDS BGD monitor's host resultset. + * + * @details Rebuilds `GloMyMon->AWS_RDS_BGD_Hosts_resultset` (and its checksum) from the + * `mysql_servers` x `mysql_aws_rds_bgd_hostgroups` join used by the RDS BGD monitor thread. + * + * @param lock When true, the monitor's `aws_rds_bgd_mutex` is taken internally. + */ + void update_aws_rds_bgd_hosts_monitor_resultset(bool lock=false); + /** + * @brief Auto-generate a runtime `mysql_aws_rds_bgd_hostgroups` entry for a server's writer hostgroup. + * + * @details Called when the read_only monitor detects a blue/green deployment. The writer/reader + * hostgroups are derived from the server's `hostgroup_server_mapping`. Green hostgroups are + * stored NULL with `auto_generated=1`. Idempotent. + * + * @param hostname Hostname of the server that exposed the blue/green topology. + * @param port Port of the server. + * + * @return true if a new entry was added; false otherwise. */ - void update_aws_rds_hosts_monitor_resultset(bool lock=false); + bool add_aws_rds_bgd_hostgroup_entry(const std::string& hostname, int port); SQLite3_result * get_stats_mysql_gtid_executed(); void generate_mysql_gtid_executed_tables(); diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index b77e49b0f8..b358782693 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -60,7 +60,7 @@ struct cmp_str { #define N_L_ASE 16 #define AWS_ENDPOINT_SUFFIX_STRING "rds.amazonaws.com" -#define QUERY_READ_ONLY_AND_AWS_TOPOLOGY_DISCOVERY "SELECT @@global.read_only read_only, id, endpoint, port from mysql.rds_topology" +#define QUERY_AWS_RDS_TOPOLOGY_DISCOVERY "SELECT * FROM mysql.rds_topology" /* @@ -204,8 +204,8 @@ enum MySQL_Monitor_State_Data_Task_Type { MON_REPLICATION_LAG, MON_GALERA, MON_AWS_AURORA, - MON_AWS_RDS, - MON_READ_ONLY__AND__AWS_RDS_TOPOLOGY_DISCOVERY + MON_AWS_RDS_BGD, + MON_AWS_RDS_TOPOLOGY_DISCOVERY }; enum class MySQL_Monitor_State_Data_Task_Result { @@ -387,9 +387,26 @@ struct mon_metrics_map_idx { }; }; -// DNS_Cache, DNS_Cache_Record, DNS_Resolve_Data and the resolver helpers now -// live in DNS_Cache.hpp (included above) so the same machinery can back the -// independent PgSQL_Monitor DNS cache. +/** + * @brief A single node (row) of a 'SELECT * FROM mysql.rds_topology' result. + */ +struct AWS_RDS_Topology_Node { + std::string id; + std::string endpoint; + int port = 0; + std::string role; ///< empty when the column is absent or NULL + std::string status; ///< empty when the column is absent or NULL +}; + +/** + * @brief Parsed representation of a 'SELECT * FROM mysql.rds_topology' result, + * shared by the read_only monitor's discovery path and the AWS RDS BGD + * monitor thread. + */ +struct AWS_RDS_Topology_Result { + bool blue_green = false; ///< 'role' and 'status' present AND non-NULL + std::vector nodes; +}; class MySQL_Monitor { @@ -399,8 +416,33 @@ class MySQL_Monitor { static bool update_dns_cache_from_mysql_conn(const MYSQL* mysql); static void trigger_dns_cache_update(); - void process_discovered_topology(const std::string& originating_server_hostname, const vector& discovered_servers, int reader_hostgroup); - bool is_aws_rds_multi_az_db_cluster_topology(const std::vector& discovered_servers); + /** + * @brief Classify the parsed mysql.rds_topology result and dispatch. + * + * @details A blue/green deployment optionally auto-generates a runtime aws_rds_bgd_hostgroups + * entry (when 'mysql-aws_blue_green_deployment_auto_discovery' is enabled); otherwise the rows + * are treated as a Multi-AZ Cluster and handed to the existing auto-discovery path. + */ + void process_aws_rds_topology(MySQL_Monitor_State_Data* mmsd); + /** + * @brief Parse a 'SELECT * FROM mysql.rds_topology' result into an AWS_RDS_Topology_Result. + * + * @details Columns are resolved by name (they may be absent or differently ordered by RDS type). + * 'blue_green' is set when the 'role'/'status' columns are present and non-NULL on the first row. + * + * @return The parsed topology; empty 'nodes' if 'result' is NULL or has no rows. The result cursor is rewound before returning. + */ + AWS_RDS_Topology_Result parse_aws_rds_topology(MYSQL_RES* result); + /** + * @brief Processes the discovered servers to eventually add them to 'runtime_mysql_servers'. + * + * @details This method takes a vector of discovered servers, compares them against the existing servers, and adds the new servers to 'runtime_mysql_servers'. + * + * @param origin_server A string which denotes the hostname of the originating server, from which the discovered servers were queried and found. + * @param discovered_servers A vector of servers discovered when querying the cluster's topology. + * @param reader_hostgroup Reader hostgroup to which we will add the discovered servers. + */ + void handle_aws_rds_multi_az_cluster(const std::string& origin_server, const std::vector& discovered_servers, int reader_hostgroup); private: std::vector *tables_defs_monitor; @@ -414,7 +456,7 @@ class MySQL_Monitor { pthread_mutex_t group_replication_mutex; // for simplicity, a mutex instead of a rwlock pthread_mutex_t galera_mutex; // for simplicity, a mutex instead of a rwlock pthread_mutex_t aws_aurora_mutex; // for simplicity, a mutex instead of a rwlock - pthread_mutex_t aws_rds_mutex; // for simplicity, a mutex instead of a rwlock + pthread_mutex_t aws_rds_bgd_mutex; pthread_mutex_t mysql_servers_mutex; // for simplicity, a mutex instead of a rwlock pthread_mutex_t proxysql_servers_mutex; //std::map Group_Replication_Hosts_Map; @@ -425,9 +467,8 @@ class MySQL_Monitor { std::map AWS_Aurora_Hosts_Map; SQLite3_result *AWS_Aurora_Hosts_resultset; uint64_t AWS_Aurora_Hosts_resultset_checksum; - // host list consumed by the AWS RDS monitor thread (join of mysql_servers x mysql_aws_rds_hostgroups) - SQLite3_result *AWS_RDS_Hosts_resultset; - uint64_t AWS_RDS_Hosts_resultset_checksum; + SQLite3_result *AWS_RDS_BGD_Hosts_resultset; + uint64_t AWS_RDS_BGD_Hosts_resultset_checksum; unsigned int num_threads; unsigned int aux_threads; unsigned int started_threads; @@ -475,11 +516,27 @@ class MySQL_Monitor { void * monitor_group_replication_2(); void * monitor_galera(); void * monitor_aws_aurora(); - void * monitor_aws_rds(); - // Invoked once the topology shape is detected on mysql.rds_topology. - // 'result' holds the fetched rows. - void rds_monitor_handle_instance_topology(unsigned int writer_hostgroup, unsigned int reader_hostgroup, int green_writer_hostgroup, int green_reader_hostgroup, MYSQL_RES* result); - void rds_monitor_handle_cluster_topology(unsigned int writer_hostgroup, unsigned int reader_hostgroup, MYSQL_RES* result); + /** + * @brief AWS RDS BGD monitor thread entry point. + * + * @details Spawns one worker (monitor_RDS_BGD_thread_HG) per writer hostgroup; each worker picks a pingable host, + * probes 'mysql.rds_topology' and dispatches to a handler based on the detected topology shape. + * Workers are (re)spawned whenever the AWS_RDS_BGD_Hosts_resultset checksum changes. + */ + void * monitor_aws_rds_bgd(); + /** + * @brief Handle a blue/green deployment topology fetched by the BGD thread. + * + * @details Invoked when the BGD thread fetches a blue/green deployment topology from + * mysql.rds_topology; performs the blue/green switchover. + * + * @param whg Writer hostgroup (blue/current writer). + * @param rhg Reader hostgroup (blue/current readers). + * @param green_whg Configured green writer hostgroup, or -1 if unset. + * @param green_rhg Configured green reader hostgroup, or -1 if unset. + * @param topology Parsed mysql.rds_topology result. + */ + void handle_aws_rds_bgd(unsigned int whg, unsigned int rhg, int green_whg, int green_rhg, const AWS_RDS_Topology_Result& topology); void * monitor_replication_lag(); void * monitor_dns_cache(); void * run(); diff --git a/include/MySQL_Thread.h b/include/MySQL_Thread.h index 2a54cb61df..0788b76209 100644 --- a/include/MySQL_Thread.h +++ b/include/MySQL_Thread.h @@ -453,6 +453,8 @@ class MySQL_Threads_Handler int monitor_ping_timeout; //! Monitor aws rds topology discovery interval. Unit: 'one discovery check per X monitor_read_only checks'. int monitor_aws_rds_topology_discovery_interval; + //! Auto-generate runtime aws_rds_bgd_hostgroups entries when the read_only monitor detects a blue/green deployment. + bool aws_blue_green_deployment_auto_discovery; //! Monitor read only timeout. Unit: 'ms'. int monitor_read_only_interval; //! Monitor read only timeout. Unit: 'ms'. diff --git a/include/ProxySQL_Admin_Tables_Definitions.h b/include/ProxySQL_Admin_Tables_Definitions.h index a50e1967f8..725514277d 100644 --- a/include/ProxySQL_Admin_Tables_Definitions.h +++ b/include/ProxySQL_Admin_Tables_Definitions.h @@ -237,9 +237,9 @@ // AWS RDS hostgroups; adds blue/green (green_*_hostgroup) over aurora. // The runtime table carries one extra runtime-only column: auto_generated. -#define ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_HOSTGROUPS "CREATE TABLE mysql_aws_rds_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , " \ - "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0) , " \ - "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0) , " \ +#define ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS "CREATE TABLE mysql_aws_rds_bgd_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , " \ + "green_writer_hostgroup INT NOT NULL CHECK (green_writer_hostgroup>=0) , " \ + "green_reader_hostgroup INT NOT NULL CHECK (green_reader_hostgroup>=0) , " \ "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , " \ "domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , " \ "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , " \ @@ -247,7 +247,7 @@ "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , " \ "comment VARCHAR NOT NULL DEFAULT '' , UNIQUE (reader_hostgroup))" -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_RDS_HOSTGROUPS "CREATE TABLE runtime_mysql_aws_rds_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , " \ +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_RDS_BGD_HOSTGROUPS "CREATE TABLE runtime_mysql_aws_rds_bgd_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , " \ "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0) , " \ "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0) , " \ "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , " \ diff --git a/include/proxysql_structs.h b/include/proxysql_structs.h index b381fb0468..0347f036cd 100644 --- a/include/proxysql_structs.h +++ b/include/proxysql_structs.h @@ -1388,6 +1388,7 @@ __thread int mysql_thread___monitor_ping_interval; __thread int mysql_thread___monitor_ping_max_failures; __thread int mysql_thread___monitor_ping_timeout; __thread int mysql_thread___monitor_aws_rds_topology_discovery_interval; +__thread int mysql_thread___aws_blue_green_deployment_auto_discovery; __thread int mysql_thread___monitor_read_only_interval; __thread int mysql_thread___monitor_read_only_timeout; __thread int mysql_thread___monitor_read_only_max_timeout_count; @@ -1729,6 +1730,7 @@ extern __thread int mysql_thread___monitor_ping_interval; extern __thread int mysql_thread___monitor_ping_max_failures; extern __thread int mysql_thread___monitor_ping_timeout; extern __thread int mysql_thread___monitor_aws_rds_topology_discovery_interval; +extern __thread int mysql_thread___aws_blue_green_deployment_auto_discovery; extern __thread int mysql_thread___monitor_read_only_interval; extern __thread int mysql_thread___monitor_read_only_timeout; extern __thread int mysql_thread___monitor_read_only_max_timeout_count; diff --git a/lib/Admin_Bootstrap.cpp b/lib/Admin_Bootstrap.cpp index 2e5f0ce38f..4573fd98db 100644 --- a/lib/Admin_Bootstrap.cpp +++ b/lib/Admin_Bootstrap.cpp @@ -753,8 +753,8 @@ bool ProxySQL_Admin::init(const bootstrap_info_t& bootstrap_info) { insert_into_tables_defs(tables_defs_admin,"runtime_mysql_galera_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GALERA_HOSTGROUPS); insert_into_tables_defs(tables_defs_admin,"mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); insert_into_tables_defs(tables_defs_admin,"runtime_mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS); - insert_into_tables_defs(tables_defs_admin,"mysql_aws_rds_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_HOSTGROUPS); - insert_into_tables_defs(tables_defs_admin,"runtime_mysql_aws_rds_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_RDS_HOSTGROUPS); + insert_into_tables_defs(tables_defs_admin,"mysql_aws_rds_bgd_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS); + insert_into_tables_defs(tables_defs_admin,"runtime_mysql_aws_rds_bgd_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_RDS_BGD_HOSTGROUPS); insert_into_tables_defs(tables_defs_admin,"mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); insert_into_tables_defs(tables_defs_admin,"runtime_mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_HOSTGROUP_ATTRIBUTES); insert_into_tables_defs(tables_defs_admin,"mysql_servers_ssl_params", ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS); @@ -839,7 +839,7 @@ bool ProxySQL_Admin::init(const bootstrap_info_t& bootstrap_info) { insert_into_tables_defs(tables_defs_config,"mysql_group_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS); insert_into_tables_defs(tables_defs_config,"mysql_galera_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS); insert_into_tables_defs(tables_defs_config,"mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); - insert_into_tables_defs(tables_defs_config,"mysql_aws_rds_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_HOSTGROUPS); + insert_into_tables_defs(tables_defs_config,"mysql_aws_rds_bgd_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS); insert_into_tables_defs(tables_defs_config,"mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); insert_into_tables_defs(tables_defs_config,"mysql_servers_ssl_params", ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS); insert_into_tables_defs(tables_defs_config,"mysql_query_rules", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES); diff --git a/lib/Admin_Handler.cpp b/lib/Admin_Handler.cpp index 275a8174ca..58bb734ee4 100644 --- a/lib/Admin_Handler.cpp +++ b/lib/Admin_Handler.cpp @@ -4273,13 +4273,13 @@ void admin_session_handler(S* sess, void *_pa, PtrSize_t *pkt) { tablename=(char *)"MYSQL AURORA HOSTGROUPS"; SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); } - if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL RDS HOSTGROUPS") && !strncasecmp("CHECKSUM MEMORY MYSQL RDS HOSTGROUPS", query_no_space, strlen(query_no_space))) + if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL RDS BGD HOSTGROUPS") && !strncasecmp("CHECKSUM MEMORY MYSQL RDS BGD HOSTGROUPS", query_no_space, strlen(query_no_space))) || - (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL RDS HOSTGROUPS") && !strncasecmp("CHECKSUM MEM MYSQL RDS HOSTGROUPS", query_no_space, strlen(query_no_space))) + (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL RDS BGD HOSTGROUPS") && !strncasecmp("CHECKSUM MEM MYSQL RDS BGD HOSTGROUPS", query_no_space, strlen(query_no_space))) || - (strlen(query_no_space)==strlen("CHECKSUM MYSQL RDS HOSTGROUPS") && !strncasecmp("CHECKSUM MYSQL RDS HOSTGROUPS", query_no_space, strlen(query_no_space)))){ - char *q=(char *)"SELECT * FROM mysql_aws_rds_hostgroups ORDER BY writer_hostgroup"; - tablename=(char *)"MYSQL RDS HOSTGROUPS"; + (strlen(query_no_space)==strlen("CHECKSUM MYSQL RDS BGD HOSTGROUPS") && !strncasecmp("CHECKSUM MYSQL RDS BGD HOSTGROUPS", query_no_space, strlen(query_no_space)))){ + char *q=(char *)"SELECT * FROM mysql_aws_rds_bgd_hostgroups ORDER BY writer_hostgroup"; + tablename=(char *)"MYSQL RDS BGD HOSTGROUPS"; SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); } if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL HOSTGROUP ATTRIBUTES") && !strncasecmp("CHECKSUM MEMORY MYSQL HOSTGROUP ATTRIBUTES", query_no_space, strlen(query_no_space))) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index a5e27d2b67..f60e08e320 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -727,7 +727,7 @@ MySQL_HostGroups_Manager::MySQL_HostGroups_Manager() { mydb->execute(MYHGM_MYSQL_GROUP_REPLICATION_HOSTGROUPS); mydb->execute(MYHGM_MYSQL_GALERA_HOSTGROUPS); mydb->execute(MYHGM_MYSQL_AWS_AURORA_HOSTGROUPS); - mydb->execute(MYHGM_MYSQL_AWS_RDS_HOSTGROUPS); + mydb->execute(MYHGM_MYSQL_AWS_RDS_BGD_HOSTGROUPS); mydb->execute(MYHGM_MYSQL_HOSTGROUP_ATTRIBUTES); mydb->execute(MYHGM_MYSQL_SERVERS_SSL_PARAMS); mydb->execute("CREATE INDEX IF NOT EXISTS idx_mysql_servers_hostname_port ON mysql_servers (hostname,port)"); @@ -737,7 +737,7 @@ MySQL_HostGroups_Manager::MySQL_HostGroups_Manager() { incoming_group_replication_hostgroups=NULL; incoming_galera_hostgroups=NULL; incoming_aws_aurora_hostgroups = NULL; - incoming_aws_rds_hostgroups = NULL; + incoming_aws_rds_bgd_hostgroups = NULL; incoming_hostgroup_attributes = NULL; incoming_mysql_servers_ssl_params = NULL; incoming_mysql_servers_v2 = NULL; @@ -1549,10 +1549,10 @@ bool MySQL_HostGroups_Manager::commit( } // AWS RDS - if (incoming_aws_rds_hostgroups) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_aws_rds_hostgroups\n"); - mydb->execute("DELETE FROM mysql_aws_rds_hostgroups"); - generate_mysql_aws_rds_hostgroups_table(); + if (incoming_aws_rds_bgd_hostgroups) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_aws_rds_bgd_hostgroups\n"); + mydb->execute("DELETE FROM mysql_aws_rds_bgd_hostgroups"); + generate_mysql_aws_rds_bgd_hostgroups_table(); } // hostgroup attributes @@ -2256,9 +2256,9 @@ SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql(const string& name) if (name == "mysql_aws_aurora_hostgroups") { query=(char *)"SELECT writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms," "check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment FROM mysql_aws_aurora_hostgroups"; - } else if (name == "mysql_aws_rds_hostgroups") { + } else if (name == "mysql_aws_rds_bgd_hostgroups") { query=(char *)"SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader," - "domain_name,check_interval_ms,check_timeout_ms,autopurge_missing_checks,comment,auto_generated FROM mysql_aws_rds_hostgroups"; + "domain_name,check_interval_ms,check_timeout_ms,autopurge_missing_checks,comment,auto_generated FROM mysql_aws_rds_bgd_hostgroups"; } else if (name == "mysql_galera_hostgroups") { query=(char *)"SELECT writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers,writer_is_also_reader,max_transactions_behind,comment FROM mysql_galera_hostgroups"; } else if (name == "mysql_group_replication_hostgroups") { @@ -3089,8 +3089,8 @@ void MySQL_HostGroups_Manager::save_incoming_mysql_table(SQLite3_result *s, cons SQLite3_result ** inc = NULL; if (name == "mysql_aws_aurora_hostgroups") { inc = &incoming_aws_aurora_hostgroups; - } else if (name == "mysql_aws_rds_hostgroups") { - inc = &incoming_aws_rds_hostgroups; + } else if (name == "mysql_aws_rds_bgd_hostgroups") { + inc = &incoming_aws_rds_bgd_hostgroups; } else if (name == "mysql_galera_hostgroups") { inc = &incoming_galera_hostgroups; } else if (name == "mysql_group_replication_hostgroups") { @@ -6280,25 +6280,28 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { } /** - * @brief Regenerates the runtime in-memory `mysql_aws_rds_hostgroups` table from `incoming_aws_rds_hostgroups`. + * @brief Regenerates the runtime in-memory `mysql_aws_rds_bgd_hostgroups` table from `incoming_aws_rds_bgd_hostgroups`. * - * The incoming resultset comes from the admin config table (11 columns, no `auto_generated`); config-loaded + * @details The incoming resultset comes from the admin config table (11 columns, no `auto_generated`); config-loaded * entries are user-defined, so `auto_generated` is stored as 0. `green_writer_hostgroup` and * `green_reader_hostgroup` are optional and bound as SQL NULL when absent. */ -void MySQL_HostGroups_Manager::generate_mysql_aws_rds_hostgroups_table() { - if (incoming_aws_rds_hostgroups==NULL) { +void MySQL_HostGroups_Manager::generate_mysql_aws_rds_bgd_hostgroups_table() { + if (incoming_aws_rds_bgd_hostgroups==NULL) { return; } + int rc; - char *query=(char *)"INSERT INTO mysql_aws_rds_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + char *query=(char *)"INSERT INTO mysql_aws_rds_bgd_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," "writer_is_also_reader,domain_name,check_interval_ms,check_timeout_ms,autopurge_missing_checks,comment,auto_generated) VALUES " "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)"; + auto [rc1, statement_unique] = mydb->prepare_v2(query); ASSERT_SQLITE_OK(rc1, mydb); sqlite3_stmt *statement = statement_unique.get(); - proxy_info("New mysql_aws_rds_hostgroups table\n"); - for (std::vector::iterator it = incoming_aws_rds_hostgroups->rows.begin() ; it != incoming_aws_rds_hostgroups->rows.end(); ++it) { + proxy_info("New mysql_aws_rds_bgd_hostgroups table\n"); + + for (std::vector::iterator it = incoming_aws_rds_bgd_hostgroups->rows.begin() ; it != incoming_aws_rds_bgd_hostgroups->rows.end(); ++it) { SQLite3_row *r=*it; int writer_hostgroup=atoi(r->fields[0]); int reader_hostgroup=atoi(r->fields[1]); @@ -6343,14 +6346,15 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_rds_hostgroups_table() { rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, mydb); rc=(*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, mydb); } - delete incoming_aws_rds_hostgroups; - incoming_aws_rds_hostgroups=NULL; + + delete incoming_aws_rds_bgd_hostgroups; + incoming_aws_rds_bgd_hostgroups=NULL; // publish the refreshed host list to the RDS monitor thread if (GloMyMon) { - pthread_mutex_lock(&GloMyMon->aws_rds_mutex); - update_aws_rds_hosts_monitor_resultset(false); - pthread_mutex_unlock(&GloMyMon->aws_rds_mutex); + pthread_mutex_lock(&GloMyMon->aws_rds_bgd_mutex); + update_aws_rds_bgd_hosts_monitor_resultset(false); + pthread_mutex_unlock(&GloMyMon->aws_rds_bgd_mutex); } } @@ -6930,17 +6934,25 @@ void MySQL_HostGroups_Manager::update_aws_aurora_hosts_monitor_resultset(bool lo } } -const char SELECT_AWS_RDS_SERVERS_FOR_MONITOR[] { +const char SELECT_AWS_RDS_BGD_SERVERS_FOR_MONITOR[] { "SELECT writer_hostgroup, reader_hostgroup, hostname, port, MAX(use_ssl) use_ssl, green_writer_hostgroup," " green_reader_hostgroup, check_interval_ms, check_timeout_ms, autopurge_missing_checks, domain_name FROM mysql_servers" - " JOIN mysql_aws_rds_hostgroups ON" + " JOIN mysql_aws_rds_bgd_hostgroups ON" " hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE active=1 AND status NOT IN (2,3)" " GROUP BY writer_hostgroup, hostname, port" }; -void MySQL_HostGroups_Manager::update_aws_rds_hosts_monitor_resultset(bool lock) { +/** + * @brief Rebuilds the AWS RDS BGD monitor's host resultset. + * + * @details Rebuilds `GloMyMon->AWS_RDS_BGD_Hosts_resultset` (and its checksum) from the + * `mysql_servers` x `mysql_aws_rds_bgd_hostgroups` join used by the RDS BGD monitor thread. + * + * @param lock When true, the monitor's `aws_rds_bgd_mutex` is taken internally. + */ +void MySQL_HostGroups_Manager::update_aws_rds_bgd_hosts_monitor_resultset(bool lock) { if (lock) { - pthread_mutex_lock(&GloMyMon->aws_rds_mutex); + pthread_mutex_lock(&GloMyMon->aws_rds_bgd_mutex); } SQLite3_result* resultset = nullptr; @@ -6948,22 +6960,95 @@ void MySQL_HostGroups_Manager::update_aws_rds_hosts_monitor_resultset(bool lock) char* error = nullptr; int cols = 0; int affected_rows = 0; - mydb->execute_statement(SELECT_AWS_RDS_SERVERS_FOR_MONITOR, &error, &cols, &affected_rows, &resultset); + mydb->execute_statement(SELECT_AWS_RDS_BGD_SERVERS_FOR_MONITOR, &error, &cols, &affected_rows, &resultset); } if (resultset) { - if (GloMyMon->AWS_RDS_Hosts_resultset) { - delete GloMyMon->AWS_RDS_Hosts_resultset; + if (GloMyMon->AWS_RDS_BGD_Hosts_resultset) { + delete GloMyMon->AWS_RDS_BGD_Hosts_resultset; } - GloMyMon->AWS_RDS_Hosts_resultset=resultset; - GloMyMon->AWS_RDS_Hosts_resultset_checksum=resultset->raw_checksum(); + GloMyMon->AWS_RDS_BGD_Hosts_resultset=resultset; + GloMyMon->AWS_RDS_BGD_Hosts_resultset_checksum=resultset->raw_checksum(); } if (lock) { - pthread_mutex_unlock(&GloMyMon->aws_rds_mutex); + pthread_mutex_unlock(&GloMyMon->aws_rds_bgd_mutex); } } +/** + * @brief Auto-generate a runtime `mysql_aws_rds_bgd_hostgroups` entry for a server's writer hostgroup. + * + * @details Called when the read_only monitor detects a blue/green deployment. The writer/reader + * hostgroups are derived from the server's `hostgroup_server_mapping`. Green hostgroups are + * stored NULL with `auto_generated=1`. Idempotent. + * + * @param hostname Hostname of the server that exposed the blue/green topology. + * @param port Port of the server. + * + * @return true if a new entry was added; false otherwise. + */ +bool MySQL_HostGroups_Manager::add_aws_rds_bgd_hostgroup_entry(const std::string& hostname, int port) { + bool added = false; + const std::string srv_id = hostname + ":::" + std::to_string(port); + + wrlock(); + + auto itr = hostgroup_server_mapping.find(srv_id); + if (itr != hostgroup_server_mapping.end() && itr->second) { + int writer_hg = -1, reader_hg = -1; + const auto& wmap = itr->second->get(HostGroup_Server_Mapping::Type::WRITER); + const auto& rmap = itr->second->get(HostGroup_Server_Mapping::Type::READER); + if (!wmap.empty()) { + writer_hg = (int)wmap[0].writer_hostgroup_id; + reader_hg = (int)wmap[0].reader_hostgroup_id; + } else if (!rmap.empty()) { + writer_hg = (int)rmap[0].writer_hostgroup_id; + reader_hg = (int)rmap[0].reader_hostgroup_id; + } + if (writer_hg >= 0 && reader_hg >= 0 && writer_hg != reader_hg) { + // only add when no runtime entry exists yet for this writer hostgroup + bool exists = false; + char* error = nullptr; + int cols = 0; + int affected_rows = 0; + SQLite3_result* res = nullptr; + + std::string sel = "SELECT 1 FROM mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + std::to_string(writer_hg); + mydb->execute_statement(sel.c_str(), &error, &cols, &affected_rows, &res); + if (res) { + exists = (res->rows_count > 0); + delete res; + } + + if (!exists) { + std::string ins = + "INSERT INTO mysql_aws_rds_bgd_hostgroups (" + "writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, " + "active, writer_is_also_reader, domain_name, check_interval_ms, check_timeout_ms, " + "autopurge_missing_checks, comment, auto_generated" + ") VALUES (" + + std::to_string(writer_hg) + ", " + std::to_string(reader_hg) + + ", NULL, NULL, 1, 0, '', 1000, 800, 0, '', 1)"; + mydb->execute(ins.c_str()); + added = true; + proxy_info( + "AWS RDS: auto-generated blue/green hostgroup entry (writer HG %d, reader HG %d) from server %s:%d\n", + writer_hg, reader_hg, hostname.c_str(), port + ); + } + } + } + + if (added) { + // publish the refreshed host list to the BGD monitor thread + update_aws_rds_bgd_hosts_monitor_resultset(true); + } + + wrunlock(); + return added; +} + MySrvC* MySQL_HostGroups_Manager::find_server_in_hg(unsigned int _hid, const std::string& addr, int port) { MySrvC* f_server = nullptr; @@ -7159,9 +7244,11 @@ MySQLServers_SslParams * MySQL_HostGroups_Manager::get_Server_SSL_Params(char *h /** * @brief Updates replication hostgroups by adding autodiscovered mysql servers. +* * @details Adds each server from 'new_servers' to the 'runtime_mysql_servers' table. * We then rebuild the 'mysql_servers' table as well as the internal 'hostname_hostgroup_mapping'. -* @param new_servers A vector of tuples where each tuple contains the values needed to add each new server. +* +* @param new_servers A vector of tuples where each tuple contains the values needed to add each new server. */ void MySQL_HostGroups_Manager::add_discovered_servers_to_mysql_servers_and_replication_hostgroups( const vector>& new_servers diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 8137b526c6..4f94d44002 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -671,8 +671,8 @@ void MySQL_Monitor_State_Data::init_async() { task_timeout_ = mysql_thread___monitor_read_only_timeout; task_handler_ = &MySQL_Monitor_State_Data::read_only_handler; break; - case MON_READ_ONLY__AND__AWS_RDS_TOPOLOGY_DISCOVERY: - query_ = QUERY_READ_ONLY_AND_AWS_TOPOLOGY_DISCOVERY; + case MON_AWS_RDS_TOPOLOGY_DISCOVERY: + query_ = QUERY_AWS_RDS_TOPOLOGY_DISCOVERY; async_state_machine_ = ASYNC_QUERY_START; task_timeout_ = mysql_thread___monitor_read_only_timeout; task_handler_ = &MySQL_Monitor_State_Data::read_only_handler; @@ -788,7 +788,7 @@ void MySQL_Monitor_State_Data::init_async() { break; case MON_AWS_AURORA: break; - case MON_AWS_RDS: + case MON_AWS_RDS_BGD: break; } } @@ -883,11 +883,14 @@ void * monitor_aws_aurora_pthread(void *arg) { return NULL; } -void * monitor_aws_rds_pthread(void *arg) { - set_thread_name("MonitorRDS", GloVars.set_thread_name); +void * monitor_aws_rds_bgd_pthread(void *arg) { + set_thread_name("MonitorRdsBgd", GloVars.set_thread_name); + // Wait for GloMTH to be initialized - if (!wait_for_glo_mth()) return NULL; // quick exit during shutdown/restart - GloMyMon->monitor_aws_rds(); + if (!wait_for_glo_mth()) + return NULL; + + GloMyMon->monitor_aws_rds_bgd(); return NULL; } @@ -1087,13 +1090,13 @@ MySQL_Monitor::MySQL_Monitor() { Galera_Hosts_resultset=NULL; pthread_mutex_init(&aws_aurora_mutex,NULL); - pthread_mutex_init(&aws_rds_mutex,NULL); + pthread_mutex_init(&aws_rds_bgd_mutex,NULL); pthread_mutex_init(&mysql_servers_mutex,NULL); pthread_mutex_init(&proxysql_servers_mutex, NULL); AWS_Aurora_Hosts_resultset=NULL; AWS_Aurora_Hosts_resultset_checksum = 0; - AWS_RDS_Hosts_resultset=NULL; - AWS_RDS_Hosts_resultset_checksum = 0; + AWS_RDS_BGD_Hosts_resultset=NULL; + AWS_RDS_BGD_Hosts_resultset_checksum = 0; shutdown=false; monitor_enabled=true; // default // create new SQLite datatabase @@ -1194,9 +1197,9 @@ MySQL_Monitor::~MySQL_Monitor() { delete AWS_Aurora_Hosts_resultset; AWS_Aurora_Hosts_resultset=NULL; } - if (AWS_RDS_Hosts_resultset) { - delete AWS_RDS_Hosts_resultset; - AWS_RDS_Hosts_resultset=NULL; + if (AWS_RDS_BGD_Hosts_resultset) { + delete AWS_RDS_BGD_Hosts_resultset; + AWS_RDS_BGD_Hosts_resultset=NULL; } std::map::iterator it2; AWS_Aurora_monitor_node *node=NULL; @@ -1712,8 +1715,8 @@ void * monitor_read_only_thread(const std::vector& da mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT @@global.read_only&@@global.innodb_read_only read_only"); } else if (mmsd->get_task_type() == MON_READ_ONLY__OR__INNODB_READ_ONLY) { mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT @@global.read_only|@@global.innodb_read_only read_only"); - } else if (mmsd->get_task_type() == MON_READ_ONLY__AND__AWS_RDS_TOPOLOGY_DISCOVERY) { - mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql, QUERY_READ_ONLY_AND_AWS_TOPOLOGY_DISCOVERY); + } else if (mmsd->get_task_type() == MON_AWS_RDS_TOPOLOGY_DISCOVERY) { + mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql, QUERY_AWS_RDS_TOPOLOGY_DISCOVERY); } else { // default mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT @@global.read_only read_only"); } @@ -3399,13 +3402,13 @@ VALGRIND_ENABLE_ERROR_REPORTING; } /** -* @brief Processes the discovered servers to eventually add them to 'runtime_mysql_servers'. -* @details This method takes a vector of discovered servers, compares them against the existing servers, and adds the new servers to 'runtime_mysql_servers'. -* @param originating_server_hostname A string which denotes the hostname of the originating server, from which the discovered servers were queried and found. +* @brief Add discovered servers to 'runtime_mysql_servers' and reader hostgroup. +* +* @param origin_server A string which denotes the hostname of the originating server, from which the discovered servers were queried and found. * @param discovered_servers A vector of servers discovered when querying the cluster's topology. -* @param reader_hostgroup Reader hostgroup to which we will add the discovered servers. +* @param reader_hostgroup Reader hostgroup to which we will add the discovered servers. */ -void MySQL_Monitor::process_discovered_topology(const std::string& originating_server_hostname, const vector& discovered_servers, int reader_hostgroup) { +void MySQL_Monitor::handle_aws_rds_multi_az_cluster(const std::string& origin_server, const std::vector& discovered_servers, int reader_hostgroup) { char *error = NULL; int cols = 0; int affected_rows = 0; @@ -3420,7 +3423,7 @@ void MySQL_Monitor::process_discovered_topology(const std::string& originating_s } else { vector> new_servers; vector saved_hostnames; - saved_hostnames.push_back(originating_server_hostname); + saved_hostnames.push_back(origin_server); // Do an initial loop through the query results to save existing runtime server hostnames for (std::vector::iterator it = runtime_mysql_servers->rows.begin(); it != runtime_mysql_servers->rows.end(); it++) { @@ -3431,20 +3434,12 @@ void MySQL_Monitor::process_discovered_topology(const std::string& originating_s } // Loop through discovered servers and process the ones we haven't saved yet - for (MYSQL_ROW s : discovered_servers) { - string current_discovered_hostname = s[2]; - string current_discovered_port_string = s[3]; - int current_discovered_port_int; - - try { - current_discovered_port_int = stoi(s[3]); - } catch (...) { - proxy_error( - "Unable to parse port value coming from '%s' during topology discovery ('%s':%s). Terminating discovery early.\n", - originating_server_hostname.c_str(), current_discovered_hostname.c_str(), current_discovered_port_string.c_str() - ); - return; + for (const AWS_RDS_Topology_Node& s : discovered_servers) { + if (s.endpoint.empty()) { + continue; } + const string& current_discovered_hostname = s.endpoint; + int current_discovered_port_int = s.port; if (find(saved_hostnames.begin(), saved_hostnames.end(), current_discovered_hostname) == saved_hostnames.end()) { tuple new_server(current_discovered_hostname, current_discovered_port_int, reader_hostgroup); @@ -3461,32 +3456,98 @@ void MySQL_Monitor::process_discovered_topology(const std::string& originating_s } /** -* @brief Check if a list of servers is matching the description of an AWS RDS Multi-AZ DB Cluster. -* @details This method takes a vector of discovered servers and checks that there are exactly three which are named "instance-[1|2|3]" respectively, as expected on an AWS RDS Multi-AZ DB Cluster. -* @param discovered_servers A vector of servers discovered when querying the cluster's topology. -* @return Returns 'true' if all conditions are met and 'false' otherwise. +* @brief Parse a 'SELECT * FROM mysql.rds_topology' result into an AWS_RDS_Topology_Result. +* +* @details Columns are resolved by name (they may be absent or differently ordered by RDS type). +* 'blue_green' is set when the 'role'/'status' columns are present and non-NULL on the first row. +* +* @return The parsed topology; empty 'nodes' if 'result' is NULL or has no rows. The result cursor is rewound before returning. */ -bool MySQL_Monitor::is_aws_rds_multi_az_db_cluster_topology(const std::vector& discovered_servers) { - if (discovered_servers.size() != 3) { - return false; +AWS_RDS_Topology_Result MySQL_Monitor::parse_aws_rds_topology(MYSQL_RES* result) { + AWS_RDS_Topology_Result out; + if (result == NULL) { + return out; } - const std::vector instance_names = {"-instance-1", "-instance-2", "-instance-3"}; - int identified_hosts = 0; - for (const std::string& instance_str : instance_names) { - for (MYSQL_ROW server : discovered_servers) { - if (server[2] == NULL || (server[2][0] == '\0')) { - continue; - } + unsigned int num_fields = mysql_num_fields(result); + MYSQL_FIELD *fields = mysql_fetch_fields(result); + int id_idx = -1, endpoint_idx = -1, port_idx = -1, role_idx = -1, status_idx = -1; + for (unsigned int i = 0; i < num_fields; i++) { + if (fields[i].name == NULL) { + continue; + } + if (strcasecmp(fields[i].name, "id") == 0) { + id_idx = (int)i; + } else if (strcasecmp(fields[i].name, "endpoint") == 0) { + endpoint_idx = (int)i; + } else if (strcasecmp(fields[i].name, "port") == 0) { + port_idx = (int)i; + } else if (strcasecmp(fields[i].name, "role") == 0) { + role_idx = (int)i; + } else if (strcasecmp(fields[i].name, "status") == 0) { + status_idx = (int)i; + } + } - std::string current_discovered_hostname = server[2]; - if (current_discovered_hostname.find(instance_str) != std::string::npos) { - ++identified_hosts; - break; + bool first = true; + MYSQL_ROW row; + while ((row = mysql_fetch_row(result))) { + AWS_RDS_Topology_Node node; + if (id_idx >= 0 && row[id_idx]) { + node.id = row[id_idx]; + } + if (endpoint_idx >= 0 && row[endpoint_idx]) { + node.endpoint = row[endpoint_idx]; + } + if (port_idx >= 0 && row[port_idx]) { + try { + node.port = std::stoi(row[port_idx]); + } catch (...) { + node.port = 0; } } + if (role_idx >= 0 && row[role_idx]) { + node.role = row[role_idx]; + } + if (status_idx >= 0 && row[status_idx]) { + node.status = row[status_idx]; + } + if (first) { + // blue/green deployment exposes non-NULL role/status; absent or NULL => Multi-AZ Cluster + out.blue_green = (role_idx >= 0 && status_idx >= 0 + && row[role_idx] != NULL && row[status_idx] != NULL); + first = false; + } + out.nodes.push_back(std::move(node)); + } + mysql_data_seek(result, 0); // rewind for any subsequent reader + return out; +} + +/** +* @brief Classify the parsed mysql.rds_topology result and dispatch. +* +* @details A blue/green deployment optionally auto-generates a runtime aws_rds_bgd_hostgroups +* entry (when 'mysql-aws_blue_green_deployment_auto_discovery' is enabled); otherwise the rows +* are treated as a Multi-AZ Cluster and handed to the existing auto-discovery path. +*/ +void MySQL_Monitor::process_aws_rds_topology(MySQL_Monitor_State_Data* mmsd) { + if (mmsd->result == NULL) { + return; + } + + AWS_RDS_Topology_Result topology = parse_aws_rds_topology(mmsd->result); + if (topology.nodes.empty()) { + return; + } + + if (topology.blue_green) { + if (mysql_thread___aws_blue_green_deployment_auto_discovery) { + MyHGM->add_aws_rds_bgd_hostgroup_entry(mmsd->hostname, mmsd->port); + } + } else { + handle_aws_rds_multi_az_cluster(mmsd->hostname, topology.nodes, mmsd->reader_hostgroup); } - return (identified_hosts == 3); } void * MySQL_Monitor::monitor_read_only() { @@ -5051,8 +5112,8 @@ void * MySQL_Monitor::run() { assert(0); // LCOV_EXCL_STOP } - pthread_t monitor_aws_rds_thread; - if (pthread_create(&monitor_aws_rds_thread, &attr, &monitor_aws_rds_pthread,NULL) != 0) { + pthread_t monitor_aws_rds_bgd_thread; + if (pthread_create(&monitor_aws_rds_bgd_thread, &attr, &monitor_aws_rds_bgd_pthread,NULL) != 0) { // LCOV_EXCL_START proxy_error("Thread creation\n"); assert(0); @@ -5157,7 +5218,7 @@ void * MySQL_Monitor::run() { pthread_join(monitor_group_replication_thread,NULL); pthread_join(monitor_galera_thread,NULL); pthread_join(monitor_aws_aurora_thread,NULL); - pthread_join(monitor_aws_rds_thread,NULL); + pthread_join(monitor_aws_rds_bgd_thread,NULL); pthread_join(monitor_replication_lag_thread,NULL); My_Conn_Pool->purge_all_connections(); @@ -6431,10 +6492,15 @@ void * MySQL_Monitor::monitor_aws_aurora() { return NULL; } -// Runs an async query + store_result on the monitor connection, honoring the -// per-check timeout and the global shutdown flag. -// Returns: 0 success, 1 timeout/query-error, 2 shutdown requested. -static int aws_rds_async_query(MySQL_Monitor_State_Data *mmsd, const char *query) { +/** +* @brief Runs an async query + store_result on the monitor connection. +* +* @param mmsd Monitor state data holding the connection, timing, and result. +* @param query SQL text to execute. +* +* @return 0 on success, 1 on timeout/query-error, 2 if shutdown was requested. +*/ +static int aws_rds_bgd_async_query(MySQL_Monitor_State_Data *mmsd, const char *query) { mmsd->t1 = monotonic_time(); mmsd->interr = 0; mmsd->async_exit_status = mysql_query_start(&mmsd->interr, mmsd->mysql, query); @@ -6445,7 +6511,9 @@ static int aws_rds_async_query(MySQL_Monitor_State_Data *mmsd, const char *query mmsd->mysql_error_msg = strdup("timeout check"); return 1; } - if (GloMyMon->shutdown == true) return 2; + if (GloMyMon->shutdown == true) { + return 2; + } if ((mmsd->async_exit_status & MYSQL_WAIT_TIMEOUT) == 0) { mmsd->async_exit_status = mysql_query_cont(&mmsd->interr, mmsd->mysql, mmsd->async_exit_status); } @@ -6458,7 +6526,9 @@ static int aws_rds_async_query(MySQL_Monitor_State_Data *mmsd, const char *query mmsd->mysql_error_msg = strdup("timeout check"); return 1; } - if (GloMyMon->shutdown == true) return 2; + if (GloMyMon->shutdown == true) { + return 2; + } if ((mmsd->async_exit_status & MYSQL_WAIT_TIMEOUT) == 0) { mmsd->async_exit_status = mysql_store_result_cont(&mmsd->result, mmsd->mysql, mmsd->async_exit_status); } @@ -6470,13 +6540,15 @@ static int aws_rds_async_query(MySQL_Monitor_State_Data *mmsd, const char *query return 0; } -// State of the per-host RDS topology probe. -enum RDS_Topology_Monitor_State { - TOPOLOGY_TABLE_CHECK, // verify mysql.rds_topology exists - TOPOLOGY_METADATA_FETCH // table confirmed present; fetch and branch on its metadata +/** +* @brief State of the per-host RDS topology probe. +*/ +enum RDS_BGD_Topology_Monitor_State { + TOPOLOGY_TABLE_CHECK, ///< verify mysql.rds_topology exists + TOPOLOGY_METADATA_FETCH ///< table confirmed present; fetch and branch on its metadata }; -void * monitor_RDS_thread_HG(void *arg) { +void * monitor_RDS_BGD_thread_HG(void *arg) { unsigned int wHG = *(unsigned int *)arg; unsigned int rHG = 0; unsigned int num_hosts = 0; @@ -6485,11 +6557,13 @@ void * monitor_RDS_thread_HG(void *arg) { unsigned int check_timeout_ms = 0; int green_writer_hostgroup = -1; int green_reader_hostgroup = -1; - set_thread_name("MonitorRDSHG", GloVars.set_thread_name); + set_thread_name("MonitorRdsBgdHG", GloVars.set_thread_name); proxy_info("Started Monitor thread for AWS RDS writer HG %u\n", wHG); // Wait for GloMTH to be initialized - if (!wait_for_glo_mth()) return NULL; // quick exit during shutdown/restart + if (!wait_for_glo_mth()) + return NULL; + unsigned int MySQL_Monitor__thread_MySQL_Thread_Variables_version; MySQL_Thread * mysql_thr = new MySQL_Thread(); mysql_thr->curtime = monotonic_time(); @@ -6498,26 +6572,35 @@ void * monitor_RDS_thread_HG(void *arg) { uint64_t initial_raw_checksum = 0; - // initial data load from the monitor resultset (columns: 0 writer_hostgroup, - // 1 reader_hostgroup, 2 hostname, 3 port, 4 use_ssl, 5 green_writer_hostgroup, - // 6 green_reader_hostgroup, 7 check_interval_ms, 8 check_timeout_ms, - // 9 autopurge_missing_checks, 10 domain_name) - pthread_mutex_lock(&GloMyMon->aws_rds_mutex); - initial_raw_checksum = GloMyMon->AWS_RDS_Hosts_resultset_checksum; - for (std::vector::iterator it = GloMyMon->AWS_RDS_Hosts_resultset->rows.begin() ; it != GloMyMon->AWS_RDS_Hosts_resultset->rows.end(); ++it) { - SQLite3_row *r=*it; + // initial data load from the monitor resultset + // Columns: + // 0 writer_hostgroup, 1 reader_hostgroup, 2 hostname, 3 port, 4 use_ssl, + // 5 green_writer_hostgroup, 6 green_reader_hostgroup, 7 check_interval_ms, + // 8 check_timeout_ms, 9 autopurge_missing_checks, 10 domain_name + pthread_mutex_lock(&GloMyMon->aws_rds_bgd_mutex); + initial_raw_checksum = GloMyMon->AWS_RDS_BGD_Hosts_resultset_checksum; + for (SQLite3_row *r : GloMyMon->AWS_RDS_BGD_Hosts_resultset->rows) { if (atoi(r->fields[0]) == (int)wHG) { num_hosts++; - if (rHG == 0) rHG = atoi(r->fields[1]); - if (green_writer_hostgroup < 0 && r->fields[5] && r->fields[5][0]) green_writer_hostgroup = atoi(r->fields[5]); - if (green_reader_hostgroup < 0 && r->fields[6] && r->fields[6][0]) green_reader_hostgroup = atoi(r->fields[6]); - if (check_interval_ms == 0) check_interval_ms = atoi(r->fields[7]); - if (check_timeout_ms == 0) check_timeout_ms = atoi(r->fields[8]); + if (rHG == 0) { + rHG = atoi(r->fields[1]); + } + if (green_writer_hostgroup < 0 && r->fields[5] && r->fields[5][0]) { + green_writer_hostgroup = atoi(r->fields[5]); + } + if (green_reader_hostgroup < 0 && r->fields[6] && r->fields[6][0]) { + green_reader_hostgroup = atoi(r->fields[6]); + } + if (check_interval_ms == 0) { + check_interval_ms = atoi(r->fields[7]); + } + if (check_timeout_ms == 0) { + check_timeout_ms = atoi(r->fields[8]); + } } } host_def_t *hpa = (host_def_t *)malloc(sizeof(host_def_t)*(num_hosts ? num_hosts : 1)); - for (std::vector::iterator it = GloMyMon->AWS_RDS_Hosts_resultset->rows.begin() ; it != GloMyMon->AWS_RDS_Hosts_resultset->rows.end(); ++it) { - SQLite3_row *r=*it; + for (SQLite3_row *r : GloMyMon->AWS_RDS_BGD_Hosts_resultset->rows) { if (atoi(r->fields[0]) == (int)wHG) { hpa[cur_host_idx].host = strdup(r->fields[2]); hpa[cur_host_idx].port = atoi(r->fields[3]); @@ -6525,8 +6608,10 @@ void * monitor_RDS_thread_HG(void *arg) { cur_host_idx++; } } - if (num_hosts && cur_host_idx >= num_hosts) cur_host_idx = num_hosts - 1; - pthread_mutex_unlock(&GloMyMon->aws_rds_mutex); + if (num_hosts && cur_host_idx >= num_hosts) { + cur_host_idx = num_hosts - 1; + } + pthread_mutex_unlock(&GloMyMon->aws_rds_bgd_mutex); bool exit_now = false; unsigned long long t1 = 0; @@ -6537,7 +6622,7 @@ void * monitor_RDS_thread_HG(void *arg) { bool found_pingable_host = false; bool rc_ping = false; MySQL_Monitor_State_Data *mmsd = NULL; - RDS_Topology_Monitor_State topology_state = TOPOLOGY_TABLE_CHECK; + RDS_BGD_Topology_Monitor_State topology_state = TOPOLOGY_TABLE_CHECK; t1 = monotonic_time(); @@ -6545,7 +6630,8 @@ void * monitor_RDS_thread_HG(void *arg) { unsigned int glover; t1 = monotonic_time(); - if (!GloMTH) goto __exit_monitor_RDS_thread_HG_now; // quick exit during shutdown/restart + if (!GloMTH) + goto __exit_monitor_RDS_BGD_thread_HG_now; // if variables changed, refresh and force a new check glover = GloMTH->get_global_version(); @@ -6556,9 +6642,9 @@ void * monitor_RDS_thread_HG(void *arg) { } // if the host list/definition changed, terminate so the dispatcher respawns - pthread_mutex_lock(&GloMyMon->aws_rds_mutex); - current_raw_checksum = GloMyMon->AWS_RDS_Hosts_resultset_checksum; - pthread_mutex_unlock(&GloMyMon->aws_rds_mutex); + pthread_mutex_lock(&GloMyMon->aws_rds_bgd_mutex); + current_raw_checksum = GloMyMon->AWS_RDS_BGD_Hosts_resultset_checksum; + pthread_mutex_unlock(&GloMyMon->aws_rds_bgd_mutex); if (current_raw_checksum != initial_raw_checksum) { exit_now = true; break; @@ -6572,7 +6658,9 @@ void * monitor_RDS_thread_HG(void *arg) { if (t1 < next_loop_at) { unsigned long long st = next_loop_at - t1; - if (st > 50000) st = 50000; + if (st > 50000) { + st = 50000; + } usleep(st); continue; } @@ -6586,7 +6674,9 @@ void * monitor_RDS_thread_HG(void *arg) { found_pingable_host = true; cur_host_idx = rnd; } else { - MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, wHG, hpa[rnd].host, hpa[rnd].port, ER_PROXYSQL_AWS_NO_PINGABLE_SRV); + MyHGM->p_update_mysql_error_counter( + p_mysql_error_type::proxysql, wHG, hpa[rnd].host, hpa[rnd].port, ER_PROXYSQL_AWS_NO_PINGABLE_SRV + ); shuffle_hosts(hpa, num_hosts); for (unsigned int i=0; (found_pingable_host == false && iserver_responds_to_ping(hpa[i].host, hpa[i].port); @@ -6594,7 +6684,9 @@ void * monitor_RDS_thread_HG(void *arg) { found_pingable_host = true; cur_host_idx = i; } else { - MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, wHG, hpa[i].host, hpa[i].port, ER_PROXYSQL_AWS_NO_PINGABLE_SRV); + MyHGM->p_update_mysql_error_counter( + p_mysql_error_type::proxysql, wHG, hpa[i].host, hpa[i].port, ER_PROXYSQL_AWS_NO_PINGABLE_SRV + ); } } } @@ -6604,20 +6696,27 @@ void * monitor_RDS_thread_HG(void *arg) { continue; } - mmsd = new MySQL_Monitor_State_Data(MON_AWS_RDS, hpa[cur_host_idx].host, hpa[cur_host_idx].port, hpa[cur_host_idx].use_ssl); + mmsd = new MySQL_Monitor_State_Data( + MON_AWS_RDS_BGD, hpa[cur_host_idx].host, hpa[cur_host_idx].port, hpa[cur_host_idx].use_ssl + ); mmsd->writer_hostgroup = wHG; - mmsd->aws_aurora_check_timeout_ms = check_timeout_ms; // reuse the generic per-check timeout field + mmsd->aws_aurora_check_timeout_ms = check_timeout_ms; mmsd->mysql = GloMyMon->My_Conn_Pool->get_connection(mmsd->hostname, mmsd->port, mmsd); mmsd->t1 = t1; crc = false; if (mmsd->mysql == NULL) { // need a new connection bool rc = mmsd->create_new_connection(); - if (mmsd->mysql) GloMyMon->My_Conn_Pool->conn_register(mmsd); + if (mmsd->mysql) { + GloMyMon->My_Conn_Pool->conn_register(mmsd); + } crc = true; if (rc == false) { proxy_error("Error on AWS RDS check for %s:%d. Unable to create a connection.\n", mmsd->hostname, mmsd->port); - MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_AWS_HEALTH_CHECK_CONN_TIMEOUT); + MyHGM->p_update_mysql_error_counter( + p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, + ER_PROXYSQL_AWS_HEALTH_CHECK_CONN_TIMEOUT + ); goto __end_of_loop; } } @@ -6627,17 +6726,31 @@ void * monitor_RDS_thread_HG(void *arg) { // we advance to TOPOLOGY_METADATA_FETCH and skip this check on subsequent // iterations, until a fetch reports the table is gone. - int qrc = aws_rds_async_query(mmsd, "SELECT 1 FROM information_schema.TABLES WHERE TABLE_SCHEMA='mysql' AND TABLE_NAME='rds_topology'"); - if (qrc == 2) goto __exit_monitor_RDS_thread_HG_now; + int qrc = aws_rds_bgd_async_query( + mmsd, + "SELECT 1 FROM information_schema.TABLES" + " WHERE TABLE_SCHEMA='mysql' AND TABLE_NAME='rds_topology'" + ); + if (qrc == 2) { + goto __exit_monitor_RDS_BGD_thread_HG_now; + } if (qrc != 0) { - proxy_error("AWS RDS topology availability check failed for %s:%d : %s\n", mmsd->hostname, mmsd->port, mmsd->mysql_error_msg ? mmsd->mysql_error_msg : "unknown"); + proxy_error( + "AWS RDS topology availability check failed for %s:%d : %s\n", + mmsd->hostname, mmsd->port, mmsd->mysql_error_msg ? mmsd->mysql_error_msg : "unknown" + ); goto __end_of_loop; } bool table_available = (mmsd->result && mysql_num_rows(mmsd->result) > 0); - if (mmsd->result) { mysql_free_result(mmsd->result); mmsd->result = NULL; } + if (mmsd->result) { + mysql_free_result(mmsd->result); + mmsd->result = NULL; + } if (!table_available) { // no blue/green deployment or multi-az cluster discovery in progress; nothing to do - proxy_debug(PROXY_DEBUG_MONITOR, 5, "mysql.rds_topology not present on %s:%d (RDS writer HG %u); skipping\n", mmsd->hostname, mmsd->port, wHG); + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "mysql.rds_topology not present on %s:%d (RDS writer HG %u); skipping\n", + mmsd->hostname, mmsd->port, wHG); goto __end_of_loop; } topology_state = TOPOLOGY_METADATA_FETCH; @@ -6646,61 +6759,55 @@ void * monitor_RDS_thread_HG(void *arg) { // differs by RDS type (the Multi-AZ Cluster topology table may not expose // 'role'/'status' at all), so dump all columns and detect what is present. - int qrc = aws_rds_async_query(mmsd, "SELECT * FROM mysql.rds_topology"); - if (qrc == 2) goto __exit_monitor_RDS_thread_HG_now; + int qrc = aws_rds_bgd_async_query(mmsd, QUERY_AWS_RDS_TOPOLOGY_DISCOVERY); + if (qrc == 2) { + goto __exit_monitor_RDS_BGD_thread_HG_now; + } if (qrc != 0) { unsigned int err = mmsd->mysql ? mysql_errno(mmsd->mysql) : 0; if (err == 1146) { // the table vanished (ER_NO_SUCH_TABLE), e.g. a blue/green deployment // was cancelled: re-check its existence on the next iteration. topology_state = TOPOLOGY_TABLE_CHECK; - proxy_debug(PROXY_DEBUG_MONITOR, 5, "mysql.rds_topology vanished on %s:%d (RDS writer HG %u); rechecking availability\n", mmsd->hostname, mmsd->port, wHG); + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "mysql.rds_topology vanished on %s:%d (RDS writer HG %u); rechecking availability\n", + mmsd->hostname, mmsd->port, wHG); } else { - proxy_error("AWS RDS topology fetch failed for %s:%d : %s\n", mmsd->hostname, mmsd->port, mmsd->mysql_error_msg ? mmsd->mysql_error_msg : "unknown"); + proxy_error( + "AWS RDS topology fetch failed for %s:%d : %s\n", + mmsd->hostname, mmsd->port, mmsd->mysql_error_msg ? mmsd->mysql_error_msg : "unknown" + ); } goto __end_of_loop; } - // locate the 'role' and 'status' columns by name; they may be absent - unsigned int num_rows = mmsd->result ? (unsigned int)mysql_num_rows(mmsd->result) : 0; - int role_idx = -1, status_idx = -1; - if (mmsd->result) { - unsigned int num_fields = mysql_num_fields(mmsd->result); - MYSQL_FIELD *fields = mysql_fetch_fields(mmsd->result); - for (unsigned int i=0; i= 0 && status_idx >= 0) { - MYSQL_ROW row = mysql_fetch_row(mmsd->result); - if (row && row[role_idx] != NULL && row[status_idx] != NULL) instance_topology = true; - mysql_data_seek(mmsd->result, 0); // rewind for the handlers + // the BGD thread only monitors blue/green hostgroups; parse the topology + // (shared with the read_only path) and hand the struct to the handler. + if (mmsd->result && mysql_num_rows(mmsd->result) > 0) { + AWS_RDS_Topology_Result topo = GloMyMon->parse_aws_rds_topology(mmsd->result); + GloMyMon->handle_aws_rds_bgd(wHG, rHG, green_writer_hostgroup, green_reader_hostgroup, topo); } - if (num_rows > 0 && instance_topology) { - GloMyMon->rds_monitor_handle_instance_topology(wHG, rHG, green_writer_hostgroup, green_reader_hostgroup, mmsd->result); - } else if (num_rows > 0) { - GloMyMon->rds_monitor_handle_cluster_topology(wHG, rHG, mmsd->result); + if (mmsd->result) { + mysql_free_result(mmsd->result); + mmsd->result = NULL; } - if (mmsd->result) { mysql_free_result(mmsd->result); mmsd->result = NULL; } } __end_of_loop: mmsd->t2 = monotonic_time(); next_loop_at = t1 + (check_interval_ms * 1000); - if (mmsd->t2 > t1) next_loop_at -= (mmsd->t2 - t1); + if (mmsd->t2 > t1) { + next_loop_at -= (mmsd->t2 - t1); + } if (mmsd->mysql) { if (mmsd->mysql_error_msg) { GloMyMon->My_Conn_Pool->destroy_mysql_connection(mmsd); } else if (crc) { - if (mmsd->set_wait_timeout()) GloMyMon->My_Conn_Pool->put_connection(mmsd->hostname, mmsd); - else GloMyMon->My_Conn_Pool->destroy_mysql_connection(mmsd); + if (mmsd->set_wait_timeout()) { + GloMyMon->My_Conn_Pool->put_connection(mmsd->hostname, mmsd); + } else { + GloMyMon->My_Conn_Pool->destroy_mysql_connection(mmsd); + } } else { GloMyMon->My_Conn_Pool->put_connection(mmsd->hostname, mmsd); } @@ -6709,45 +6816,53 @@ void * monitor_RDS_thread_HG(void *arg) { mmsd = NULL; } -__exit_monitor_RDS_thread_HG_now: - if (mmsd) { delete mmsd; mmsd = NULL; } +__exit_monitor_RDS_BGD_thread_HG_now: + if (mmsd) { + delete mmsd; + mmsd = NULL; + } for (unsigned int i=0; icurtime = monotonic_time(); @@ -6761,7 +6876,8 @@ void * MySQL_Monitor::monitor_aws_rds() { while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true) { unsigned int glover; - if (!GloMTH) return NULL; // quick exit during shutdown/restart + if (!GloMTH) + return NULL; glover = GloMTH->get_global_version(); if (MySQL_Monitor__thread_MySQL_Thread_Variables_version < glover) { @@ -6770,9 +6886,9 @@ void * MySQL_Monitor::monitor_aws_rds() { } // respawn the per-writer-HG workers when the host list/definition changes - pthread_mutex_lock(&aws_rds_mutex); - uint64_t new_raw_checksum = AWS_RDS_Hosts_resultset->raw_checksum(); - pthread_mutex_unlock(&aws_rds_mutex); + pthread_mutex_lock(&aws_rds_bgd_mutex); + uint64_t new_raw_checksum = AWS_RDS_BGD_Hosts_resultset->raw_checksum(); + pthread_mutex_unlock(&aws_rds_bgd_mutex); if (new_raw_checksum != last_raw_checksum) { proxy_info("Detected new/changed definition for AWS RDS monitoring\n"); last_raw_checksum = new_raw_checksum; @@ -6786,17 +6902,19 @@ void * MySQL_Monitor::monitor_aws_rds() { pthreads_array = NULL; hgs_array = NULL; } + hgs_num = 0; - pthread_mutex_lock(&aws_rds_mutex); - unsigned int num_rows = AWS_RDS_Hosts_resultset->rows_count; + pthread_mutex_lock(&aws_rds_bgd_mutex); + unsigned int num_rows = AWS_RDS_BGD_Hosts_resultset->rows_count; if (num_rows) { unsigned int *tmp_hgs_array = (unsigned int *)malloc(sizeof(unsigned int)*num_rows); - for (std::vector::iterator it = AWS_RDS_Hosts_resultset->rows.begin() ; it != AWS_RDS_Hosts_resultset->rows.end(); ++it) { - SQLite3_row *r=*it; + for (SQLite3_row *r : AWS_RDS_BGD_Hosts_resultset->rows) { int wHG = atoi(r->fields[0]); bool found = false; for (unsigned int i=0; i < hgs_num; i++) { - if (tmp_hgs_array[i] == (unsigned int)wHG) found = true; + if (tmp_hgs_array[i] == (unsigned int)wHG) { + found = true; + } } if (found == false) { tmp_hgs_array[hgs_num] = wHG; @@ -6809,7 +6927,7 @@ void * MySQL_Monitor::monitor_aws_rds() { for (unsigned int i=0; i < hgs_num; i++) { hgs_array[i] = tmp_hgs_array[i]; proxy_info("Starting Monitor thread for AWS RDS writer HG %u\n", hgs_array[i]); - if (pthread_create(&pthreads_array[i], NULL, monitor_RDS_thread_HG, &hgs_array[i]) != 0) { + if (pthread_create(&pthreads_array[i], NULL, monitor_RDS_BGD_thread_HG, &hgs_array[i]) != 0) { // LCOV_EXCL_START proxy_error("Thread creation\n"); assert(0); @@ -6818,7 +6936,7 @@ void * MySQL_Monitor::monitor_aws_rds() { } free(tmp_hgs_array); } - pthread_mutex_unlock(&aws_rds_mutex); + pthread_mutex_unlock(&aws_rds_bgd_mutex); } usleep(10000); @@ -7790,6 +7908,42 @@ bool MySQL_Monitor::monitor_read_only_process_ready_tasks(const std::vectorget_task_type() == MON_AWS_RDS_TOPOLOGY_DISCOVERY) { + if (task_result == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_SUCCESS) { + __sync_fetch_and_add(&read_only_check_OK, 1); + if (mmsd->interr == 0 && mmsd->result) { + process_aws_rds_topology(mmsd); + } + if (mmsd->result) { + mysql_free_result(mmsd->result); + mmsd->result = NULL; + } + My_Conn_Pool->put_connection(mmsd->hostname, mmsd); + } else { + __sync_fetch_and_add(&read_only_check_ERR, 1); + unsigned int err = mmsd->mysql ? mysql_errno(mmsd->mysql) : 0; + if (err == 1146) { + // mysql.rds_topology absent (no active blue/green deployment); expected, skip quietly + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "mysql.rds_topology not present on %s:%d; skipping blue/green discovery\n", + mmsd->hostname, mmsd->port); + } else { + MyHGM->p_update_mysql_error_counter( + p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, + err ? err : ER_PROXYSQL_READ_ONLY_CHECK_TIMEOUT + ); + proxy_error( + "Error on AWS RDS blue/green topology discovery for %s:%d : %s\n", + mmsd->hostname, mmsd->port, (mmsd->mysql_error_msg ? mmsd->mysql_error_msg : "") + ); + } + My_Conn_Pool->destroy_mysql_connection(mmsd); + } + continue; + } + if (task_result == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_SUCCESS) { __sync_fetch_and_add(&read_only_check_OK, 1); My_Conn_Pool->put_connection(mmsd->hostname, mmsd); @@ -7852,38 +8006,6 @@ VALGRIND_ENABLE_ERROR_REPORTING; } rc = (*proxy_sqlite3_bind_int64)(statement, 5, read_only); ASSERT_SQLITE_OK(rc, mmsd->mondb); - } else if (fields && mmsd->get_task_type() == MON_READ_ONLY__AND__AWS_RDS_TOPOLOGY_DISCOVERY) { - // Process the read_only field as above and store the first server - vector discovered_servers; - for (k = 0; k < num_fields; k++) { - if (strcmp((char*)"read_only", (char*)fields[k].name) == 0) { - j = k; - } - } - if (j > -1) { - MYSQL_ROW row = mysql_fetch_row(mmsd->result); - if (row) { - discovered_servers.push_back(row); -VALGRIND_DISABLE_ERROR_REPORTING; - if (row[j]) { - if (!strcmp(row[j], "0") || !strcasecmp(row[j], "OFF")) - read_only = 0; - } -VALGRIND_ENABLE_ERROR_REPORTING; - } - } - - // Store the remaining servers - int num_rows = mysql_num_rows(mmsd->result); - for (int i = 1; i < num_rows; i++) { - MYSQL_ROW row = mysql_fetch_row(mmsd->result); - discovered_servers.push_back(row); - } - - // Process the discovered servers and add them to 'runtime_mysql_servers' (process only for AWS RDS Multi-AZ DB Clusters) - if (!discovered_servers.empty() && is_aws_rds_multi_az_db_cluster_topology(discovered_servers)) { - process_discovered_topology(originating_server_hostname, discovered_servers, mmsd->reader_hostgroup); - } } else { proxy_error("mysql_fetch_fields returns NULL, or mysql_num_fields is incorrect. Server %s:%d . See bug #1994\n", mmsd->hostname, mmsd->port); rc = (*proxy_sqlite3_bind_null)(statement, 5); ASSERT_SQLITE_OK(rc, mmsd->mondb); @@ -7967,11 +8089,6 @@ void MySQL_Monitor::monitor_read_only_async(SQLite3_result* resultset, bool do_d task_type = MON_READ_ONLY__OR__INNODB_READ_ONLY; } - // Change task type if it's time to do discovery check. Only for aws rds endpoints - string hostname = r->fields[0]; - if (do_discovery_check && hostname.find(AWS_ENDPOINT_SUFFIX_STRING) != std::string::npos) { - task_type = MON_READ_ONLY__AND__AWS_RDS_TOPOLOGY_DISCOVERY; - } } std::unique_ptr mmsd( @@ -7986,10 +8103,29 @@ void MySQL_Monitor::monitor_read_only_async(SQLite3_result* resultset, bool do_d monitor_poll.add((POLLIN|POLLOUT|POLLPRI), mmsd.get()); mmsds.push_back(std::move(mmsd)); } else { - WorkItem* item = + WorkItem* item = new WorkItem(mmsd.release(), monitor_read_only_thread); queue->add(item); } + + // On discovery cycles, enqueue an additional standalone topology-discovery + // task for AWS RDS endpoints. The read_only check above is unaffected. + string hostname = r->fields[0]; + if (do_discovery_check && hostname.find(AWS_ENDPOINT_SUFFIX_STRING) != std::string::npos) { + std::unique_ptr tmmsd( + new MySQL_Monitor_State_Data(MON_AWS_RDS_TOPOLOGY_DISCOVERY, r->fields[0], atoi(r->fields[1]), atoi(r->fields[2]))); + tmmsd->reader_hostgroup = atoi(r->fields[4]); + tmmsd->mondb = monitordb; + tmmsd->mysql = My_Conn_Pool->get_connection(tmmsd->hostname, tmmsd->port, tmmsd.get()); + if (tmmsd->mysql) { + monitor_poll.add((POLLIN|POLLOUT|POLLPRI), tmmsd.get()); + mmsds.push_back(std::move(tmmsd)); + } else { + WorkItem* item = + new WorkItem(tmmsd.release(), monitor_read_only_thread); + queue->add(item); + } + } } if (shutdown) return; diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index f39e2000df..acfe2b3da2 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -400,6 +400,7 @@ static char * mysql_thread_variables_names[]= { (char *)"monitor_ping_max_failures", (char *)"monitor_ping_timeout", (char *)"monitor_aws_rds_topology_discovery_interval", + (char *)"aws_blue_green_deployment_auto_discovery", (char *)"monitor_read_only_interval", (char *)"monitor_read_only_timeout", (char *)"monitor_read_only_max_timeout_count", @@ -1274,6 +1275,7 @@ MySQL_Threads_Handler::MySQL_Threads_Handler() { variables.monitor_ping_max_failures=3; variables.monitor_ping_timeout=1000; variables.monitor_aws_rds_topology_discovery_interval=0; + variables.aws_blue_green_deployment_auto_discovery=true; variables.monitor_read_only_interval=1000; variables.monitor_read_only_timeout=800; variables.monitor_read_only_max_timeout_count=3; @@ -2530,6 +2532,7 @@ char ** MySQL_Threads_Handler::get_variables_list() { VariablesPointers_bool["log_mysql_warnings_enabled"] = make_tuple(&variables.log_mysql_warnings_enabled, false); VariablesPointers_bool["log_unhealthy_connections"] = make_tuple(&variables.log_unhealthy_connections, false); VariablesPointers_bool["monitor_enabled"] = make_tuple(&variables.monitor_enabled, false); + VariablesPointers_bool["aws_blue_green_deployment_auto_discovery"] = make_tuple(&variables.aws_blue_green_deployment_auto_discovery, false); VariablesPointers_bool["monitor_replication_lag_group_by_host"] = make_tuple(&variables.monitor_replication_lag_group_by_host, false); VariablesPointers_bool["monitor_wait_timeout"] = make_tuple(&variables.monitor_wait_timeout, false); VariablesPointers_bool["monitor_writer_is_also_reader"] = make_tuple(&variables.monitor_writer_is_also_reader, false); @@ -4654,6 +4657,7 @@ void MySQL_Thread::refresh_variables() { REFRESH_VARIABLE_INT(monitor_ping_max_failures); REFRESH_VARIABLE_INT(monitor_ping_timeout); REFRESH_VARIABLE_INT(monitor_aws_rds_topology_discovery_interval); + REFRESH_VARIABLE_BOOL(aws_blue_green_deployment_auto_discovery); REFRESH_VARIABLE_INT(monitor_read_only_interval); REFRESH_VARIABLE_INT(monitor_read_only_timeout); REFRESH_VARIABLE_INT(monitor_read_only_max_timeout_count); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index a61c3fb6e7..ee7c3f90d5 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -147,7 +147,7 @@ static const vector mysql_servers_tablenames = { "mysql_group_replication_hostgroups", "mysql_galera_hostgroups", "mysql_aws_aurora_hostgroups", - "mysql_aws_rds_hostgroups", + "mysql_aws_rds_bgd_hostgroups", "mysql_hostgroup_attributes", "mysql_servers_ssl_params", }; @@ -1510,7 +1510,7 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign || strstr(query_no_space,"runtime_mysql_aws_aurora_hostgroups") || - strstr(query_no_space,"runtime_mysql_aws_rds_hostgroups") + strstr(query_no_space,"runtime_mysql_aws_rds_bgd_hostgroups") || strstr(query_no_space,"runtime_mysql_hostgroup_attributes") || @@ -7550,29 +7550,29 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { if(resultset) delete resultset; resultset=NULL; - // dump mysql_aws_rds_hostgroups + // dump mysql_aws_rds_bgd_hostgroups // The runtime table carries the extra runtime-only 'auto_generated' column; the config table // does not. 'dump_table_mysql' always returns 12 columns (last is 'auto_generated'); we bind // 12 for the runtime table and only the first 11 for the config table. 'green_writer_hostgroup' // and 'green_reader_hostgroup' (fields 2,3) are nullable and bound as NULL when absent. if (_runtime) { - query=(char *)"DELETE FROM main.runtime_mysql_aws_rds_hostgroups"; + query=(char *)"DELETE FROM main.runtime_mysql_aws_rds_bgd_hostgroups"; } else { - query=(char *)"DELETE FROM main.mysql_aws_rds_hostgroups"; + query=(char *)"DELETE FROM main.mysql_aws_rds_bgd_hostgroups"; } proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); admindb->execute(query); - resultset=MyHGM->dump_table_mysql("mysql_aws_rds_hostgroups"); + resultset=MyHGM->dump_table_mysql("mysql_aws_rds_bgd_hostgroups"); if (resultset) { int rc; sqlite3_stmt *statement=NULL; char *query=NULL; if (_runtime) { - query=(char *)"INSERT INTO runtime_mysql_aws_rds_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader,domain_name,check_interval_ms,check_timeout_ms,autopurge_missing_checks,comment,auto_generated) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)"; + query=(char *)"INSERT INTO runtime_mysql_aws_rds_bgd_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader,domain_name,check_interval_ms,check_timeout_ms,autopurge_missing_checks,comment,auto_generated) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)"; } else { - query=(char *)"INSERT INTO mysql_aws_rds_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader,domain_name,check_interval_ms,check_timeout_ms,autopurge_missing_checks,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"; + query=(char *)"INSERT INTO mysql_aws_rds_bgd_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader,domain_name,check_interval_ms,check_timeout_ms,autopurge_missing_checks,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"; } auto [rc1, statement_unique] = admindb->prepare_v2(query); @@ -7952,7 +7952,7 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc SQLite3_result *resultset_group_replication=NULL; SQLite3_result *resultset_galera=NULL; SQLite3_result *resultset_aws_aurora=NULL; - SQLite3_result *resultset_aws_rds=NULL; + SQLite3_result *resultset_aws_rds_bgd=NULL; SQLite3_result *resultset_hostgroup_attributes=NULL; SQLite3_result *resultset_mysql_servers_ssl_params=NULL; @@ -8116,15 +8116,15 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc MyHGM->save_incoming_mysql_table(resultset_aws_aurora,"mysql_aws_aurora_hostgroups"); } - // support for AWS RDS, table mysql_aws_rds_hostgroups - query=(char *)"SELECT a.* FROM mysql_aws_rds_hostgroups a LEFT JOIN mysql_aws_rds_hostgroups b ON (a.writer_hostgroup=b.reader_hostgroup) WHERE b.reader_hostgroup IS NULL ORDER BY writer_hostgroup"; + // support for AWS RDS, table mysql_aws_rds_bgd_hostgroups + query=(char *)"SELECT a.* FROM mysql_aws_rds_bgd_hostgroups a LEFT JOIN mysql_aws_rds_bgd_hostgroups b ON (a.writer_hostgroup=b.reader_hostgroup) WHERE b.reader_hostgroup IS NULL ORDER BY writer_hostgroup"; proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); - admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_aws_rds); + admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_aws_rds_bgd); if (error) { proxy_error("Error on %s : %s\n", query, error); } else { // Pass the resultset to MyHGM - MyHGM->save_incoming_mysql_table(resultset_aws_rds,"mysql_aws_rds_hostgroups"); + MyHGM->save_incoming_mysql_table(resultset_aws_rds_bgd,"mysql_aws_rds_bgd_hostgroups"); } // support for hostgroup attributes, table mysql_hostgroup_attributes @@ -8186,9 +8186,9 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc //delete resultset_aws_aurora; // do not delete, resultset is stored in MyHGM resultset_aws_aurora=NULL; } - if (resultset_aws_rds) { - //delete resultset_aws_rds; // do not delete, resultset is stored in MyHGM - resultset_aws_rds=NULL; + if (resultset_aws_rds_bgd) { + //delete resultset_aws_rds_bgd; // do not delete, resultset is stored in MyHGM + resultset_aws_rds_bgd=NULL; } if (resultset_hostgroup_attributes) { resultset_hostgroup_attributes = NULL; diff --git a/lib/ProxySQL_Config.cpp b/lib/ProxySQL_Config.cpp index 31e106a977..63ead27333 100644 --- a/lib/ProxySQL_Config.cpp +++ b/lib/ProxySQL_Config.cpp @@ -1097,14 +1097,14 @@ int ProxySQL_Config::Write_MySQL_Servers_to_configfile(std::string& data) { if (sqlite_resultset) delete sqlite_resultset; - query=(char *)"SELECT * FROM mysql_aws_rds_hostgroups"; + query=(char *)"SELECT * FROM mysql_aws_rds_bgd_hostgroups"; admindb->execute_statement(query, &error, &cols, &affected_rows, &sqlite_resultset); if (error) { - proxy_error("Error on read from mysql_aws_rds_hostgroups: %s\n", error); + proxy_error("Error on read from mysql_aws_rds_bgd_hostgroups: %s\n", error); return -1; } else { if (sqlite_resultset) { - data += "mysql_aws_rds_hostgroups:\n(\n"; + data += "mysql_aws_rds_bgd_hostgroups:\n(\n"; bool isNext = false; for (auto r : sqlite_resultset->rows) { if (isNext) @@ -1488,13 +1488,13 @@ int ProxySQL_Config::Read_MySQL_Servers_from_configfile(std::string& error) { } } - if (root.exists("mysql_aws_rds_hostgroups")==true) { - const Setting &mysql_aws_rds_hostgroups = root["mysql_aws_rds_hostgroups"]; - int count = mysql_aws_rds_hostgroups.getLength(); + if (root.exists("mysql_aws_rds_bgd_hostgroups")==true) { + const Setting &mysql_aws_rds_bgd_hostgroups = root["mysql_aws_rds_bgd_hostgroups"]; + int count = mysql_aws_rds_bgd_hostgroups.getLength(); // green_writer_hostgroup / green_reader_hostgroup are nullable -> passed as %s ("NULL" or an integer) - char *q=(char *)"INSERT OR REPLACE INTO mysql_aws_rds_hostgroups (writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active, writer_is_also_reader, domain_name, check_interval_ms, check_timeout_ms, autopurge_missing_checks, comment ) VALUES (%d, %d, %s, %s, %d, %d, '%s', %d, %d, %d, '%s')"; + char *q=(char *)"INSERT OR REPLACE INTO mysql_aws_rds_bgd_hostgroups (writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active, writer_is_also_reader, domain_name, check_interval_ms, check_timeout_ms, autopurge_missing_checks, comment ) VALUES (%d, %d, %s, %s, %d, %d, '%s', %d, %d, %d, '%s')"; for (i=0; i< count; i++) { - const Setting &line = mysql_aws_rds_hostgroups[i]; + const Setting &line = mysql_aws_rds_bgd_hostgroups[i]; int writer_hostgroup; int reader_hostgroup; int green_writer_hostgroup; @@ -1507,11 +1507,11 @@ int ProxySQL_Config::Read_MySQL_Servers_from_configfile(std::string& error) { std::string comment=""; std::string domain_name=""; if (line.lookupValue("writer_hostgroup", writer_hostgroup)==false) { - proxy_error("Admin: detected a mysql_aws_rds_hostgroups in config file without a mandatory writer_hostgroup\n"); + proxy_error("Admin: detected a mysql_aws_rds_bgd_hostgroups in config file without a mandatory writer_hostgroup\n"); continue; } if (line.lookupValue("reader_hostgroup", reader_hostgroup)==false) { - proxy_error("Admin: detected a mysql_aws_rds_hostgroups in config file without a mandatory reader_hostgroup\n"); + proxy_error("Admin: detected a mysql_aws_rds_bgd_hostgroups in config file without a mandatory reader_hostgroup\n"); continue; } char green_writer_str[24]; From c4aa572e52a8938a76025f97c0da57ab8aabb650 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sun, 21 Jun 2026 14:54:36 +0000 Subject: [PATCH 04/81] feat: Add DNS cache IP pinning - Add support for IP pinning in `DNS_Cache`; Pinned IP entries override resolved IPs in during `lookup()` until they are unpinned. - Add a helper - `dns_resolve()` and reuse it in `monitor_dns_resolver_thread()`. Signed-off-by: Wazir Ahmed --- include/DNS_Cache.hpp | 40 +++++++- lib/DNS_Cache.cpp | 211 ++++++++++++++++++++++++++++-------------- 2 files changed, 179 insertions(+), 72 deletions(-) diff --git a/include/DNS_Cache.hpp b/include/DNS_Cache.hpp index a994a23b7f..e24023af6e 100644 --- a/include/DNS_Cache.hpp +++ b/include/DNS_Cache.hpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include "thread.h" @@ -84,9 +85,28 @@ class DNS_Cache { bool empty() const; std::string lookup(const std::string& hostname, size_t* ip_count) const; + /** + * @brief Pin a hostname to a fixed set of IPs that override resolution until unpin(). + * + * @param hostname Hostname whose resolution is overridden. + * @param ips IP addresses to serve for 'hostname' (moved into the cache). + */ + void pin(const std::string& hostname, std::vector&& ips); + + /** + * @brief Remove a pin set by pin(), restoring normal resolution (no-op if not pinned). + * + * @param hostname Hostname to unpin. + */ + void unpin(const std::string& hostname); + private: struct IP_ADDR { std::vector ips; + // Pinned override: when non-empty, get_next_ip()/lookup() serve these + // instead of 'ips'. Set by pin(), cleared by unpin(); untouched by add(), + // so it is preserved across resolver-thread TTL refreshes. + std::vector pinned_ips; // 'counter' is bumped by get_next_ip() (a const method) for // round-robin selection; the logical state of the cache record is // unchanged, so mutable is the right tool here and lets us drop a @@ -94,7 +114,15 @@ class DNS_Cache { mutable unsigned long counter = 0; }; - std::string get_next_ip(const IP_ADDR& ip_addr) const; + /** + * @brief Next round-robin IP for 'ip_addr' and the size of the served set. + * + * @param ip_addr Cache record to select from. + * + * @return { ip, set_size }, or { "", 0 } when the set is empty. + */ + std::pair get_next_ip(const IP_ADDR& ip_addr) const; + std::unordered_map records; std::atomic_bool enabled; mutable pthread_rwlock_t rwlock_; @@ -154,6 +182,16 @@ bool validate_ip(const std::string& ip); // failure / non-IP families. std::string get_connected_peer_ip_from_socket(int socket_fd); +/** +* @brief Resolve a hostname to its IP(s) via getaddrinfo. +* +* @param hostname Hostname to resolve. +* @param ai_family Address family for getaddrinfo (an AF_* value; AF_UNSPEC for OS default). +* +* @return The resolved IPs, or an empty vector on failure. +*/ +std::vector dns_resolve(const std::string& hostname, int ai_family); + // Helper: stringify a list of IPs for debug logging. Defined inline because // it's templated over the iterable type used by the various call sites. template diff --git a/lib/DNS_Cache.cpp b/lib/DNS_Cache.cpp index 689e0a0279..f182723ed4 100644 --- a/lib/DNS_Cache.cpp +++ b/lib/DNS_Cache.cpp @@ -61,12 +61,18 @@ std::string get_connected_peer_ip_from_socket(int socket_fd) { return result; } -void* monitor_dns_resolver_thread(const std::vector& dns_resolve_data_list) { - assert(!dns_resolve_data_list.empty()); - DNS_Resolve_Data* dns_resolve_data = dns_resolve_data_list.front(); +/** +* @brief Resolve a hostname to its IP(s) via getaddrinfo. +* +* @param hostname Hostname to resolve. +* @param ai_family Address family for getaddrinfo (an AF_* value; AF_UNSPEC for OS default). +* +* @return The resolved IPs, or an empty vector on failure. +*/ +std::vector dns_resolve(const std::string& hostname, int ai_family) { + std::vector ips; struct addrinfo hints, *res = NULL; - memset(&hints, 0, sizeof(hints)); hints.ai_protocol = IPPROTO_TCP; hints.ai_socktype = SOCK_STREAM; @@ -76,86 +82,79 @@ void* monitor_dns_resolver_thread(const std::vector& dns_reso // purpose. Useful on IPv4-only hosts so getaddrinfo() doesn't return IPv6 // addresses that connect/bind would always fail on. hints.ai_flags = AI_ADDRCONFIG; - hints.ai_family = dns_resolve_data->ai_family; - proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, - "Resolving hostname:[%s] to its mapped IP address.\n", - dns_resolve_data->hostname.c_str()); - int gai_rc = getaddrinfo(dns_resolve_data->hostname.c_str(), NULL, &hints, &res); + hints.ai_family = ai_family; + int gai_rc = getaddrinfo(hostname.c_str(), NULL, &hints, &res); if (gai_rc != 0 || !res) { - proxy_error("An error occurred while resolving hostname: %s [%d]\n", - dns_resolve_data->hostname.c_str(), gai_rc); - goto __error; + proxy_error("An error occurred while resolving hostname: %s [%d]\n", hostname.c_str(), gai_rc); + return ips; } - try { - std::vector ips; - ips.reserve(64); + char ip_addr[INET6_ADDRSTRLEN]; + for (auto p = res; p != NULL; p = p->ai_next) { + if (p->ai_family == AF_INET) { + struct sockaddr_in* ipv4 = (struct sockaddr_in*)p->ai_addr; + inet_ntop(p->ai_addr->sa_family, &ipv4->sin_addr, ip_addr, INET_ADDRSTRLEN); + ips.push_back(ip_addr); + } + else { + struct sockaddr_in6* ipv6 = (struct sockaddr_in6*)p->ai_addr; + inet_ntop(p->ai_addr->sa_family, &ipv6->sin6_addr, ip_addr, INET6_ADDRSTRLEN); + ips.push_back(ip_addr); + } + } - char ip_addr[INET6_ADDRSTRLEN]; + freeaddrinfo(res); + return ips; +} - for (auto p = res; p != NULL; p = p->ai_next) { - if (p->ai_family == AF_INET) { - struct sockaddr_in* ipv4 = (struct sockaddr_in*)p->ai_addr; - inet_ntop(p->ai_addr->sa_family, &ipv4->sin_addr, ip_addr, INET_ADDRSTRLEN); - ips.push_back(ip_addr); - } - else { - struct sockaddr_in6* ipv6 = (struct sockaddr_in6*)p->ai_addr; - inet_ntop(p->ai_addr->sa_family, &ipv6->sin6_addr, ip_addr, INET6_ADDRSTRLEN); - ips.push_back(ip_addr); - } - } +void* monitor_dns_resolver_thread(const std::vector& dns_resolve_data_list) { + assert(!dns_resolve_data_list.empty()); + DNS_Resolve_Data* data = dns_resolve_data_list.front(); - freeaddrinfo(res); + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Resolving hostname:[%s] to its mapped IP address.\n", + data->hostname.c_str()); + try { + std::vector ips = dns_resolve(data->hostname, data->ai_family); if (!ips.empty()) { - - bool to_update_cache = false; - int cache_ttl = dns_resolve_data->ttl; - if (dns_resolve_data->ttl > dns_resolve_data->refresh_intv) { + unsigned int cache_ttl = data->ttl; + if (data->ttl > data->refresh_intv) { // NOSONAR cpp:S2245 — mt19937 used here only as a DNS-cache // TTL jitter source (non-cryptographic timing tweak); no // security boundary. Inline annotation on the construction // line because Sonar attributes the hotspot to it. thread_local std::mt19937 gen(std::random_device{}()); // NOSONAR cpp:S2245 - const int jitter = static_cast(dns_resolve_data->ttl * 0.025); + const int jitter = static_cast(data->ttl * 0.025); std::uniform_int_distribution dis(-jitter, jitter); cache_ttl += dis(gen); } - if (!dns_resolve_data->cached_ips.empty()) { - - if (dns_resolve_data->cached_ips.size() == ips.size()) { - for (const std::string& ip : ips) { - if (dns_resolve_data->cached_ips.find(ip) == dns_resolve_data->cached_ips.end()) { - to_update_cache = true; - break; - } - } - } - else - to_update_cache = true; - - if (!to_update_cache) { + bool to_update_cache = true; + unsigned long long expiry = monotonic_time() + (1000ULL * (unsigned long long)cache_ttl); + + if (!data->cached_ips.empty() + && data->cached_ips.size() == ips.size()) { + bool match_all = std::all_of( + ips.begin(), + ips.end(), + [&](const std::string& ip) { return data->cached_ips.count(ip) != 0; } + ); + if (match_all) { + // keep the existing record, just refresh its expiry + to_update_cache = false; proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "DNS cache record already up-to-date. (Hostname:[%s] IP:[%s])\n", - dns_resolve_data->hostname.c_str(), - debug_iplisttostring(ips).c_str()); - dns_resolve_data->result.set_value(std::make_tuple<>(true, - DNS_Cache_Record(dns_resolve_data->hostname, - std::move(dns_resolve_data->cached_ips), - monotonic_time() + (1000ULL * static_cast(cache_ttl))))); + data->hostname.c_str(), debug_iplisttostring(ips).c_str()); + data->result.set_value(std::make_tuple<>(true, + DNS_Cache_Record(data->hostname, std::move(data->cached_ips), expiry))); } } - else - to_update_cache = true; if (to_update_cache) { - dns_resolve_data->result.set_value(std::make_tuple<>(true, - DNS_Cache_Record(dns_resolve_data->hostname, ips, - monotonic_time() + (1000ULL * static_cast(cache_ttl))))); - dns_resolve_data->dns_cache->add(dns_resolve_data->hostname, std::move(ips)); + data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(data->hostname, ips, expiry))); + data->dns_cache->add(data->hostname, std::move(ips)); } return NULL; @@ -163,15 +162,14 @@ void* monitor_dns_resolver_thread(const std::vector& dns_reso } catch (std::exception& ex) { proxy_error("An exception occurred while resolving hostname: %s [%s]\n", - dns_resolve_data->hostname.c_str(), ex.what()); + data->hostname.c_str(), ex.what()); } catch (...) { proxy_error("An unknown exception has occurred while resolving hostname: %s\n", - dns_resolve_data->hostname.c_str()); + data->hostname.c_str()); } -__error: - dns_resolve_data->result.set_value(std::make_tuple<>(false, DNS_Cache_Record())); + data->result.set_value(std::make_tuple<>(false, DNS_Cache_Record())); return NULL; } @@ -253,14 +251,27 @@ bool DNS_Cache::add_if_not_exist(const std::string& hostname, std::vector DNS_Cache::get_next_ip(const IP_ADDR& ip_addr) const { + // A pinned record overrides the resolved IPs. + const std::vector& src = + ip_addr.pinned_ips.empty() ? ip_addr.ips : ip_addr.pinned_ips; + + if (src.empty()) + return { "", 0 }; const auto counter_val = __sync_fetch_and_add(&ip_addr.counter, 1); - return ip_addr.ips[counter_val % ip_addr.ips.size()]; + size_t ip_count = src.size(); + auto ip = src[counter_val % ip_count]; + + return { ip, ip_count }; } std::string DNS_Cache::lookup(const std::string& hostname, size_t* ip_count) const { @@ -280,10 +291,10 @@ std::string DNS_Cache::lookup(const std::string& hostname, size_t* ip_count) con auto itr = records.find(hostname); if (itr != records.end()) { - ip = get_next_ip(itr->second); + auto [ip, count] = get_next_ip(itr->second); if (ip_count) - *ip_count = itr->second.ips.size(); + *ip_count = count; proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "DNS cache lookup success. (Hostname:[%s] IP returned:[%s])\n", @@ -302,6 +313,64 @@ std::string DNS_Cache::lookup(const std::string& hostname, size_t* ip_count) con return ip; } +/** +* @brief Pin a hostname to a fixed set of IPs that override resolution until unpin(). +* +* @param hostname Hostname whose resolution is overridden. +* @param ips IP addresses to serve for 'hostname' (moved into the cache). +*/ +void DNS_Cache::pin(const std::string& hostname, std::vector&& ips) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Pinning DNS cache record. (Hostname:[%s] IP:[%s])\n", + hostname.c_str(), debug_iplisttostring(ips).c_str()); + + int rc = pthread_rwlock_wrlock(&rwlock_); + assert(rc == 0); + + // Store on the record's 'pinned_ips' so a concurrent resolver add() (which + // only rewrites 'ips') cannot drop the override on a TTL refresh. + auto& ip_addr = records[hostname]; + ip_addr.pinned_ips = std::move(ips); + __sync_fetch_and_and(&ip_addr.counter, 0); + + rc = pthread_rwlock_unlock(&rwlock_); + assert(rc == 0); + + if (counter_record_updated_) + counter_record_updated_->fetch_add(1, std::memory_order_relaxed); +} + +/** +* @brief Remove a pin set by pin(), restoring normal resolution (no-op if not pinned). +* +* @param hostname Hostname to unpin. +*/ +void DNS_Cache::unpin(const std::string& hostname) { + bool item_removed = false; + + int rc = pthread_rwlock_wrlock(&rwlock_); + assert(rc == 0); + + auto itr = records.find(hostname); + if (itr != records.end() && !itr->second.pinned_ips.empty()) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Unpinning DNS cache record. (Hostname:[%s] IP:[%s])\n", + hostname.c_str(), debug_iplisttostring(itr->second.pinned_ips).c_str()); + itr->second.pinned_ips.clear(); + // drop the record entirely if pinning was the only thing keeping it alive + // (e.g. the host is not otherwise resolved into the cache). + if (itr->second.ips.empty()) + records.erase(itr); + item_removed = true; + } + + rc = pthread_rwlock_unlock(&rwlock_); + assert(rc == 0); + + if (item_removed && counter_record_updated_) + counter_record_updated_->fetch_add(1, std::memory_order_relaxed); +} + void DNS_Cache::remove(const std::string& hostname) { bool item_removed = false; From 1fd472ec0bb5dbfce8d9ab1a8aae43de10848776 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Mon, 22 Jun 2026 03:47:51 +0000 Subject: [PATCH 05/81] feat: Add helper to shun a server with shun recovery disabled - Add a new helper `set_server_shun()` to shun/unshun a server that takes shun auto-recovery into account. It allows shunning a server while preventing the server-selection path from automatically unshunning it after shun_recovery_time. - Make `IsServerOffline()` / `async_send_simple_command()` act based on `shunned_and_kill_all_connections` alone, without considering `shunned_automatic`. Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 17 ++++++++++++ lib/MySQL_HostGroups_Manager.cpp | 43 ++++++++++++++++++++++++++++++ lib/mysql_connection.cpp | 4 +-- 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 0491c7ca36..086c2a38ac 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -214,6 +214,9 @@ class MySrvC { // MySQL Server Container unsigned long long queries_gtid_sync; unsigned long long bytes_sent; unsigned long long bytes_recv; + // shunned_automatic acts as a guard for server auto-recovery. When true, the shun recovery path + // (MyHGC::get_random_MySrvC) brings the server back online after shun_recovery_time; when false, + // the shun is held until an explicit unshun. bool shunned_automatic; bool shunned_and_kill_all_connections; // if a serious failure is detected, this will cause all connections to die even if the server is just shunned int32_t use_ssl; @@ -1046,6 +1049,20 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { bool shun_and_killall(char *hostname, int port); void set_server_current_latency_us(char *hostname, int port, unsigned int _current_latency_us); void set_Readyset_status(char *hostname, int port, enum MySerStatus status); + /** + * @brief Shun or release a server across all hostgroups. + * + * @details Shunning sets shunned_and_kill_all_connections and takes 'shunned_automatic' from + * auto_recover (false = held until released; true = enables shun recovery). Releasing does + * NOT unshun directly: it only enables shun recovery (shunned_automatic=true) and leaves the + * actual unshun to the shun recovery path (MyHGC::get_random_MySrvC). + * + * @param hostname Address of the server to match. + * @param port Port of the server to match. + * @param shun true to shun the server, false to release it. + * @param auto_recover When shunning, whether the server is eligible for auto-recovery; ignored on release. + */ + void set_server_shun(char *hostname, int port, bool shun, bool auto_recover); unsigned long long Get_Memory_Stats(); void add_discovered_servers_to_mysql_servers_and_replication_hostgroups(const vector>& new_servers); diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index f60e08e320..da4489d451 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -3834,6 +3834,49 @@ void MySQL_HostGroups_Manager::set_Readyset_status(char *hostname, int port, enu wrunlock(); } +void MySQL_HostGroups_Manager::set_server_shun(char *hostname, int port, bool shun, bool auto_recover) { + wrlock(); + + MySrvC *mysrvc = NULL; + + for (unsigned int i = 0; i < MyHostGroups->len; i++) { + MyHGC *myhgc = (MyHGC *)MyHostGroups->index(i); + unsigned int l = myhgc->mysrvs->cnt(); + + for (unsigned int j = 0; j < l; j++) { + mysrvc = myhgc->mysrvs->idx(j); + + if (mysrvc->port == port && strcmp(mysrvc->address,hostname) == 0) { + if (shun) { + if (mysrvc->get_status() == MYSQL_SERVER_STATUS_ONLINE) { + mysrvc->set_status(MYSQL_SERVER_STATUS_SHUNNED); + } + // 'shunned_automatic' is the auto-recovery guard: the shun recovery path + // (MyHGC::get_random_MySrvC) only brings back servers that have it set. + // Passing auto_recover=false holds the shun until an explicit unshun. + mysrvc->shunned_automatic = auto_recover; + mysrvc->shunned_and_kill_all_connections = true; + // TODO: Check if last_detected_error should be set to a time in future, + // similar to MySQL_HostGroups_Manager::shun_and_killall() + mysrvc->time_last_detected_error = time(NULL); + mysrvc->ConnectionsFree->drop_all_connections(); + proxy_warning("Shunning server %s:%d in HG %u with auto-recovery %s\n", + hostname, port, myhgc->hid, (auto_recover) ? "enabled" : "disabled"); + } else { + // We don't unshun directly. Keep the server SHUNNED (with kill_all_connections set) + // and only enable auto-recovery; the shun recovery path (MyHGC::get_random_MySrvC) + // then brings it back online once all its old connections have drained. + // The actual unshunning work is done by MySQL_HostGroups_Manager::unshun_server_all_hostgroups + mysrvc->shunned_automatic = true; + proxy_warning("Enabling shun recovery for server %s:%d in HG %u\n", hostname, port, myhgc->hid); + } + } + } + } + + wrunlock(); +} + void MySQL_HostGroups_Manager::p_update_metrics() { p_update_counter(status.p_counter_array[p_hg_counter::servers_table_version], status.servers_table_version); // Update *server_connections* related metrics diff --git a/lib/mysql_connection.cpp b/lib/mysql_connection.cpp index a48c36f40e..a38842ce5a 100644 --- a/lib/mysql_connection.cpp +++ b/lib/mysql_connection.cpp @@ -2167,7 +2167,7 @@ bool MySQL_Connection::IsServerOffline() { if ( (server_status==MYSQL_SERVER_STATUS_OFFLINE_HARD) // the server is OFFLINE as specific by the user || - (server_status==MYSQL_SERVER_STATUS_SHUNNED && parent->shunned_automatic==true && parent->shunned_and_kill_all_connections==true) // the server is SHUNNED due to a serious issue + (server_status==MYSQL_SERVER_STATUS_SHUNNED && parent->shunned_and_kill_all_connections==true) // the server is SHUNNED due to a serious issue || (server_status==MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG) // slave is lagging! see #774 || @@ -3034,7 +3034,7 @@ int MySQL_Connection::async_send_simple_command(short event, char *stmt, unsigne if ( (parent->get_status()==MYSQL_SERVER_STATUS_OFFLINE_HARD) // the server is OFFLINE as specific by the user || - (parent->get_status()==MYSQL_SERVER_STATUS_SHUNNED && parent->shunned_automatic==true && parent->shunned_and_kill_all_connections==true) // the server is SHUNNED due to a serious issue + (parent->get_status()==MYSQL_SERVER_STATUS_SHUNNED && parent->shunned_and_kill_all_connections==true) // the server is SHUNNED due to a serious issue ) { return -1; } From cd89f4253749f1468e4afdc15c649450dd2406f4 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Mon, 22 Jun 2026 07:15:09 +0000 Subject: [PATCH 06/81] feat: Perform the AWS RDS blue/green switchover in the RDS monitor Implement `handle_aws_rds_bgd()` as a state machine over the `mysql.rds_topology` switchover status: - `AVAILABLE`: Build the `blue<->green` host map by hostname. - `SWITCHOVER_INITIATED` / `IN_PROGRESS`: Resolve the green IPs and keep them warm. - `SWITCHOVER_IN_POST_PROCESSING`: Pin each blue host onto its green IP in the DNS cache and drop the blue free connections; shun blue readers that have no green counterpart, or enforce writer_is_also_reader, to funnel reads to the green writer. - `SWITCHOVER_COMPLETED`: Undo the above and let server selection recover the readers once their connections drain. Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 2 +- include/MySQL_Monitor.hpp | 74 +++++- lib/MySQL_HostGroups_Manager.cpp | 2 +- lib/MySQL_Monitor.cpp | 402 ++++++++++++++++++++++++++--- 4 files changed, 437 insertions(+), 43 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 086c2a38ac..5ba8634afe 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -1062,7 +1062,7 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { * @param shun true to shun the server, false to release it. * @param auto_recover When shunning, whether the server is eligible for auto-recovery; ignored on release. */ - void set_server_shun(char *hostname, int port, bool shun, bool auto_recover); + void set_server_shun(const char *hostname, int port, bool shun, bool auto_recover); unsigned long long Get_Memory_Stats(); void add_discovered_servers_to_mysql_servers_and_replication_hostgroups(const vector>& new_servers); diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index b358782693..b1d348ca43 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -408,6 +408,60 @@ struct AWS_RDS_Topology_Result { std::vector nodes; }; +/** + * @brief Per-deployment switchover state carried by one RDS BGD worker thread. + * + * @details One worker (monitor_RDS_BGD_thread_HG) owns one writer hostgroup == + * one blue/green deployment, so this struct lives on the worker's stack and is + * single-owner (no locking on the struct itself). It is passed by reference to + * handle_aws_rds_bgd, which runs the status-driven switchover FSM and mutates it + * across poll cycles. Config-derived fields are loaded once from the resultset; + * the rest carries topology, resolved IPs, and one-shot enforcement bookkeeping. + */ +struct AWS_RDS_BGD_State { + unsigned int writer_hg = 0; ///< blue/current writer hostgroup + unsigned int reader_hg = 0; ///< blue/current reader hostgroup + int green_writer_hg = -1; ///< -1 when NULL (auto-discovery path) + int green_reader_hg = -1; ///< -1 when NULL + + /// One blue host and its name-matched green counterpart. + struct BlueGreenPair { + std::string blue_host; ///< blue host (from writer/reader HG) + std::string green_host; ///< matched green host (-green-) + int port = 0; ///< shared blue/green port (HGM keys on host+port) + int64_t blue_weight = 1; ///< blue server's connection settings, mirrored onto green when added + int64_t blue_max_conns = 1000; + int32_t blue_use_ssl = 0; + std::string green_ip; ///< green host IP, resolved at SWITCHOVER_INITIATED and held warm + unsigned long long green_ip_ttl = 0; ///< expiry of a green_ip if it is resolved by BGD thread; 0 => DNS_Cache-sourced + bool is_writer = false; ///< true => maps the blue writer + }; + std::vector bg_map; ///< [writer] always; [readers] only when green_reader_hg is configured + std::vector> shunned_readers; ///< (host,port) we shunned + std::string last_status; ///< status from the previous poll, to act only when it changes + + bool writer_is_also_reader_enforced = false; ///< whether POST_PROCESSING added the writer to the reader HG + + unsigned int next_check_interval_ms = 0; ///< FSM-controlled interval; 0 => baseline +}; + +/** +* @brief State of the per-host RDS topology probe. +*/ +enum RDS_BGD_Topology_Monitor_State { + TOPOLOGY_TABLE_CHECK, ///< verify mysql.rds_topology exists + TOPOLOGY_METADATA_FETCH ///< table confirmed present; fetch and branch on its metadata +}; + +// AWS RDS blue/green role and switchover-status column values (mysql.rds_topology). +inline const char* const BGD_ROLE_SOURCE = "BLUE_GREEN_DEPLOYMENT_SOURCE"; // blue +inline const char* const BGD_ROLE_TARGET = "BLUE_GREEN_DEPLOYMENT_TARGET"; // green +inline const char* const BGD_STATUS_AVAILABLE = "AVAILABLE"; +inline const char* const BGD_STATUS_INITIATED = "SWITCHOVER_INITIATED"; +inline const char* const BGD_STATUS_IN_PROGRESS = "SWITCHOVER_IN_PROGRESS"; +inline const char* const BGD_STATUS_POST_PROC = "SWITCHOVER_IN_POST_PROCESSING"; +inline const char* const BGD_STATUS_COMPLETED = "SWITCHOVER_COMPLETED"; + class MySQL_Monitor { public: @@ -525,18 +579,20 @@ class MySQL_Monitor { */ void * monitor_aws_rds_bgd(); /** - * @brief Handle a blue/green deployment topology fetched by the BGD thread. + * @brief Run the status-driven blue/green switchover FSM for one deployment. * - * @details Invoked when the BGD thread fetches a blue/green deployment topology from - * mysql.rds_topology; performs the blue/green switchover. + * @details Invoked each poll cycle by the BGD worker after it fetches the + * mysql.rds_topology result. Dispatches on the deployment's switchover status + * (AVAILABLE -> SWITCHOVER_INITIATED -> IN_PROGRESS -> IN_POST_PROCESSING -> + * COMPLETED): builds the blue<->green map, pre-resolves green IPs, repoints the + * blue hostnames onto the green IPs in the DNS cache, drains blue free + * connections, and shuns/enforces reader handling. State carried across cycles + * lives in @p st. * - * @param whg Writer hostgroup (blue/current writer). - * @param rhg Reader hostgroup (blue/current readers). - * @param green_whg Configured green writer hostgroup, or -1 if unset. - * @param green_rhg Configured green reader hostgroup, or -1 if unset. - * @param topology Parsed mysql.rds_topology result. + * @param st Per-deployment switchover state (worker-owned, mutated here). + * @param topology Parsed mysql.rds_topology result for this cycle. */ - void handle_aws_rds_bgd(unsigned int whg, unsigned int rhg, int green_whg, int green_rhg, const AWS_RDS_Topology_Result& topology); + void handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_Result& topology); void * monitor_replication_lag(); void * monitor_dns_cache(); void * run(); diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index da4489d451..29eab93098 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -3834,7 +3834,7 @@ void MySQL_HostGroups_Manager::set_Readyset_status(char *hostname, int port, enu wrunlock(); } -void MySQL_HostGroups_Manager::set_server_shun(char *hostname, int port, bool shun, bool auto_recover) { +void MySQL_HostGroups_Manager::set_server_shun(const char *hostname, int port, bool shun, bool auto_recover) { wrlock(); MySrvC *mysrvc = NULL; diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 4f94d44002..08751d0c3b 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6540,23 +6540,12 @@ static int aws_rds_bgd_async_query(MySQL_Monitor_State_Data *mmsd, const char *q return 0; } -/** -* @brief State of the per-host RDS topology probe. -*/ -enum RDS_BGD_Topology_Monitor_State { - TOPOLOGY_TABLE_CHECK, ///< verify mysql.rds_topology exists - TOPOLOGY_METADATA_FETCH ///< table confirmed present; fetch and branch on its metadata -}; - void * monitor_RDS_BGD_thread_HG(void *arg) { unsigned int wHG = *(unsigned int *)arg; - unsigned int rHG = 0; unsigned int num_hosts = 0; unsigned int cur_host_idx = 0; unsigned int check_interval_ms = 0; unsigned int check_timeout_ms = 0; - int green_writer_hostgroup = -1; - int green_reader_hostgroup = -1; set_thread_name("MonitorRdsBgdHG", GloVars.set_thread_name); proxy_info("Started Monitor thread for AWS RDS writer HG %u\n", wHG); @@ -6564,6 +6553,9 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { if (!wait_for_glo_mth()) return NULL; + AWS_RDS_BGD_State st; + st.writer_hg = wHG; + unsigned int MySQL_Monitor__thread_MySQL_Thread_Variables_version; MySQL_Thread * mysql_thr = new MySQL_Thread(); mysql_thr->curtime = monotonic_time(); @@ -6582,14 +6574,14 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { for (SQLite3_row *r : GloMyMon->AWS_RDS_BGD_Hosts_resultset->rows) { if (atoi(r->fields[0]) == (int)wHG) { num_hosts++; - if (rHG == 0) { - rHG = atoi(r->fields[1]); + if (st.reader_hg == 0) { + st.reader_hg = atoi(r->fields[1]); } - if (green_writer_hostgroup < 0 && r->fields[5] && r->fields[5][0]) { - green_writer_hostgroup = atoi(r->fields[5]); + if (st.green_writer_hg < 0 && r->fields[5] && r->fields[5][0]) { + st.green_writer_hg = atoi(r->fields[5]); } - if (green_reader_hostgroup < 0 && r->fields[6] && r->fields[6][0]) { - green_reader_hostgroup = atoi(r->fields[6]); + if (st.green_reader_hg < 0 && r->fields[6] && r->fields[6][0]) { + st.green_reader_hg = atoi(r->fields[6]); } if (check_interval_ms == 0) { check_interval_ms = atoi(r->fields[7]); @@ -6599,6 +6591,7 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { } } } + host_def_t *hpa = (host_def_t *)malloc(sizeof(host_def_t)*(num_hosts ? num_hosts : 1)); for (SQLite3_row *r : GloMyMon->AWS_RDS_BGD_Hosts_resultset->rows) { if (atoi(r->fields[0]) == (int)wHG) { @@ -6767,8 +6760,10 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { unsigned int err = mmsd->mysql ? mysql_errno(mmsd->mysql) : 0; if (err == 1146) { // the table vanished (ER_NO_SUCH_TABLE), e.g. a blue/green deployment - // was cancelled: re-check its existence on the next iteration. + // was cancelled: re-check its existence on the next iteration and + // return to the baseline poll interval. topology_state = TOPOLOGY_TABLE_CHECK; + st.next_check_interval_ms = 0; proxy_debug(PROXY_DEBUG_MONITOR, 5, "mysql.rds_topology vanished on %s:%d (RDS writer HG %u); rechecking availability\n", mmsd->hostname, mmsd->port, wHG); @@ -6785,8 +6780,12 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { // (shared with the read_only path) and hand the struct to the handler. if (mmsd->result && mysql_num_rows(mmsd->result) > 0) { AWS_RDS_Topology_Result topo = GloMyMon->parse_aws_rds_topology(mmsd->result); - GloMyMon->handle_aws_rds_bgd(wHG, rHG, green_writer_hostgroup, green_reader_hostgroup, topo); + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "AWS RDS BGD [wHG=%u]: topology probe on %s:%d (blue_green=%d, nodes=%zu)\n", + wHG, mmsd->hostname, mmsd->port, topo.blue_green ? 1 : 0, topo.nodes.size()); + GloMyMon->handle_aws_rds_bgd(st, topo); } + if (mmsd->result) { mysql_free_result(mmsd->result); mmsd->result = NULL; @@ -6795,7 +6794,10 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { __end_of_loop: mmsd->t2 = monotonic_time(); - next_loop_at = t1 + (check_interval_ms * 1000); + // the FSM tightens the interval to 100ms while a switchover is in flight + // (st.next_check_interval_ms); otherwise fall back to the configured baseline. + unsigned int eff = st.next_check_interval_ms ? st.next_check_interval_ms : check_interval_ms; + next_loop_at = t1 + (eff * 1000); if (mmsd->t2 > t1) { next_loop_at -= (mmsd->t2 - t1); } @@ -6833,22 +6835,358 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { return NULL; } +// Split "." into the host (part before the first dot) and the remaining domain. +static void aws_rds_bgd_split_hostname(const std::string& hostname, std::string& host, std::string& domain) { + size_t dot = hostname.find('.'); + if (dot == std::string::npos) { + host = hostname; + domain.clear(); + return; + } + host = hostname.substr(0, dot); + domain = hostname.substr(dot + 1); +} + +// Given a green host "-green-", return ""; +// returns empty when the "-green-" suffix is absent. +static std::string aws_rds_bgd_strip_green_host_suffix(const std::string& green_host) { + size_t pos = green_host.find("-green-"); + if (pos == std::string::npos) { + return ""; + } + return green_host.substr(0, pos); +} + +// True when the green hostname is the blue/green TARGET counterpart of the blue hostname, i.e. +// green "-green-." maps to blue ".". +static bool aws_rds_bgd_match_host(const std::string& blue_hostname, const std::string& green_hostname) { + std::string b_host, b_domain, g_host, g_domain; + aws_rds_bgd_split_hostname(blue_hostname, b_host, b_domain); + aws_rds_bgd_split_hostname(green_hostname, g_host, g_domain); + std::string g_host_stripped = aws_rds_bgd_strip_green_host_suffix(g_host); + if (g_host_stripped.empty()) { + return false; + } + return g_host_stripped == b_host && g_domain == b_domain; +} + +// Build the blue<->green map once (refreshed only after a switchover completes and +// clears it, which also covers a worker that starts mid-switchover). The topology +// exposes only primaries, so the writer pair is always present; reader pairs exist +// only when green_reader_hostgroup is configured (the user populated it). +static void aws_rds_bgd_build_map(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_Result& topo) { + if (!st.bg_map.empty()) { + return; + } + + std::string green_writer_host; + for (const AWS_RDS_Topology_Node& n : topo.nodes) { + if (strcasecmp(n.role.c_str(), BGD_ROLE_TARGET) == 0) { + green_writer_host = n.endpoint; + break; + } + } + if (green_writer_host.empty()) { + return; + } + + MyHGM->wrlock(); + + // blue writer: the writer_hostgroup member whose name matches the green TARGET. + MyHGC* whgc = MyHGM->MyHGC_lookup(st.writer_hg); + if (whgc && whgc->mysrvs) { + for (unsigned int j = 0; j < whgc->mysrvs->cnt(); j++) { + MySrvC* s = whgc->mysrvs->idx(j); + if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + continue; + } + if (aws_rds_bgd_match_host(s->address, green_writer_host)) { + AWS_RDS_BGD_State::BlueGreenPair p; + p.blue_host = s->address; + p.port = s->port; + p.green_host = green_writer_host; + p.blue_weight = s->weight; + p.blue_max_conns = s->max_connections; + p.blue_use_ssl = s->use_ssl; + p.is_writer = true; + proxy_debug(PROXY_DEBUG_MONITOR, 7, + "AWS RDS BGD [wHG=%u]: mapped blue writer '%s:%d' <-> green '%s'\n", + st.writer_hg, p.blue_host.c_str(), p.port, p.green_host.c_str()); + st.bg_map.push_back(std::move(p)); + break; + } + } + } + + // reader pairs: match blue readers to user-added green readers by name. + if (st.green_reader_hg >= 0) { + std::vector green_reader_hosts; + MyHGC* grhgc = MyHGM->MyHGC_lookup((unsigned int)st.green_reader_hg); + if (grhgc && grhgc->mysrvs) { + for (unsigned int j = 0; j < grhgc->mysrvs->cnt(); j++) { + MySrvC* s = grhgc->mysrvs->idx(j); + if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + continue; + } + green_reader_hosts.push_back(s->address); + } + } + + MyHGC* rhgc = MyHGM->MyHGC_lookup(st.reader_hg); + if (rhgc && rhgc->mysrvs) { + for (unsigned int j = 0; j < rhgc->mysrvs->cnt(); j++) { + MySrvC* s = rhgc->mysrvs->idx(j); + if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + continue; + } + for (const std::string& green_reader_host : green_reader_hosts) { + if (aws_rds_bgd_match_host(s->address, green_reader_host)) { + AWS_RDS_BGD_State::BlueGreenPair p; + p.blue_host = s->address; + p.port = s->port; + p.green_host = green_reader_host; + p.blue_weight = s->weight; + p.blue_max_conns = s->max_connections; + p.blue_use_ssl = s->use_ssl; + p.is_writer = false; + proxy_debug(PROXY_DEBUG_MONITOR, 7, + "AWS RDS BGD [wHG=%u]: mapped blue reader '%s:%d' <-> green '%s'\n", + st.writer_hg, p.blue_host.c_str(), p.port, p.green_host.c_str()); + st.bg_map.push_back(std::move(p)); + break; + } + } + } + } + } + + MyHGM->wrunlock(); +} + /** -* @brief Handle a blue/green deployment topology fetched by the BGD thread. +* @brief Run the status-driven blue/green switchover FSM for one deployment. +* +* @details Dispatches on the deployment switchover status read from the SOURCE row +* of mysql.rds_topology. State carried across poll cycles (the blue<->green map, +* resolved green IPs, enforcement bookkeeping, and the next poll interval) lives in +* 'st', owned by the calling worker thread. * -* @param whg Writer hostgroup (blue/current writer). -* @param rhg Reader hostgroup (blue/current readers). -* @param green_whg Configured green writer hostgroup, or -1 if unset. -* @param green_rhg Configured green reader hostgroup, or -1 if unset. -* @param topology Parsed mysql.rds_topology result. +* @param st Per-deployment switchover state (worker-owned, mutated here). +* @param topology Parsed mysql.rds_topology result for this cycle. */ -void MySQL_Monitor::handle_aws_rds_bgd(unsigned int whg, unsigned int rhg, int green_whg, int green_rhg, const AWS_RDS_Topology_Result& topology) { - proxy_debug(PROXY_DEBUG_MONITOR, 5, - "AWS RDS BGD: blue/green topology fetched for writer HG %u (reader HG %u, green w/r HG %d/%d)," - " %zu nodes; no action taken\n", - whg, rhg, green_whg, green_rhg, topology.nodes.size()); +void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_Result& topology) { + if (!topology.blue_green) { + st.next_check_interval_ms = 0; + return; + } + + // deployment switchover status comes from the SOURCE (blue) row + std::string status; + for (const AWS_RDS_Topology_Node& n : topology.nodes) { + if (strcasecmp(n.role.c_str(), BGD_ROLE_SOURCE) == 0) { + status = n.status; + break; + } + } + if (status.empty()) { + st.next_check_interval_ms = 0; + return; + } + + if (strcasecmp(status.c_str(), st.last_status.c_str()) != 0) { + proxy_info( + "AWS RDS BGD [wHG=%u rHG=%u]: switchover status '%s' -> '%s'\n", + st.writer_hg, st.reader_hg, + st.last_status.empty() ? "(none)" : st.last_status.c_str(), status.c_str()); + } + + if (strcasecmp(status.c_str(), BGD_STATUS_AVAILABLE) == 0) { + aws_rds_bgd_build_map(st, topology); + + // Add the green writer to green_writer_hostgroup when configured, mirroring the blue + // writer's connection settings (weight/max_connections/use_ssl) onto it. + if (st.green_writer_hg >= 0) { + for (const AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { + if (p.is_writer) { + srv_info_t srv_info { p.green_host, (uint16_t)p.port, "AWS RDS BGD green writer" }; + srv_opts_t srv_opts { p.blue_weight, p.blue_max_conns, p.blue_use_ssl }; + MyHGM->wrlock(); + MyHGM->create_new_server_in_hg((uint32_t)st.green_writer_hg, srv_info, srv_opts); + MyHGM->wrunlock(); + break; + } + } + } + + st.next_check_interval_ms = 0; + } + else if (strcasecmp(status.c_str(), BGD_STATUS_INITIATED) == 0 + || strcasecmp(status.c_str(), BGD_STATUS_IN_PROGRESS) == 0) { + aws_rds_bgd_build_map(st, topology); + + // Resolve the green IPs and keep them warm for an instant POST_PROCESSING repoint. + int ai_family = mysql_resolution_family_to_ai_family(mysql_thread___resolution_family); + for (auto &p : st.bg_map) { + // Always check the cache first: a green host that is a monitored server may be there. + size_t n = 0; + std::string ip = MySQL_Monitor::dns_lookup(p.green_host, false, &n); + if (!ip.empty()) { + p.green_ip = ip; + p.green_ip_ttl = 0; + proxy_debug(PROXY_DEBUG_MONITOR, 7, + "AWS RDS BGD [wHG=%u]: green '%s' IP %s (DNS_Cache)\n", + st.writer_hg, p.green_host.c_str(), p.green_ip.c_str()); + continue; + } + // Cache miss (green is not a monitored server): resolve DNS now and track its TTL. + if (p.green_ip.empty() || __builtin_expect(p.green_ip_ttl != 0 && monotonic_time() > p.green_ip_ttl, 0)) { + std::vector ips = dns_resolve(p.green_host, ai_family); + if (!ips.empty()) { + p.green_ip = ips.front(); + p.green_ip_ttl = monotonic_time() + + (1000ULL * (unsigned long long)mysql_thread___monitor_local_dns_cache_ttl); + proxy_debug(PROXY_DEBUG_MONITOR, 7, + "AWS RDS BGD [wHG=%u]: green '%s' IP %s (resolved, ttl=%lus)\n", + st.writer_hg, p.green_host.c_str(), p.green_ip.c_str(), + (unsigned long)mysql_thread___monitor_local_dns_cache_ttl); + } + } + } + st.next_check_interval_ms = 100; + } + else if (strcasecmp(status.c_str(), BGD_STATUS_POST_PROC) == 0) { + st.next_check_interval_ms = 100; + + // Run the steps below only on the first poll after entering POST_PROCESSING; + // later polls see the same status, so there is nothing to redo. + if (strcasecmp(status.c_str(), st.last_status.c_str()) == 0) { + return; + } + + aws_rds_bgd_build_map(st, topology); + + // TODO: Handle the case where thread observes POST_PROCESSING status directly, + // without observing SWITCHOVER_INITIATED first (resolve green IPs). + + // Repoint each mapped blue host onto its green IP and drain the blue free + // pool so new connections resolve to green. + bool any_reader_mapped = false; + for (AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { + if (!p.is_writer) { + any_reader_mapped = true; + } + if (p.green_ip.empty()) { + proxy_warning( + "AWS RDS BGD [wHG=%u rHG=%u]: no green IP for blue '%s:%d'; cannot repoint\n", + st.writer_hg, st.reader_hg, p.blue_host.c_str(), p.port); + continue; + } + dns_cache->pin(p.blue_host, { p.green_ip }); + proxy_info( + "AWS RDS BGD [wHG=%u rHG=%u]: repointed blue '%s' to green IP %s\n", + st.writer_hg, st.reader_hg, p.blue_host.c_str(), p.green_ip.c_str()); + + // TODO: Draining blue free connection pool is not enough + // Kill used connection without SHUNNING the server + unsigned int hid = p.is_writer ? st.writer_hg : st.reader_hg; + MyHGM->wrlock(); + MySrvC* s = MyHGM->find_server_in_hg(hid, p.blue_host, p.port); + if (s) { + s->ConnectionsFree->drop_all_connections(); + } + MyHGM->wrunlock(); + } + + // Blue readers without a green counterpart must stop serving reads. + std::vector> blue_readers; + MyHGM->wrlock(); + MyHGC* rhgc = MyHGM->MyHGC_lookup(st.reader_hg); + if (rhgc && rhgc->mysrvs) { + for (unsigned int j = 0; j < rhgc->mysrvs->cnt(); j++) { + MySrvC* s = rhgc->mysrvs->idx(j); + if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + continue; + } + blue_readers.push_back({ std::string(s->address), s->port }); + } + } + MyHGM->wrunlock(); + for (const std::pair& br : blue_readers) { + bool mapped = false; + for (const AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { + if (!p.is_writer && p.blue_host == br.first && p.port == br.second) { + mapped = true; + break; + } + } + if (!mapped) { + MyHGM->set_server_shun(br.first.c_str(), br.second, true, false); + st.shunned_readers.push_back(br); + } + } + + // If no reader is mapped, funnel reads to the (now green) writer by + // enforcing writer_is_also_reader for the duration of the switchover. + if (!any_reader_mapped) { + for (const AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { + if (p.is_writer) { + srv_info_t srv_info { p.blue_host, (uint16_t)p.port, "AWS RDS BGD writer_is_also_reader" }; + srv_opts_t srv_opts { p.blue_weight, p.blue_max_conns, p.blue_use_ssl }; + MyHGM->wrlock(); + MyHGM->create_new_server_in_hg(st.reader_hg, srv_info, srv_opts); + MyHGM->wrunlock(); + st.writer_is_also_reader_enforced = true; + break; + } + } + } + } + else if (strcasecmp(status.c_str(), BGD_STATUS_COMPLETED) == 0) { + st.next_check_interval_ms = 0; + // Undo everything only on the first poll after entering COMPLETED; + // later polls see the same status, so there is nothing to redo. + if (strcasecmp(status.c_str(), st.last_status.c_str()) == 0) { + return; + } + + if (st.writer_is_also_reader_enforced) { + for (const AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { + if (p.is_writer) { + MyHGM->wrlock(); + MyHGM->remove_server_in_hg(st.reader_hg, p.blue_host, (uint16_t)p.port); + MyHGM->wrunlock(); + break; + } + } + st.writer_is_also_reader_enforced = false; + } + + if (!st.shunned_readers.empty()) { + for (const std::pair& br : st.shunned_readers) { + MyHGM->set_server_shun(br.first.c_str(), br.second, false, false); + // purge so the blue reader hostname re-resolves to the promoted instance + dns_cache->remove(br.first); + } + st.shunned_readers.clear(); + } + + for (const AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { + dns_cache->unpin(p.blue_host); + } + + // TODO: Drain Green HGs + + proxy_info( + "AWS RDS BGD [wHG=%u rHG=%u]: switchover complete; state cleared\n", + st.writer_hg, st.reader_hg); + st.bg_map.clear(); + } + else { + // unknown status: take no action, stay at baseline interval + st.next_check_interval_ms = 0; + } - // TODO: Complete this + st.last_status = status; } /** From 3b2fe9a4ac5ee6351f462b749c985943fbb15860 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Mon, 22 Jun 2026 12:58:18 +0000 Subject: [PATCH 07/81] fix: Ignore `Table doesn't exist` error for `mysql.rds_topology` Signed-off-by: Wazir Ahmed --- include/ProxySQL_Admin_Tables_Definitions.h | 4 ++-- lib/MySQL_Monitor.cpp | 13 +++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/include/ProxySQL_Admin_Tables_Definitions.h b/include/ProxySQL_Admin_Tables_Definitions.h index 725514277d..8c18870af6 100644 --- a/include/ProxySQL_Admin_Tables_Definitions.h +++ b/include/ProxySQL_Admin_Tables_Definitions.h @@ -241,7 +241,7 @@ "green_writer_hostgroup INT NOT NULL CHECK (green_writer_hostgroup>=0) , " \ "green_reader_hostgroup INT NOT NULL CHECK (green_reader_hostgroup>=0) , " \ "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , " \ - "domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , " \ + "domain_name VARCHAR NOT NULL CHECK (domain_name = '' OR SUBSTR(domain_name,1,1) = '.') , " \ "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , " \ "check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , " \ "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , " \ @@ -251,7 +251,7 @@ "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0) , " \ "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0) , " \ "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , " \ - "domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , " \ + "domain_name VARCHAR NOT NULL CHECK (domain_name = '' OR SUBSTR(domain_name,1,1) = '.') , " \ "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , " \ "check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , " \ "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , " \ diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 08751d0c3b..caab9ad4ff 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -1892,8 +1892,17 @@ VALGRIND_ENABLE_ERROR_REPORTING; } if (mmsd->interr || mmsd->mysql_error_msg) { // check failed if (mmsd->mysql) { - proxy_error("Got error: mmsd %p , MYSQL %p , FD %d : %s\n", mmsd, mmsd->mysql, mmsd->mysql->net.fd, mmsd->mysql_error_msg); - MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, mysql_errno(mmsd->mysql)); + // AWS RDS topology discovery probes every replication-hostgroup member, but + // mysql.rds_topology exists only where a blue/green deployment is active. + // Treat ER_NO_SUCH_TABLE (1146) as "no topology here" and skip quietly. + if (mmsd->get_task_type() == MON_AWS_RDS_TOPOLOGY_DISCOVERY && mysql_errno(mmsd->mysql) == 1146) { + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "mysql.rds_topology not present on %s:%d; skipping blue/green discovery\n", + mmsd->hostname, mmsd->port); + } else { + proxy_error("Got error: mmsd %p , MYSQL %p , FD %d : %s\n", mmsd, mmsd->mysql, mmsd->mysql->net.fd, mmsd->mysql_error_msg); + MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, mysql_errno(mmsd->mysql)); + } GloMyMon->My_Conn_Pool->destroy_mysql_connection(mmsd); } } else { From 198e32352d6b36be390a717c807aaf7f7df23aa0 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Mon, 22 Jun 2026 15:54:17 +0000 Subject: [PATCH 08/81] refactor: Use green writer IP in AWS RDS to poll for topology switchover status Signed-off-by: Wazir Ahmed --- include/MySQL_Monitor.hpp | 3 + lib/MySQL_Monitor.cpp | 245 +++++++++++++++++++++++--------------- 2 files changed, 149 insertions(+), 99 deletions(-) diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index b1d348ca43..80198196ef 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -440,9 +440,12 @@ struct AWS_RDS_BGD_State { std::vector> shunned_readers; ///< (host,port) we shunned std::string last_status; ///< status from the previous poll, to act only when it changes + bool green_writer_added_in_hg = false; ///< whether green writer added to green_writer_hg bool writer_is_also_reader_enforced = false; ///< whether POST_PROCESSING added the writer to the reader HG unsigned int next_check_interval_ms = 0; ///< FSM-controlled interval; 0 => baseline + std::string next_check_host; ///< FSM-pinned probe host; when set (the green IP), the worker + ///< polls it directly instead of selecting among the blue hosts }; /** diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index caab9ad4ff..6f18840282 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6667,39 +6667,53 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { continue; } - // pick a pingable host: random first, then shuffle and scan - found_pingable_host = false; - rnd = (size_t) rand(); - rnd %= num_hosts; - rc_ping = GloMyMon->server_responds_to_ping(hpa[rnd].host, hpa[rnd].port); - if (rc_ping) { - found_pingable_host = true; - cur_host_idx = rnd; + // Determine the host to probe. If the FSM pinned a host (the green IP, during a + // switchover), poll it directly and skip ping/random selection; otherwise pick a + // pingable host (random first, then shuffle and scan). + const char* poll_host; + int poll_port; + bool poll_use_ssl; + if (!st.next_check_host.empty()) { + poll_host = st.next_check_host.c_str(); + poll_port = hpa[0].port; // port/use_ssl are uniform across the deployment + poll_use_ssl = hpa[0].use_ssl; } else { - MyHGM->p_update_mysql_error_counter( - p_mysql_error_type::proxysql, wHG, hpa[rnd].host, hpa[rnd].port, ER_PROXYSQL_AWS_NO_PINGABLE_SRV - ); - shuffle_hosts(hpa, num_hosts); - for (unsigned int i=0; (found_pingable_host == false && iserver_responds_to_ping(hpa[i].host, hpa[i].port); - if (rc_ping) { - found_pingable_host = true; - cur_host_idx = i; - } else { - MyHGM->p_update_mysql_error_counter( - p_mysql_error_type::proxysql, wHG, hpa[i].host, hpa[i].port, ER_PROXYSQL_AWS_NO_PINGABLE_SRV - ); + found_pingable_host = false; + rnd = (size_t) rand(); + rnd %= num_hosts; + rc_ping = GloMyMon->server_responds_to_ping(hpa[rnd].host, hpa[rnd].port); + if (rc_ping) { + found_pingable_host = true; + cur_host_idx = rnd; + } else { + MyHGM->p_update_mysql_error_counter( + p_mysql_error_type::proxysql, wHG, hpa[rnd].host, hpa[rnd].port, ER_PROXYSQL_AWS_NO_PINGABLE_SRV + ); + shuffle_hosts(hpa, num_hosts); + for (unsigned int i=0; (found_pingable_host == false && iserver_responds_to_ping(hpa[i].host, hpa[i].port); + if (rc_ping) { + found_pingable_host = true; + cur_host_idx = i; + } else { + MyHGM->p_update_mysql_error_counter( + p_mysql_error_type::proxysql, wHG, hpa[i].host, hpa[i].port, ER_PROXYSQL_AWS_NO_PINGABLE_SRV + ); + } } } - } - if (found_pingable_host == false) { - proxy_error("No node is pingable for AWS RDS cluster with writer HG %u\n", wHG); - next_loop_at = t1 + check_interval_ms * 1000; - continue; + if (found_pingable_host == false) { + proxy_error("No node is pingable for AWS RDS cluster with writer HG %u\n", wHG); + next_loop_at = t1 + check_interval_ms * 1000; + continue; + } + poll_host = hpa[cur_host_idx].host; + poll_port = hpa[cur_host_idx].port; + poll_use_ssl = hpa[cur_host_idx].use_ssl; } mmsd = new MySQL_Monitor_State_Data( - MON_AWS_RDS_BGD, hpa[cur_host_idx].host, hpa[cur_host_idx].port, hpa[cur_host_idx].use_ssl + MON_AWS_RDS_BGD, (char*)poll_host, poll_port, poll_use_ssl ); mmsd->writer_hostgroup = wHG; mmsd->aws_aurora_check_timeout_ms = check_timeout_ms; @@ -6972,13 +6986,90 @@ static void aws_rds_bgd_build_map(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_ MyHGM->wrunlock(); } +/** +* @brief Resolve the green IPs and pin the worker's probe to the green writer's IP. +* +* @details Resolves each pair's green host (DNS_Cache first, then a live lookup tracking TTL), +* then sets 'st.next_check_host' to the green writer's IP. From then on the worker polls the +* green primary BY IP: green stays reachable through the entire cutover (blue has a connectivity +* gap), and the green IP survives the post-COMPLETED name swap (it becomes the promoted primary), +* whereas the green DNS name is retired. 'next_check_host' is cleared at COMPLETED. +*/ +static void aws_rds_bgd_resolve_green_ips(AWS_RDS_BGD_State& st) { + int ai_family = mysql_resolution_family_to_ai_family(mysql_thread___resolution_family); + for (auto &p : st.bg_map) { + // Always check the cache first: a green host that is a monitored server may be there. + size_t n = 0; + std::string ip = MySQL_Monitor::dns_lookup(p.green_host, false, &n); + if (!ip.empty()) { + p.green_ip = ip; + p.green_ip_ttl = 0; + proxy_debug(PROXY_DEBUG_MONITOR, 7, + "AWS RDS BGD [wHG=%u]: green '%s' IP %s (DNS_Cache)\n", + st.writer_hg, p.green_host.c_str(), p.green_ip.c_str()); + continue; + } + // Cache miss (green is not a monitored server): resolve DNS now and track its TTL. + if (p.green_ip.empty() || (p.green_ip_ttl != 0 && monotonic_time() > p.green_ip_ttl)) { + std::vector ips = dns_resolve(p.green_host, ai_family); + if (!ips.empty()) { + p.green_ip = ips.front(); + p.green_ip_ttl = monotonic_time() + + (1000ULL * (unsigned long long)mysql_thread___monitor_local_dns_cache_ttl); + proxy_debug(PROXY_DEBUG_MONITOR, 7, + "AWS RDS BGD [wHG=%u]: green '%s' IP %s (resolved, ttl=%lus)\n", + st.writer_hg, p.green_host.c_str(), p.green_ip.c_str(), + (unsigned long)mysql_thread___monitor_local_dns_cache_ttl); + } + } + } + + // Pin the worker's next probe to the green writer's IP (observe the switchover from green). + for (const AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { + if (p.is_writer && !p.green_ip.empty()) { + if (st.next_check_host != p.green_ip) { + st.next_check_host = p.green_ip; + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "AWS RDS BGD [wHG=%u]: pinning topology probe to green IP %s\n", + st.writer_hg, p.green_ip.c_str()); + } + break; + } + } +} + +/** +* @brief Add the green writer to green_writer_hostgroup, when that hostgroup is configured. +* +* @details Mirrors the blue writer's connection settings (weight/max_connections/use_ssl) onto +* the green writer. No-op when green_writer_hostgroup is NULL (the auto-discovery path) or when +* the green writer was already added on a prior poll. +*/ +static void aws_rds_bgd_add_green_writer_in_hg(AWS_RDS_BGD_State& st) { + if (st.green_writer_hg < 0 || st.green_writer_added_in_hg) { + return; + } + for (const AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { + if (p.is_writer) { + srv_info_t srv_info { p.green_host, (uint16_t)p.port, "AWS RDS BGD green writer" }; + srv_opts_t srv_opts { p.blue_weight, p.blue_max_conns, p.blue_use_ssl }; + MyHGM->wrlock(); + MyHGM->create_new_server_in_hg((uint32_t)st.green_writer_hg, srv_info, srv_opts); + MyHGM->wrunlock(); + st.green_writer_added_in_hg = true; + break; + } + } +} + /** * @brief Run the status-driven blue/green switchover FSM for one deployment. * -* @details Dispatches on the deployment switchover status read from the SOURCE row -* of mysql.rds_topology. State carried across poll cycles (the blue<->green map, -* resolved green IPs, enforcement bookkeeping, and the next poll interval) lives in -* 'st', owned by the calling worker thread. +* @details Dispatches on the deployment switchover status read from the TARGET (green) row +* of mysql.rds_topology (the TARGET row carries the status in every phase, including +* COMPLETED where the SOURCE row is absent). State carried across poll cycles (the +* blue<->green map, resolved green IPs, the pinned probe host, enforcement bookkeeping, +* and the next poll interval) lives in 'st', owned by the calling worker thread. * * @param st Per-deployment switchover state (worker-owned, mutated here). * @param topology Parsed mysql.rds_topology result for this cycle. @@ -6989,10 +7080,9 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo return; } - // deployment switchover status comes from the SOURCE (blue) row std::string status; for (const AWS_RDS_Topology_Node& n : topology.nodes) { - if (strcasecmp(n.role.c_str(), BGD_ROLE_SOURCE) == 0) { + if (strcasecmp(n.role.c_str(), BGD_ROLE_TARGET) == 0) { status = n.status; break; } @@ -7002,80 +7092,40 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo return; } - if (strcasecmp(status.c_str(), st.last_status.c_str()) != 0) { - proxy_info( - "AWS RDS BGD [wHG=%u rHG=%u]: switchover status '%s' -> '%s'\n", - st.writer_hg, st.reader_hg, - st.last_status.empty() ? "(none)" : st.last_status.c_str(), status.c_str()); + if (strcasecmp(status.c_str(), st.last_status.c_str()) == 0) { + // no state change + return; } - if (strcasecmp(status.c_str(), BGD_STATUS_AVAILABLE) == 0) { - aws_rds_bgd_build_map(st, topology); + proxy_info( + "AWS RDS BGD [wHG=%u rHG=%u]: switchover status '%s' -> '%s'\n", + st.writer_hg, st.reader_hg, + st.last_status.empty() ? "(none)" : st.last_status.c_str(), status.c_str()); - // Add the green writer to green_writer_hostgroup when configured, mirroring the blue - // writer's connection settings (weight/max_connections/use_ssl) onto it. - if (st.green_writer_hg >= 0) { - for (const AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { - if (p.is_writer) { - srv_info_t srv_info { p.green_host, (uint16_t)p.port, "AWS RDS BGD green writer" }; - srv_opts_t srv_opts { p.blue_weight, p.blue_max_conns, p.blue_use_ssl }; - MyHGM->wrlock(); - MyHGM->create_new_server_in_hg((uint32_t)st.green_writer_hg, srv_info, srv_opts); - MyHGM->wrunlock(); - break; - } - } - } + if (strcasecmp(status.c_str(), BGD_STATUS_AVAILABLE) == 0) { + st.next_check_interval_ms = 250; - st.next_check_interval_ms = 0; + aws_rds_bgd_build_map(st, topology); + aws_rds_bgd_resolve_green_ips(st); + aws_rds_bgd_add_green_writer_in_hg(st); } else if (strcasecmp(status.c_str(), BGD_STATUS_INITIATED) == 0 || strcasecmp(status.c_str(), BGD_STATUS_IN_PROGRESS) == 0) { - aws_rds_bgd_build_map(st, topology); - - // Resolve the green IPs and keep them warm for an instant POST_PROCESSING repoint. - int ai_family = mysql_resolution_family_to_ai_family(mysql_thread___resolution_family); - for (auto &p : st.bg_map) { - // Always check the cache first: a green host that is a monitored server may be there. - size_t n = 0; - std::string ip = MySQL_Monitor::dns_lookup(p.green_host, false, &n); - if (!ip.empty()) { - p.green_ip = ip; - p.green_ip_ttl = 0; - proxy_debug(PROXY_DEBUG_MONITOR, 7, - "AWS RDS BGD [wHG=%u]: green '%s' IP %s (DNS_Cache)\n", - st.writer_hg, p.green_host.c_str(), p.green_ip.c_str()); - continue; - } - // Cache miss (green is not a monitored server): resolve DNS now and track its TTL. - if (p.green_ip.empty() || __builtin_expect(p.green_ip_ttl != 0 && monotonic_time() > p.green_ip_ttl, 0)) { - std::vector ips = dns_resolve(p.green_host, ai_family); - if (!ips.empty()) { - p.green_ip = ips.front(); - p.green_ip_ttl = monotonic_time() - + (1000ULL * (unsigned long long)mysql_thread___monitor_local_dns_cache_ttl); - proxy_debug(PROXY_DEBUG_MONITOR, 7, - "AWS RDS BGD [wHG=%u]: green '%s' IP %s (resolved, ttl=%lus)\n", - st.writer_hg, p.green_host.c_str(), p.green_ip.c_str(), - (unsigned long)mysql_thread___monitor_local_dns_cache_ttl); - } - } - } st.next_check_interval_ms = 100; + + aws_rds_bgd_build_map(st, topology); + aws_rds_bgd_resolve_green_ips(st); + aws_rds_bgd_add_green_writer_in_hg(st); } else if (strcasecmp(status.c_str(), BGD_STATUS_POST_PROC) == 0) { st.next_check_interval_ms = 100; - // Run the steps below only on the first poll after entering POST_PROCESSING; - // later polls see the same status, so there is nothing to redo. - if (strcasecmp(status.c_str(), st.last_status.c_str()) == 0) { - return; - } - + // Run setup here too: the thread may observe POST_PROCESSING directly, without having + // seen AVAILABLE/INITIATED first. All three are idempotent, so the repoint/shun logic + // below always runs against a built map, resolved green IPs, and an added green writer. aws_rds_bgd_build_map(st, topology); - - // TODO: Handle the case where thread observes POST_PROCESSING status directly, - // without observing SWITCHOVER_INITIATED first (resolve green IPs). + aws_rds_bgd_resolve_green_ips(st); + aws_rds_bgd_add_green_writer_in_hg(st); // Repoint each mapped blue host onto its green IP and drain the blue free // pool so new connections resolve to green. @@ -7152,11 +7202,6 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo } else if (strcasecmp(status.c_str(), BGD_STATUS_COMPLETED) == 0) { st.next_check_interval_ms = 0; - // Undo everything only on the first poll after entering COMPLETED; - // later polls see the same status, so there is nothing to redo. - if (strcasecmp(status.c_str(), st.last_status.c_str()) == 0) { - return; - } if (st.writer_is_also_reader_enforced) { for (const AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { @@ -7189,6 +7234,8 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo "AWS RDS BGD [wHG=%u rHG=%u]: switchover complete; state cleared\n", st.writer_hg, st.reader_hg); st.bg_map.clear(); + // Stop polling green by IP; revert to the configured (blue) name, now the promoted primary. + st.next_check_host.clear(); } else { // unknown status: take no action, stay at baseline interval From be9187d29a1dad389454ca6f642619c0ed2b6f20 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 23 Jun 2026 09:23:01 +0000 Subject: [PATCH 09/81] fix: Suspend read_only monitor actions during AWS RDS BGD switchover Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 22 +++++++++++++++++ include/MySQL_Monitor.hpp | 1 + lib/MySQL_HostGroups_Manager.cpp | 29 ++++++++++++++++++++++- lib/MySQL_Monitor.cpp | 38 +++++++++++++++++++++++++++++- 4 files changed, 88 insertions(+), 2 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 5ba8634afe..16f90a7ea6 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -601,6 +601,21 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { return readonly_flag; } + inline + void set_aws_rds_bgd_in_progress() { + aws_rds_bgd_in_progress = true; + } + + inline + bool is_aws_rds_bgd_in_progress() { + return aws_rds_bgd_in_progress; + } + + inline + void clear_aws_rds_bgd_in_progress() { + aws_rds_bgd_in_progress = false; + } + private: unsigned int get_hostgroup_id(Type type, const Node& node) const; MySrvC* insert_HGM(unsigned int hostgroup_id, const MySrvC* srv); @@ -609,6 +624,7 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { std::array, TYPE_SIZE_> mapping; // index 0 contains reader and 1 contains writer hostgroups int readonly_flag; MySQL_HostGroups_Manager* myHGM; + bool aws_rds_bgd_in_progress = false; }; /** @@ -1063,6 +1079,12 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { * @param auto_recover When shunning, whether the server is eligible for auto-recovery; ignored on release. */ void set_server_shun(const char *hostname, int port, bool shun, bool auto_recover); + /** + * @brief Flag/unflag every server in the writer and reader hostgroups of an AWS RDS blue/green + * deployment as "switchover in progress", so the read_only monitor (read_only_action_v2) takes + * no action on them while the BGD FSM is driving the switchover. + */ + void set_aws_rds_bgd_in_progress(unsigned int writer_hg, unsigned int reader_hg, bool in_progress); unsigned long long Get_Memory_Stats(); void add_discovered_servers_to_mysql_servers_and_replication_hostgroups(const vector>& new_servers); diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 80198196ef..847eb4cd49 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -442,6 +442,7 @@ struct AWS_RDS_BGD_State { bool green_writer_added_in_hg = false; ///< whether green writer added to green_writer_hg bool writer_is_also_reader_enforced = false; ///< whether POST_PROCESSING added the writer to the reader HG + bool bgd_in_progress_set = false; ///< whether we flagged the deployment's servers as switchover-in-progress (set once at INITIATED+, cleared at COMPLETED) so read_only_action_v2 leaves them alone unsigned int next_check_interval_ms = 0; ///< FSM-controlled interval; 0 => baseline std::string next_check_host; ///< FSM-pinned probe host; when set (the green IP), the worker diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 29eab93098..622675c0fd 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -3591,7 +3591,7 @@ void MySQL_HostGroups_Manager::read_only_action_v2(const std::listsecond.get(); - if (!host_server_mapping) { + if (!host_server_mapping || host_server_mapping->is_aws_rds_bgd_in_progress()) { continue; } @@ -3877,6 +3877,33 @@ void MySQL_HostGroups_Manager::set_server_shun(const char *hostname, int port, b wrunlock(); } +void MySQL_HostGroups_Manager::set_aws_rds_bgd_in_progress(unsigned int writer_hg, unsigned int reader_hg, bool in_progress) { + wrlock(); + + unsigned int hgs[2] = { writer_hg, reader_hg }; + for (unsigned int i = 0; i < 2; i++) { + // MyHGC_find (not MyHGC_lookup, which creates on miss) so we never materialize an empty HG. + MyHGC* myhgc = MyHGC_find(hgs[i]); + if (myhgc == nullptr || myhgc->mysrvs == nullptr) { + continue; + } + for (unsigned int j = 0; j < myhgc->mysrvs->cnt(); j++) { + MySrvC* s = myhgc->mysrvs->idx(j); + const std::string srv_id = std::string(s->address) + ":::" + std::to_string(s->port); + auto itr = hostgroup_server_mapping.find(srv_id); + if (itr != hostgroup_server_mapping.end() && itr->second) { + if (in_progress) { + itr->second->set_aws_rds_bgd_in_progress(); + } else { + itr->second->clear_aws_rds_bgd_in_progress(); + } + } + } + } + + wrunlock(); +} + void MySQL_HostGroups_Manager::p_update_metrics() { p_update_counter(status.p_counter_array[p_hg_counter::servers_table_version], status.servers_table_version); // Update *server_connections* related metrics diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 6f18840282..2ca96113d8 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6839,6 +6839,8 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { } delete mmsd; mmsd = NULL; + + // TODO: call aws_rds_bgd_clear_bgd_in_progress() } __exit_monitor_RDS_BGD_thread_HG_now: @@ -7062,6 +7064,34 @@ static void aws_rds_bgd_add_green_writer_in_hg(AWS_RDS_BGD_State& st) { } } +/** +* @brief Flag the deployment's servers as switchover-in-progress so the read_only monitor leaves them alone. +* +* @details During a switchover AWS makes the blue writer read-only; without this, read_only_action_v2 would +* demote/relocate it and fight the BGD FSM. Set once (guarded by st.bgd_in_progress_set) when status reaches +* INITIATED and held through POST_PROCESSING; cleared at COMPLETED by aws_rds_bgd_clear_bgd_in_progress. +*/ +static void aws_rds_bgd_set_bgd_in_progress(AWS_RDS_BGD_State& st) { + if (st.bgd_in_progress_set) { + return; + } + MyHGM->set_aws_rds_bgd_in_progress(st.writer_hg, st.reader_hg, true); + st.bgd_in_progress_set = true; + proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: switchover in progress, suspending read_only monitor actions on these hostgroups until SWITCHOVER_COMPLETED\n", + st.writer_hg, st.reader_hg); +} + +/** +* @brief Re-enable read_only monitor action on the deployment's servers (undo aws_rds_bgd_set_bgd_in_progress). +*/ +static void aws_rds_bgd_clear_bgd_in_progress(AWS_RDS_BGD_State& st) { + if (!st.bgd_in_progress_set) { + return; + } + MyHGM->set_aws_rds_bgd_in_progress(st.writer_hg, st.reader_hg, false); + st.bgd_in_progress_set = false; +} + /** * @brief Run the status-driven blue/green switchover FSM for one deployment. * @@ -7116,6 +7146,7 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo aws_rds_bgd_build_map(st, topology); aws_rds_bgd_resolve_green_ips(st); aws_rds_bgd_add_green_writer_in_hg(st); + aws_rds_bgd_set_bgd_in_progress(st); } else if (strcasecmp(status.c_str(), BGD_STATUS_POST_PROC) == 0) { st.next_check_interval_ms = 100; @@ -7126,6 +7157,7 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo aws_rds_bgd_build_map(st, topology); aws_rds_bgd_resolve_green_ips(st); aws_rds_bgd_add_green_writer_in_hg(st); + aws_rds_bgd_set_bgd_in_progress(st); // Repoint each mapped blue host onto its green IP and drain the blue free // pool so new connections resolve to green. @@ -7225,7 +7257,9 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo } for (const AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { - dns_cache->unpin(p.blue_host); + // remove the cache record (pin + resolved IPs) so the blue name + // re-resolves to the promoted (green) instance + dns_cache->remove(p.blue_host); } // TODO: Drain Green HGs @@ -7236,6 +7270,8 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo st.bg_map.clear(); // Stop polling green by IP; revert to the configured (blue) name, now the promoted primary. st.next_check_host.clear(); + // Re-enable the read_only monitor on these servers now that the switchover is done. + aws_rds_bgd_clear_bgd_in_progress(st); } else { // unknown status: take no action, stay at baseline interval From 1f530aba68f1597286309392e980c7b627502f6e Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Wed, 24 Jun 2026 12:20:01 +0000 Subject: [PATCH 10/81] fix: Rename shadowed variable in `DNS_Cache::lookup()` Signed-off-by: Wazir Ahmed --- lib/DNS_Cache.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/DNS_Cache.cpp b/lib/DNS_Cache.cpp index f182723ed4..b82384b78d 100644 --- a/lib/DNS_Cache.cpp +++ b/lib/DNS_Cache.cpp @@ -291,7 +291,8 @@ std::string DNS_Cache::lookup(const std::string& hostname, size_t* ip_count) con auto itr = records.find(hostname); if (itr != records.end()) { - auto [ip, count] = get_next_ip(itr->second); + auto [next_ip, count] = get_next_ip(itr->second); + ip = next_ip; if (ip_count) *ip_count = count; From 8fcda28359aba3fcc5f2a9e49d9baf631252132e Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Fri, 26 Jun 2026 07:30:56 +0000 Subject: [PATCH 11/81] fix: Avoid calling `read_only_action_v2` for unexpected resultset - Log and dump malformed `read_only` check resultsets for diagnostics and avoid falling back to the `read_only=1` default path. - Keep AWS RDS topology discovery out of `read_only` result processing. Signed-off-by: Wazir Ahmed --- include/MySQL_Monitor.hpp | 5 ++ include/proxysql_utils.h | 25 +++++++++ lib/MySQL_Monitor.cpp | 56 ++++++++++++++------ lib/proxysql_utils.cpp | 104 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 175 insertions(+), 15 deletions(-) diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 847eb4cd49..5ae6471876 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -297,6 +297,11 @@ class MySQL_Monitor_State_Data { return task_result_; } + inline + const char* get_query() const { + return query_.c_str(); + } + private: std::string query_; unsigned long long task_expiry_time_; // task expiry time (t1 + task_timeout_ * 1000) diff --git a/include/proxysql_utils.h b/include/proxysql_utils.h index 7937635f76..24bb05bf02 100644 --- a/include/proxysql_utils.h +++ b/include/proxysql_utils.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -14,6 +15,7 @@ #include #include +#include "mysql.h" #include "../deps/json/json.hpp" #ifndef ProxySQL_Checksum_Value_LENGTH @@ -424,4 +426,27 @@ static inline bool wait_for_glo_mth() { return false; } +/** + * @brief Pretty-print a MySQL result set into a string. + * + * @details Formats the full buffered result set as an ASCII table. The current row cursor is preserved: + * the function seeks to the first row for formatting and restores the original cursor before returning. + * + * @param result MySQL result set to format. + * + * @return Pretty-printed result set, or an empty string if the result is NULL or has no fields. + */ +std::string mysql_result_to_string(MYSQL_RES* result); + +/** + * @brief Pretty-print a MySQL result set to a file stream. + * + * @details Uses mysql_result_to_string() for formatting and writes the resulting string to the supplied + * file stream. The result set row cursor is preserved. + * + * @param file Destination file stream. + * @param result MySQL result set to format. + */ +void dump_mysql_result(FILE* file, MYSQL_RES* result); + #endif diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 2ca96113d8..da60a2f661 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -1670,6 +1670,7 @@ void * monitor_read_only_thread(const std::vector& da mysql_close(mysql_init(NULL)); bool timeout_reached = false; MySQL_Monitor_State_Data *mmsd = data.front(); + std::string monitor_query; // Wait for GloMTH to be initialized if (!wait_for_glo_mth()) return NULL; // quick exit during shutdown/restart MySQL_Thread * mysql_thr = new MySQL_Thread(); @@ -1708,23 +1709,24 @@ void * monitor_read_only_thread(const std::vector& da mmsd->interr=0; // reset the value #ifndef TEST_READONLY if (mmsd->get_task_type() == MON_INNODB_READ_ONLY) { - mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT @@global.innodb_read_only read_only"); + monitor_query = "SELECT @@global.innodb_read_only read_only"; } else if (mmsd->get_task_type() == MON_SUPER_READ_ONLY) { - mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT @@global.super_read_only read_only"); + monitor_query = "SELECT @@global.super_read_only read_only"; } else if (mmsd->get_task_type() == MON_READ_ONLY__AND__INNODB_READ_ONLY) { - mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT @@global.read_only&@@global.innodb_read_only read_only"); + monitor_query = "SELECT @@global.read_only&@@global.innodb_read_only read_only"; } else if (mmsd->get_task_type() == MON_READ_ONLY__OR__INNODB_READ_ONLY) { - mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT @@global.read_only|@@global.innodb_read_only read_only"); + monitor_query = "SELECT @@global.read_only|@@global.innodb_read_only read_only"; } else if (mmsd->get_task_type() == MON_AWS_RDS_TOPOLOGY_DISCOVERY) { - mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql, QUERY_AWS_RDS_TOPOLOGY_DISCOVERY); + monitor_query = QUERY_AWS_RDS_TOPOLOGY_DISCOVERY; } else { // default - mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT @@global.read_only read_only"); + monitor_query = "SELECT @@global.read_only read_only"; } + mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql, monitor_query.c_str()); #else // TEST_READONLY { - std::string s = "SELECT @@global.read_only read_only"; - s += " " + std::string(mmsd->hostname) + ":" + std::to_string(mmsd->port); - mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,s.c_str()); + monitor_query = "SELECT @@global.read_only read_only"; + monitor_query += " " + std::string(mmsd->hostname) + ":" + std::to_string(mmsd->port); + mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,monitor_query.c_str()); } #endif // TEST_READONLY while (mmsd->async_exit_status) { @@ -1789,7 +1791,14 @@ void * monitor_read_only_thread(const std::vector& da __exit_monitor_read_only_thread: mmsd->t2=monotonic_time(); - { + if (mmsd->get_task_type() == MON_AWS_RDS_TOPOLOGY_DISCOVERY) { + if (mmsd->interr == 0 && mmsd->result) { + GloMyMon->process_aws_rds_topology(mmsd); + mysql_free_result(mmsd->result); + mmsd->result = NULL; + read_only_success = true; + } + } else { /* handle read_only checks */ char *query=NULL; query=(char *)"INSERT OR REPLACE INTO mysql_server_read_only_log VALUES (?1 , ?2 , ?3 , ?4 , ?5 , ?6)"; auto [rc1, statement_unique] = mmsd->mondb->prepare_v2(query); @@ -1797,6 +1806,7 @@ void * monitor_read_only_thread(const std::vector& da sqlite3_stmt *statement = statement_unique.get(); int rc; int read_only=1; // as a safety mechanism , read_only=1 is the default + bool valid_result = true; rc=(*proxy_sqlite3_bind_text)(statement, 1, mmsd->hostname, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mmsd->mondb); rc=(*proxy_sqlite3_bind_int)(statement, 2, mmsd->port); ASSERT_SQLITE_OK(rc, mmsd->mondb); unsigned long long time_now=realtime_time(); @@ -1833,8 +1843,11 @@ VALGRIND_ENABLE_ERROR_REPORTING; // rc=(*proxy_sqlite3_bind_null)(statement, 5); ASSERT_SQLITE_OK(rc, mmsd->mondb); // } } else { - proxy_error("mysql_fetch_fields returns NULL, or mysql_num_fields is incorrect. Server %s:%d . See bug #1994\n", mmsd->hostname, mmsd->port); + valid_result = false; rc=(*proxy_sqlite3_bind_null)(statement, 5); ASSERT_SQLITE_OK(rc, mmsd->mondb); + proxy_error("mysql_fetch_fields returns NULL, or mysql_num_fields is incorrect. Server %s:%d . See bug #1994\n", mmsd->hostname, mmsd->port); + proxy_info("Dumping read_only result for server %s:%d, query: %s\n", mmsd->hostname, mmsd->port, monitor_query.c_str()); + dump_mysql_result(stderr, mmsd->result); } mysql_free_result(mmsd->result); mmsd->result=NULL; @@ -1851,11 +1864,13 @@ VALGRIND_ENABLE_ERROR_REPORTING; rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, mmsd->mondb); rc=(*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, mmsd->mondb); - if (mmsd->mysql_error_msg == NULL) { + if (valid_result && mmsd->mysql_error_msg == NULL) { read_only_success = true; } - if (timeout_reached == false && mmsd->interr == 0) { + if (!valid_result) { + // Ignore malformed read_only resultsets: do not infer backend state. + } else if (timeout_reached == false && mmsd->interr == 0) { MyHGM->read_only_action_v2( std::list { read_only_server_t { mmsd->hostname, mmsd->port, read_only } } ); // default behavior @@ -1890,6 +1905,8 @@ VALGRIND_ENABLE_ERROR_REPORTING; free(buff); } } + + /* error handling for both read_only and rds_topology checks */ if (mmsd->interr || mmsd->mysql_error_msg) { // check failed if (mmsd->mysql) { // AWS RDS topology discovery probes every replication-hostgroup member, but @@ -1912,6 +1929,7 @@ VALGRIND_ENABLE_ERROR_REPORTING; } } } + __fast_exit_monitor_read_only_thread: if (mmsd->mysql) { // if we reached here we didn't put the connection back @@ -1936,11 +1954,13 @@ VALGRIND_ENABLE_ERROR_REPORTING; } } } + if (read_only_success) { __sync_fetch_and_add(&GloMyMon->read_only_check_OK,1); } else { __sync_fetch_and_add(&GloMyMon->read_only_check_ERR,1); } + delete mysql_thr; return NULL; } @@ -8404,6 +8424,7 @@ bool MySQL_Monitor::monitor_read_only_process_ready_tasks(const std::vectorhostname, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mmsd->mondb); rc = (*proxy_sqlite3_bind_int)(statement, 2, mmsd->port); ASSERT_SQLITE_OK(rc, mmsd->mondb); unsigned long long time_now = realtime_time(); @@ -8437,8 +8458,11 @@ VALGRIND_ENABLE_ERROR_REPORTING; rc = (*proxy_sqlite3_bind_int64)(statement, 5, read_only); ASSERT_SQLITE_OK(rc, mmsd->mondb); } else { - proxy_error("mysql_fetch_fields returns NULL, or mysql_num_fields is incorrect. Server %s:%d . See bug #1994\n", mmsd->hostname, mmsd->port); + valid_result = false; rc = (*proxy_sqlite3_bind_null)(statement, 5); ASSERT_SQLITE_OK(rc, mmsd->mondb); + proxy_error("mysql_fetch_fields returns NULL, or mysql_num_fields is incorrect. Server %s:%d . See bug #1994\n", mmsd->hostname, mmsd->port); + proxy_info("Dumping read_only result for server %s:%d, query: %s\n", mmsd->hostname, mmsd->port, mmsd->get_query()); + dump_mysql_result(stderr, mmsd->result); } mysql_free_result(mmsd->result); mmsd->result = NULL; @@ -8455,7 +8479,9 @@ VALGRIND_ENABLE_ERROR_REPORTING; rc = (*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, mmsd->mondb); rc = (*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, mmsd->mondb); - if (task_result == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_SUCCESS) { + if (!valid_result) { + // Ignore malformed read_only resultsets: do not infer backend state. + } else if (task_result == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_SUCCESS) { //MyHGM->read_only_action_v2(mmsd->hostname, mmsd->port, read_only); // default behavior mysql_servers.push_back( std::tuple { mmsd->hostname, mmsd->port, read_only }); } else { diff --git a/lib/proxysql_utils.cpp b/lib/proxysql_utils.cpp index 331902586c..3e92bb338b 100644 --- a/lib/proxysql_utils.cpp +++ b/lib/proxysql_utils.cpp @@ -835,3 +835,107 @@ int calculate_percentile_from_histogram( return thresholds.back(); } + +/** + * @brief Pretty-print a MySQL result set into a string. + * + * @details Formats the full buffered result set as an ASCII table. The current row cursor is preserved: + * the function seeks to the first row for formatting and restores the original cursor before returning. + * + * @param result MySQL result set to format. + * + * @return Pretty-printed result set, or an empty string if the result is NULL or has no fields. + */ +std::string mysql_result_to_string(MYSQL_RES* result) { + if (!result) return ""; + + MYSQL_ROW_OFFSET original_row = mysql_row_tell(result); + mysql_data_seek(result, 0); + + int num_fields = mysql_num_fields(result); + MYSQL_FIELD* fields = mysql_fetch_fields(result); + if (!fields || num_fields == 0) { + mysql_row_seek(result, original_row); + return ""; + } + + std::vector> rows; + MYSQL_ROW row; + while ((row = mysql_fetch_row(result))) { + unsigned long* lens = mysql_fetch_lengths(result); + std::vector r; + r.reserve(num_fields); + for (int i = 0; i < num_fields; i++) { + r.emplace_back(row[i] ? std::string(row[i], lens[i]) : "NULL"); + } + rows.push_back(std::move(r)); + } + + std::vector widths(num_fields); + for (int i = 0; i < num_fields; i++) { + widths[i] = strlen(fields[i].name); + } + for (const auto& r : rows) { + for (int i = 0; i < num_fields; i++) { + if (r[i].size() > widths[i]) widths[i] = r[i].size(); + } + } + + std::string s; + std::string out; + + auto append_border = [&]() { + s = "+"; + for (int i = 0; i < num_fields; i++) { + for (size_t j = 0; j < widths[i] + 2; j++) s += "-"; + s += "+"; + } + out += s; + out += "\n"; + }; + + append_border(); + s = "|"; + for (int i = 0; i < num_fields; i++) { + size_t len = strlen(fields[i].name); + s += " "; s += fields[i].name; + for (size_t j = 0; j < widths[i] - len + 1; j++) s += " "; + s += "|"; + } + out += s; + out += "\n"; + append_border(); + + for (const auto& r : rows) { + s = "|"; + for (int i = 0; i < num_fields; i++) { + s += " "; s += r[i]; + for (size_t j = 0; j < widths[i] - r[i].size() + 1; j++) s += " "; + s += "|"; + } + out += s; + out += "\n"; + } + append_border(); + + mysql_row_seek(result, original_row); + return out; +} + +/** + * @brief Pretty-print a MySQL result set to a file stream. + * + * @details Uses mysql_result_to_string() for formatting and writes the resulting string to the supplied + * file stream. The result set row cursor is preserved. + * + * @param file Destination file stream. + * @param result MySQL result set to format. + */ +void dump_mysql_result(FILE* file, MYSQL_RES* result) { + if (!file) return; + + std::string result_string = mysql_result_to_string(result); + if (!result_string.empty()) { + fputs(result_string.c_str(), file); + } +} From d425ec39697451de53f98e6096fc330a0527e77e Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Fri, 26 Jun 2026 17:05:04 +0000 Subject: [PATCH 12/81] fix: Ignore monitor query timeouts after resolved IP changes - Classify monitor query timeouts separately when the connection target no longer matches the current DNS cache entry. - Keep the timeout visible in monitor logs, but avoid applying backend state changes based on the stale connection. Signed-off-by: Wazir Ahmed --- include/DNS_Cache.hpp | 1 + include/MySQL_Monitor.hpp | 2 + lib/DNS_Cache.cpp | 21 ++++++ lib/MySQL_Monitor.cpp | 153 ++++++++++++++++++++++++++++---------- 4 files changed, 139 insertions(+), 38 deletions(-) diff --git a/include/DNS_Cache.hpp b/include/DNS_Cache.hpp index e24023af6e..d3c7d73bf7 100644 --- a/include/DNS_Cache.hpp +++ b/include/DNS_Cache.hpp @@ -83,6 +83,7 @@ class DNS_Cache { void remove(const std::string& hostname); void clear(); bool empty() const; + bool is_ip_valid(const std::string& hostname, const std::string& ip) const; std::string lookup(const std::string& hostname, size_t* ip_count) const; /** diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 5ae6471876..6981d659d1 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -211,6 +211,7 @@ enum MySQL_Monitor_State_Data_Task_Type { enum class MySQL_Monitor_State_Data_Task_Result { TASK_RESULT_UNKNOWN, TASK_RESULT_TIMEOUT, + TASK_RESULT_TIMEOUT_STALE_IP, TASK_RESULT_FAILED, TASK_RESULT_SUCCESS, TASK_RESULT_PENDING @@ -478,6 +479,7 @@ class MySQL_Monitor { static std::string dns_lookup(const char* hostname, bool return_hostname_if_lookup_fails = true, size_t* ip_count = nullptr); static bool update_dns_cache_from_mysql_conn(const MYSQL* mysql); static void trigger_dns_cache_update(); + bool timeout_validate_ip_change(const MySQL_Monitor_State_Data* mmsd) const; /** * @brief Classify the parsed mysql.rds_topology result and dispatch. diff --git a/lib/DNS_Cache.cpp b/lib/DNS_Cache.cpp index b82384b78d..3611387bfb 100644 --- a/lib/DNS_Cache.cpp +++ b/lib/DNS_Cache.cpp @@ -205,6 +205,27 @@ void* DNSResolverWorker::run() { return nullptr; } +bool DNS_Cache::is_ip_valid(const std::string& hostname, const std::string& ip) const { + if (!enabled || hostname.empty() || ip.empty()) { + return false; + } + + int rc = pthread_rwlock_rdlock(&rwlock_); + assert(rc == 0); + + bool valid = false; + auto itr = records.find(hostname); + if (itr != records.end()) { + const std::vector& src = + itr->second.pinned_ips.empty() ? itr->second.ips : itr->second.pinned_ips; + valid = std::find(src.begin(), src.end(), ip) != src.end(); + } + + rc = pthread_rwlock_unlock(&rwlock_); + assert(rc == 0); + + return valid; +} bool DNS_Cache::add(const std::string& hostname, std::vector&& ips) { diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index da60a2f661..3c69b88a94 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -795,7 +795,10 @@ void MySQL_Monitor_State_Data::init_async() { void MySQL_Monitor_State_Data::mark_task_as_timeout(unsigned long long time) { - task_result_ = MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_TIMEOUT; + const bool stale_ip_timeout = GloMyMon && GloMyMon->timeout_validate_ip_change(this); + task_result_ = stale_ip_timeout + ? MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_TIMEOUT_STALE_IP + : MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_TIMEOUT; t2 = time; if (mysql_error_msg) @@ -803,10 +806,15 @@ void MySQL_Monitor_State_Data::mark_task_as_timeout(unsigned long long time) { if (task_id_ == MON_PING) { async_state_machine_ = ASYNC_PING_TIMEOUT; - mysql_error_msg = strdup("timeout during ping"); + mysql_error_msg = strdup(stale_ip_timeout ? "resolved IP no longer valid" : "timeout during ping"); } else { async_state_machine_ = (async_state_machine_ == ASYNC_QUERY_CONT) ? ASYNC_QUERY_TIMEOUT : ASYNC_STORE_RESULT_TIMEOUT; - mysql_error_msg = strdup("timeout check"); + mysql_error_msg = strdup(stale_ip_timeout ? "resolved IP no longer valid" : "timeout check"); + } + if (stale_ip_timeout) { + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "Ignoring monitor timeout for %s:%d because resolved IP is no longer valid\n", + hostname, port); } } @@ -1671,6 +1679,7 @@ void * monitor_read_only_thread(const std::vector& da bool timeout_reached = false; MySQL_Monitor_State_Data *mmsd = data.front(); std::string monitor_query; + bool stale_ip_timeout = false; // Wait for GloMTH to be initialized if (!wait_for_glo_mth()) return NULL; // quick exit during shutdown/restart MySQL_Thread * mysql_thr = new MySQL_Thread(); @@ -1737,9 +1746,16 @@ void * monitor_read_only_thread(const std::vector& da const unsigned long long now = monotonic_time(); #endif if (now > mmsd->t1 + mysql_thread___monitor_read_only_timeout * 1000) { - mmsd->mysql_error_msg=strdup("timeout check"); - proxy_error("Timeout on read_only check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_read_only_timeout.\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); - MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_READ_ONLY_CHECK_TIMEOUT); + stale_ip_timeout = GloMyMon->timeout_validate_ip_change(mmsd); + mmsd->mysql_error_msg=strdup(stale_ip_timeout ? "resolved IP no longer valid" : "timeout check"); + if (stale_ip_timeout) { + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "Ignoring read_only timeout for %s:%d because resolved IP is no longer valid\n", + mmsd->hostname, mmsd->port); + } else { + proxy_error("Timeout on read_only check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_read_only_timeout.\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); + MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_READ_ONLY_CHECK_TIMEOUT); + } timeout_reached = true; goto __exit_monitor_read_only_thread; } @@ -1771,9 +1787,16 @@ void * monitor_read_only_thread(const std::vector& da const unsigned long long now = monotonic_time(); #endif if (now > mmsd->t1 + mysql_thread___monitor_read_only_timeout * 1000) { - mmsd->mysql_error_msg=strdup("timeout check"); - proxy_error("Timeout on read_only check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_read_only_timeout.\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); - MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_READ_ONLY_CHECK_TIMEOUT); + stale_ip_timeout = GloMyMon->timeout_validate_ip_change(mmsd); + mmsd->mysql_error_msg=strdup(stale_ip_timeout ? "resolved IP no longer valid" : "timeout check"); + if (stale_ip_timeout) { + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "Ignoring read_only timeout for %s:%d because resolved IP is no longer valid\n", + mmsd->hostname, mmsd->port); + } else { + proxy_error("Timeout on read_only check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_read_only_timeout.\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); + MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_READ_ONLY_CHECK_TIMEOUT); + } timeout_reached = true; goto __exit_monitor_read_only_thread; } @@ -1868,8 +1891,8 @@ VALGRIND_ENABLE_ERROR_REPORTING; read_only_success = true; } - if (!valid_result) { - // Ignore malformed read_only resultsets: do not infer backend state. + if (!valid_result || stale_ip_timeout) { + // Ignore; do not infer backend state. } else if (timeout_reached == false && mmsd->interr == 0) { MyHGM->read_only_action_v2( std::list { read_only_server_t { mmsd->hostname, mmsd->port, read_only } @@ -1969,6 +1992,7 @@ void * monitor_group_replication_thread(const std::vector mmsd->t1 + mysql_thread___monitor_groupreplication_healthcheck_timeout * 1000) { - mmsd->mysql_error_msg=strdup("timeout check"); - proxy_error("Timeout on group replication health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_groupreplication_healthcheck_timeout. Assuming viable_candidate=NO and read_only=YES\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); - MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_GR_HEALTH_CHECK_TIMEOUT); + stale_ip_timeout = GloMyMon->timeout_validate_ip_change(mmsd); + mmsd->mysql_error_msg=strdup(stale_ip_timeout ? "resolved IP no longer valid" : "timeout check"); + if (stale_ip_timeout) { + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "Ignoring group replication timeout for %s:%d because resolved IP is no longer valid\n", + mmsd->hostname, mmsd->port); + } else { + proxy_error("Timeout on group replication health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_groupreplication_healthcheck_timeout. Assuming viable_candidate=NO and read_only=YES\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); + MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_GR_HEALTH_CHECK_TIMEOUT); + } goto __exit_monitor_group_replication_thread; } if (mmsd->interr) { @@ -2049,9 +2080,16 @@ void * monitor_group_replication_thread(const std::vector mmsd->t1 + mysql_thread___monitor_groupreplication_healthcheck_timeout * 1000) { - mmsd->mysql_error_msg=strdup("timeout check"); - proxy_error("Timeout on group replication health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_groupreplication_healthcheck_timeout. Assuming viable_candidate=NO and read_only=YES\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); - MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_GR_HEALTH_CHECK_TIMEOUT); + stale_ip_timeout = GloMyMon->timeout_validate_ip_change(mmsd); + mmsd->mysql_error_msg=strdup(stale_ip_timeout ? "resolved IP no longer valid" : "timeout check"); + if (stale_ip_timeout) { + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "Ignoring group replication timeout for %s:%d because resolved IP is no longer valid\n", + mmsd->hostname, mmsd->port); + } else { + proxy_error("Timeout on group replication health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_groupreplication_healthcheck_timeout. Assuming viable_candidate=NO and read_only=YES\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); + MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_GR_HEALTH_CHECK_TIMEOUT); + } goto __exit_monitor_group_replication_thread; } if (GloMyMon->shutdown==true) { @@ -2158,7 +2196,9 @@ void * monitor_group_replication_thread(const std::vectorgroup_replication_mutex); // NOTE: we update MyHGM outside the mutex group_replication_mutex - if (mmsd->mysql_error_msg) { // there was an error checking the status of the server, surely we need to reconfigure GR + if (stale_ip_timeout) { + // Logged/counted; do not change GR state for stale DNS targets. + } else if (mmsd->mysql_error_msg) { // there was an error checking the status of the server, surely we need to reconfigure GR if (num_timeouts == 0) { // it wasn't a timeout, reconfigure immediately MyHGM->update_group_replication_set_offline(mmsd->hostname, mmsd->port, mmsd->writer_hostgroup, mmsd->mysql_error_msg); @@ -2310,6 +2350,7 @@ void * monitor_galera_thread(const std::vector& data) assert(!data.empty()); mysql_close(mysql_init(NULL)); MySQL_Monitor_State_Data *mmsd = data.front(); + bool stale_ip_timeout = false; // Wait for GloMTH to be initialized if (!wait_for_glo_mth()) return NULL; // quick exit during shutdown/restart MySQL_Thread * mysql_thr = new MySQL_Thread(); @@ -2393,9 +2434,16 @@ void * monitor_galera_thread(const std::vector& data) const unsigned long long now = monotonic_time(); #endif if (now > mmsd->t1 + mysql_thread___monitor_galera_healthcheck_timeout * 1000) { - mmsd->mysql_error_msg=strdup("timeout check"); - proxy_error("Timeout on Galera health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_galera_healthcheck_timeout.\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); - MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_GALERA_HEALTH_CHECK_TIMEOUT); + stale_ip_timeout = GloMyMon->timeout_validate_ip_change(mmsd); + mmsd->mysql_error_msg=strdup(stale_ip_timeout ? "resolved IP no longer valid" : "timeout check"); + if (stale_ip_timeout) { + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "Ignoring Galera timeout for %s:%d because resolved IP is no longer valid\n", + mmsd->hostname, mmsd->port); + } else { + proxy_error("Timeout on Galera health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_galera_healthcheck_timeout.\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); + MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_GALERA_HEALTH_CHECK_TIMEOUT); + } goto __exit_monitor_galera_thread; } if (GloMyMon->shutdown==true) { @@ -2414,9 +2462,16 @@ void * monitor_galera_thread(const std::vector& data) const unsigned long long now = monotonic_time(); #endif if (now > mmsd->t1 + mysql_thread___monitor_galera_healthcheck_timeout * 1000) { - mmsd->mysql_error_msg=strdup("timeout check"); - proxy_error("Timeout on Galera health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_galera_healthcheck_timeout.\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); - MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_GALERA_HEALTH_CHECK_TIMEOUT); + stale_ip_timeout = GloMyMon->timeout_validate_ip_change(mmsd); + mmsd->mysql_error_msg=strdup(stale_ip_timeout ? "resolved IP no longer valid" : "timeout check"); + if (stale_ip_timeout) { + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "Ignoring Galera timeout for %s:%d because resolved IP is no longer valid\n", + mmsd->hostname, mmsd->port); + } else { + proxy_error("Timeout on Galera health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_galera_healthcheck_timeout.\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); + MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_GALERA_HEALTH_CHECK_TIMEOUT); + } goto __exit_monitor_galera_thread; } if (GloMyMon->shutdown==true) { @@ -2593,7 +2648,9 @@ void * monitor_galera_thread(const std::vector& data) pthread_mutex_unlock(&GloMyMon->galera_mutex); // NOTE: we update MyHGM outside the mutex galera_mutex - if (mmsd->mysql_error_msg) { // there was an error checking the status of the server, surely we need to reconfigure Galera + if (stale_ip_timeout) { + // Logged/counted; do not change Galera state for stale DNS targets. + } else if (mmsd->mysql_error_msg) { // there was an error checking the status of the server, surely we need to reconfigure Galera if (num_timeouts == 0) { // it wasn't a timeout, reconfigure immediately MyHGM->update_galera_set_offline(mmsd->hostname, mmsd->port, mmsd->writer_hostgroup, mmsd->mysql_error_msg); @@ -7668,6 +7725,23 @@ std::string MySQL_Monitor::dns_lookup(const char* hostname, bool return_hostname return MySQL_Monitor::dns_lookup(std::string(hostname), return_hostname_if_lookup_fails, ip_count); } +bool MySQL_Monitor::timeout_validate_ip_change(const MySQL_Monitor_State_Data* mmsd) const { + if (!mmsd || !mmsd->mysql || !mmsd->hostname || !dns_cache) { + return false; + } + + if (mmsd->port == 0 || validate_ip(mmsd->hostname)) { + return false; + } + + const std::string connected_ip = get_connected_peer_ip_from_socket(mmsd->mysql->net.fd); + if (connected_ip.empty()) { + return false; + } + + return !dns_cache->is_ip_valid(mmsd->hostname, connected_ip); +} + bool MySQL_Monitor::update_dns_cache_from_mysql_conn(const MYSQL* mysql) { assert(mysql); @@ -8018,9 +8092,10 @@ MySQL_Monitor_State_Data_Task_Result MySQL_Monitor_State_Data::task_handler(shor assert(task_handler_); if (event_ != -1) { - - if (task_result_ == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_TIMEOUT) - return MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_TIMEOUT; + if (task_result_ == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_TIMEOUT || + task_result_ == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_TIMEOUT_STALE_IP) { + return task_result_; + } #ifdef DEBUG const unsigned long long now = (GloMyMon->proxytest_forced_timeout == false) ? monotonic_time() : ULLONG_MAX; #else @@ -8349,12 +8424,12 @@ MySQL_Monitor_State_Data_Task_Result MySQL_Monitor_State_Data::generic_handler(s } bool MySQL_Monitor::monitor_read_only_process_ready_tasks(const std::vector& mmsds) { - std::list mysql_servers; for (auto& mmsd : mmsds) { string originating_server_hostname = mmsd->hostname; const auto task_result = mmsd->get_task_result(); + const bool stale_ip_timeout = task_result == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_TIMEOUT_STALE_IP; assert(task_result != MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_PENDING); @@ -8479,8 +8554,8 @@ VALGRIND_ENABLE_ERROR_REPORTING; rc = (*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, mmsd->mondb); rc = (*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, mmsd->mondb); - if (!valid_result) { - // Ignore malformed read_only resultsets: do not infer backend state. + if (!valid_result || stale_ip_timeout) { + // Ignore; do not infer backend state. } else if (task_result == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_SUCCESS) { //MyHGM->read_only_action_v2(mmsd->hostname, mmsd->port, read_only); // default behavior mysql_servers.push_back( std::tuple { mmsd->hostname, mmsd->port, read_only }); @@ -8605,11 +8680,10 @@ void MySQL_Monitor::monitor_read_only_async(SQLite3_result* resultset, bool do_d } bool MySQL_Monitor::monitor_group_replication_process_ready_tasks(const std::vector& mmsds) { - for (auto& mmsd : mmsds) { - const auto task_result = mmsd->get_task_result(); - + const bool stale_ip_timeout = task_result == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_TIMEOUT_STALE_IP; + assert(task_result != MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_PENDING); if (task_result == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_SUCCESS) { @@ -8710,7 +8784,9 @@ bool MySQL_Monitor::monitor_group_replication_process_ready_tasks(const std::vec pthread_mutex_unlock(&group_replication_mutex); // NOTE: we update MyHGM outside the mutex group_replication_mutex - if (mmsd->mysql_error_msg) { // there was an error checking the status of the server, surely we need to reconfigure GR + if (stale_ip_timeout) { + // Logged/counted; do not change GR state for stale DNS targets. + } else if (mmsd->mysql_error_msg) { // there was an error checking the status of the server, surely we need to reconfigure GR if (num_timeouts == 0) { // it wasn't a timeout, reconfigure immediately MyHGM->update_group_replication_set_offline(mmsd->hostname, mmsd->port, mmsd->writer_hostgroup, mmsd->mysql_error_msg); @@ -9090,10 +9166,9 @@ void MySQL_Monitor::monitor_replication_lag_async(SQLite3_result* resultset) { } bool MySQL_Monitor::monitor_galera_process_ready_tasks(const std::vector& mmsds) { - for (auto& mmsd : mmsds) { - const auto task_result = mmsd->get_task_result(); + const bool stale_ip_timeout = task_result == MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_TIMEOUT_STALE_IP; assert(task_result != MySQL_Monitor_State_Data_Task_Result::TASK_RESULT_PENDING); @@ -9271,7 +9346,9 @@ bool MySQL_Monitor::monitor_galera_process_ready_tasks(const std::vectormysql_error_msg) { // there was an error checking the status of the server, surely we need to reconfigure Galera + if (stale_ip_timeout) { + // Logged/counted; do not change Galera state for stale DNS targets. + } else if (mmsd->mysql_error_msg) { // there was an error checking the status of the server, surely we need to reconfigure Galera if (num_timeouts == 0) { // it wasn't a timeout, reconfigure immediately MyHGM->update_galera_set_offline(mmsd->hostname, mmsd->port, mmsd->writer_hostgroup, mmsd->mysql_error_msg); From 4f7ed9ab121d421482db0a4a9dc64d5181f1aa5b Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sat, 27 Jun 2026 15:33:56 +0000 Subject: [PATCH 13/81] feat: Add expiry for pinned IPs in `DNS_Cache` Signed-off-by: Wazir Ahmed --- include/DNS_Cache.hpp | 42 ++++++++---- lib/DNS_Cache.cpp | 147 +++++++++++++++++++++++++++++++++--------- lib/MySQL_Monitor.cpp | 2 +- 3 files changed, 147 insertions(+), 44 deletions(-) diff --git a/include/DNS_Cache.hpp b/include/DNS_Cache.hpp index d3c7d73bf7..ee1614e6ba 100644 --- a/include/DNS_Cache.hpp +++ b/include/DNS_Cache.hpp @@ -87,12 +87,25 @@ class DNS_Cache { std::string lookup(const std::string& hostname, size_t* ip_count) const; /** - * @brief Pin a hostname to a fixed set of IPs that override resolution until unpin(). + * @brief Pin a hostname to a fixed IP until it is explicitly unpinned. * - * @param hostname Hostname whose resolution is overridden. - * @param ips IP addresses to serve for 'hostname' (moved into the cache). + * @param hostname Hostname whose cached resolution is overridden. + * @param ip IP address to serve for 'hostname' while pinned. */ - void pin(const std::string& hostname, std::vector&& ips); + void pin(const std::string& hostname, const std::string& ip); + + /** + * @brief Pin a hostname to a fixed IP for a bounded time. + * + * @details While the pin is active, lookup() serves 'ip' instead of the resolved + * address set. Once ttl_ms expires, lookup() serves the resolved address and + * clears the expired pin before returning. + * + * @param hostname Hostname whose cached resolution is overridden. + * @param ip IP address to serve for 'hostname' while pinned. + * @param ttl_ms Pin lifetime in milliseconds; 0 means no expiry. + */ + void pin(const std::string& hostname, const std::string& ip, unsigned long long ttl_ms); /** * @brief Remove a pin set by pin(), restoring normal resolution (no-op if not pinned). @@ -104,10 +117,10 @@ class DNS_Cache { private: struct IP_ADDR { std::vector ips; - // Pinned override: when non-empty, get_next_ip()/lookup() serve these - // instead of 'ips'. Set by pin(), cleared by unpin(); untouched by add(), - // so it is preserved across resolver-thread TTL refreshes. - std::vector pinned_ips; + // Pinned override: when non-empty, lookup() serves it instead of 'ips' + // while pinned_until is not expired. + std::string pinned_ip; + unsigned long long pinned_until = 0; // 'counter' is bumped by get_next_ip() (a const method) for // round-robin selection; the logical state of the cache record is // unchanged, so mutable is the right tool here and lets us drop a @@ -115,16 +128,23 @@ class DNS_Cache { mutable unsigned long counter = 0; }; + struct lookup_result_t { + std::string resolved_ip; + size_t ip_count = 0; + std::string pinned_ip; + unsigned long long pinned_until = 0; + }; + /** * @brief Next round-robin IP for 'ip_addr' and the size of the served set. * * @param ip_addr Cache record to select from. * - * @return { ip, set_size }, or { "", 0 } when the set is empty. + * @return Selected resolved IP details plus current pin metadata. */ - std::pair get_next_ip(const IP_ADDR& ip_addr) const; + lookup_result_t get_next_ip(const IP_ADDR& ip_addr) const; - std::unordered_map records; + mutable std::unordered_map records; std::atomic_bool enabled; mutable pthread_rwlock_t rwlock_; diff --git a/lib/DNS_Cache.cpp b/lib/DNS_Cache.cpp index 3611387bfb..def9ffca97 100644 --- a/lib/DNS_Cache.cpp +++ b/lib/DNS_Cache.cpp @@ -216,9 +216,13 @@ bool DNS_Cache::is_ip_valid(const std::string& hostname, const std::string& ip) bool valid = false; auto itr = records.find(hostname); if (itr != records.end()) { - const std::vector& src = - itr->second.pinned_ips.empty() ? itr->second.ips : itr->second.pinned_ips; - valid = std::find(src.begin(), src.end(), ip) != src.end(); + const unsigned long long now = monotonic_time(); + const bool pin_active = itr->second.pinned_until != 0 && now <= itr->second.pinned_until; + if (pin_active) { + valid = ip == itr->second.pinned_ip; + } else { + valid = std::find(itr->second.ips.begin(), itr->second.ips.end(), ip) != itr->second.ips.end(); + } } rc = pthread_rwlock_unlock(&rwlock_); @@ -228,17 +232,30 @@ bool DNS_Cache::is_ip_valid(const std::string& hostname, const std::string& ip) } bool DNS_Cache::add(const std::string& hostname, std::vector&& ips) { - if (!enabled) return false; proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Updating DNS cache. (Hostname:[%s] IP:[%s])\n", hostname.c_str(), debug_iplisttostring(ips).c_str()); + int rc = pthread_rwlock_wrlock(&rwlock_); assert(rc == 0); + auto& ip_addr = records[hostname]; ip_addr.ips = std::move(ips); + + // Check if IP pinning is no longer necessary. + if (!ip_addr.pinned_ip.empty() && + std::find(ip_addr.ips.begin(), ip_addr.ips.end(), ip_addr.pinned_ip) != ip_addr.ips.end()) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Unpinning DNS cache record because resolved IP matches pinned IP. (Hostname:[%s] IP:[%s])\n", + hostname.c_str(), ip_addr.pinned_ip.c_str()); + ip_addr.pinned_ip.clear(); + ip_addr.pinned_until = 0; + } + __sync_fetch_and_and(&ip_addr.counter, 0); + rc = pthread_rwlock_unlock(&rwlock_); assert(rc == 0); @@ -254,15 +271,29 @@ bool DNS_Cache::add_if_not_exist(const std::string& hostname, std::vectorsecond.ips.empty()) { proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Updating DNS cache. (Hostname:[%s] IP:[%s])\n", hostname.c_str(), debug_iplisttostring(ips).c_str()); auto& ip_addr = records[hostname]; ip_addr.ips = std::move(ips); + + // Check if IP pinning is no longer necessary. + if (!ip_addr.pinned_ip.empty() && + std::find(ip_addr.ips.begin(), ip_addr.ips.end(), ip_addr.pinned_ip) != ip_addr.ips.end()) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Unpinning DNS cache record because resolved IP matches pinned IP. (Hostname:[%s] IP:[%s])\n", + hostname.c_str(), ip_addr.pinned_ip.c_str()); + ip_addr.pinned_ip.clear(); + ip_addr.pinned_until = 0; + } + __sync_fetch_and_and(&ip_addr.counter, 0); inserted = true; } + rc = pthread_rwlock_unlock(&rwlock_); assert(rc == 0); @@ -277,22 +308,21 @@ bool DNS_Cache::add_if_not_exist(const std::string& hostname, std::vector DNS_Cache::get_next_ip(const IP_ADDR& ip_addr) const { - // A pinned record overrides the resolved IPs. - const std::vector& src = - ip_addr.pinned_ips.empty() ? ip_addr.ips : ip_addr.pinned_ips; +DNS_Cache::lookup_result_t DNS_Cache::get_next_ip(const IP_ADDR& ip_addr) const { + lookup_result_t result; - if (src.empty()) - return { "", 0 }; - - const auto counter_val = __sync_fetch_and_add(&ip_addr.counter, 1); + if (!ip_addr.ips.empty()) { + const auto counter_val = __sync_fetch_and_add(&ip_addr.counter, 1); + result.ip_count = ip_addr.ips.size(); + result.resolved_ip = ip_addr.ips[counter_val % result.ip_count]; + } - size_t ip_count = src.size(); - auto ip = src[counter_val % ip_count]; + result.pinned_ip = ip_addr.pinned_ip; + result.pinned_until = ip_addr.pinned_until; - return { ip, ip_count }; + return result; } std::string DNS_Cache::lookup(const std::string& hostname, size_t* ip_count) const { @@ -303,6 +333,7 @@ std::string DNS_Cache::lookup(const std::string& hostname, size_t* ip_count) con } std::string ip; + bool clear_expired_pin = false; if (counter_queried_) counter_queried_->fetch_add(1, std::memory_order_relaxed); @@ -312,11 +343,21 @@ std::string DNS_Cache::lookup(const std::string& hostname, size_t* ip_count) con auto itr = records.find(hostname); if (itr != records.end()) { - auto [next_ip, count] = get_next_ip(itr->second); - ip = next_ip; - - if (ip_count) - *ip_count = count; + lookup_result_t result = get_next_ip(itr->second); + + const unsigned long long now = monotonic_time(); + const bool pin_active = result.pinned_until != 0 && now <= result.pinned_until; + clear_expired_pin = result.pinned_until != 0 && now > result.pinned_until; + + if (pin_active) { + ip = result.pinned_ip; + if (ip_count) + *ip_count = 1; + } else { + ip = result.resolved_ip; + if (ip_count) + *ip_count = result.ip_count; + } proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "DNS cache lookup success. (Hostname:[%s] IP returned:[%s])\n", @@ -332,27 +373,68 @@ std::string DNS_Cache::lookup(const std::string& hostname, size_t* ip_count) con if (!ip.empty() && counter_lookup_success_) counter_lookup_success_->fetch_add(1, std::memory_order_relaxed); + // cleanup expired pinned IP + if (clear_expired_pin) { + rc = pthread_rwlock_wrlock(&rwlock_); + assert(rc == 0); + auto itr2 = records.find(hostname); + if (itr2 != records.end() && itr2->second.pinned_until != 0 && + monotonic_time() > itr2->second.pinned_until) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Unpinning expired DNS cache record. (Hostname:[%s] IP:[%s])\n", + hostname.c_str(), itr2->second.pinned_ip.c_str()); + itr2->second.pinned_ip.clear(); + itr2->second.pinned_until = 0; + __sync_fetch_and_and(&itr2->second.counter, 0); + if (itr2->second.ips.empty()) { + records.erase(itr2); + } + if (counter_record_updated_) + counter_record_updated_->fetch_add(1, std::memory_order_relaxed); + } + rc = pthread_rwlock_unlock(&rwlock_); + assert(rc == 0); + } + return ip; } /** -* @brief Pin a hostname to a fixed set of IPs that override resolution until unpin(). +* @brief Pin a hostname to a fixed IP until it is explicitly unpinned. * -* @param hostname Hostname whose resolution is overridden. -* @param ips IP addresses to serve for 'hostname' (moved into the cache). +* @param hostname Hostname whose cached resolution is overridden. +* @param ip IP address to serve for 'hostname' while pinned. */ -void DNS_Cache::pin(const std::string& hostname, std::vector&& ips) { +void DNS_Cache::pin(const std::string& hostname, const std::string& ip) { + pin(hostname, ip, 0); +} + +/** +* @brief Pin a hostname to a fixed IP for a bounded time. +* +* @details While the pin is active, lookup() serves 'ip' instead of the resolved +* address set. Once ttl_ms expires, lookup() serves the resolved address and +* clears the expired pin before returning. +* +* @param hostname Hostname whose cached resolution is overridden. +* @param ip IP address to serve for 'hostname' while pinned. +* @param ttl_ms Pin lifetime in milliseconds; 0 means no expiry. +*/ +void DNS_Cache::pin(const std::string& hostname, const std::string& ip, unsigned long long ttl_ms) { + if (!enabled || hostname.empty() || ip.empty()) return; + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Pinning DNS cache record. (Hostname:[%s] IP:[%s])\n", - hostname.c_str(), debug_iplisttostring(ips).c_str()); + hostname.c_str(), ip.c_str()); int rc = pthread_rwlock_wrlock(&rwlock_); assert(rc == 0); - // Store on the record's 'pinned_ips' so a concurrent resolver add() (which + // Store on the record's 'pinned_ip' so a concurrent resolver add() (which // only rewrites 'ips') cannot drop the override on a TTL refresh. auto& ip_addr = records[hostname]; - ip_addr.pinned_ips = std::move(ips); + ip_addr.pinned_ip = ip; + ip_addr.pinned_until = ttl_ms ? monotonic_time() + (ttl_ms * 1000) : 0; __sync_fetch_and_and(&ip_addr.counter, 0); rc = pthread_rwlock_unlock(&rwlock_); @@ -374,11 +456,12 @@ void DNS_Cache::unpin(const std::string& hostname) { assert(rc == 0); auto itr = records.find(hostname); - if (itr != records.end() && !itr->second.pinned_ips.empty()) { + if (itr != records.end() && !itr->second.pinned_ip.empty()) { proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Unpinning DNS cache record. (Hostname:[%s] IP:[%s])\n", - hostname.c_str(), debug_iplisttostring(itr->second.pinned_ips).c_str()); - itr->second.pinned_ips.clear(); + hostname.c_str(), itr->second.pinned_ip.c_str()); + itr->second.pinned_ip.clear(); + itr->second.pinned_until = 0; // drop the record entirely if pinning was the only thing keeping it alive // (e.g. the host is not otherwise resolved into the cache). if (itr->second.ips.empty()) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 3c69b88a94..b6e907e6bd 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -7249,7 +7249,7 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo st.writer_hg, st.reader_hg, p.blue_host.c_str(), p.port); continue; } - dns_cache->pin(p.blue_host, { p.green_ip }); + dns_cache->pin(p.blue_host, p.green_ip); proxy_info( "AWS RDS BGD [wHG=%u rHG=%u]: repointed blue '%s' to green IP %s\n", st.writer_hg, st.reader_hg, p.blue_host.c_str(), p.green_ip.c_str()); From 409279760266df7d089621a2b795efcaf75a0b29 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Mon, 29 Jun 2026 05:06:03 +0000 Subject: [PATCH 14/81] feat: Add new server status `SHUNNED_AWS_BGD` Signed-off-by: Wazir Ahmed --- include/Base_HostGroups_Manager.h | 5 +- include/MySQL_HostGroups_Manager.h | 22 +++--- include/ProxySQL_Admin_Tables_Definitions.h | 2 +- include/ServerSelection.h | 3 +- include/proxysql_structs.h | 3 +- lib/Base_HostGroups_Manager.cpp | 6 ++ lib/MySQL_HostGroups_Manager.cpp | 80 +++++++++++++-------- lib/MySQL_Monitor.cpp | 9 +-- lib/mysql_connection.cpp | 23 +++--- 9 files changed, 95 insertions(+), 58 deletions(-) diff --git a/include/Base_HostGroups_Manager.h b/include/Base_HostGroups_Manager.h index 4a93047ac4..35d2a2e630 100644 --- a/include/Base_HostGroups_Manager.h +++ b/include/Base_HostGroups_Manager.h @@ -86,7 +86,7 @@ class MetricsCollector; "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , " \ "comment VARCHAR , UNIQUE (reader_hostgroup))" -#define MYHGM_GEN_ADMIN_RUNTIME_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers ORDER BY hostgroup_id, hostname, port" +#define MYHGM_GEN_ADMIN_RUNTIME_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" WHEN 5 THEN \"SHUNNED_AWS_BGD\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers ORDER BY hostgroup_id, hostname, port" #define MYHGM_MYSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , hostgroup_settings VARCHAR CHECK (JSON_VALID(hostgroup_settings) OR hostgroup_settings = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" @@ -95,7 +95,7 @@ class MetricsCollector; /* * @brief Generates the 'runtime_mysql_servers' resultset exposed to other ProxySQL cluster members. - * @details Makes 'SHUNNED' and 'SHUNNED_REPLICATION_LAG' statuses equivalent to 'ONLINE'. 'SHUNNED' states + * @details Makes 'SHUNNED', 'SHUNNED_REPLICATION_LAG' and 'SHUNNED_AWS_BGD' statuses equivalent to 'ONLINE'. 'SHUNNED' states * are by definition local transitory states, this is why a 'mysql_servers' table reconfiguration isn't * normally performed when servers are internally imposed with these statuses. This means, that propagating * this state to other cluster members is undesired behavior, and so it's generating a different checksum, @@ -117,6 +117,7 @@ class MetricsCollector; " WHEN 2 THEN \"OFFLINE_SOFT\"" \ " WHEN 3 THEN \"OFFLINE_HARD\"" \ " WHEN 4 THEN \"ONLINE\" " \ + " WHEN 5 THEN \"ONLINE\" " \ "END status," \ "weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment " \ "FROM mysql_servers " \ diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 16f90a7ea6..9fc9300cc1 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -76,7 +76,7 @@ "auto_generated INT CHECK (auto_generated IN (0,1)) NOT NULL DEFAULT 0," \ "UNIQUE (reader_hostgroup))" -#define MYHGM_GEN_ADMIN_RUNTIME_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers ORDER BY hostgroup_id, hostname, port" +#define MYHGM_GEN_ADMIN_RUNTIME_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" WHEN 5 THEN \"SHUNNED_AWS_BGD\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers ORDER BY hostgroup_id, hostname, port" #define MYHGM_MYSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , hostgroup_settings VARCHAR CHECK (JSON_VALID(hostgroup_settings) OR hostgroup_settings = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" @@ -85,7 +85,7 @@ /* * @brief Generates the 'runtime_mysql_servers' resultset exposed to other ProxySQL cluster members. - * @details Makes 'SHUNNED' and 'SHUNNED_REPLICATION_LAG' statuses equivalent to 'ONLINE'. 'SHUNNED' states + * @details Makes 'SHUNNED', 'SHUNNED_REPLICATION_LAG' and 'SHUNNED_AWS_BGD' statuses equivalent to 'ONLINE'. 'SHUNNED' states * are by definition local transitory states, this is why a 'mysql_servers' table reconfiguration isn't * normally performed when servers are internally imposed with these statuses. This means, that propagating * this state to other cluster members is undesired behavior, and so it's generating a different checksum, @@ -107,6 +107,7 @@ " WHEN 2 THEN \"OFFLINE_SOFT\"" \ " WHEN 3 THEN \"OFFLINE_HARD\"" \ " WHEN 4 THEN \"ONLINE\" " \ + " WHEN 5 THEN \"ONLINE\" " \ "END status," \ "weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment " \ "FROM mysql_servers " \ @@ -1066,19 +1067,20 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { void set_server_current_latency_us(char *hostname, int port, unsigned int _current_latency_us); void set_Readyset_status(char *hostname, int port, enum MySerStatus status); /** - * @brief Shun or release a server across all hostgroups. + * @brief Set or clear AWS BGD shun state for a matching server. * - * @details Shunning sets shunned_and_kill_all_connections and takes 'shunned_automatic' from - * auto_recover (false = held until released; true = enables shun recovery). Releasing does - * NOT unshun directly: it only enables shun recovery (shunned_automatic=true) and leaves the - * actual unshun to the shun recovery path (MyHGC::get_random_MySrvC). + * @details When shunning, transitions an ONLINE server to SHUNNED_AWS_BGD, + * enables shun metadata, and drops free connections. When unshunning, + * transitions only SHUNNED_AWS_BGD back to ONLINE and clears shun metadata. + * Servers in other statuses are left unchanged. * + * @param hostgroup_id Hostgroup to search. * @param hostname Address of the server to match. * @param port Port of the server to match. - * @param shun true to shun the server, false to release it. - * @param auto_recover When shunning, whether the server is eligible for auto-recovery; ignored on release. + * @param shun true to shun the server, false to unshun it. + * @return true if this call changed a server's status. */ - void set_server_shun(const char *hostname, int port, bool shun, bool auto_recover); + bool aws_rds_bgd_set_shun_server(unsigned int hostgroup_id, const char *hostname, int port, bool shun); /** * @brief Flag/unflag every server in the writer and reader hostgroups of an AWS RDS blue/green * deployment as "switchover in progress", so the read_only monitor (read_only_action_v2) takes diff --git a/include/ProxySQL_Admin_Tables_Definitions.h b/include/ProxySQL_Admin_Tables_Definitions.h index 8c18870af6..9cc5cebfd4 100644 --- a/include/ProxySQL_Admin_Tables_Definitions.h +++ b/include/ProxySQL_Admin_Tables_Definitions.h @@ -147,7 +147,7 @@ #define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS "CREATE TABLE runtime_mysql_firewall_whitelist_sqli_fingerprints (active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , fingerprint VARCHAR NOT NULL , PRIMARY KEY (fingerprint) )" -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS "CREATE TABLE runtime_mysql_servers (hostgroup_id INT CHECK (hostgroup_id>=0) NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT CHECK (port >= 0 AND port <= 65535) NOT NULL DEFAULT 3306 , gtid_port INT CHECK ((gtid_port <> port OR gtid_port=0) AND gtid_port >= 0 AND gtid_port <= 65535) NOT NULL DEFAULT 0 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0 AND weight <=10000000) NOT NULL DEFAULT 1 , compression INT CHECK (compression IN(0,1)) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS "CREATE TABLE runtime_mysql_servers (hostgroup_id INT CHECK (hostgroup_id>=0) NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT CHECK (port >= 0 AND port <= 65535) NOT NULL DEFAULT 3306 , gtid_port INT CHECK ((gtid_port <> port OR gtid_port=0) AND gtid_port >= 0 AND gtid_port <= 65535) NOT NULL DEFAULT 0 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','SHUNNED_AWS_BGD','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0 AND weight <=10000000) NOT NULL DEFAULT 1 , compression INT CHECK (compression IN(0,1)) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" #define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS_SSL_PARAMS "CREATE TABLE runtime_mysql_servers_ssl_params (hostname VARCHAR NOT NULL , port INT CHECK (port >= 0 AND port <= 65535) NOT NULL DEFAULT 3306 , username VARCHAR NOT NULL DEFAULT '' , ssl_ca VARCHAR NOT NULL DEFAULT '' , ssl_cert VARCHAR NOT NULL DEFAULT '' , ssl_key VARCHAR NOT NULL DEFAULT '' , ssl_capath VARCHAR NOT NULL DEFAULT '' , ssl_crl VARCHAR NOT NULL DEFAULT '' , ssl_crlpath VARCHAR NOT NULL DEFAULT '' , ssl_cipher VARCHAR NOT NULL DEFAULT '' , tls_version VARCHAR NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostname, port, username) )" diff --git a/include/ServerSelection.h b/include/ServerSelection.h index 58a8352461..606381c43f 100644 --- a/include/ServerSelection.h +++ b/include/ServerSelection.h @@ -25,7 +25,8 @@ enum ServerSelectionStatus { SERVER_SHUNNED = 1, SERVER_OFFLINE_SOFT = 2, SERVER_OFFLINE_HARD = 3, - SERVER_SHUNNED_REPLICATION_LAG = 4 + SERVER_SHUNNED_REPLICATION_LAG = 4, + SERVER_SHUNNED_AWS_BGD = 5 }; /** diff --git a/include/proxysql_structs.h b/include/proxysql_structs.h index 0347f036cd..85b739a848 100644 --- a/include/proxysql_structs.h +++ b/include/proxysql_structs.h @@ -19,7 +19,8 @@ enum MySerStatus { MYSQL_SERVER_STATUS_SHUNNED, MYSQL_SERVER_STATUS_OFFLINE_SOFT, MYSQL_SERVER_STATUS_OFFLINE_HARD, - MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG + MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG, + MYSQL_SERVER_STATUS_SHUNNED_AWS_BGD }; enum log_event_type { diff --git a/lib/Base_HostGroups_Manager.cpp b/lib/Base_HostGroups_Manager.cpp index 99a85d207f..a150e88340 100644 --- a/lib/Base_HostGroups_Manager.cpp +++ b/lib/Base_HostGroups_Manager.cpp @@ -1778,6 +1778,9 @@ void MySQL_HostGroups_Manager::generate_mysql_servers_table(int *_onlyhg) { case 4: st=(char *)"SHUNNED"; break; + case 5: + st=(char *)"SHUNNED_AWS_BGD"; + break; } fprintf(stderr,"HID: %d , address: %s , port: %d , gtid_port: %d , weight: %ld , status: %s , max_connections: %ld , max_replication_lag: %u , use_ssl: %u , max_latency_ms: %u , comment: %s\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, mysrvc->gtid_port, mysrvc->weight, st, mysrvc->max_connections, mysrvc->max_replication_lag, mysrvc->use_ssl, mysrvc->max_latency_us*1000, mysrvc->comment); } @@ -3140,6 +3143,9 @@ SQLite3_result * MySQL_HostGroups_Manager::SQL3_Connection_Pool(bool _reset, int case 4: pta[3]=strdup("SHUNNED_REPLICATION_LAG"); break; + case 5: + pta[3]=strdup("SHUNNED_AWS_BGD"); + break; default: // LCOV_EXCL_START assert(0); diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 622675c0fd..5a04140308 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -660,7 +660,7 @@ hg_metrics_map = std::make_tuple( std::make_tuple ( p_hg_dyn_gauge::connection_pool_status, "proxysql_connpool_conns_status", - "The status of the backend server (1 - ONLINE, 2 - SHUNNED, 3 - OFFLINE_SOFT, 4 - OFFLINE_HARD, 5 - SHUNNED_REPLICATION_LAG).", + "The status of the backend server (1 - ONLINE, 2 - SHUNNED, 3 - OFFLINE_SOFT, 4 - OFFLINE_HARD, 5 - SHUNNED_REPLICATION_LAG, 6 - SHUNNED_AWS_BGD).", metric_tags { { "protocol", "mysql" } } @@ -1884,6 +1884,9 @@ void MySQL_HostGroups_Manager::generate_mysql_servers_table(int *_onlyhg) { case 4: st=(char *)"SHUNNED"; break; + case 5: + st=(char *)"SHUNNED_AWS_BGD"; + break; } fprintf(stderr,"HID: %u , address: %s , port: %d , gtid_port: %d , weight: %ld , status: %s , max_connections: %ld , max_replication_lag: %u , use_ssl: %d , max_latency_ms: %u , comment: %s\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, mysrvc->gtid_port, mysrvc->weight, st, mysrvc->max_connections, mysrvc->max_replication_lag, mysrvc->use_ssl, mysrvc->max_latency_us*1000, mysrvc->comment); } @@ -3502,6 +3505,9 @@ SQLite3_result * MySQL_HostGroups_Manager::SQL3_Connection_Pool(bool _reset, int case 4: pta[3]=strdup("SHUNNED_REPLICATION_LAG"); break; + case 5: + pta[3]=strdup("SHUNNED_AWS_BGD"); + break; default: // LCOV_EXCL_START assert(0); @@ -3834,47 +3840,61 @@ void MySQL_HostGroups_Manager::set_Readyset_status(char *hostname, int port, enu wrunlock(); } -void MySQL_HostGroups_Manager::set_server_shun(const char *hostname, int port, bool shun, bool auto_recover) { - wrlock(); - - MySrvC *mysrvc = NULL; +/** +* @brief Set or clear AWS BGD shun state for a matching server. +* +* @details When shunning, transitions an ONLINE server to SHUNNED_AWS_BGD, +* enables shun metadata, and drops free connections. When unshunning, +* transitions only SHUNNED_AWS_BGD back to ONLINE and clears shun metadata. +* Servers in other statuses are left unchanged. +* +* @param hostgroup_id Hostgroup to search. +* @param hostname Address of the server to match. +* @param port Port of the server to match. +* @param shun true to shun the server, false to unshun it. +* @return true if this call changed a server's status. +*/ +bool MySQL_HostGroups_Manager::aws_rds_bgd_set_shun_server(unsigned int hostgroup_id, const char *hostname, int port, bool shun) { + bool changed = false; - for (unsigned int i = 0; i < MyHostGroups->len; i++) { - MyHGC *myhgc = (MyHGC *)MyHostGroups->index(i); - unsigned int l = myhgc->mysrvs->cnt(); + wrlock(); - for (unsigned int j = 0; j < l; j++) { - mysrvc = myhgc->mysrvs->idx(j); + MyHGC *myhgc = MyHGC_find(hostgroup_id); + if (myhgc && myhgc->mysrvs) { + for (unsigned int j = 0; j < myhgc->mysrvs->cnt(); j++) { + MySrvC *mysrvc = myhgc->mysrvs->idx(j); + if (mysrvc->port != port || strcmp(mysrvc->address, hostname) != 0) { + continue; + } - if (mysrvc->port == port && strcmp(mysrvc->address,hostname) == 0) { - if (shun) { - if (mysrvc->get_status() == MYSQL_SERVER_STATUS_ONLINE) { - mysrvc->set_status(MYSQL_SERVER_STATUS_SHUNNED); - } - // 'shunned_automatic' is the auto-recovery guard: the shun recovery path - // (MyHGC::get_random_MySrvC) only brings back servers that have it set. - // Passing auto_recover=false holds the shun until an explicit unshun. - mysrvc->shunned_automatic = auto_recover; + if (shun) { + if (mysrvc->get_status() == MYSQL_SERVER_STATUS_ONLINE) { + mysrvc->set_status(MYSQL_SERVER_STATUS_SHUNNED_AWS_BGD); + mysrvc->shunned_automatic = true; mysrvc->shunned_and_kill_all_connections = true; - // TODO: Check if last_detected_error should be set to a time in future, - // similar to MySQL_HostGroups_Manager::shun_and_killall() mysrvc->time_last_detected_error = time(NULL); mysrvc->ConnectionsFree->drop_all_connections(); - proxy_warning("Shunning server %s:%d in HG %u with auto-recovery %s\n", - hostname, port, myhgc->hid, (auto_recover) ? "enabled" : "disabled"); - } else { - // We don't unshun directly. Keep the server SHUNNED (with kill_all_connections set) - // and only enable auto-recovery; the shun recovery path (MyHGC::get_random_MySrvC) - // then brings it back online once all its old connections have drained. - // The actual unshunning work is done by MySQL_HostGroups_Manager::unshun_server_all_hostgroups - mysrvc->shunned_automatic = true; - proxy_warning("Enabling shun recovery for server %s:%d in HG %u\n", hostname, port, myhgc->hid); + proxy_warning("AWS RDS BGD shunning server %s:%d in HG %u\n", + hostname, port, myhgc->hid); + changed = true; + } + } else { + if (mysrvc->get_status() == MYSQL_SERVER_STATUS_SHUNNED_AWS_BGD) { + mysrvc->set_status(MYSQL_SERVER_STATUS_ONLINE); + mysrvc->shunned_automatic = false; + mysrvc->shunned_and_kill_all_connections = false; + mysrvc->connect_ERR_at_time_last_detected_error = 0; + mysrvc->time_last_detected_error = 0; + proxy_warning("AWS RDS BGD unshunning server %s:%d in HG %u\n", + hostname, port, myhgc->hid); + changed = true; } } } } wrunlock(); + return changed; } void MySQL_HostGroups_Manager::set_aws_rds_bgd_in_progress(unsigned int writer_hg, unsigned int reader_hg, bool in_progress) { diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index b6e907e6bd..2792ed19ac 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -7282,14 +7282,15 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo for (const std::pair& br : blue_readers) { bool mapped = false; for (const AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { - if (!p.is_writer && p.blue_host == br.first && p.port == br.second) { + if (p.blue_host == br.first && p.port == br.second) { mapped = true; break; } } if (!mapped) { - MyHGM->set_server_shun(br.first.c_str(), br.second, true, false); - st.shunned_readers.push_back(br); + if (MyHGM->aws_rds_bgd_set_shun_server(st.reader_hg, br.first.c_str(), br.second, true)) { + st.shunned_readers.push_back(br); + } } } @@ -7326,7 +7327,7 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo if (!st.shunned_readers.empty()) { for (const std::pair& br : st.shunned_readers) { - MyHGM->set_server_shun(br.first.c_str(), br.second, false, false); + MyHGM->aws_rds_bgd_set_shun_server(st.reader_hg, br.first.c_str(), br.second, false); // purge so the blue reader hostname re-resolves to the promoted instance dns_cache->remove(br.first); } diff --git a/lib/mysql_connection.cpp b/lib/mysql_connection.cpp index a38842ce5a..1ba8c10547 100644 --- a/lib/mysql_connection.cpp +++ b/lib/mysql_connection.cpp @@ -2160,21 +2160,25 @@ int MySQL_Connection::async_connect(short event) { bool MySQL_Connection::IsServerOffline() { - bool ret=false; - if (parent==NULL) + bool ret = false; + if (parent == NULL) return ret; - server_status=parent->get_status(); // we copy it here to avoid race condition. The caller will see this + + server_status = parent->get_status(); // we copy it here to avoid race condition. The caller will see this + bool server_shunned = (server_status == MYSQL_SERVER_STATUS_SHUNNED) || (server_status == MYSQL_SERVER_STATUS_SHUNNED_AWS_BGD); + if ( - (server_status==MYSQL_SERVER_STATUS_OFFLINE_HARD) // the server is OFFLINE as specific by the user + (server_status == MYSQL_SERVER_STATUS_OFFLINE_HARD) // the server is OFFLINE as specific by the user || - (server_status==MYSQL_SERVER_STATUS_SHUNNED && parent->shunned_and_kill_all_connections==true) // the server is SHUNNED due to a serious issue + (server_shunned && parent->shunned_automatic == true && parent->shunned_and_kill_all_connections == true) // the server is SHUNNED due to a serious issue || - (server_status==MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG) // slave is lagging! see #774 + (server_status == MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG) // slave is lagging! see #774 || (parent->myhgc->online_servers_within_threshold() == false) // number of online servers in a hostgroup exceeds the configured maximum servers ) { - ret=true; + ret = true; } + return ret; } @@ -3031,10 +3035,11 @@ int MySQL_Connection::async_send_simple_command(short event, char *stmt, unsigne assert(mysql); assert(ret_mysql); server_status=parent->get_status(); // we copy it here to avoid race condition. The caller will see this + bool server_shunned = (server_status == MYSQL_SERVER_STATUS_SHUNNED) || (server_status == MYSQL_SERVER_STATUS_SHUNNED_AWS_BGD); if ( - (parent->get_status()==MYSQL_SERVER_STATUS_OFFLINE_HARD) // the server is OFFLINE as specific by the user + (server_status==MYSQL_SERVER_STATUS_OFFLINE_HARD) // the server is OFFLINE as specific by the user || - (parent->get_status()==MYSQL_SERVER_STATUS_SHUNNED && parent->shunned_and_kill_all_connections==true) // the server is SHUNNED due to a serious issue + (server_shunned && parent->shunned_automatic == true && parent->shunned_and_kill_all_connections==true) // the server is SHUNNED due to a serious issue ) { return -1; } From 63dca4bb5cfddb9b75da9caea742a2d49d2924e4 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Mon, 29 Jun 2026 08:01:29 +0000 Subject: [PATCH 15/81] feat: Drain backend connections during AWS BGD switchover - During switchover, drop free backend connections for affected servers and mark used connections with `MySQL_Connection::healthy=false` and `reusable=false`. - Treat unhealthy backend connections as offline in active backend paths, and prevent reset-algorithm-2 from resetting drained connections. - Purge idle monitor pool entries when BGD repoints or unpins blue hosts, preventing monitor checks from reusing stale connections after DNS/cache changes. Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 13 +++++++ include/mysql_connection.h | 1 + lib/Base_Session.cpp | 2 +- lib/MySQL_HostGroups_Manager.cpp | 57 ++++++++++++++++++++++++----- lib/MySQL_Monitor.cpp | 58 ++++++++++++++++++++++++------ lib/MySQL_Session.cpp | 6 ++-- lib/MySQL_Thread.cpp | 5 +++ lib/MySrvConnList.cpp | 9 ++++- lib/mysql_connection.cpp | 7 ++++ lib/mysql_data_stream.cpp | 8 +++-- 10 files changed, 141 insertions(+), 25 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 9fc9300cc1..f7563bb0c5 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -186,6 +186,7 @@ class MySrvConnList { void get_random_MyConn_inner_search(unsigned int start, unsigned int end, unsigned int& conn_found_idx, unsigned int& connection_quality_level, unsigned int& number_of_matching_session_variables, const MySQL_Connection * client_conn); unsigned int conns_length() { return conns->len; } void drop_all_connections(); + void mark_connections_unhealthy(); MySQL_Connection *index(unsigned int); }; @@ -1081,6 +1082,18 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { * @return true if this call changed a server's status. */ bool aws_rds_bgd_set_shun_server(unsigned int hostgroup_id, const char *hostname, int port, bool shun); + /** + * @brief Drain existing backend connections for a server. + * + * @details Drops free connections immediately and marks used connections as unhealthy and non-reusable, + * so in-flight operations fail on their next backend step and the connection is never pooled again. + * + * @param hostgroup_id Hostgroup to search. + * @param hostname Address of the server to match. + * @param port Port of the server to match. + * @return true if a matching server was found. + */ + bool drain_server_connections(unsigned int hostgroup_id, const char *hostname, int port); /** * @brief Flag/unflag every server in the writer and reader hostgroups of an AWS RDS blue/green * deployment as "switchover in progress", so the read_only monitor (read_only_action_v2) takes diff --git a/include/mysql_connection.h b/include/mysql_connection.h index 25588e6d35..0e34da620a 100644 --- a/include/mysql_connection.h +++ b/include/mysql_connection.h @@ -166,6 +166,7 @@ class MySQL_Connection { my_bool ret_bool; bool async_fetch_row_start; bool send_quit; + bool healthy; bool reusable; bool processing_multi_statement; bool multiplex_delayed; diff --git a/lib/Base_Session.cpp b/lib/Base_Session.cpp index 6a8a956e70..7e6e980cac 100644 --- a/lib/Base_Session.cpp +++ b/lib/Base_Session.cpp @@ -511,7 +511,7 @@ void Base_Session::housekeeping_before_pkts() { DS * myds = mybe->server_myds; if constexpr (std::is_same_v) { if (mysql_thread___autocommit_false_not_reusable && myds->myconn->IsAutoCommit() == false) { - if (mysql_thread___reset_connection_algorithm == 2) { + if (mysql_thread___reset_connection_algorithm == 2 && myds->myconn->healthy) { create_new_session_and_reset_connection(myds); } else { myds->destroy_MySQL_Connection_From_Pool(true); diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 5a04140308..abb559588c 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -2408,9 +2408,14 @@ void MySQL_HostGroups_Manager::push_MyConn_to_pool_array(MySQL_Connection **ca, wrlock(); // Iterate through the array of connections - while (ireusable) { + c->send_quit = false; + destroy_MyConn_from_pool(c, false); + } else { + // Push the current connection back to the pool without acquiring a lock for each individual push + push_MyConn_to_pool(c, false); + } i++; if (iparent; - if (mysrvc->get_status() == MYSQL_SERVER_STATUS_ONLINE && c->send_quit && queue.size() < __sync_fetch_and_add(&GloMTH->variables.connpoll_reset_queue_length, 0)) { + if (c->healthy && mysrvc->get_status() == MYSQL_SERVER_STATUS_ONLINE && c->send_quit && queue.size() < __sync_fetch_and_add(&GloMTH->variables.connpoll_reset_queue_length, 0)) { if (c->async_state_machine==ASYNC_IDLE) { // overall, the backend seems healthy and so it is the connection. Try to reset it int myerr=mysql_errno(c->mysql); @@ -3843,10 +3848,9 @@ void MySQL_HostGroups_Manager::set_Readyset_status(char *hostname, int port, enu /** * @brief Set or clear AWS BGD shun state for a matching server. * -* @details When shunning, transitions an ONLINE server to SHUNNED_AWS_BGD, -* enables shun metadata, and drops free connections. When unshunning, -* transitions only SHUNNED_AWS_BGD back to ONLINE and clears shun metadata. -* Servers in other statuses are left unchanged. +* @details When shunning, transitions an ONLINE server to SHUNNED_AWS_BGD, enables shun metadata, +* drops free connections, and marks used connections unhealthy. When unshunning, transitions only +* SHUNNED_AWS_BGD back to ONLINE and clears shun metadata. Servers in other statuses are left unchanged. * * @param hostgroup_id Hostgroup to search. * @param hostname Address of the server to match. @@ -3874,6 +3878,7 @@ bool MySQL_HostGroups_Manager::aws_rds_bgd_set_shun_server(unsigned int hostgrou mysrvc->shunned_and_kill_all_connections = true; mysrvc->time_last_detected_error = time(NULL); mysrvc->ConnectionsFree->drop_all_connections(); + mysrvc->ConnectionsUsed->mark_connections_unhealthy(); proxy_warning("AWS RDS BGD shunning server %s:%d in HG %u\n", hostname, port, myhgc->hid); changed = true; @@ -3897,6 +3902,42 @@ bool MySQL_HostGroups_Manager::aws_rds_bgd_set_shun_server(unsigned int hostgrou return changed; } +/** + * @brief Drain existing backend connections for a server. + * + * @details Drops free connections immediately and marks used connections as unhealthy and non-reusable, + * so in-flight operations fail on their next backend step and the connection is never pooled again. + * + * @param hostgroup_id Hostgroup to search. + * @param hostname Address of the server to match. + * @param port Port of the server to match. + * @return true if a matching server was found. + */ +bool MySQL_HostGroups_Manager::drain_server_connections(unsigned int hostgroup_id, const char *hostname, int port) { + bool found = false; + + wrlock(); + + MyHGC *myhgc = MyHGC_find(hostgroup_id); + if (myhgc && myhgc->mysrvs) { + for (unsigned int j = 0; j < myhgc->mysrvs->cnt(); j++) { + MySrvC *mysrvc = myhgc->mysrvs->idx(j); + if (mysrvc->port != port || strcmp(mysrvc->address, hostname) != 0) { + continue; + } + + mysrvc->ConnectionsFree->drop_all_connections(); + mysrvc->ConnectionsUsed->mark_connections_unhealthy(); + proxy_warning("Draining existing connections for server %s:%d in HG %u\n", + hostname, port, myhgc->hid); + found = true; + } + } + + wrunlock(); + return found; +} + void MySQL_HostGroups_Manager::set_aws_rds_bgd_in_progress(unsigned int writer_hg, unsigned int reader_hg, bool in_progress) { wrlock(); diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 2792ed19ac..6de87f78e4 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -244,6 +244,16 @@ class MySQL_Monitor_Connection_Pool { MYSQL * get_connection(char *hostname, int port, MySQL_Monitor_State_Data *mmsd); void put_connection(char *hostname, MySQL_Monitor_State_Data* mmsd); void purge_some_connections(); + /** + * @brief Purge idle monitor connections for a server. + * + * @details Removes the idle monitor connection pool entry matching the supplied hostname and port. + * Active monitor tasks are not affected. + * + * @param hostname Server hostname to match. + * @param port Server port to match. + */ + void purge_connections(const char* hostname, int port); void purge_all_connections(); void destroy_mysql_connection(MySQL_Monitor_State_Data* mmsd); MySQL_Monitor_Connection_Pool() { @@ -337,6 +347,38 @@ void MySQL_Monitor_Connection_Pool::purge_all_connections() { #endif } +/** + * @brief Purge idle monitor connections for a server. + * + * @details Removes the idle monitor connection pool entry matching the supplied hostname and port. + * Active monitor tasks are not affected. + * + * @param hostname Server hostname to match. + * @param port Server port to match. + */ +void MySQL_Monitor_Connection_Pool::purge_connections(const char* hostname, int port) { + std::lock_guard lock(mutex); +#ifdef DEBUG + pthread_mutex_lock(&m2); +#endif + if (servers) { + for (unsigned int i = 0; i < servers->len; i++) { + MonMySrvC* srv = static_cast(servers->index(i)); + if (srv && srv->port == port && strcmp(hostname, srv->address) == 0) { + proxy_debug(PROXY_DEBUG_MONITOR, 7, + "Purging %u idle monitor connections for server %s:%d\n", + srv->conns->len, hostname, port); + delete srv; + servers->remove_index_fast(i); + break; + } + } + } +#ifdef DEBUG + pthread_mutex_unlock(&m2); +#endif +} + void MySQL_Monitor_Connection_Pool::destroy_mysql_connection(MySQL_Monitor_State_Data* mmsd) { if (mmsd->mysql) { #ifdef DEBUG @@ -7236,8 +7278,8 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo aws_rds_bgd_add_green_writer_in_hg(st); aws_rds_bgd_set_bgd_in_progress(st); - // Repoint each mapped blue host onto its green IP and drain the blue free - // pool so new connections resolve to green. + // Repoint each mapped blue host onto its green IP and drain existing + // connections so new backend work resolves to green. bool any_reader_mapped = false; for (AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { if (!p.is_writer) { @@ -7254,15 +7296,9 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo "AWS RDS BGD [wHG=%u rHG=%u]: repointed blue '%s' to green IP %s\n", st.writer_hg, st.reader_hg, p.blue_host.c_str(), p.green_ip.c_str()); - // TODO: Draining blue free connection pool is not enough - // Kill used connection without SHUNNING the server unsigned int hid = p.is_writer ? st.writer_hg : st.reader_hg; - MyHGM->wrlock(); - MySrvC* s = MyHGM->find_server_in_hg(hid, p.blue_host, p.port); - if (s) { - s->ConnectionsFree->drop_all_connections(); - } - MyHGM->wrunlock(); + MyHGM->drain_server_connections(hid, p.blue_host.c_str(), p.port); + My_Conn_Pool->purge_connections(p.blue_host.c_str(), p.port); } // Blue readers without a green counterpart must stop serving reads. @@ -7330,6 +7366,7 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo MyHGM->aws_rds_bgd_set_shun_server(st.reader_hg, br.first.c_str(), br.second, false); // purge so the blue reader hostname re-resolves to the promoted instance dns_cache->remove(br.first); + My_Conn_Pool->purge_connections(br.first.c_str(), br.second); } st.shunned_readers.clear(); } @@ -7338,6 +7375,7 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo // remove the cache record (pin + resolved IPs) so the blue name // re-resolves to the promoted (green) instance dns_cache->remove(p.blue_host); + My_Conn_Pool->purge_connections(p.blue_host.c_str(), p.port); } // TODO: Drain Green HGs diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index 131ad27c9b..acb987a74d 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -4987,7 +4987,7 @@ bool MySQL_Session::handler_minus1_HandleErrorCodes(MySQL_Data_Stream *myds, int myds->destroy_MySQL_Connection_From_Pool(false); break; default: - if (mysql_thread___reset_connection_algorithm == 2) { + if (mysql_thread___reset_connection_algorithm == 2 && myds->myconn->healthy) { create_new_session_and_reset_connection(myds); } else { myds->destroy_MySQL_Connection_From_Pool(true); @@ -5082,7 +5082,7 @@ void MySQL_Session::handler_minus1_HandleBackendConnection(MySQL_Data_Stream *my if (mysql_thread___multiplexing && (myds->myconn->reusable==true) && myds->myconn->IsActiveTransaction()==false && myds->myconn->MultiplexDisabled()==false) { myds->DSS=STATE_NOT_INITIALIZED; if (mysql_thread___autocommit_false_not_reusable && myds->myconn->IsAutoCommit()==false) { - if (mysql_thread___reset_connection_algorithm == 2) { + if (mysql_thread___reset_connection_algorithm == 2 && myds->myconn->healthy) { create_new_session_and_reset_connection(myds); } else { myds->destroy_MySQL_Connection_From_Pool(true); @@ -8556,7 +8556,7 @@ void MySQL_Session::finishQuery(MySQL_Data_Stream *myds, MySQL_Connection *mycon myds->wait_until=0; myds->DSS=STATE_NOT_INITIALIZED; if (mysql_thread___autocommit_false_not_reusable && myds->myconn->IsAutoCommit()==false) { - if (mysql_thread___reset_connection_algorithm == 2) { + if (mysql_thread___reset_connection_algorithm == 2 && myds->myconn->healthy) { create_new_session_and_reset_connection(myds); } else { myds->destroy_MySQL_Connection_From_Pool(true); diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index acfe2b3da2..a738fc1c89 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -6398,6 +6398,11 @@ MySQL_Connection * MySQL_Thread::get_MyConn_local(unsigned int _hid, MySQL_Sessi for (i=0; ilen; i++) { c = (MySQL_Connection *) cached_connections->index(i); + // Skip unhealthy or non-reusable connections + if (!c->healthy || !c->reusable) { + continue; + } + // Skip cached connections whose parent server is inside the session-tracking // capability backoff window. See 'MySrvC::session_track_backoff_until' for the // full rationale; reads are relaxed because the deadline is compared against diff --git a/lib/MySrvConnList.cpp b/lib/MySrvConnList.cpp index 2ee3bf1db7..779fd804c1 100644 --- a/lib/MySrvConnList.cpp +++ b/lib/MySrvConnList.cpp @@ -48,6 +48,14 @@ void MySrvConnList::drop_all_connections() { } } +void MySrvConnList::mark_connections_unhealthy() { + for (unsigned int i = 0; i < conns_length(); i++) { + MySQL_Connection *conn = index(i); + conn->healthy=false; + conn->reusable=false; + } +} + unsigned int calculate_eviction_count(unsigned int conns_free, unsigned int conns_used, unsigned int max_connections) { if (conns_free < 1) return 0; unsigned int pct_max_connections = (3 * max_connections) / 4; @@ -303,4 +311,3 @@ MySQL_Connection * MySrvConnList::get_random_MyConn(MySQL_Session *sess, bool ff } return NULL; // never reach here } - diff --git a/lib/mysql_connection.cpp b/lib/mysql_connection.cpp index 1ba8c10547..20470ec714 100644 --- a/lib/mysql_connection.cpp +++ b/lib/mysql_connection.cpp @@ -428,6 +428,7 @@ MySQL_Connection::MySQL_Connection() { async_state_machine=ASYNC_CONNECT_START; ret_mysql=NULL; send_quit=true; + healthy=true; myds=NULL; inserted_into_pool=0; reusable=false; @@ -2164,6 +2165,9 @@ bool MySQL_Connection::IsServerOffline() { if (parent == NULL) return ret; + if (healthy == false) + return true; + server_status = parent->get_status(); // we copy it here to avoid race condition. The caller will see this bool server_shunned = (server_status == MYSQL_SERVER_STATUS_SHUNNED) || (server_status == MYSQL_SERVER_STATUS_SHUNNED_AWS_BGD); @@ -3037,6 +3041,8 @@ int MySQL_Connection::async_send_simple_command(short event, char *stmt, unsigne server_status=parent->get_status(); // we copy it here to avoid race condition. The caller will see this bool server_shunned = (server_status == MYSQL_SERVER_STATUS_SHUNNED) || (server_status == MYSQL_SERVER_STATUS_SHUNNED_AWS_BGD); if ( + (healthy == false) + || (server_status==MYSQL_SERVER_STATUS_OFFLINE_HARD) // the server is OFFLINE as specific by the user || (server_shunned && parent->shunned_automatic == true && parent->shunned_and_kill_all_connections==true) // the server is SHUNNED due to a serious issue @@ -3092,6 +3098,7 @@ void MySQL_Connection::reset() { bool old_no_multiplex_hg = get_status(STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); bool old_compress = get_status(STATUS_MYSQL_CONNECTION_COMPRESSION); status_flags=0; + healthy=true; // reconfigure STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG set_status(old_no_multiplex_hg,STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); // reconfigure STATUS_MYSQL_CONNECTION_COMPRESSION diff --git a/lib/mysql_data_stream.cpp b/lib/mysql_data_stream.cpp index 451d63367a..96792e52b7 100644 --- a/lib/mysql_data_stream.cpp +++ b/lib/mysql_data_stream.cpp @@ -1764,6 +1764,10 @@ void MySQL_Data_Stream::setDSS_STATE_QUERY_SENT_NET() { void MySQL_Data_Stream::return_MySQL_Connection_To_Pool() { MySQL_Connection *mc=myconn; mc->last_time_used=sess->thread->curtime; + if (!mc->reusable) { + destroy_MySQL_Connection_From_Pool(true); + return; + } // before detaching, check if last_HG_affected_rows matches . if yes, set it back to -1 if (mybe) { if (mybe->hostgroup_id == sess->last_HG_affected_rows) { @@ -1784,7 +1788,7 @@ void MySQL_Data_Stream::return_MySQL_Connection_To_Pool() { // is used outside 'PINGING_SERVER' operation. For more context see #3502. sess->status != PINGING_SERVER ) { - if (mysql_thread___reset_connection_algorithm == 2) { + if (mysql_thread___reset_connection_algorithm == 2 && mc->healthy) { sess->create_new_session_and_reset_connection(this); } else { destroy_MySQL_Connection_From_Pool(true); @@ -1829,7 +1833,7 @@ bool MySQL_Data_Stream::data_in_rbio() { void MySQL_Data_Stream::reset_connection() { if (myconn) { - if (mysql_thread___multiplexing && (DSS == STATE_MARIADB_GENERIC || DSS == STATE_READY) && myconn->reusable == true && myconn->IsActiveTransaction() == false && myconn->MultiplexDisabled() == false && myconn->async_state_machine == ASYNC_IDLE) { + if (mysql_thread___multiplexing && (DSS == STATE_MARIADB_GENERIC || DSS == STATE_READY) && myconn->healthy == true && myconn->reusable == true && myconn->IsActiveTransaction() == false && myconn->MultiplexDisabled() == false && myconn->async_state_machine == ASYNC_IDLE) { myconn->last_time_used = sess->thread->curtime; return_MySQL_Connection_To_Pool(); } From d0dbcd39829ba453819105ae9097e77fe85997d5 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 30 Jun 2026 05:54:59 +0000 Subject: [PATCH 16/81] fix: Make `DNS_Cache` honor IP pins with no expire time Signed-off-by: Wazir Ahmed --- include/DNS_Cache.hpp | 2 +- lib/DNS_Cache.cpp | 33 ++++++++++----------------------- 2 files changed, 11 insertions(+), 24 deletions(-) diff --git a/include/DNS_Cache.hpp b/include/DNS_Cache.hpp index ee1614e6ba..44fb6dad44 100644 --- a/include/DNS_Cache.hpp +++ b/include/DNS_Cache.hpp @@ -84,7 +84,7 @@ class DNS_Cache { void clear(); bool empty() const; bool is_ip_valid(const std::string& hostname, const std::string& ip) const; - std::string lookup(const std::string& hostname, size_t* ip_count) const; + std::string lookup(const std::string& hostname, size_t* ip_count); /** * @brief Pin a hostname to a fixed IP until it is explicitly unpinned. diff --git a/lib/DNS_Cache.cpp b/lib/DNS_Cache.cpp index def9ffca97..8face0834c 100644 --- a/lib/DNS_Cache.cpp +++ b/lib/DNS_Cache.cpp @@ -217,7 +217,8 @@ bool DNS_Cache::is_ip_valid(const std::string& hostname, const std::string& ip) auto itr = records.find(hostname); if (itr != records.end()) { const unsigned long long now = monotonic_time(); - const bool pin_active = itr->second.pinned_until != 0 && now <= itr->second.pinned_until; + const bool pin_active = !itr->second.pinned_ip.empty() + && (itr->second.pinned_until == 0 || now <= itr->second.pinned_until); if (pin_active) { valid = ip == itr->second.pinned_ip; } else { @@ -325,7 +326,7 @@ DNS_Cache::lookup_result_t DNS_Cache::get_next_ip(const IP_ADDR& ip_addr) const return result; } -std::string DNS_Cache::lookup(const std::string& hostname, size_t* ip_count) const { +std::string DNS_Cache::lookup(const std::string& hostname, size_t* ip_count) { if (!enabled) { if (ip_count) *ip_count = 0; @@ -346,8 +347,10 @@ std::string DNS_Cache::lookup(const std::string& hostname, size_t* ip_count) con lookup_result_t result = get_next_ip(itr->second); const unsigned long long now = monotonic_time(); - const bool pin_active = result.pinned_until != 0 && now <= result.pinned_until; - clear_expired_pin = result.pinned_until != 0 && now > result.pinned_until; + const bool pin_active = !result.pinned_ip.empty() + && (result.pinned_until == 0 || now <= result.pinned_until); + clear_expired_pin = !result.pinned_ip.empty() + && result.pinned_until != 0 && now > result.pinned_until; if (pin_active) { ip = result.pinned_ip; @@ -375,25 +378,9 @@ std::string DNS_Cache::lookup(const std::string& hostname, size_t* ip_count) con // cleanup expired pinned IP if (clear_expired_pin) { - rc = pthread_rwlock_wrlock(&rwlock_); - assert(rc == 0); - auto itr2 = records.find(hostname); - if (itr2 != records.end() && itr2->second.pinned_until != 0 && - monotonic_time() > itr2->second.pinned_until) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, - "Unpinning expired DNS cache record. (Hostname:[%s] IP:[%s])\n", - hostname.c_str(), itr2->second.pinned_ip.c_str()); - itr2->second.pinned_ip.clear(); - itr2->second.pinned_until = 0; - __sync_fetch_and_and(&itr2->second.counter, 0); - if (itr2->second.ips.empty()) { - records.erase(itr2); - } - if (counter_record_updated_) - counter_record_updated_->fetch_add(1, std::memory_order_relaxed); - } - rc = pthread_rwlock_unlock(&rwlock_); - assert(rc == 0); + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "Removing expired DNS cache pin. (Hostname:[%s])\n", hostname.c_str()); + unpin(hostname); } return ip; From 252897cff5c46d4f3de6b5075a5caa6aa7b15489 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 30 Jun 2026 06:04:28 +0000 Subject: [PATCH 17/81] fix: Clear affected-row hostgroup before destroying backend Signed-off-by: Wazir Ahmed --- lib/mysql_data_stream.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/mysql_data_stream.cpp b/lib/mysql_data_stream.cpp index 96792e52b7..2645fd406a 100644 --- a/lib/mysql_data_stream.cpp +++ b/lib/mysql_data_stream.cpp @@ -1764,16 +1764,19 @@ void MySQL_Data_Stream::setDSS_STATE_QUERY_SENT_NET() { void MySQL_Data_Stream::return_MySQL_Connection_To_Pool() { MySQL_Connection *mc=myconn; mc->last_time_used=sess->thread->curtime; - if (!mc->reusable) { - destroy_MySQL_Connection_From_Pool(true); - return; - } + // before detaching, check if last_HG_affected_rows matches . if yes, set it back to -1 if (mybe) { if (mybe->hostgroup_id == sess->last_HG_affected_rows) { sess->last_HG_affected_rows = -1; } } + + if (!mc->reusable) { + destroy_MySQL_Connection_From_Pool(true); + return; + } + unsigned long long intv = mysql_thread___connection_max_age_ms; intv *= 1000; if ( From 9b3903f75ac52338cb7858082240c77eafdd4288 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Wed, 1 Jul 2026 07:03:12 +0000 Subject: [PATCH 18/81] feat: Fast polling for RDS blue/green servers in read_only monitor Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 50 +++++---- include/MySQL_Monitor.hpp | 20 +++- lib/MySQL_HostGroups_Manager.cpp | 70 +++++++------ lib/MySQL_Monitor.cpp | 161 ++++++++++++++++++----------- 4 files changed, 185 insertions(+), 116 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index f7563bb0c5..0212e20a41 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -603,21 +603,6 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { return readonly_flag; } - inline - void set_aws_rds_bgd_in_progress() { - aws_rds_bgd_in_progress = true; - } - - inline - bool is_aws_rds_bgd_in_progress() { - return aws_rds_bgd_in_progress; - } - - inline - void clear_aws_rds_bgd_in_progress() { - aws_rds_bgd_in_progress = false; - } - private: unsigned int get_hostgroup_id(Type type, const Node& node) const; MySrvC* insert_HGM(unsigned int hostgroup_id, const MySrvC* srv); @@ -626,7 +611,6 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { std::array, TYPE_SIZE_> mapping; // index 0 contains reader and 1 contains writer hostgroups int readonly_flag; MySQL_HostGroups_Manager* myHGM; - bool aws_rds_bgd_in_progress = false; }; /** @@ -1079,9 +1063,20 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { * @param hostname Address of the server to match. * @param port Port of the server to match. * @param shun true to shun the server, false to unshun it. + * * @return true if this call changed a server's status. + * + * @note Caller must hold wrlock(). */ bool aws_rds_bgd_set_shun_server(unsigned int hostgroup_id, const char *hostname, int port, bool shun); + /** + * @brief Shun or unshun multiple servers in a hostgroup, then publish the change to the runtime tables. + * + * @param hostgroup_id Hostgroup to search. + * @param servers (hostname, port) pairs to act on. + * @param shun true to shun, false to unshun. + */ + void aws_rds_bgd_shun_servers(unsigned int hostgroup_id, const std::vector>& servers, bool shun); /** * @brief Drain existing backend connections for a server. * @@ -1095,11 +1090,26 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { */ bool drain_server_connections(unsigned int hostgroup_id, const char *hostname, int port); /** - * @brief Flag/unflag every server in the writer and reader hostgroups of an AWS RDS blue/green - * deployment as "switchover in progress", so the read_only monitor (read_only_action_v2) takes - * no action on them while the BGD FSM is driving the switchover. + * @brief Number of AWS RDS blue/green deployments currently mid-switchover. + * + * @details Each BGD worker increments the count while its deployment is switching over and + * decrements it once the deployment leaves the switchover states. While the count is non-zero + * the read_only monitor fast-polls the BGD servers; it resumes the full-fleet cadence at zero. */ - void set_aws_rds_bgd_in_progress(unsigned int writer_hg, unsigned int reader_hg, bool in_progress); + std::atomic aws_rds_bgd_in_progress_count{0}; + + void set_aws_rds_bgd_in_progress(bool in_progress) { + if (in_progress) { + aws_rds_bgd_in_progress_count.fetch_add(1, std::memory_order_relaxed); + } else { + aws_rds_bgd_in_progress_count.fetch_sub(1, std::memory_order_relaxed); + } + } + + bool is_aws_rds_bgd_in_progress() const { + return aws_rds_bgd_in_progress_count.load(std::memory_order_relaxed) > 0; + } + unsigned long long Get_Memory_Stats(); void add_discovered_servers_to_mysql_servers_and_replication_hostgroups(const vector>& new_servers); diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 6981d659d1..09984f5583 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -61,6 +61,7 @@ struct cmp_str { #define AWS_ENDPOINT_SUFFIX_STRING "rds.amazonaws.com" #define QUERY_AWS_RDS_TOPOLOGY_DISCOVERY "SELECT * FROM mysql.rds_topology" +#define QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK "SELECT 1 FROM information_schema.TABLES WHERE TABLE_SCHEMA='mysql' AND TABLE_NAME='rds_topology'" /* @@ -446,9 +447,9 @@ struct AWS_RDS_BGD_State { std::vector> shunned_readers; ///< (host,port) we shunned std::string last_status; ///< status from the previous poll, to act only when it changes - bool green_writer_added_in_hg = false; ///< whether green writer added to green_writer_hg - bool writer_is_also_reader_enforced = false; ///< whether POST_PROCESSING added the writer to the reader HG - bool bgd_in_progress_set = false; ///< whether we flagged the deployment's servers as switchover-in-progress (set once at INITIATED+, cleared at COMPLETED) so read_only_action_v2 leaves them alone + bool green_writer_added_in_hg = false; ///< green writer added to green_writer_hg + bool writer_is_also_reader_enforced = false; ///< POST_PROCESSING added the writer to the reader HG + bool bgd_in_progress_set = false; ///< MyHGM's in-progress switchover count is incremented unsigned int next_check_interval_ms = 0; ///< FSM-controlled interval; 0 => baseline std::string next_check_host; ///< FSM-pinned probe host; when set (the green IP), the worker @@ -472,6 +473,19 @@ inline const char* const BGD_STATUS_IN_PROGRESS = "SWITCHOVER_IN_PROGRESS"; inline const char* const BGD_STATUS_POST_PROC = "SWITCHOVER_IN_POST_PROCESSING"; inline const char* const BGD_STATUS_COMPLETED = "SWITCHOVER_COMPLETED"; +// While any AWS RDS blue/green deployment is mid-switchover, the read_only monitor polls just that +// deployment's servers at this tightened interval (250ms) so it detects the writer's read_only flips +// quickly; matches the BGD FSM's own fast poll tiers. The full-fleet pass stays at +// mysql-monitor_read_only_interval. +#define READ_ONLY_BGD_LOOP_INTERVAL_US 250000 +#define READ_ONLY_NEXT_LOOP_INTERVAL_US 500000 + +// read_only monitor server-enumeration queries. +// Every server that belongs to a replication hostgroup and status NOT IN (2,3,5) +#define SELECT_SERVERS_FOR_READ_ONLY "SELECT hostname, port, MAX(use_ssl) use_ssl, check_type, reader_hostgroup FROM mysql_servers JOIN mysql_replication_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE status NOT IN (2,3,5) GROUP BY hostname, port ORDER BY RANDOM()" +// Fast pass: only servers in an AWS RDS blue/green deployment +#define SELECT_RDS_BGD_SERVERS_FOR_READ_ONLY "SELECT hostname, port, MAX(use_ssl) use_ssl, 'read_only' check_type, reader_hostgroup FROM mysql_servers JOIN mysql_aws_rds_bgd_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE active=1 AND status NOT IN (2,3,5) GROUP BY hostname, port ORDER BY RANDOM()" + class MySQL_Monitor { public: diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index abb559588c..606394a3b4 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -3602,7 +3602,7 @@ void MySQL_HostGroups_Manager::read_only_action_v2(const std::listsecond.get(); - if (!host_server_mapping || host_server_mapping->is_aws_rds_bgd_in_progress()) { + if (!host_server_mapping) { continue; } @@ -3861,8 +3861,6 @@ void MySQL_HostGroups_Manager::set_Readyset_status(char *hostname, int port, enu bool MySQL_HostGroups_Manager::aws_rds_bgd_set_shun_server(unsigned int hostgroup_id, const char *hostname, int port, bool shun) { bool changed = false; - wrlock(); - MyHGC *myhgc = MyHGC_find(hostgroup_id); if (myhgc && myhgc->mysrvs) { for (unsigned int j = 0; j < myhgc->mysrvs->cnt(); j++) { @@ -3898,10 +3896,47 @@ bool MySQL_HostGroups_Manager::aws_rds_bgd_set_shun_server(unsigned int hostgrou } } - wrunlock(); return changed; } +void MySQL_HostGroups_Manager::aws_rds_bgd_shun_servers( + unsigned int hostgroup_id, const std::vector>& servers, bool shun +) { + bool changed = false; + + wrlock(); + + for (const std::pair& s : servers) { + if (aws_rds_bgd_set_shun_server(hostgroup_id, s.first.c_str(), s.second, shun)) { + changed = true; + } + } + + if (!changed) { + wrunlock(); + return; + } + + // Publish the new in-memory statuses into the runtime mysql_servers table + purge_mysql_servers_table(); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers\n"); + mydb->execute("DELETE FROM mysql_servers"); + generate_mysql_servers_table(); + + // Update the global checksums after 'mysql_servers' regeneration + unique_ptr resultset { get_admin_runtime_mysql_servers(mydb) }; + uint64_t raw_checksum = resultset ? resultset->raw_checksum() : 0; + hgsm_mysql_servers_checksum = raw_checksum; + string mysrvs_checksum { get_checksum_from_hash(raw_checksum) }; + save_runtime_mysql_servers(resultset.release()); + proxy_info("Checksum for table %s is %s\n", "mysql_servers", mysrvs_checksum.c_str()); + pthread_mutex_lock(&GloVars.checksum_mutex); + update_glovars_mysql_servers_checksum(mysrvs_checksum); + pthread_mutex_unlock(&GloVars.checksum_mutex); + + wrunlock(); +} + /** * @brief Drain existing backend connections for a server. * @@ -3938,33 +3973,6 @@ bool MySQL_HostGroups_Manager::drain_server_connections(unsigned int hostgroup_i return found; } -void MySQL_HostGroups_Manager::set_aws_rds_bgd_in_progress(unsigned int writer_hg, unsigned int reader_hg, bool in_progress) { - wrlock(); - - unsigned int hgs[2] = { writer_hg, reader_hg }; - for (unsigned int i = 0; i < 2; i++) { - // MyHGC_find (not MyHGC_lookup, which creates on miss) so we never materialize an empty HG. - MyHGC* myhgc = MyHGC_find(hgs[i]); - if (myhgc == nullptr || myhgc->mysrvs == nullptr) { - continue; - } - for (unsigned int j = 0; j < myhgc->mysrvs->cnt(); j++) { - MySrvC* s = myhgc->mysrvs->idx(j); - const std::string srv_id = std::string(s->address) + ":::" + std::to_string(s->port); - auto itr = hostgroup_server_mapping.find(srv_id); - if (itr != hostgroup_server_mapping.end() && itr->second) { - if (in_progress) { - itr->second->set_aws_rds_bgd_in_progress(); - } else { - itr->second->clear_aws_rds_bgd_in_progress(); - } - } - } - } - - wrunlock(); -} - void MySQL_HostGroups_Manager::p_update_metrics() { p_update_counter(status.p_counter_array[p_hg_counter::servers_table_version], status.servers_table_version); // Update *server_connections* related metrics diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 6de87f78e4..0b0aebc088 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -3691,18 +3691,25 @@ void * MySQL_Monitor::monitor_read_only() { unsigned long long t1; unsigned long long t2; + // next loop iteration time for regular read_only checks unsigned long long next_loop_at=0; - int topology_loop = 0; + // next loop iteration time for read_only checks on RDS blue/green deployment servers + unsigned long long next_bgd_loop_at = 0; + int rds_topology_check_counter = 0; while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true) { - int topology_loop_max = mysql_thread___monitor_aws_rds_topology_discovery_interval; - bool do_discovery_check = false; + // whether to run read_only checks on RDS blue/green deployment servers only + bool rds_bgd_only_loop = false; + + // whether to run mysql.rds_topology check for RDS servers + // in addition to regular read_only checks for all servers + bool rds_topology_check = false; + int rds_topology_check_interval = mysql_thread___monitor_aws_rds_topology_discovery_interval; unsigned int glover; char *error=NULL; SQLite3_result *resultset=NULL; - // add support for SSL - char *query=(char *)"SELECT hostname, port, MAX(use_ssl) use_ssl, check_type, reader_hostgroup FROM mysql_servers JOIN mysql_replication_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE status NOT IN (2,3) GROUP BY hostname, port ORDER BY RANDOM()"; + const char *query = NULL; t1=monotonic_time(); if (!GloMTH) return NULL; // quick exit during shutdown/restart @@ -3713,13 +3720,28 @@ void * MySQL_Monitor::monitor_read_only() { next_loop_at=0; } + bool bgd_active = MyHGM->is_aws_rds_bgd_in_progress(); + if (bgd_active) { + if (t1 < next_loop_at && t1 >= next_bgd_loop_at) { + rds_bgd_only_loop = true; + } + next_bgd_loop_at = t1 + READ_ONLY_BGD_LOOP_INTERVAL_US; + } else { + // BGD is not active && regular read_only interval time has not elapsed + if (t1 < next_loop_at) { + goto __sleep_monitor_read_only; + } + } - if (t1 < next_loop_at) { - goto __sleep_monitor_read_only; + if (rds_bgd_only_loop) { + query = SELECT_RDS_BGD_SERVERS_FOR_READ_ONLY; + } else { + query = SELECT_SERVERS_FOR_READ_ONLY; + next_loop_at = t1 + 1000ULL * (unsigned int) mysql_thread___monitor_read_only_interval; } - next_loop_at=t1+1000*mysql_thread___monitor_read_only_interval; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); - resultset = MyHGM->execute_query(query, &error); + resultset = MyHGM->execute_query((char*) query, &error); assert(resultset); if (error) { proxy_error("Error on %s : %s\n", query, error); @@ -3730,20 +3752,20 @@ void * MySQL_Monitor::monitor_read_only() { goto __end_monitor_read_only_loop; } - if (topology_loop_max > 0) { // if the discovery interval is set to zero, do not query for the topology - if (topology_loop >= topology_loop_max) { - do_discovery_check = true; - topology_loop = 0; + if (!rds_bgd_only_loop && rds_topology_check_interval > 0) { + if (rds_topology_check_counter >= rds_topology_check_interval) { + rds_topology_check = true; + rds_topology_check_counter = 0; } - topology_loop += 1; + rds_topology_check_counter += 1; } // resultset must be initialized before calling monitor_read_only_async - monitor_read_only_async(resultset, do_discovery_check); + monitor_read_only_async(resultset, rds_topology_check); if (shutdown) return NULL; __end_monitor_read_only_loop: - if (mysql_thread___monitor_enabled==true) { + if (!rds_bgd_only_loop && mysql_thread___monitor_enabled) { char *query=NULL; query=(char *)"DELETE FROM mysql_server_read_only_log WHERE time_start_us < ?1"; auto [rc1, statement_unique] = monitordb->prepare_v2(query); @@ -3765,16 +3787,24 @@ void * MySQL_Monitor::monitor_read_only() { delete resultset; __sleep_monitor_read_only: - t2=monotonic_time(); - if (t2 500000) { - st = 500000; + t2 = monotonic_time(); + unsigned long long st = 0; + if (bgd_active) { + if (t2 < next_bgd_loop_at) { + st = next_bgd_loop_at - t2; + usleep(st); + } + } else { + if (t2 < next_loop_at) { + st = next_loop_at - t2; + if (st > READ_ONLY_NEXT_LOOP_INTERVAL_US) { + st = READ_ONLY_NEXT_LOOP_INTERVAL_US; + } + usleep(st); } - usleep(st); } } + if (mysql_thr) { delete mysql_thr; mysql_thr=NULL; @@ -6668,6 +6698,37 @@ static int aws_rds_bgd_async_query(MySQL_Monitor_State_Data *mmsd, const char *q return 0; } +/** +* @brief Mark a switchover in progress so the read_only monitor fast-polls this deployment's servers. +*/ +static void aws_rds_bgd_set_bgd_in_progress(AWS_RDS_BGD_State& st) { + if (st.bgd_in_progress_set) { + return; + } + + MyHGM->set_aws_rds_bgd_in_progress(true); + st.bgd_in_progress_set = true; + + proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: Enabling fast-poll of BGD servers on read_only monitor.\n", + st.writer_hg, st.reader_hg); +} + +/** +* @brief Drop this deployment's in-progress count so the read_only monitor can return to baseline +* polling cadence (undo aws_rds_bgd_set_bgd_in_progress). +*/ +static void aws_rds_bgd_clear_bgd_in_progress(AWS_RDS_BGD_State& st) { + if (!st.bgd_in_progress_set) { + return; + } + + MyHGM->set_aws_rds_bgd_in_progress(false); + st.bgd_in_progress_set = false; + + proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: Disabling fast-poll of BGD servers on read_only monitor.\n", + st.writer_hg, st.reader_hg); +} + void * monitor_RDS_BGD_thread_HG(void *arg) { unsigned int wHG = *(unsigned int *)arg; unsigned int num_hosts = 0; @@ -6861,11 +6922,7 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { // we advance to TOPOLOGY_METADATA_FETCH and skip this check on subsequent // iterations, until a fetch reports the table is gone. - int qrc = aws_rds_bgd_async_query( - mmsd, - "SELECT 1 FROM information_schema.TABLES" - " WHERE TABLE_SCHEMA='mysql' AND TABLE_NAME='rds_topology'" - ); + int qrc = aws_rds_bgd_async_query(mmsd, QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK); if (qrc == 2) { goto __exit_monitor_RDS_BGD_thread_HG_now; } @@ -6883,6 +6940,7 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { } if (!table_available) { // no blue/green deployment or multi-az cluster discovery in progress; nothing to do + aws_rds_bgd_clear_bgd_in_progress(st); proxy_debug(PROXY_DEBUG_MONITOR, 5, "mysql.rds_topology not present on %s:%d (RDS writer HG %u); skipping\n", mmsd->hostname, mmsd->port, wHG); @@ -6906,6 +6964,7 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { // return to the baseline poll interval. topology_state = TOPOLOGY_TABLE_CHECK; st.next_check_interval_ms = 0; + aws_rds_bgd_clear_bgd_in_progress(st); proxy_debug(PROXY_DEBUG_MONITOR, 5, "mysql.rds_topology vanished on %s:%d (RDS writer HG %u); rechecking availability\n", mmsd->hostname, mmsd->port, wHG); @@ -6958,23 +7017,26 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { } delete mmsd; mmsd = NULL; - - // TODO: call aws_rds_bgd_clear_bgd_in_progress() } __exit_monitor_RDS_BGD_thread_HG_now: + aws_rds_bgd_clear_bgd_in_progress(st); + if (mmsd) { delete mmsd; mmsd = NULL; } + for (unsigned int i=0; iset_aws_rds_bgd_in_progress(st.writer_hg, st.reader_hg, true); - st.bgd_in_progress_set = true; - proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: switchover in progress, suspending read_only monitor actions on these hostgroups until SWITCHOVER_COMPLETED\n", - st.writer_hg, st.reader_hg); -} - -/** -* @brief Re-enable read_only monitor action on the deployment's servers (undo aws_rds_bgd_set_bgd_in_progress). -*/ -static void aws_rds_bgd_clear_bgd_in_progress(AWS_RDS_BGD_State& st) { - if (!st.bgd_in_progress_set) { - return; - } - MyHGM->set_aws_rds_bgd_in_progress(st.writer_hg, st.reader_hg, false); - st.bgd_in_progress_set = false; -} - /** * @brief Run the status-driven blue/green switchover FSM for one deployment. * @@ -7226,6 +7260,7 @@ static void aws_rds_bgd_clear_bgd_in_progress(AWS_RDS_BGD_State& st) { void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_Result& topology) { if (!topology.blue_green) { st.next_check_interval_ms = 0; + aws_rds_bgd_clear_bgd_in_progress(st); return; } @@ -7238,6 +7273,7 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo } if (status.empty()) { st.next_check_interval_ms = 0; + aws_rds_bgd_clear_bgd_in_progress(st); return; } @@ -7315,6 +7351,7 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo } } MyHGM->wrunlock(); + std::vector> unmapped_readers; for (const std::pair& br : blue_readers) { bool mapped = false; for (const AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { @@ -7324,11 +7361,11 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo } } if (!mapped) { - if (MyHGM->aws_rds_bgd_set_shun_server(st.reader_hg, br.first.c_str(), br.second, true)) { - st.shunned_readers.push_back(br); - } + unmapped_readers.push_back(br); } } + MyHGM->aws_rds_bgd_shun_servers(st.reader_hg, unmapped_readers, true); + st.shunned_readers.insert(st.shunned_readers.end(), unmapped_readers.begin(), unmapped_readers.end()); // If no reader is mapped, funnel reads to the (now green) writer by // enforcing writer_is_also_reader for the duration of the switchover. @@ -7362,8 +7399,8 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo } if (!st.shunned_readers.empty()) { + MyHGM->aws_rds_bgd_shun_servers(st.reader_hg, st.shunned_readers, false); for (const std::pair& br : st.shunned_readers) { - MyHGM->aws_rds_bgd_set_shun_server(st.reader_hg, br.first.c_str(), br.second, false); // purge so the blue reader hostname re-resolves to the promoted instance dns_cache->remove(br.first); My_Conn_Pool->purge_connections(br.first.c_str(), br.second); From 4eac77dc907d33d51f281cf9587236c824296313 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 2 Jul 2026 04:24:41 +0000 Subject: [PATCH 19/81] Fix gaps in handling new server status `SHUNNED_AWS_BGD` Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 4 ++-- include/MySQL_Monitor.hpp | 2 +- lib/MySQL_Monitor.cpp | 36 ++++++------------------------ lib/ProxySQL_Admin.cpp | 8 +++++-- lib/ProxySQL_Cluster.cpp | 6 ++++- 5 files changed, 21 insertions(+), 35 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 0212e20a41..aec3d278d8 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -39,8 +39,8 @@ // we have 2 versions of the same tables: with (debug) and without (no debug) checks #ifdef DEBUG -#define MYHGM_MYSQL_SERVERS "CREATE TABLE mysql_servers ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT NOT NULL DEFAULT 0 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , status INT CHECK (status IN (0, 1, 2, 3, 4)) NOT NULL DEFAULT 0 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , mem_pointer INT NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" -#define MYHGM_MYSQL_SERVERS_INCOMING "CREATE TABLE mysql_servers_incoming ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT NOT NULL DEFAULT 0 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , status INT CHECK (status IN (0, 1, 2, 3, 4)) NOT NULL DEFAULT 0 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port))" +#define MYHGM_MYSQL_SERVERS "CREATE TABLE mysql_servers ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT NOT NULL DEFAULT 0 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , status INT CHECK (status IN (0, 1, 2, 3, 4, 5)) NOT NULL DEFAULT 0 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , mem_pointer INT NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" +#define MYHGM_MYSQL_SERVERS_INCOMING "CREATE TABLE mysql_servers_incoming ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT NOT NULL DEFAULT 0 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , status INT CHECK (status IN (0, 1, 2, 3, 4, 5)) NOT NULL DEFAULT 0 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port))" #else #define MYHGM_MYSQL_SERVERS "CREATE TABLE mysql_servers ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT NOT NULL DEFAULT 0 , weight INT NOT NULL DEFAULT 1 , status INT NOT NULL DEFAULT 0 , compression INT NOT NULL DEFAULT 0 , max_connections INT NOT NULL DEFAULT 1000 , max_replication_lag INT NOT NULL DEFAULT 0 , use_ssl INT NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , mem_pointer INT NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" #define MYHGM_MYSQL_SERVERS_INCOMING "CREATE TABLE mysql_servers_incoming ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT NOT NULL DEFAULT 0 , weight INT NOT NULL DEFAULT 1 , status INT NOT NULL DEFAULT 0 , compression INT NOT NULL DEFAULT 0 , max_connections INT NOT NULL DEFAULT 1000 , max_replication_lag INT NOT NULL DEFAULT 0 , use_ssl INT NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port))" diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 09984f5583..8c1d48d847 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -40,7 +40,7 @@ #define MONITOR_SQLITE_TABLE_MYSQL_SERVER_AWS_AURORA_FAILOVERS "CREATE TABLE mysql_server_aws_aurora_failovers (writer_hostgroup INT NOT NULL , hostname VARCHAR NOT NULL , inserted_at VARCHAR NOT NULL)" -#define MONITOR_SQLITE_TABLE_MYSQL_SERVERS "CREATE TABLE mysql_servers (hostname VARCHAR NOT NULL , port INT NOT NULL , status INT CHECK (status IN (0, 1, 2, 3, 4)) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , PRIMARY KEY (hostname, port) )" +#define MONITOR_SQLITE_TABLE_MYSQL_SERVERS "CREATE TABLE mysql_servers (hostname VARCHAR NOT NULL , port INT NOT NULL , status INT CHECK (status IN (0, 1, 2, 3, 4, 5)) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , PRIMARY KEY (hostname, port) )" #define MONITOR_SQLITE_TABLE_PROXYSQL_SERVERS "CREATE TABLE proxysql_servers (hostname VARCHAR NOT NULL , port INT NOT NULL , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostname, port) )" diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 0b0aebc088..2815a2c2bc 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -7316,11 +7316,7 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo // Repoint each mapped blue host onto its green IP and drain existing // connections so new backend work resolves to green. - bool any_reader_mapped = false; for (AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { - if (!p.is_writer) { - any_reader_mapped = true; - } if (p.green_ip.empty()) { proxy_warning( "AWS RDS BGD [wHG=%u rHG=%u]: no green IP for blue '%s:%d'; cannot repoint\n", @@ -7366,36 +7362,18 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo } MyHGM->aws_rds_bgd_shun_servers(st.reader_hg, unmapped_readers, true); st.shunned_readers.insert(st.shunned_readers.end(), unmapped_readers.begin(), unmapped_readers.end()); - - // If no reader is mapped, funnel reads to the (now green) writer by - // enforcing writer_is_also_reader for the duration of the switchover. - if (!any_reader_mapped) { - for (const AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { - if (p.is_writer) { - srv_info_t srv_info { p.blue_host, (uint16_t)p.port, "AWS RDS BGD writer_is_also_reader" }; - srv_opts_t srv_opts { p.blue_weight, p.blue_max_conns, p.blue_use_ssl }; - MyHGM->wrlock(); - MyHGM->create_new_server_in_hg(st.reader_hg, srv_info, srv_opts); - MyHGM->wrunlock(); - st.writer_is_also_reader_enforced = true; - break; - } - } - } } else if (strcasecmp(status.c_str(), BGD_STATUS_COMPLETED) == 0) { st.next_check_interval_ms = 0; - if (st.writer_is_also_reader_enforced) { - for (const AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { - if (p.is_writer) { - MyHGM->wrlock(); - MyHGM->remove_server_in_hg(st.reader_hg, p.blue_host, (uint16_t)p.port); - MyHGM->wrunlock(); - break; - } + // if writer is in reader_hg, remove it + for (const AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { + if (p.is_writer) { + MyHGM->wrlock(); + MyHGM->remove_server_in_hg(st.reader_hg, p.blue_host, (uint16_t)p.port); + MyHGM->wrunlock(); + break; } - st.writer_is_also_reader_enforced = false; } if (!st.shunned_readers.empty()) { diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index ee7c3f90d5..aa6b5ebd79 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -7330,13 +7330,17 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { max_bulk_row_idx=max_bulk_row_idx*32; for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r1=*it; + const char *status = r1->fields[4]; + if (_runtime == false && (strcmp(status,"SHUNNED") == 0 || strcmp(status,"SHUNNED_AWS_BGD") == 0)) { + status = "ONLINE"; + } int idx=row_idx%32; if (row_idxfields[0])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_text)(statement32, (idx*12)+2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+3, atoi(r1->fields[2])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+4, atoi(r1->fields[3])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*12)+5, ( _runtime ? r1->fields[4] : ( strcmp(r1->fields[4],"SHUNNED")==0 ? "ONLINE" : r1->fields[4] ) ), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*12)+5, status, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+6, atoi(r1->fields[5])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+7, atoi(r1->fields[6])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+8, atoi(r1->fields[7])); ASSERT_SQLITE_OK(rc, admindb); @@ -7354,7 +7358,7 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { rc=(*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement1, 3, atoi(r1->fields[2])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atoi(r1->fields[3])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement1, 5, ( _runtime ? r1->fields[4] : ( strcmp(r1->fields[4],"SHUNNED")==0 ? "ONLINE" : r1->fields[4] ) ), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement1, 5, status, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement1, 6, atoi(r1->fields[5])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoi(r1->fields[6])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atoi(r1->fields[7])); ASSERT_SQLITE_OK(rc, admindb); diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 55fbcb26a9..1e54d0b361 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -2248,7 +2248,11 @@ void ProxySQL_Cluster::pull_mysql_servers_v2_from_peer(const mysql_servers_v2_ch char* o = escape_string_single_quotes(row[11], false); char* query = (char*)malloc(strlen(q) + l + strlen(o) + 64); - sprintf(query, q, row[0], row[1], row[2], row[3], (strcmp(row[4], "SHUNNED") == 0 ? "ONLINE" : row[4]), row[5], row[6], row[7], row[8], row[9], row[10], o); + const char *status = row[4]; + if (strcmp(status, "SHUNNED") == 0 || strcmp(status, "SHUNNED_AWS_BGD") == 0) { + status = "ONLINE"; + } + sprintf(query, q, row[0], row[1], row[2], row[3], status, row[5], row[6], row[7], row[8], row[9], row[10], o); if (o != row[11]) { // there was a copy free(o); } From 085785df201a62a187b2ee74641b9b25d3940b59 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 2 Jul 2026 08:08:13 +0000 Subject: [PATCH 20/81] fix: Move AWS RDS blue readers to SHUNNED after switchover - Move the blue readers to SHUNNED with a fixed recovery time after BGD status `SWITCHOVER_COMPLETED` Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 4 ++++ include/MySQL_Monitor.hpp | 34 ++++++++++++++++++------------ lib/MySQL_HostGroups_Manager.cpp | 15 +++++++------ lib/MySQL_Monitor.cpp | 16 +++++++------- 4 files changed, 42 insertions(+), 27 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index aec3d278d8..fe12db0ef6 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -37,6 +37,10 @@ //#define STRESSTEST_POOL #endif // DEBUG +// Seconds to keep a blue reader SHUNNED after an AWS RDS blue/green switchover, +// giving AWS time to migrate it before it rejoins the reader hostgroup. +#define AWS_RDS_BGD_UNSHUN_DELAY_SEC 15 + // we have 2 versions of the same tables: with (debug) and without (no debug) checks #ifdef DEBUG #define MYHGM_MYSQL_SERVERS "CREATE TABLE mysql_servers ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT NOT NULL DEFAULT 0 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , status INT CHECK (status IN (0, 1, 2, 3, 4, 5)) NOT NULL DEFAULT 0 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , mem_pointer INT NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 8c1d48d847..beed5908d9 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -415,6 +415,26 @@ struct AWS_RDS_Topology_Result { std::vector nodes; }; +/** + * @brief Mapping between one blue host and its name-matched green counterpart. + * + * @details The RDS BGD worker builds these pairs from the current blue + * writer/reader hostgroups and the discovered green topology. Each entry + * carries the blue server attributes needed to move the matching green + * server during switchover handling. + */ +struct AWS_RDS_BlueGreenPair { + std::string blue_host; ///< Blue hostname from the writer or reader hostgroup. + std::string green_host; ///< Matched green hostname using the RDS "-green-" naming pattern. + int port = 0; ///< Shared blue/green port; hostgroup manager keys servers by host and port. + int64_t blue_weight = 1; ///< Blue server weight mirrored onto the green server when it is added. + int64_t blue_max_conns = 1000; ///< Blue server max_connections mirrored onto the green server when it is added. + int32_t blue_use_ssl = 0; ///< Blue server SSL setting mirrored onto the green server when it is added. + std::string green_ip; ///< Green host IP resolved at SWITCHOVER_INITIATED and held warm. + unsigned long long green_ip_ttl = 0; ///< Expiry for green_ip when resolved by the BGD thread; 0 means DNS_Cache-sourced. + bool is_writer = false; ///< True when this pair maps the blue writer. +}; + /** * @brief Per-deployment switchover state carried by one RDS BGD worker thread. * @@ -431,19 +451,7 @@ struct AWS_RDS_BGD_State { int green_writer_hg = -1; ///< -1 when NULL (auto-discovery path) int green_reader_hg = -1; ///< -1 when NULL - /// One blue host and its name-matched green counterpart. - struct BlueGreenPair { - std::string blue_host; ///< blue host (from writer/reader HG) - std::string green_host; ///< matched green host (-green-) - int port = 0; ///< shared blue/green port (HGM keys on host+port) - int64_t blue_weight = 1; ///< blue server's connection settings, mirrored onto green when added - int64_t blue_max_conns = 1000; - int32_t blue_use_ssl = 0; - std::string green_ip; ///< green host IP, resolved at SWITCHOVER_INITIATED and held warm - unsigned long long green_ip_ttl = 0; ///< expiry of a green_ip if it is resolved by BGD thread; 0 => DNS_Cache-sourced - bool is_writer = false; ///< true => maps the blue writer - }; - std::vector bg_map; ///< [writer] always; [readers] only when green_reader_hg is configured + std::vector bg_map; ///< [writer] always; [readers] only when green_reader_hg is configured std::vector> shunned_readers; ///< (host,port) we shunned std::string last_status; ///< status from the previous poll, to act only when it changes diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 606394a3b4..c0911d7eeb 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -3869,12 +3869,14 @@ bool MySQL_HostGroups_Manager::aws_rds_bgd_set_shun_server(unsigned int hostgrou continue; } + time_t now = time(NULL); + if (shun) { if (mysrvc->get_status() == MYSQL_SERVER_STATUS_ONLINE) { mysrvc->set_status(MYSQL_SERVER_STATUS_SHUNNED_AWS_BGD); mysrvc->shunned_automatic = true; mysrvc->shunned_and_kill_all_connections = true; - mysrvc->time_last_detected_error = time(NULL); + mysrvc->time_last_detected_error = now; mysrvc->ConnectionsFree->drop_all_connections(); mysrvc->ConnectionsUsed->mark_connections_unhealthy(); proxy_warning("AWS RDS BGD shunning server %s:%d in HG %u\n", @@ -3883,12 +3885,13 @@ bool MySQL_HostGroups_Manager::aws_rds_bgd_set_shun_server(unsigned int hostgrou } } else { if (mysrvc->get_status() == MYSQL_SERVER_STATUS_SHUNNED_AWS_BGD) { - mysrvc->set_status(MYSQL_SERVER_STATUS_ONLINE); - mysrvc->shunned_automatic = false; + // Don't move back to ONLINE immediately. Move SHUNNED_AWS_BGD -> SHUNNED, + // set time_last_detected_error to a future time and let the server-selection + // logic auto-recover this server. + mysrvc->set_status(MYSQL_SERVER_STATUS_SHUNNED); mysrvc->shunned_and_kill_all_connections = false; - mysrvc->connect_ERR_at_time_last_detected_error = 0; - mysrvc->time_last_detected_error = 0; - proxy_warning("AWS RDS BGD unshunning server %s:%d in HG %u\n", + mysrvc->time_last_detected_error = now + AWS_RDS_BGD_UNSHUN_DELAY_SEC; + proxy_warning("AWS RDS BGD changing server status from SHUNNED_AWS_BGD to SHUNNED for %s:%d in HG %u\n", hostname, port, myhgc->hid); changed = true; } diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 2815a2c2bc..d390796ac2 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -7107,7 +7107,7 @@ static void aws_rds_bgd_build_map(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_ continue; } if (aws_rds_bgd_match_host(s->address, green_writer_host)) { - AWS_RDS_BGD_State::BlueGreenPair p; + AWS_RDS_BlueGreenPair p; p.blue_host = s->address; p.port = s->port; p.green_host = green_writer_host; @@ -7147,7 +7147,7 @@ static void aws_rds_bgd_build_map(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_ } for (const std::string& green_reader_host : green_reader_hosts) { if (aws_rds_bgd_match_host(s->address, green_reader_host)) { - AWS_RDS_BGD_State::BlueGreenPair p; + AWS_RDS_BlueGreenPair p; p.blue_host = s->address; p.port = s->port; p.green_host = green_reader_host; @@ -7208,7 +7208,7 @@ static void aws_rds_bgd_resolve_green_ips(AWS_RDS_BGD_State& st) { } // Pin the worker's next probe to the green writer's IP (observe the switchover from green). - for (const AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { + for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { if (p.is_writer && !p.green_ip.empty()) { if (st.next_check_host != p.green_ip) { st.next_check_host = p.green_ip; @@ -7232,7 +7232,7 @@ static void aws_rds_bgd_add_green_writer_in_hg(AWS_RDS_BGD_State& st) { if (st.green_writer_hg < 0 || st.green_writer_added_in_hg) { return; } - for (const AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { + for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { if (p.is_writer) { srv_info_t srv_info { p.green_host, (uint16_t)p.port, "AWS RDS BGD green writer" }; srv_opts_t srv_opts { p.blue_weight, p.blue_max_conns, p.blue_use_ssl }; @@ -7316,7 +7316,7 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo // Repoint each mapped blue host onto its green IP and drain existing // connections so new backend work resolves to green. - for (AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { + for (AWS_RDS_BlueGreenPair& p : st.bg_map) { if (p.green_ip.empty()) { proxy_warning( "AWS RDS BGD [wHG=%u rHG=%u]: no green IP for blue '%s:%d'; cannot repoint\n", @@ -7350,7 +7350,7 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo std::vector> unmapped_readers; for (const std::pair& br : blue_readers) { bool mapped = false; - for (const AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { + for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { if (p.blue_host == br.first && p.port == br.second) { mapped = true; break; @@ -7367,7 +7367,7 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo st.next_check_interval_ms = 0; // if writer is in reader_hg, remove it - for (const AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { + for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { if (p.is_writer) { MyHGM->wrlock(); MyHGM->remove_server_in_hg(st.reader_hg, p.blue_host, (uint16_t)p.port); @@ -7386,7 +7386,7 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo st.shunned_readers.clear(); } - for (const AWS_RDS_BGD_State::BlueGreenPair& p : st.bg_map) { + for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { // remove the cache record (pin + resolved IPs) so the blue name // re-resolves to the promoted (green) instance dns_cache->remove(p.blue_host); From e2cfb42d1eb6389dc68729b4dc0e5bae42818c9c Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 2 Jul 2026 18:40:33 +0000 Subject: [PATCH 21/81] fix: Remove unused columns from `mysql_aws_rds_bgd_hostgroups` table - `domain_name` - `autopurge_missing_checks` Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 2 -- include/ProxySQL_Admin_Tables_Definitions.h | 4 --- lib/MySQL_HostGroups_Manager.cpp | 39 ++++++++++----------- lib/MySQL_Monitor.cpp | 2 +- lib/ProxySQL_Admin.cpp | 20 +++++------ lib/ProxySQL_Config.cpp | 22 ++++-------- 6 files changed, 34 insertions(+), 55 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index fe12db0ef6..9cd755926f 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -72,10 +72,8 @@ "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0), " \ "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0), " \ "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , " \ - "domain_name VARCHAR NOT NULL DEFAULT '', " \ "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000, " \ "check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800, " \ - "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0, " \ "comment VARCHAR NOT NULL DEFAULT '', " \ "auto_generated INT CHECK (auto_generated IN (0,1)) NOT NULL DEFAULT 0," \ "UNIQUE (reader_hostgroup))" diff --git a/include/ProxySQL_Admin_Tables_Definitions.h b/include/ProxySQL_Admin_Tables_Definitions.h index 9cc5cebfd4..24bb1ad8cc 100644 --- a/include/ProxySQL_Admin_Tables_Definitions.h +++ b/include/ProxySQL_Admin_Tables_Definitions.h @@ -241,20 +241,16 @@ "green_writer_hostgroup INT NOT NULL CHECK (green_writer_hostgroup>=0) , " \ "green_reader_hostgroup INT NOT NULL CHECK (green_reader_hostgroup>=0) , " \ "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , " \ - "domain_name VARCHAR NOT NULL CHECK (domain_name = '' OR SUBSTR(domain_name,1,1) = '.') , " \ "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , " \ "check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , " \ - "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , " \ "comment VARCHAR NOT NULL DEFAULT '' , UNIQUE (reader_hostgroup))" #define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_RDS_BGD_HOSTGROUPS "CREATE TABLE runtime_mysql_aws_rds_bgd_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , " \ "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0) , " \ "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0) , " \ "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , " \ - "domain_name VARCHAR NOT NULL CHECK (domain_name = '' OR SUBSTR(domain_name,1,1) = '.') , " \ "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , " \ "check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , " \ - "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , " \ "comment VARCHAR NOT NULL DEFAULT '' , " \ "auto_generated INT CHECK (auto_generated IN (0,1)) NOT NULL DEFAULT 0 , UNIQUE (reader_hostgroup))" diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index c0911d7eeb..f2be8be7e1 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -2261,7 +2261,7 @@ SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql(const string& name) "check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment FROM mysql_aws_aurora_hostgroups"; } else if (name == "mysql_aws_rds_bgd_hostgroups") { query=(char *)"SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader," - "domain_name,check_interval_ms,check_timeout_ms,autopurge_missing_checks,comment,auto_generated FROM mysql_aws_rds_bgd_hostgroups"; + "check_interval_ms,check_timeout_ms,comment,auto_generated FROM mysql_aws_rds_bgd_hostgroups"; } else if (name == "mysql_galera_hostgroups") { query=(char *)"SELECT writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers,writer_is_also_reader,max_transactions_behind,comment FROM mysql_galera_hostgroups"; } else if (name == "mysql_group_replication_hostgroups") { @@ -6435,8 +6435,8 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_rds_bgd_hostgroups_table() { int rc; char *query=(char *)"INSERT INTO mysql_aws_rds_bgd_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," - "writer_is_also_reader,domain_name,check_interval_ms,check_timeout_ms,autopurge_missing_checks,comment,auto_generated) VALUES " - "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)"; + "writer_is_also_reader,check_interval_ms,check_timeout_ms,comment,auto_generated) VALUES " + "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)"; auto [rc1, statement_unique] = mydb->prepare_v2(query); ASSERT_SQLITE_OK(rc1, mydb); @@ -6453,14 +6453,13 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_rds_bgd_hostgroups_table() { int green_reader_hostgroup = (gr_str && gr_str[0]) ? atoi(gr_str) : -1; int active=atoi(r->fields[4]); int writer_is_also_reader = atoi(r->fields[5]); - int check_interval_ms = atoi(r->fields[7]); - int check_timeout_ms = atoi(r->fields[8]); - int autopurge_missing_checks = atoi(r->fields[9]); + int check_interval_ms = atoi(r->fields[6]); + int check_timeout_ms = atoi(r->fields[7]); // entries loaded from the admin config table are always user-defined int auto_generated = 0; - proxy_info("Loading AWS RDS info for (%d,%d,%d,%d,%s,%d,\"%s\",%d,%d,%d,%d,\"%s\")\n", writer_hostgroup,reader_hostgroup, - green_writer_hostgroup,green_reader_hostgroup,(active ? "on" : "off"),writer_is_also_reader,r->fields[6], - check_interval_ms,check_timeout_ms,autopurge_missing_checks,auto_generated,r->fields[10]); + proxy_info("Loading AWS RDS info for (%d,%d,%d,%d,%s,%d,%d,%d,%d,\"%s\")\n", writer_hostgroup,reader_hostgroup, + green_writer_hostgroup,green_reader_hostgroup,(active ? "on" : "off"),writer_is_also_reader, + check_interval_ms,check_timeout_ms,auto_generated,r->fields[8]); rc=(*proxy_sqlite3_bind_int64)(statement, 1, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); rc=(*proxy_sqlite3_bind_int64)(statement, 2, reader_hostgroup); ASSERT_SQLITE_OK(rc, mydb); if (green_writer_hostgroup >= 0) { @@ -6477,12 +6476,10 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_rds_bgd_hostgroups_table() { ASSERT_SQLITE_OK(rc, mydb); rc=(*proxy_sqlite3_bind_int64)(statement, 5, active); ASSERT_SQLITE_OK(rc, mydb); rc=(*proxy_sqlite3_bind_int64)(statement, 6, writer_is_also_reader); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_text)(statement, 7, r->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 8, check_interval_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 9, check_timeout_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 10, autopurge_missing_checks); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_text)(statement, 11, r->fields[10], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 12, auto_generated); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 7, check_interval_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 8, check_timeout_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 9, r->fields[8], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 10, auto_generated); ASSERT_SQLITE_OK(rc, mydb); SAFE_SQLITE3_STEP2(statement); rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, mydb); @@ -7078,10 +7075,10 @@ void MySQL_HostGroups_Manager::update_aws_aurora_hosts_monitor_resultset(bool lo const char SELECT_AWS_RDS_BGD_SERVERS_FOR_MONITOR[] { "SELECT writer_hostgroup, reader_hostgroup, hostname, port, MAX(use_ssl) use_ssl, green_writer_hostgroup," - " green_reader_hostgroup, check_interval_ms, check_timeout_ms, autopurge_missing_checks, domain_name FROM mysql_servers" - " JOIN mysql_aws_rds_bgd_hostgroups ON" + " green_reader_hostgroup, check_interval_ms, check_timeout_ms FROM mysql_servers" + " JOIN mysql_aws_rds_bgd_hostgroups ON" " hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE active=1 AND status NOT IN (2,3)" - " GROUP BY writer_hostgroup, hostname, port" + " GROUP BY writer_hostgroup, hostname, port" }; /** @@ -7167,11 +7164,11 @@ bool MySQL_HostGroups_Manager::add_aws_rds_bgd_hostgroup_entry(const std::string std::string ins = "INSERT INTO mysql_aws_rds_bgd_hostgroups (" "writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, " - "active, writer_is_also_reader, domain_name, check_interval_ms, check_timeout_ms, " - "autopurge_missing_checks, comment, auto_generated" + "active, writer_is_also_reader, check_interval_ms, check_timeout_ms, " + "comment, auto_generated" ") VALUES (" + std::to_string(writer_hg) + ", " + std::to_string(reader_hg) - + ", NULL, NULL, 1, 0, '', 1000, 800, 0, '', 1)"; + + ", NULL, NULL, 1, 0, 1000, 800, '', 1)"; mydb->execute(ins.c_str()); added = true; proxy_info( diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index d390796ac2..e140f6b7eb 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6757,7 +6757,7 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { // Columns: // 0 writer_hostgroup, 1 reader_hostgroup, 2 hostname, 3 port, 4 use_ssl, // 5 green_writer_hostgroup, 6 green_reader_hostgroup, 7 check_interval_ms, - // 8 check_timeout_ms, 9 autopurge_missing_checks, 10 domain_name + // 8 check_timeout_ms pthread_mutex_lock(&GloMyMon->aws_rds_bgd_mutex); initial_raw_checksum = GloMyMon->AWS_RDS_BGD_Hosts_resultset_checksum; for (SQLite3_row *r : GloMyMon->AWS_RDS_BGD_Hosts_resultset->rows) { diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index aa6b5ebd79..d5fdf99e19 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -7556,8 +7556,8 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { // dump mysql_aws_rds_bgd_hostgroups // The runtime table carries the extra runtime-only 'auto_generated' column; the config table - // does not. 'dump_table_mysql' always returns 12 columns (last is 'auto_generated'); we bind - // 12 for the runtime table and only the first 11 for the config table. 'green_writer_hostgroup' + // does not. 'dump_table_mysql' always returns 10 columns (last is 'auto_generated'); we bind + // 10 for the runtime table and only the first 9 for the config table. 'green_writer_hostgroup' // and 'green_reader_hostgroup' (fields 2,3) are nullable and bound as NULL when absent. if (_runtime) { @@ -7574,9 +7574,9 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { char *query=NULL; if (_runtime) { - query=(char *)"INSERT INTO runtime_mysql_aws_rds_bgd_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader,domain_name,check_interval_ms,check_timeout_ms,autopurge_missing_checks,comment,auto_generated) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)"; + query=(char *)"INSERT INTO runtime_mysql_aws_rds_bgd_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment,auto_generated) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)"; } else { - query=(char *)"INSERT INTO mysql_aws_rds_bgd_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader,domain_name,check_interval_ms,check_timeout_ms,autopurge_missing_checks,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"; + query=(char *)"INSERT INTO mysql_aws_rds_bgd_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)"; } auto [rc1, statement_unique] = admindb->prepare_v2(query); @@ -7586,10 +7586,10 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r=*it; - // auto_generated (field 11) entries are created at runtime by the monitor; they are NOT + // auto_generated (field 9) entries are created at runtime by the monitor; they are NOT // user configuration, so they must not be persisted to the memory config table. They are // still written to the runtime table. - if (!_runtime && r->fields[11] && atoi(r->fields[11]) != 0) { + if (!_runtime && r->fields[9] && atoi(r->fields[9]) != 0) { continue; } rc=(*proxy_sqlite3_bind_int64)(statement, 1, atoi(r->fields[0])); ASSERT_SQLITE_OK(rc, admindb); @@ -7608,13 +7608,11 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement, 5, atoi(r->fields[4])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement, 6, atoi(r->fields[5])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement, 7, r->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 7, atoi(r->fields[6])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement, 8, atoi(r->fields[7])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 9, atoi(r->fields[8])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 10, atoi(r->fields[9])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement, 11, r->fields[10], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement, 9, r->fields[8], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); if (_runtime) { - rc=(*proxy_sqlite3_bind_int64)(statement, 12, atoi(r->fields[11])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 10, atoi(r->fields[9])); ASSERT_SQLITE_OK(rc, admindb); } SAFE_SQLITE3_STEP2(statement); diff --git a/lib/ProxySQL_Config.cpp b/lib/ProxySQL_Config.cpp index 63ead27333..5beb21dfaa 100644 --- a/lib/ProxySQL_Config.cpp +++ b/lib/ProxySQL_Config.cpp @@ -1117,11 +1117,9 @@ int ProxySQL_Config::Write_MySQL_Servers_to_configfile(std::string& data) { addField(data, "green_reader_hostgroup", r->fields[3], ""); addField(data, "active", r->fields[4], ""); addField(data, "writer_is_also_reader", r->fields[5], ""); - addField(data, "domain_name", r->fields[6]); - addField(data, "check_interval_ms", r->fields[7], ""); - addField(data, "check_timeout_ms", r->fields[8], ""); - addField(data, "autopurge_missing_checks", r->fields[9], ""); - addField(data, "comment", r->fields[10]); + addField(data, "check_interval_ms", r->fields[6], ""); + addField(data, "check_timeout_ms", r->fields[7], ""); + addField(data, "comment", r->fields[8]); data += "\t}"; isNext = true; @@ -1492,7 +1490,7 @@ int ProxySQL_Config::Read_MySQL_Servers_from_configfile(std::string& error) { const Setting &mysql_aws_rds_bgd_hostgroups = root["mysql_aws_rds_bgd_hostgroups"]; int count = mysql_aws_rds_bgd_hostgroups.getLength(); // green_writer_hostgroup / green_reader_hostgroup are nullable -> passed as %s ("NULL" or an integer) - char *q=(char *)"INSERT OR REPLACE INTO mysql_aws_rds_bgd_hostgroups (writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active, writer_is_also_reader, domain_name, check_interval_ms, check_timeout_ms, autopurge_missing_checks, comment ) VALUES (%d, %d, %s, %s, %d, %d, '%s', %d, %d, %d, '%s')"; + char *q=(char *)"INSERT OR REPLACE INTO mysql_aws_rds_bgd_hostgroups (writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active, writer_is_also_reader, check_interval_ms, check_timeout_ms, comment ) VALUES (%d, %d, %s, %s, %d, %d, %d, %d, '%s')"; for (i=0; i< count; i++) { const Setting &line = mysql_aws_rds_bgd_hostgroups[i]; int writer_hostgroup; @@ -1503,9 +1501,7 @@ int ProxySQL_Config::Read_MySQL_Servers_from_configfile(std::string& error) { int writer_is_also_reader; int check_interval_ms; int check_timeout_ms; - int autopurge_missing_checks; std::string comment=""; - std::string domain_name=""; if (line.lookupValue("writer_hostgroup", writer_hostgroup)==false) { proxy_error("Admin: detected a mysql_aws_rds_bgd_hostgroups in config file without a mandatory writer_hostgroup\n"); continue; @@ -1530,20 +1526,14 @@ int ProxySQL_Config::Read_MySQL_Servers_from_configfile(std::string& error) { if (line.lookupValue("writer_is_also_reader", writer_is_also_reader)==false) writer_is_also_reader=0; if (line.lookupValue("check_interval_ms", check_interval_ms)==false) check_interval_ms=1000; if (line.lookupValue("check_timeout_ms", check_timeout_ms)==false) check_timeout_ms=800; - if (line.lookupValue("autopurge_missing_checks", autopurge_missing_checks)==false) autopurge_missing_checks=0; line.lookupValue("comment", comment); - line.lookupValue("domain_name", domain_name); char *o1=strdup(comment.c_str()); char *o=escape_string_single_quotes(o1, false); - char *p1=strdup(domain_name.c_str()); - char *p=escape_string_single_quotes(p1, false); - char *query=(char *)malloc(strlen(q)+strlen(o)+strlen(p)+256); // 128 vs sizeof(int)*8 - sprintf(query,q, writer_hostgroup, reader_hostgroup, green_writer_str, green_reader_str, active, writer_is_also_reader, p, check_interval_ms, check_timeout_ms, autopurge_missing_checks, o); + char *query=(char *)malloc(strlen(q)+strlen(o)+256); // 128 vs sizeof(int)*8 + sprintf(query,q, writer_hostgroup, reader_hostgroup, green_writer_str, green_reader_str, active, writer_is_also_reader, check_interval_ms, check_timeout_ms, o); admindb->execute(query); if (o!=o1) free(o); free(o1); - if (p!=p1) free(p); - free(p1); free(query); rows++; } From 4e04ee8889a7e186119391faf02e46715a883a4b Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Fri, 3 Jul 2026 06:40:55 +0000 Subject: [PATCH 22/81] fix: Defer AWS RDS BGD switchover teardown until the rds_topology table drains - Add `AWS_RDS_BGD_Status` phase enum (1:1 with rds_topology status) plus an inferred `READER_SWITCHOVER_IN_PROGRESS` status. - After `SWITCHOVER_COMPLETED`, keep blue readers shunned and their DNS pinned; run cleanup only once `mysql.rds_topology` drains to empty. - Restore shunned blue readers straight to ONLINE after the topology table drains. - Reset the worker's FSM state on completion. Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 4 - include/MySQL_Monitor.hpp | 37 ++++- lib/MySQL_HostGroups_Manager.cpp | 14 +- lib/MySQL_Monitor.cpp | 219 ++++++++++++++++++++++------- 4 files changed, 214 insertions(+), 60 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 9cd755926f..1a3fd006a4 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -37,10 +37,6 @@ //#define STRESSTEST_POOL #endif // DEBUG -// Seconds to keep a blue reader SHUNNED after an AWS RDS blue/green switchover, -// giving AWS time to migrate it before it rejoins the reader hostgroup. -#define AWS_RDS_BGD_UNSHUN_DELAY_SEC 15 - // we have 2 versions of the same tables: with (debug) and without (no debug) checks #ifdef DEBUG #define MYHGM_MYSQL_SERVERS "CREATE TABLE mysql_servers ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT NOT NULL DEFAULT 0 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , status INT CHECK (status IN (0, 1, 2, 3, 4, 5)) NOT NULL DEFAULT 0 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , mem_pointer INT NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index beed5908d9..d75f412832 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -435,6 +435,35 @@ struct AWS_RDS_BlueGreenPair { bool is_writer = false; ///< True when this pair maps the blue writer. }; +/** + * @brief Switchover phase for an RDS blue/green deployment. + * + * @details AWS's mysql.rds_topology status only captures the writer switchover. As of 2026/07/03 + * the table exposes no read-replica switchover status; ProxySQL infers that the replicas have + * switched over from the table draining to empty (or disappearing) after it last reported + * SWITCHOVER_COMPLETED. + * + * Observed table lifecycle across one switchover: + * - Steady state: two rows (SOURCE = blue, TARGET = green), both AVAILABLE. + * - Switching: both rows step through SWITCHOVER_INITIATED -> _IN_PROGRESS -> _IN_POST_PROCESSING. + * - Writer done: the SOURCE row drops; a lone TARGET row reports SWITCHOVER_COMPLETED. + * - Replicas done: the table drains to empty (blue-reader DNS has propagated). + * + * The WRITER_SWITCHOVER_* values map 1:1 onto the mysql.rds_topology status strings. + * READER_SWITCHOVER_IN_PROGRESS is a ProxySQL inferred status with no topology-string mapping: we + * enter it after WRITER_SWITCHOVER_COMPLETED, deferring reader/DNS cleanup until the table drains + * to empty. + */ +enum class AWS_RDS_BGD_Status { + NONE, ///< no BGD topology / baseline + AVAILABLE, ///< "AVAILABLE" + WRITER_SWITCHOVER_INITIATED, ///< "SWITCHOVER_INITIATED" + WRITER_SWITCHOVER_IN_PROGRESS, ///< "SWITCHOVER_IN_PROGRESS" + WRITER_SWITCHOVER_POST_PROCESSING, ///< "SWITCHOVER_IN_POST_PROCESSING" + WRITER_SWITCHOVER_COMPLETED, ///< "SWITCHOVER_COMPLETED" + READER_SWITCHOVER_IN_PROGRESS, ///< ProxySQL inferred reader status; awaiting topology drain + deferred cleanup +}; + /** * @brief Per-deployment switchover state carried by one RDS BGD worker thread. * @@ -451,9 +480,10 @@ struct AWS_RDS_BGD_State { int green_writer_hg = -1; ///< -1 when NULL (auto-discovery path) int green_reader_hg = -1; ///< -1 when NULL + std::string last_topology_status; ///< raw mysql.rds_topology TARGET status from the previous poll (verbatim) std::vector bg_map; ///< [writer] always; [readers] only when green_reader_hg is configured std::vector> shunned_readers; ///< (host,port) we shunned - std::string last_status; ///< status from the previous poll, to act only when it changes + AWS_RDS_BGD_Status bgd_status = AWS_RDS_BGD_Status::NONE; ///< drives the FSM and the deferred cleanup bool green_writer_added_in_hg = false; ///< green writer added to green_writer_hg bool writer_is_also_reader_enforced = false; ///< POST_PROCESSING added the writer to the reader HG @@ -626,6 +656,11 @@ class MySQL_Monitor { * @param topology Parsed mysql.rds_topology result for this cycle. */ void handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_Result& topology); + // Deferred teardown: runs once mysql.rds_topology drains after SWITCHOVER_COMPLETED. + void handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st); + // Called by the BGD worker when the topology table is absent/empty/vanished; routes to the + // deferred cleanup when bgd_status is READER_SWITCHOVER_IN_PROGRESS, else preserves the baseline release. + void aws_rds_bgd_handle_topology_absent(AWS_RDS_BGD_State& st); void * monitor_replication_lag(); void * monitor_dns_cache(); void * run(); diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index f2be8be7e1..240149e40e 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -3849,8 +3849,8 @@ void MySQL_HostGroups_Manager::set_Readyset_status(char *hostname, int port, enu * @brief Set or clear AWS BGD shun state for a matching server. * * @details When shunning, transitions an ONLINE server to SHUNNED_AWS_BGD, enables shun metadata, -* drops free connections, and marks used connections unhealthy. When unshunning, transitions only -* SHUNNED_AWS_BGD back to ONLINE and clears shun metadata. Servers in other statuses are left unchanged. +* drops free connections, and marks used connections unhealthy. When unshunning, transitions +* back to ONLINE and clears shun metadata. Servers in other statuses are left unchanged. * * @param hostgroup_id Hostgroup to search. * @param hostname Address of the server to match. @@ -3885,13 +3885,11 @@ bool MySQL_HostGroups_Manager::aws_rds_bgd_set_shun_server(unsigned int hostgrou } } else { if (mysrvc->get_status() == MYSQL_SERVER_STATUS_SHUNNED_AWS_BGD) { - // Don't move back to ONLINE immediately. Move SHUNNED_AWS_BGD -> SHUNNED, - // set time_last_detected_error to a future time and let the server-selection - // logic auto-recover this server. - mysrvc->set_status(MYSQL_SERVER_STATUS_SHUNNED); + mysrvc->set_status(MYSQL_SERVER_STATUS_ONLINE); + mysrvc->shunned_automatic = false; mysrvc->shunned_and_kill_all_connections = false; - mysrvc->time_last_detected_error = now + AWS_RDS_BGD_UNSHUN_DELAY_SEC; - proxy_warning("AWS RDS BGD changing server status from SHUNNED_AWS_BGD to SHUNNED for %s:%d in HG %u\n", + mysrvc->time_last_detected_error = 0; + proxy_info("AWS RDS BGD unshunning server %s:%d in HG %u\n", hostname, port, myhgc->hid); changed = true; } diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index e140f6b7eb..e528056cf1 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6939,8 +6939,9 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { mmsd->result = NULL; } if (!table_available) { - // no blue/green deployment or multi-az cluster discovery in progress; nothing to do - aws_rds_bgd_clear_bgd_in_progress(st); + // no blue/green deployment or multi-az cluster discovery in progress, or the + // post-switchover topology has fully drained; run any pending deferred cleanup. + GloMyMon->aws_rds_bgd_handle_topology_absent(st); proxy_debug(PROXY_DEBUG_MONITOR, 5, "mysql.rds_topology not present on %s:%d (RDS writer HG %u); skipping\n", mmsd->hostname, mmsd->port, wHG); @@ -6960,11 +6961,11 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { unsigned int err = mmsd->mysql ? mysql_errno(mmsd->mysql) : 0; if (err == 1146) { // the table vanished (ER_NO_SUCH_TABLE), e.g. a blue/green deployment - // was cancelled: re-check its existence on the next iteration and - // return to the baseline poll interval. + // was cancelled or a post-switchover topology fully drained: re-check its + // existence on the next iteration and return to the baseline poll interval. topology_state = TOPOLOGY_TABLE_CHECK; st.next_check_interval_ms = 0; - aws_rds_bgd_clear_bgd_in_progress(st); + GloMyMon->aws_rds_bgd_handle_topology_absent(st); proxy_debug(PROXY_DEBUG_MONITOR, 5, "mysql.rds_topology vanished on %s:%d (RDS writer HG %u); rechecking availability\n", mmsd->hostname, mmsd->port, wHG); @@ -6985,6 +6986,10 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { "AWS RDS BGD [wHG=%u]: topology probe on %s:%d (blue_green=%d, nodes=%zu)\n", wHG, mmsd->hostname, mmsd->port, topo.blue_green ? 1 : 0, topo.nodes.size()); GloMyMon->handle_aws_rds_bgd(st, topo); + } else { + // Query succeeded with no rows: mysql.rds_topology has drained (blue-reader + // DNS fully propagated). Run post-switchover cleanup. + GloMyMon->aws_rds_bgd_handle_topology_absent(st); } if (mmsd->result) { @@ -7245,6 +7250,44 @@ static void aws_rds_bgd_add_green_writer_in_hg(AWS_RDS_BGD_State& st) { } } +// Map a raw mysql.rds_topology TARGET status string onto BGD phase enum. +static AWS_RDS_BGD_Status aws_rds_bgd_status_from_topology(const std::string& status) { + if (strcasecmp(status.c_str(), BGD_STATUS_AVAILABLE) == 0) { + return AWS_RDS_BGD_Status::AVAILABLE; + } else if (strcasecmp(status.c_str(), BGD_STATUS_INITIATED) == 0) { + return AWS_RDS_BGD_Status::WRITER_SWITCHOVER_INITIATED; + } else if (strcasecmp(status.c_str(), BGD_STATUS_IN_PROGRESS) == 0) { + return AWS_RDS_BGD_Status::WRITER_SWITCHOVER_IN_PROGRESS; + } else if (strcasecmp(status.c_str(), BGD_STATUS_POST_PROC) == 0) { + return AWS_RDS_BGD_Status::WRITER_SWITCHOVER_POST_PROCESSING; + } else if (strcasecmp(status.c_str(), BGD_STATUS_COMPLETED) == 0) { + return AWS_RDS_BGD_Status::WRITER_SWITCHOVER_COMPLETED; + } else { + return AWS_RDS_BGD_Status::NONE; + } +} + +// Human-readable name for a phase enum, for logging and (later) the runtime status column. +static const char* aws_rds_bgd_status_str(AWS_RDS_BGD_Status s) { + switch (s) { + case AWS_RDS_BGD_Status::NONE: + return "NONE"; + case AWS_RDS_BGD_Status::AVAILABLE: + return "AVAILABLE"; + case AWS_RDS_BGD_Status::WRITER_SWITCHOVER_INITIATED: + return "WRITER_SWITCHOVER_INITIATED"; + case AWS_RDS_BGD_Status::WRITER_SWITCHOVER_IN_PROGRESS: + return "WRITER_SWITCHOVER_IN_PROGRESS"; + case AWS_RDS_BGD_Status::WRITER_SWITCHOVER_POST_PROCESSING: + return "WRITER_SWITCHOVER_POST_PROCESSING"; + case AWS_RDS_BGD_Status::WRITER_SWITCHOVER_COMPLETED: + return "WRITER_SWITCHOVER_COMPLETED"; + case AWS_RDS_BGD_Status::READER_SWITCHOVER_IN_PROGRESS: + return "READER_SWITCHOVER_IN_PROGRESS"; + } + return "UNKNOWN"; +} + /** * @brief Run the status-driven blue/green switchover FSM for one deployment. * @@ -7276,26 +7319,40 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo aws_rds_bgd_clear_bgd_in_progress(st); return; } + st.last_topology_status = status; - if (strcasecmp(status.c_str(), st.last_status.c_str()) == 0) { - // no state change + AWS_RDS_BGD_Status topology_status = aws_rds_bgd_status_from_topology(status); + + // Once we advance to READER_SWITCHOVER_IN_PROGRESS phase, AWS keeps reporting + // WRITER_SWITCHOVER_COMPLETED (a single green row) until mysql.rds_topology drains. Ignore + // those repeats: the deferred cleanup fires from aws_rds_bgd_handle_topology_absent() when + // the table empties/vanishes, not from a status change here. + if (topology_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_COMPLETED + && st.bgd_status == AWS_RDS_BGD_Status::READER_SWITCHOVER_IN_PROGRESS) { return; } - proxy_info( - "AWS RDS BGD [wHG=%u rHG=%u]: switchover status '%s' -> '%s'\n", + if (topology_status == st.bgd_status) { + // no phase change + return; + } + + proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: switchover status '%s' -> '%s'\n", st.writer_hg, st.reader_hg, - st.last_status.empty() ? "(none)" : st.last_status.c_str(), status.c_str()); + aws_rds_bgd_status_str(st.bgd_status), + aws_rds_bgd_status_str(topology_status)); - if (strcasecmp(status.c_str(), BGD_STATUS_AVAILABLE) == 0) { + st.bgd_status = topology_status; + + if (st.bgd_status == AWS_RDS_BGD_Status::AVAILABLE) { st.next_check_interval_ms = 250; aws_rds_bgd_build_map(st, topology); aws_rds_bgd_resolve_green_ips(st); aws_rds_bgd_add_green_writer_in_hg(st); } - else if (strcasecmp(status.c_str(), BGD_STATUS_INITIATED) == 0 - || strcasecmp(status.c_str(), BGD_STATUS_IN_PROGRESS) == 0) { + else if (st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_INITIATED + || st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_IN_PROGRESS) { st.next_check_interval_ms = 100; aws_rds_bgd_build_map(st, topology); @@ -7303,7 +7360,7 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo aws_rds_bgd_add_green_writer_in_hg(st); aws_rds_bgd_set_bgd_in_progress(st); } - else if (strcasecmp(status.c_str(), BGD_STATUS_POST_PROC) == 0) { + else if (st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_POST_PROCESSING) { st.next_check_interval_ms = 100; // Run setup here too: the thread may observe POST_PROCESSING directly, without having @@ -7363,53 +7420,121 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo MyHGM->aws_rds_bgd_shun_servers(st.reader_hg, unmapped_readers, true); st.shunned_readers.insert(st.shunned_readers.end(), unmapped_readers.begin(), unmapped_readers.end()); } - else if (strcasecmp(status.c_str(), BGD_STATUS_COMPLETED) == 0) { - st.next_check_interval_ms = 0; + else if (st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_COMPLETED) { + // Writer switchover done, but the topology table lingers with a single green row until the blue + // readers' DNS propagates and it drains. Defer reader teardown: advance to READER_SWITCHOVER phase + // and let the drain (aws_rds_bgd_handle_topology_absent) trigger it. + st.bgd_status = AWS_RDS_BGD_Status::READER_SWITCHOVER_IN_PROGRESS; - // if writer is in reader_hg, remove it + // Drop the writer's DNS_Cache entry: this clears the IP pin and lets regular DNS + // resolution take over from here. Readers stay pinned until their DNS propagates. for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { if (p.is_writer) { - MyHGM->wrlock(); - MyHGM->remove_server_in_hg(st.reader_hg, p.blue_host, (uint16_t)p.port); - MyHGM->wrunlock(); + dns_cache->remove(p.blue_host); break; } } - if (!st.shunned_readers.empty()) { - MyHGM->aws_rds_bgd_shun_servers(st.reader_hg, st.shunned_readers, false); - for (const std::pair& br : st.shunned_readers) { - // purge so the blue reader hostname re-resolves to the promoted instance - dns_cache->remove(br.first); - My_Conn_Pool->purge_connections(br.first.c_str(), br.second); - } - st.shunned_readers.clear(); - } - - for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { - // remove the cache record (pin + resolved IPs) so the blue name - // re-resolves to the promoted (green) instance - dns_cache->remove(p.blue_host); - My_Conn_Pool->purge_connections(p.blue_host.c_str(), p.port); - } - - // TODO: Drain Green HGs + // release BGD monitor worker from fast-polling + st.next_check_interval_ms = 0; - proxy_info( - "AWS RDS BGD [wHG=%u rHG=%u]: switchover complete; state cleared\n", - st.writer_hg, st.reader_hg); - st.bg_map.clear(); - // Stop polling green by IP; revert to the configured (blue) name, now the promoted primary. - st.next_check_host.clear(); - // Re-enable the read_only monitor on these servers now that the switchover is done. + // release read_only monitor from fast-polling aws_rds_bgd_clear_bgd_in_progress(st); + + proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: switchover status '%s' -> '%s'\n", + st.writer_hg, st.reader_hg, + aws_rds_bgd_status_str(AWS_RDS_BGD_Status::WRITER_SWITCHOVER_COMPLETED), + aws_rds_bgd_status_str(AWS_RDS_BGD_Status::READER_SWITCHOVER_IN_PROGRESS)); } else { - // unknown status: take no action, stay at baseline interval + // NONE / unrecognized status: take no action, stay at baseline interval st.next_check_interval_ms = 0; } +} + +/** +* @brief Deferred switchover teardown: run once mysql.rds_topology has drained after COMPLETED. +* +* @details Blue-reader DNS has propagated to the promoted instances by the time the topology +* table goes empty, so this: returns the writer to a writer-only role, restores the shunned +* blue readers to ONLINE immediately (no recovery delay), drops the blue->green DNS pins, and +* clears the per-worker switchover state so a future switchover starts from a clean FSM. +*/ +void MySQL_Monitor::handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st) { + if (st.bgd_status != AWS_RDS_BGD_Status::READER_SWITCHOVER_IN_PROGRESS) { + return; + } - st.last_status = status; + proxy_info( + "AWS RDS BGD [wHG=%u rHG=%u]: switchover status '%s' -> '%s'; running post-switchover cleanup\n", + st.writer_hg, st.reader_hg, aws_rds_bgd_status_str(st.bgd_status), "SWITCHOVER_COMPLETED" + ); + + // Return the writer to a writer-only role by removing it from the reader HG. + // TODO: make this conditional on the replication_hostgroups writer_is_also_reader + // flag for this writer/reader pair. + for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { + if (p.is_writer) { + MyHGM->wrlock(); + MyHGM->remove_server_in_hg(st.reader_hg, p.blue_host, (uint16_t)p.port); + MyHGM->wrunlock(); + break; + } + } + + // unshun blue readers which are previously shunned + if (!st.shunned_readers.empty()) { + MyHGM->aws_rds_bgd_shun_servers(st.reader_hg, st.shunned_readers, false); + for (const std::pair& br : st.shunned_readers) { + // Drop DNS cache for the unmapped readers so their blue names re-resolve + // to the promoted (green) instances. + dns_cache->remove(br.first); + My_Conn_Pool->purge_connections(br.first.c_str(), br.second); + } + st.shunned_readers.clear(); + } + + // Drop DNS pins for the mapped readers so their blue names resolve natively + // to the promoted (green) instances. The writer pin was already cleared at + // WRITER_SWITCHOVER_COMPLETED, so skip it here. + for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { + if (p.is_writer) { + continue; + } + dns_cache->remove(p.blue_host); + My_Conn_Pool->purge_connections(p.blue_host.c_str(), p.port); + } + + // TODO: Drain Green HGs + + // state cleanup + st.bg_map.clear(); + st.last_topology_status.clear(); + st.bgd_status = AWS_RDS_BGD_Status::NONE; + st.next_check_host.clear(); + st.next_check_interval_ms = 0; + st.green_writer_added_in_hg = false; + st.writer_is_also_reader_enforced = false; + aws_rds_bgd_clear_bgd_in_progress(st); + + proxy_info( + "AWS RDS BGD [wHG=%u rHG=%u]: post-switchover cleanup complete; state cleared\n", + st.writer_hg, st.reader_hg); +} + +/** +* @brief Invoked by the BGD worker whenever mysql.rds_topology is absent/empty/vanished. +* +* @details When a post-switchover teardown is pending, an empty topology table means blue-reader +* DNS has propagated, so run the deferred cleanup. Otherwise keep the pre-existing behavior of +* releasing the read_only fast-poll engagement for this deployment. +*/ +void MySQL_Monitor::aws_rds_bgd_handle_topology_absent(AWS_RDS_BGD_State& st) { + if (st.bgd_status == AWS_RDS_BGD_Status::READER_SWITCHOVER_IN_PROGRESS) { + handle_aws_rds_bgd_post_switchover(st); + } else { + aws_rds_bgd_clear_bgd_in_progress(st); + } } /** From e8905ffb68ef3de4d66ca491af3ff8bb88352d8f Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Mon, 6 Jul 2026 06:55:51 +0000 Subject: [PATCH 23/81] refactor: Reconfigure reader HG properly during AWS RDS BGD switchover - If read_only monitor missed the event (read_only=0) during `SWITCHOVER_IN_PROGRESS`, add writer to reader HG when FSM reaches `SWITCHOVER_POST_PROCESSING` state. - While removing the writer from HG after `SWITCHOVER_COMPLETED`, honor `writer_is_also_reader` configuration in `mysql_aws_rds_bgd_hostgroups`. Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 15 +++-- include/MySQL_Monitor.hpp | 15 ++++- lib/MySQL_HostGroups_Manager.cpp | 33 ++++------ lib/MySQL_Monitor.cpp | 100 ++++++++++++++++++++++++----- 4 files changed, 117 insertions(+), 46 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 1a3fd006a4..2696a269f8 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -1068,13 +1068,14 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { */ bool aws_rds_bgd_set_shun_server(unsigned int hostgroup_id, const char *hostname, int port, bool shun); /** - * @brief Shun or unshun multiple servers in a hostgroup, then publish the change to the runtime tables. - * - * @param hostgroup_id Hostgroup to search. - * @param servers (hostname, port) pairs to act on. - * @param shun true to shun, false to unshun. - */ - void aws_rds_bgd_shun_servers(unsigned int hostgroup_id, const std::vector>& servers, bool shun); + * @brief Aligns the runtime 'mysql_servers' table + checksums with the server state in MyHGM. + * + * @details One-way alignment (in-memory -> runtime): regenerates the runtime 'mysql_servers' table + * from the current in-memory MyHGM state and recomputes/republishes the global checksum. + * + * @note Caller must hold wrlock(). + */ + void publish_mysql_servers_to_runtime(); /** * @brief Drain existing backend connections for a server. * diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index d75f412832..79b636565b 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -479,6 +479,7 @@ struct AWS_RDS_BGD_State { unsigned int reader_hg = 0; ///< blue/current reader hostgroup int green_writer_hg = -1; ///< -1 when NULL (auto-discovery path) int green_reader_hg = -1; ///< -1 when NULL + int writer_is_also_reader = 0; ///< drives post-switchover writer cleanup std::string last_topology_status; ///< raw mysql.rds_topology TARGET status from the previous poll (verbatim) std::vector bg_map; ///< [writer] always; [readers] only when green_reader_hg is configured @@ -486,7 +487,6 @@ struct AWS_RDS_BGD_State { AWS_RDS_BGD_Status bgd_status = AWS_RDS_BGD_Status::NONE; ///< drives the FSM and the deferred cleanup bool green_writer_added_in_hg = false; ///< green writer added to green_writer_hg - bool writer_is_also_reader_enforced = false; ///< POST_PROCESSING added the writer to the reader HG bool bgd_in_progress_set = false; ///< MyHGM's in-progress switchover count is incremented unsigned int next_check_interval_ms = 0; ///< FSM-controlled interval; 0 => baseline @@ -524,6 +524,11 @@ inline const char* const BGD_STATUS_COMPLETED = "SWITCHOVER_COMPLETED"; // Fast pass: only servers in an AWS RDS blue/green deployment #define SELECT_RDS_BGD_SERVERS_FOR_READ_ONLY "SELECT hostname, port, MAX(use_ssl) use_ssl, 'read_only' check_type, reader_hostgroup FROM mysql_servers JOIN mysql_aws_rds_bgd_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE active=1 AND status NOT IN (2,3,5) GROUP BY hostname, port ORDER BY RANDOM()" +// Defined in MySQL_HostGroups_Manager.h; forward-declared here because the include cycle +// (Monitor.hpp -> HGM.h -> cpp.h -> Monitor.hpp) can leave them undefined at this point. Only +// used below via pointer, so a forward declaration is sufficient. +struct srv_info_t; +struct srv_opts_t; class MySQL_Monitor { public: @@ -661,6 +666,14 @@ class MySQL_Monitor { // Called by the BGD worker when the topology table is absent/empty/vanished; routes to the // deferred cleanup when bgd_status is READER_SWITCHOVER_IN_PROGRESS, else preserves the baseline release. void aws_rds_bgd_handle_topology_absent(AWS_RDS_BGD_State& st); + // Apply one switchover step's reader-HG mutations, with the action derived from bgd_status: + // POST_PROCESSING shuns the readers and (when writer_info is set) adds the writer as a reader; + // READER_SWITCHOVER_IN_PROGRESS unshuns the readers and (when writer_info is set) removes the writer. + void aws_rds_bgd_reconfigure_reader_hg( + AWS_RDS_BGD_Status bgd_status, unsigned int reader_hg, + std::vector>& unmapped_readers, + srv_info_t* writer_info, srv_opts_t* writer_opts); + void * monitor_replication_lag(); void * monitor_dns_cache(); void * run(); diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 240149e40e..8f24145a34 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -3900,25 +3900,16 @@ bool MySQL_HostGroups_Manager::aws_rds_bgd_set_shun_server(unsigned int hostgrou return changed; } -void MySQL_HostGroups_Manager::aws_rds_bgd_shun_servers( - unsigned int hostgroup_id, const std::vector>& servers, bool shun -) { - bool changed = false; - - wrlock(); - - for (const std::pair& s : servers) { - if (aws_rds_bgd_set_shun_server(hostgroup_id, s.first.c_str(), s.second, shun)) { - changed = true; - } - } - - if (!changed) { - wrunlock(); - return; - } - - // Publish the new in-memory statuses into the runtime mysql_servers table +/** + * @brief Aligns the runtime 'mysql_servers' table + checksums with the server state in MyHGM. + * + * @details One-way alignment (in-memory -> runtime): regenerates the runtime 'mysql_servers' table + * from the current in-memory 'MyHGC'/'MySrvC' structures and recomputes/republishes the global + * 'mysql_servers' checksum for cluster sync. + * + * @note the caller MUST already hold 'wrlock()'. + */ +void MySQL_HostGroups_Manager::publish_mysql_servers_to_runtime() { purge_mysql_servers_table(); proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers\n"); mydb->execute("DELETE FROM mysql_servers"); @@ -3934,8 +3925,6 @@ void MySQL_HostGroups_Manager::aws_rds_bgd_shun_servers( pthread_mutex_lock(&GloVars.checksum_mutex); update_glovars_mysql_servers_checksum(mysrvs_checksum); pthread_mutex_unlock(&GloVars.checksum_mutex); - - wrunlock(); } /** @@ -7073,7 +7062,7 @@ void MySQL_HostGroups_Manager::update_aws_aurora_hosts_monitor_resultset(bool lo const char SELECT_AWS_RDS_BGD_SERVERS_FOR_MONITOR[] { "SELECT writer_hostgroup, reader_hostgroup, hostname, port, MAX(use_ssl) use_ssl, green_writer_hostgroup," - " green_reader_hostgroup, check_interval_ms, check_timeout_ms FROM mysql_servers" + " green_reader_hostgroup, check_interval_ms, check_timeout_ms, writer_is_also_reader FROM mysql_servers" " JOIN mysql_aws_rds_bgd_hostgroups ON" " hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE active=1 AND status NOT IN (2,3)" " GROUP BY writer_hostgroup, hostname, port" diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index e528056cf1..786d533601 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6757,7 +6757,7 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { // Columns: // 0 writer_hostgroup, 1 reader_hostgroup, 2 hostname, 3 port, 4 use_ssl, // 5 green_writer_hostgroup, 6 green_reader_hostgroup, 7 check_interval_ms, - // 8 check_timeout_ms + // 8 check_timeout_ms, 9 writer_is_also_reader pthread_mutex_lock(&GloMyMon->aws_rds_bgd_mutex); initial_raw_checksum = GloMyMon->AWS_RDS_BGD_Hosts_resultset_checksum; for (SQLite3_row *r : GloMyMon->AWS_RDS_BGD_Hosts_resultset->rows) { @@ -6778,6 +6778,9 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { if (check_timeout_ms == 0) { check_timeout_ms = atoi(r->fields[8]); } + if (r->fields[9] && r->fields[9][0]) { + st.writer_is_also_reader = atoi(r->fields[9]); + } } } @@ -7417,7 +7420,28 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo unmapped_readers.push_back(br); } } - MyHGM->aws_rds_bgd_shun_servers(st.reader_hg, unmapped_readers, true); + + // If shunning the unmapped readers would leave the reader HG with no serving readers, + // add the writer into the reader HG so reads keep flowing during the switchover. + // This a no-ops when read_only monitor has added the writer to reader HG already. + std::unique_ptr writer_info; + std::unique_ptr writer_opts; + if (!unmapped_readers.empty() && unmapped_readers.size() == blue_readers.size()) { + for (AWS_RDS_BlueGreenPair& p : st.bg_map) { + if (p.is_writer) { + writer_info = std::make_unique(srv_info_t{ p.blue_host, (uint16_t)p.port, "AWS RDS BGD writer as reader" }); + writer_opts = std::make_unique(srv_opts_t{ p.blue_weight, p.blue_max_conns, p.blue_use_ssl }); + proxy_info( + "AWS RDS BGD [wHG=%u rHG=%u]: reader HG would be emptied by shun; adding writer '%s:%d' to reader HG\n", + st.writer_hg, st.reader_hg, p.blue_host.c_str(), p.port); + break; + } + } + } + + // Shun the unmapped blue readers. + aws_rds_bgd_reconfigure_reader_hg( + st.bgd_status, st.reader_hg, unmapped_readers, writer_info.get(), writer_opts.get()); st.shunned_readers.insert(st.shunned_readers.end(), unmapped_readers.begin(), unmapped_readers.end()); } else if (st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_COMPLETED) { @@ -7452,6 +7476,51 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo } } +/** +* @brief Apply one switchover step's reader-HG mutations, with the action derived from bgd_status. +* +* @details POST_PROCESSING shuns the unmapped readers and, when writer_info is set, adds the writer as +* a reader (using writer_opts). READER_SWITCHOVER_IN_PROGRESS unshuns the readers and, when writer_info +* is set, removes the writer. +*/ +void MySQL_Monitor::aws_rds_bgd_reconfigure_reader_hg( + AWS_RDS_BGD_Status bgd_status, unsigned int reader_hg, + std::vector>& unmapped_readers, + srv_info_t* writer_info, srv_opts_t* writer_opts) +{ + bool post_proc = (bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_POST_PROCESSING); + // WRITER_SWITCHOVER_POST_PROCESSING => shun; + // READER_SWITCHOVER_IN_PROGRESS => unshun + bool shun = post_proc; + bool changed = false; + + MyHGM->wrlock(); + + if (writer_info) { + if (post_proc && writer_info && writer_opts) { + if (MyHGM->create_new_server_in_hg(reader_hg, *writer_info, *writer_opts) == 0) { + changed = true; + } + } else { + if (MyHGM->remove_server_in_hg(reader_hg, writer_info->addr, writer_info->port) == 0) { + changed = true; + } + } + } + + for (std::pair& s : unmapped_readers) { + if (MyHGM->aws_rds_bgd_set_shun_server(reader_hg, s.first.c_str(), s.second, shun)) { + changed = true; + } + } + + if (changed) { + MyHGM->publish_mysql_servers_to_runtime(); + } + + MyHGM->wrunlock(); +} + /** * @brief Deferred switchover teardown: run once mysql.rds_topology has drained after COMPLETED. * @@ -7470,24 +7539,24 @@ void MySQL_Monitor::handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st) { st.writer_hg, st.reader_hg, aws_rds_bgd_status_str(st.bgd_status), "SWITCHOVER_COMPLETED" ); - // Return the writer to a writer-only role by removing it from the reader HG. - // TODO: make this conditional on the replication_hostgroups writer_is_also_reader - // flag for this writer/reader pair. - for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { - if (p.is_writer) { - MyHGM->wrlock(); - MyHGM->remove_server_in_hg(st.reader_hg, p.blue_host, (uint16_t)p.port); - MyHGM->wrunlock(); - break; + // Restore the writer's original role in the reader HG based on writer_is_also_reader config and + // unshun the previously shunned blue readers. + std::unique_ptr writer_info; + if (st.writer_is_also_reader == 0) { + for (AWS_RDS_BlueGreenPair& p : st.bg_map) { + if (p.is_writer) { + writer_info = std::make_unique(srv_info_t{ p.blue_host, (uint16_t)p.port, "AWS RDS BGD writer" }); + break; + } } } - // unshun blue readers which are previously shunned + aws_rds_bgd_reconfigure_reader_hg(st.bgd_status, st.reader_hg, st.shunned_readers, writer_info.get(), NULL); + + // Drop DNS cache + purge connections for the previously shunned readers + // so their blue names re-resolve to the promoted (green) instances. if (!st.shunned_readers.empty()) { - MyHGM->aws_rds_bgd_shun_servers(st.reader_hg, st.shunned_readers, false); for (const std::pair& br : st.shunned_readers) { - // Drop DNS cache for the unmapped readers so their blue names re-resolve - // to the promoted (green) instances. dns_cache->remove(br.first); My_Conn_Pool->purge_connections(br.first.c_str(), br.second); } @@ -7514,7 +7583,6 @@ void MySQL_Monitor::handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st) { st.next_check_host.clear(); st.next_check_interval_ms = 0; st.green_writer_added_in_hg = false; - st.writer_is_also_reader_enforced = false; aws_rds_bgd_clear_bgd_in_progress(st); proxy_info( From 696607fbf9418e2e8f620bb4d2212024e2031ceb Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Mon, 6 Jul 2026 08:09:29 +0000 Subject: [PATCH 24/81] feat: Add runtime column `status` to `mysql_aws_rds_bgd_hostgroups` table Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 41 ++++++------- include/MySQL_Monitor.hpp | 18 +++--- include/ProxySQL_Admin_Tables_Definitions.h | 18 ++++-- lib/MySQL_HostGroups_Manager.cpp | 35 ++++++++++- lib/MySQL_Monitor.cpp | 65 ++++++--------------- lib/ProxySQL_Admin.cpp | 6 +- 6 files changed, 98 insertions(+), 85 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 2696a269f8..d7f866149d 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -64,14 +64,18 @@ "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , " \ "comment VARCHAR , UNIQUE (reader_hostgroup))" -#define MYHGM_MYSQL_AWS_RDS_BGD_HOSTGROUPS "CREATE TABLE mysql_aws_rds_bgd_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0), " \ +#define MYHGM_MYSQL_AWS_RDS_BGD_HOSTGROUPS "CREATE TABLE mysql_aws_rds_bgd_hostgroups ("\ + "writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , "\ + "reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0), " \ "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0), " \ "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0), " \ - "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , " \ + "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , " \ + "writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , " \ "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000, " \ "check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800, " \ "comment VARCHAR NOT NULL DEFAULT '', " \ - "auto_generated INT CHECK (auto_generated IN (0,1)) NOT NULL DEFAULT 0," \ + "auto_generated INT CHECK (auto_generated IN (0,1)) NOT NULL DEFAULT 0, " \ + "status INT NOT NULL DEFAULT 0, " \ "UNIQUE (reader_hostgroup))" #define MYHGM_GEN_ADMIN_RUNTIME_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" WHEN 5 THEN \"SHUNNED_AWS_BGD\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers ORDER BY hostgroup_id, hostname, port" @@ -1067,6 +1071,17 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { * @note Caller must hold wrlock(). */ bool aws_rds_bgd_set_shun_server(unsigned int hostgroup_id, const char *hostname, int port, bool shun); + /** + * @brief Persist BGD switchover status into the runtime mysql_aws_rds_bgd_hostgroups table. + * + * @details Called by the BGD worker on every FSM status transition. For the read_only monitor, + * it both gates the fast-poll cadence (is_aws_rds_bgd_in_progress) and filters which servers the + * fast poll selects. + * + * @param writer_hg Writer hostgroup identifying the deployment. + * @param status AWS_RDS_BGD_Status underlying value. + */ + void aws_rds_bgd_set_switchover_status(unsigned int writer_hg, int status); /** * @brief Aligns the runtime 'mysql_servers' table + checksums with the server state in MyHGM. * @@ -1089,25 +1104,11 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { */ bool drain_server_connections(unsigned int hostgroup_id, const char *hostname, int port); /** - * @brief Number of AWS RDS blue/green deployments currently mid-switchover. + * @brief Whether any AWS RDS blue/green deployment is currently in a active switchover status. * - * @details Each BGD worker increments the count while its deployment is switching over and - * decrements it once the deployment leaves the switchover states. While the count is non-zero - * the read_only monitor fast-polls the BGD servers; it resumes the full-fleet cadence at zero. + * @details Derived from the runtime mysql_aws_rds_bgd_hostgroups 'status' column. */ - std::atomic aws_rds_bgd_in_progress_count{0}; - - void set_aws_rds_bgd_in_progress(bool in_progress) { - if (in_progress) { - aws_rds_bgd_in_progress_count.fetch_add(1, std::memory_order_relaxed); - } else { - aws_rds_bgd_in_progress_count.fetch_sub(1, std::memory_order_relaxed); - } - } - - bool is_aws_rds_bgd_in_progress() const { - return aws_rds_bgd_in_progress_count.load(std::memory_order_relaxed) > 0; - } + bool is_aws_rds_bgd_in_progress(); unsigned long long Get_Memory_Stats(); diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 79b636565b..a68cd57574 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -455,15 +455,18 @@ struct AWS_RDS_BlueGreenPair { * to empty. */ enum class AWS_RDS_BGD_Status { - NONE, ///< no BGD topology / baseline - AVAILABLE, ///< "AVAILABLE" - WRITER_SWITCHOVER_INITIATED, ///< "SWITCHOVER_INITIATED" - WRITER_SWITCHOVER_IN_PROGRESS, ///< "SWITCHOVER_IN_PROGRESS" - WRITER_SWITCHOVER_POST_PROCESSING, ///< "SWITCHOVER_IN_POST_PROCESSING" - WRITER_SWITCHOVER_COMPLETED, ///< "SWITCHOVER_COMPLETED" - READER_SWITCHOVER_IN_PROGRESS, ///< ProxySQL inferred reader status; awaiting topology drain + deferred cleanup + NONE = 0, ///< no BGD topology / baseline + AVAILABLE = 1, ///< "AVAILABLE" + WRITER_SWITCHOVER_INITIATED = 2, ///< "SWITCHOVER_INITIATED" + WRITER_SWITCHOVER_IN_PROGRESS = 3, ///< "SWITCHOVER_IN_PROGRESS" + WRITER_SWITCHOVER_POST_PROCESSING = 4, ///< "SWITCHOVER_IN_POST_PROCESSING" + WRITER_SWITCHOVER_COMPLETED = 5, ///< "SWITCHOVER_COMPLETED" + READER_SWITCHOVER_IN_PROGRESS = 6, ///< ProxySQL inferred reader status; awaiting topology drain + deferred cleanup }; +// Maps a switchover status enum to its stored/display string. +const char* aws_rds_bgd_status_str(AWS_RDS_BGD_Status s); + /** * @brief Per-deployment switchover state carried by one RDS BGD worker thread. * @@ -487,7 +490,6 @@ struct AWS_RDS_BGD_State { AWS_RDS_BGD_Status bgd_status = AWS_RDS_BGD_Status::NONE; ///< drives the FSM and the deferred cleanup bool green_writer_added_in_hg = false; ///< green writer added to green_writer_hg - bool bgd_in_progress_set = false; ///< MyHGM's in-progress switchover count is incremented unsigned int next_check_interval_ms = 0; ///< FSM-controlled interval; 0 => baseline std::string next_check_host; ///< FSM-pinned probe host; when set (the green IP), the worker diff --git a/include/ProxySQL_Admin_Tables_Definitions.h b/include/ProxySQL_Admin_Tables_Definitions.h index 24bb1ad8cc..6031ac5598 100644 --- a/include/ProxySQL_Admin_Tables_Definitions.h +++ b/include/ProxySQL_Admin_Tables_Definitions.h @@ -237,22 +237,30 @@ // AWS RDS hostgroups; adds blue/green (green_*_hostgroup) over aurora. // The runtime table carries one extra runtime-only column: auto_generated. -#define ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS "CREATE TABLE mysql_aws_rds_bgd_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , " \ +#define ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS "CREATE TABLE mysql_aws_rds_bgd_hostgroups ("\ + "writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , "\ + "reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , " \ "green_writer_hostgroup INT NOT NULL CHECK (green_writer_hostgroup>=0) , " \ "green_reader_hostgroup INT NOT NULL CHECK (green_reader_hostgroup>=0) , " \ - "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , " \ + "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , " \ + "writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , " \ "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , " \ "check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , " \ "comment VARCHAR NOT NULL DEFAULT '' , UNIQUE (reader_hostgroup))" -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_RDS_BGD_HOSTGROUPS "CREATE TABLE runtime_mysql_aws_rds_bgd_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , " \ +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_RDS_BGD_HOSTGROUPS "CREATE TABLE runtime_mysql_aws_rds_bgd_hostgroups ("\ + "writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , "\ + "reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , " \ "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0) , " \ "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0) , " \ - "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , " \ + "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , "\ + "writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , " \ "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , " \ "check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , " \ "comment VARCHAR NOT NULL DEFAULT '' , " \ - "auto_generated INT CHECK (auto_generated IN (0,1)) NOT NULL DEFAULT 0 , UNIQUE (reader_hostgroup))" + "auto_generated INT CHECK (auto_generated IN (0,1)) NOT NULL DEFAULT 0 , " \ + "status VARCHAR NOT NULL DEFAULT 'NONE' , "\ + "UNIQUE (reader_hostgroup))" #define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_0 "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 8f24145a34..e8e9ce03c3 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -2261,7 +2261,7 @@ SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql(const string& name) "check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment FROM mysql_aws_aurora_hostgroups"; } else if (name == "mysql_aws_rds_bgd_hostgroups") { query=(char *)"SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader," - "check_interval_ms,check_timeout_ms,comment,auto_generated FROM mysql_aws_rds_bgd_hostgroups"; + "check_interval_ms,check_timeout_ms,comment,auto_generated,status FROM mysql_aws_rds_bgd_hostgroups"; } else if (name == "mysql_galera_hostgroups") { query=(char *)"SELECT writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers,writer_is_also_reader,max_transactions_behind,comment FROM mysql_galera_hostgroups"; } else if (name == "mysql_group_replication_hostgroups") { @@ -3900,6 +3900,35 @@ bool MySQL_HostGroups_Manager::aws_rds_bgd_set_shun_server(unsigned int hostgrou return changed; } +void MySQL_HostGroups_Manager::aws_rds_bgd_set_switchover_status(unsigned int writer_hg, int status) { + char query[128]; + snprintf(query, sizeof(query), + "UPDATE mysql_aws_rds_bgd_hostgroups SET status=%d WHERE writer_hostgroup=%u", status, writer_hg); + wrlock(); + mydb->execute(query); + wrunlock(); +} + +bool MySQL_HostGroups_Manager::is_aws_rds_bgd_in_progress() { + bool in_progress = false; + char *error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result *resultset = NULL; + wrlock(); + mydb->execute_statement( + (char *)"SELECT EXISTS(SELECT 1 FROM mysql_aws_rds_bgd_hostgroups WHERE status!=0)", + &error, &cols, &affected_rows, &resultset); + wrunlock(); + if (resultset) { + if (resultset->rows_count && resultset->rows[0]->fields[0]) { + in_progress = (atoi(resultset->rows[0]->fields[0]) != 0); + } + delete resultset; + } + return in_progress; +} + /** * @brief Aligns the runtime 'mysql_servers' table + checksums with the server state in MyHGM. * @@ -7063,8 +7092,8 @@ void MySQL_HostGroups_Manager::update_aws_aurora_hosts_monitor_resultset(bool lo const char SELECT_AWS_RDS_BGD_SERVERS_FOR_MONITOR[] { "SELECT writer_hostgroup, reader_hostgroup, hostname, port, MAX(use_ssl) use_ssl, green_writer_hostgroup," " green_reader_hostgroup, check_interval_ms, check_timeout_ms, writer_is_also_reader FROM mysql_servers" - " JOIN mysql_aws_rds_bgd_hostgroups ON" - " hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE active=1 AND status NOT IN (2,3)" + " JOIN mysql_aws_rds_bgd_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup" + " WHERE active=1 AND mysql_servers.status NOT IN (2,3)" " GROUP BY writer_hostgroup, hostname, port" }; diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 786d533601..6b61fcd739 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6699,34 +6699,19 @@ static int aws_rds_bgd_async_query(MySQL_Monitor_State_Data *mmsd, const char *q } /** -* @brief Mark a switchover in progress so the read_only monitor fast-polls this deployment's servers. +* @brief Set the deployment's switchover status and persist it in runtime table. */ -static void aws_rds_bgd_set_bgd_in_progress(AWS_RDS_BGD_State& st) { - if (st.bgd_in_progress_set) { +static void aws_rds_bgd_set_status(AWS_RDS_BGD_State& st, AWS_RDS_BGD_Status status) { + if (st.bgd_status == status) { return; } - MyHGM->set_aws_rds_bgd_in_progress(true); - st.bgd_in_progress_set = true; - - proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: Enabling fast-poll of BGD servers on read_only monitor.\n", - st.writer_hg, st.reader_hg); -} - -/** -* @brief Drop this deployment's in-progress count so the read_only monitor can return to baseline -* polling cadence (undo aws_rds_bgd_set_bgd_in_progress). -*/ -static void aws_rds_bgd_clear_bgd_in_progress(AWS_RDS_BGD_State& st) { - if (!st.bgd_in_progress_set) { - return; - } - - MyHGM->set_aws_rds_bgd_in_progress(false); - st.bgd_in_progress_set = false; + proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: switchover status '%s' -> '%s'\n", + st.writer_hg, st.reader_hg, + aws_rds_bgd_status_str(st.bgd_status), aws_rds_bgd_status_str(status)); - proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: Disabling fast-poll of BGD servers on read_only monitor.\n", - st.writer_hg, st.reader_hg); + st.bgd_status = status; + MyHGM->aws_rds_bgd_set_switchover_status(st.writer_hg, static_cast(status)); } void * monitor_RDS_BGD_thread_HG(void *arg) { @@ -6745,6 +6730,8 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { AWS_RDS_BGD_State st; st.writer_hg = wHG; + MyHGM->aws_rds_bgd_set_switchover_status(wHG, static_cast(AWS_RDS_BGD_Status::NONE)); + unsigned int MySQL_Monitor__thread_MySQL_Thread_Variables_version; MySQL_Thread * mysql_thr = new MySQL_Thread(); mysql_thr->curtime = monotonic_time(); @@ -7028,8 +7015,6 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { } __exit_monitor_RDS_BGD_thread_HG_now: - aws_rds_bgd_clear_bgd_in_progress(st); - if (mmsd) { delete mmsd; mmsd = NULL; @@ -7271,7 +7256,7 @@ static AWS_RDS_BGD_Status aws_rds_bgd_status_from_topology(const std::string& st } // Human-readable name for a phase enum, for logging and (later) the runtime status column. -static const char* aws_rds_bgd_status_str(AWS_RDS_BGD_Status s) { +const char* aws_rds_bgd_status_str(AWS_RDS_BGD_Status s) { switch (s) { case AWS_RDS_BGD_Status::NONE: return "NONE"; @@ -7306,7 +7291,7 @@ static const char* aws_rds_bgd_status_str(AWS_RDS_BGD_Status s) { void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_Result& topology) { if (!topology.blue_green) { st.next_check_interval_ms = 0; - aws_rds_bgd_clear_bgd_in_progress(st); + aws_rds_bgd_set_status(st, AWS_RDS_BGD_Status::NONE); return; } @@ -7319,7 +7304,7 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo } if (status.empty()) { st.next_check_interval_ms = 0; - aws_rds_bgd_clear_bgd_in_progress(st); + aws_rds_bgd_set_status(st, AWS_RDS_BGD_Status::NONE); return; } st.last_topology_status = status; @@ -7340,12 +7325,7 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo return; } - proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: switchover status '%s' -> '%s'\n", - st.writer_hg, st.reader_hg, - aws_rds_bgd_status_str(st.bgd_status), - aws_rds_bgd_status_str(topology_status)); - - st.bgd_status = topology_status; + aws_rds_bgd_set_status(st, topology_status); if (st.bgd_status == AWS_RDS_BGD_Status::AVAILABLE) { st.next_check_interval_ms = 250; @@ -7361,7 +7341,6 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo aws_rds_bgd_build_map(st, topology); aws_rds_bgd_resolve_green_ips(st); aws_rds_bgd_add_green_writer_in_hg(st); - aws_rds_bgd_set_bgd_in_progress(st); } else if (st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_POST_PROCESSING) { st.next_check_interval_ms = 100; @@ -7372,7 +7351,6 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo aws_rds_bgd_build_map(st, topology); aws_rds_bgd_resolve_green_ips(st); aws_rds_bgd_add_green_writer_in_hg(st); - aws_rds_bgd_set_bgd_in_progress(st); // Repoint each mapped blue host onto its green IP and drain existing // connections so new backend work resolves to green. @@ -7448,7 +7426,7 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo // Writer switchover done, but the topology table lingers with a single green row until the blue // readers' DNS propagates and it drains. Defer reader teardown: advance to READER_SWITCHOVER phase // and let the drain (aws_rds_bgd_handle_topology_absent) trigger it. - st.bgd_status = AWS_RDS_BGD_Status::READER_SWITCHOVER_IN_PROGRESS; + aws_rds_bgd_set_status(st, AWS_RDS_BGD_Status::READER_SWITCHOVER_IN_PROGRESS); // Drop the writer's DNS_Cache entry: this clears the IP pin and lets regular DNS // resolution take over from here. Readers stay pinned until their DNS propagates. @@ -7461,14 +7439,6 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo // release BGD monitor worker from fast-polling st.next_check_interval_ms = 0; - - // release read_only monitor from fast-polling - aws_rds_bgd_clear_bgd_in_progress(st); - - proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: switchover status '%s' -> '%s'\n", - st.writer_hg, st.reader_hg, - aws_rds_bgd_status_str(AWS_RDS_BGD_Status::WRITER_SWITCHOVER_COMPLETED), - aws_rds_bgd_status_str(AWS_RDS_BGD_Status::READER_SWITCHOVER_IN_PROGRESS)); } else { // NONE / unrecognized status: take no action, stay at baseline interval @@ -7579,11 +7549,10 @@ void MySQL_Monitor::handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st) { // state cleanup st.bg_map.clear(); st.last_topology_status.clear(); - st.bgd_status = AWS_RDS_BGD_Status::NONE; st.next_check_host.clear(); st.next_check_interval_ms = 0; st.green_writer_added_in_hg = false; - aws_rds_bgd_clear_bgd_in_progress(st); + aws_rds_bgd_set_status(st, AWS_RDS_BGD_Status::NONE); proxy_info( "AWS RDS BGD [wHG=%u rHG=%u]: post-switchover cleanup complete; state cleared\n", @@ -7601,7 +7570,7 @@ void MySQL_Monitor::aws_rds_bgd_handle_topology_absent(AWS_RDS_BGD_State& st) { if (st.bgd_status == AWS_RDS_BGD_Status::READER_SWITCHOVER_IN_PROGRESS) { handle_aws_rds_bgd_post_switchover(st); } else { - aws_rds_bgd_clear_bgd_in_progress(st); + aws_rds_bgd_set_status(st, AWS_RDS_BGD_Status::NONE); } } diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index d5fdf99e19..c968f1e212 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -7574,7 +7574,7 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { char *query=NULL; if (_runtime) { - query=(char *)"INSERT INTO runtime_mysql_aws_rds_bgd_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment,auto_generated) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)"; + query=(char *)"INSERT INTO runtime_mysql_aws_rds_bgd_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment,auto_generated,status) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"; } else { query=(char *)"INSERT INTO mysql_aws_rds_bgd_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)"; } @@ -7613,6 +7613,10 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { rc=(*proxy_sqlite3_bind_text)(statement, 9, r->fields[8], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); if (_runtime) { rc=(*proxy_sqlite3_bind_int64)(statement, 10, atoi(r->fields[9])); ASSERT_SQLITE_OK(rc, admindb); + // 'status' (field 10) is the AWS_RDS_BGD_Status underlying int; we store it as text in the runtime table. + const char *bgd_status_str = + aws_rds_bgd_status_str(static_cast(r->fields[10] ? atoi(r->fields[10]) : 0)); + rc=(*proxy_sqlite3_bind_text)(statement, 11, bgd_status_str, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); } SAFE_SQLITE3_STEP2(statement); From 912b03a659b3daab6847e49762ea6f4cdbfdbc13 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Mon, 6 Jul 2026 09:26:46 +0000 Subject: [PATCH 25/81] feat: Drain AWS RDS BGD green hostgroups after switchover In post-switchover phase, evict the DNS cache entries and drain connections for servers in the green writer/reader hostgroups. Servers are left in place and monitor shuns them once the retired green DNS stops resolving. Signed-off-by: Wazir Ahmed --- include/MySQL_Monitor.hpp | 13 +++++++++ lib/MySQL_Monitor.cpp | 55 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index a68cd57574..849c51c877 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -665,6 +665,19 @@ class MySQL_Monitor { void handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_Result& topology); // Deferred teardown: runs once mysql.rds_topology drains after SWITCHOVER_COMPLETED. void handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st); + /** + * @brief Evict stale DNS and drain connections for the deployment's green hostgroups after switchover. + * + * @details No-op unless the green writer/reader hostgroups are configured (the explicit green-HG path). + * For every non-OFFLINE_HARD member of each green hostgroup this drops the DNS cache entry, drains the + * server's backend connections and purges the monitor connection pool. + * + * The servers are left in place; the monitor shuns them on ping/connect errors once the retired green + * DNS names stop resolving to an IP. + * + * @param st Switchover state. + */ + void aws_rds_bgd_drain_green_hg(AWS_RDS_BGD_State& st); // Called by the BGD worker when the topology table is absent/empty/vanished; routes to the // deferred cleanup when bgd_status is READER_SWITCHOVER_IN_PROGRESS, else preserves the baseline release. void aws_rds_bgd_handle_topology_absent(AWS_RDS_BGD_State& st); diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 6b61fcd739..d387268efc 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -7544,7 +7544,7 @@ void MySQL_Monitor::handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st) { My_Conn_Pool->purge_connections(p.blue_host.c_str(), p.port); } - // TODO: Drain Green HGs + aws_rds_bgd_drain_green_hg(st); // state cleanup st.bg_map.clear(); @@ -7559,6 +7559,59 @@ void MySQL_Monitor::handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st) { st.writer_hg, st.reader_hg); } +/** +* @brief Evict stale DNS and drain connections for the deployment's green hostgroups after switchover. +* +* @details No-op unless the green writer/reader hostgroups are configured (the explicit green-HG path). +* For every non-OFFLINE_HARD member of each green hostgroup this drops the DNS cache entry, drains the +* server's backend connections and purges the monitor connection pool. +* +* The servers are left in place; the monitor shuns them on ping/connect errors once the retired green +* DNS names stop resolving to an IP. +* +* @param st Switchover state. +*/ +void MySQL_Monitor::aws_rds_bgd_drain_green_hg(AWS_RDS_BGD_State& st) { + std::vector green_hgs; + if (st.green_writer_hg >= 0) { + green_hgs.push_back((unsigned int)st.green_writer_hg); + } + if (st.green_reader_hg >= 0) { + green_hgs.push_back((unsigned int)st.green_reader_hg); + } + if (green_hgs.empty()) { + return; + } + + for (unsigned int hg : green_hgs) { + std::vector> servers; + + MyHGM->wrlock(); + MyHGC* hgc = MyHGM->MyHGC_lookup(hg); + if (hgc && hgc->mysrvs) { + for (unsigned int j = 0; j < hgc->mysrvs->cnt(); j++) { + MySrvC* s = hgc->mysrvs->idx(j); + if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + continue; + } + servers.push_back({ std::string(s->address), s->port }); + } + } + MyHGM->wrunlock(); + + for (std::pair& srv : servers) { + std::string& host = srv.first; + int port = srv.second; + dns_cache->remove(host); + MyHGM->drain_server_connections(hg, host.c_str(), port); + My_Conn_Pool->purge_connections(host.c_str(), port); + proxy_info( + "AWS RDS BGD [wHG=%u rHG=%u]: connections drained from green HG %u server '%s:%d'\n", + st.writer_hg, st.reader_hg, hg, host.c_str(), port); + } + } +} + /** * @brief Invoked by the BGD worker whenever mysql.rds_topology is absent/empty/vanished. * From 72516ed9a482189e579af7bfbcb5e4330abaa9a0 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 7 Jul 2026 19:35:15 +0000 Subject: [PATCH 26/81] fix: Drain blue server connections across all hostgroups during switchover Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 5 ++--- lib/MySQL_HostGroups_Manager.cpp | 13 ++++++++----- lib/MySQL_Monitor.cpp | 5 ++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index d7f866149d..55f472d828 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -1092,17 +1092,16 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { */ void publish_mysql_servers_to_runtime(); /** - * @brief Drain existing backend connections for a server. + * @brief Drain existing backend connections for a server in all hostgroups. * * @details Drops free connections immediately and marks used connections as unhealthy and non-reusable, * so in-flight operations fail on their next backend step and the connection is never pooled again. * - * @param hostgroup_id Hostgroup to search. * @param hostname Address of the server to match. * @param port Port of the server to match. * @return true if a matching server was found. */ - bool drain_server_connections(unsigned int hostgroup_id, const char *hostname, int port); + bool drain_server_connections(const char *hostname, int port); /** * @brief Whether any AWS RDS blue/green deployment is currently in a active switchover status. * diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index e8e9ce03c3..13556afbb6 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -3957,23 +3957,26 @@ void MySQL_HostGroups_Manager::publish_mysql_servers_to_runtime() { } /** - * @brief Drain existing backend connections for a server. + * @brief Drain existing backend connections for a server in all hostgroups. * * @details Drops free connections immediately and marks used connections as unhealthy and non-reusable, * so in-flight operations fail on their next backend step and the connection is never pooled again. * - * @param hostgroup_id Hostgroup to search. * @param hostname Address of the server to match. * @param port Port of the server to match. * @return true if a matching server was found. */ -bool MySQL_HostGroups_Manager::drain_server_connections(unsigned int hostgroup_id, const char *hostname, int port) { +bool MySQL_HostGroups_Manager::drain_server_connections(const char *hostname, int port) { bool found = false; wrlock(); - MyHGC *myhgc = MyHGC_find(hostgroup_id); - if (myhgc && myhgc->mysrvs) { + for (unsigned int i = 0; i < MyHostGroups->len; i++) { + MyHGC *myhgc = (MyHGC *)MyHostGroups->index(i); + if (!myhgc || !myhgc->mysrvs) { + continue; + } + for (unsigned int j = 0; j < myhgc->mysrvs->cnt(); j++) { MySrvC *mysrvc = myhgc->mysrvs->idx(j); if (mysrvc->port != port || strcmp(mysrvc->address, hostname) != 0) { diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index d387268efc..48566b23b8 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -7366,8 +7366,7 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo "AWS RDS BGD [wHG=%u rHG=%u]: repointed blue '%s' to green IP %s\n", st.writer_hg, st.reader_hg, p.blue_host.c_str(), p.green_ip.c_str()); - unsigned int hid = p.is_writer ? st.writer_hg : st.reader_hg; - MyHGM->drain_server_connections(hid, p.blue_host.c_str(), p.port); + MyHGM->drain_server_connections(p.blue_host.c_str(), p.port); My_Conn_Pool->purge_connections(p.blue_host.c_str(), p.port); } @@ -7603,7 +7602,7 @@ void MySQL_Monitor::aws_rds_bgd_drain_green_hg(AWS_RDS_BGD_State& st) { std::string& host = srv.first; int port = srv.second; dns_cache->remove(host); - MyHGM->drain_server_connections(hg, host.c_str(), port); + MyHGM->drain_server_connections(host.c_str(), port); My_Conn_Pool->purge_connections(host.c_str(), port); proxy_info( "AWS RDS BGD [wHG=%u rHG=%u]: connections drained from green HG %u server '%s:%d'\n", From 8bdb921f80d226428e7f5130d9febbf2769f6624 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Wed, 8 Jul 2026 14:34:16 +0000 Subject: [PATCH 27/81] refactor: Drive hostgroup migration from AWS RDS BGD monitor - Gate the `read_only` monitor actions with a per-server `bgd_in_progress` flag so it backs off during a switchover. - Drive writer/reader hostgroup placement from the BGD FSM and the `writer_is_also_reader` configuration. - Remove the `read_only` fast-poll loop. Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 63 ++++-- include/MySQL_Monitor.hpp | 82 +++++--- lib/MySQL_HostGroups_Manager.cpp | 114 +++++++++-- lib/MySQL_Monitor.cpp | 309 ++++++++++++++++------------- 4 files changed, 363 insertions(+), 205 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 55f472d828..3119803cc3 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -605,6 +605,21 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { return readonly_flag; } + inline + void set_aws_rds_bgd_in_progress() { + aws_rds_bgd_in_progress = true; + } + + inline + bool is_aws_rds_bgd_in_progress() { + return aws_rds_bgd_in_progress; + } + + inline + void clear_aws_rds_bgd_in_progress() { + aws_rds_bgd_in_progress = false; + } + private: unsigned int get_hostgroup_id(Type type, const Node& node) const; MySrvC* insert_HGM(unsigned int hostgroup_id, const MySrvC* srv); @@ -613,6 +628,7 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { std::array, TYPE_SIZE_> mapping; // index 0 contains reader and 1 contains writer hostgroups int readonly_flag; MySQL_HostGroups_Manager* myHGM; + bool aws_rds_bgd_in_progress = false; }; /** @@ -1046,15 +1062,27 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { void replication_lag_action_inner(MyHGC *, const char*, unsigned int, int, bool); void replication_lag_action(const std::list& mysql_servers); -// void read_only_action(char *hostname, int port, int read_only); - void read_only_action_v2(const std::list& mysql_servers); + /** + * @brief Reconcile writer/reader hostgroup placement from read_only monitor results. + * + * @details New implementation of the read_only_action that does not depend on the admin table. + * Checks each server in the provided list and adjusts writer/reader hostgroup placement + * according to the corresponding read_only value. If any change occurs, the runtime + * mysql_servers table and checksum are regenerated. When `force` is false, + * servers flagged as AWS RDS BGD switchover-in-progress are skipped; when true, the supplied + * state is applied even for those servers. + * + * @param mysql_servers Servers and their observed/read-only state. + * @param force Force state changes regardless of AWS RDS BGD switchover state. + */ + void read_only_action_v2(const std::list& mysql_servers, bool force = false); unsigned int get_servers_table_version(); void wait_servers_table_version(unsigned, unsigned); bool shun_and_killall(char *hostname, int port); void set_server_current_latency_us(char *hostname, int port, unsigned int _current_latency_us); void set_Readyset_status(char *hostname, int port, enum MySerStatus status); /** - * @brief Set or clear AWS BGD shun state for a matching server. + * @brief Set or clear AWS RDS BGD shun state for a matching server. * * @details When shunning, transitions an ONLINE server to SHUNNED_AWS_BGD, * enables shun metadata, and drops free connections. When unshunning, @@ -1072,16 +1100,27 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { */ bool aws_rds_bgd_set_shun_server(unsigned int hostgroup_id, const char *hostname, int port, bool shun); /** - * @brief Persist BGD switchover status into the runtime mysql_aws_rds_bgd_hostgroups table. + * @brief Configure the AWS RDS BGD writer's writer/reader hostgroup membership. + * + * @details Ensures the writer is present in its writer hostgroup, with optional reader + * hostgroup membership controlled by writer_is_also_reader. + * + * @param hostname Server hostname to configure. + * @param port Server port to configure. + * @param writer_is_also_reader Whether the writer should also be present in reader hostgroup. * - * @details Called by the BGD worker on every FSM status transition. For the read_only monitor, - * it both gates the fast-poll cadence (is_aws_rds_bgd_in_progress) and filters which servers the - * fast poll selects. + * @return true if hostgroup membership changed. + * + * @note Caller must hold wrlock(). + */ + bool aws_rds_bgd_configure_writer(const char *hostname, int port, bool writer_is_also_reader); + /** + * @brief Set AWS RDS BGD switchover status in runtime mysql_aws_rds_bgd_hostgroups table * * @param writer_hg Writer hostgroup identifying the deployment. * @param status AWS_RDS_BGD_Status underlying value. */ - void aws_rds_bgd_set_switchover_status(unsigned int writer_hg, int status); + void aws_rds_bgd_set_runtime_status(unsigned int writer_hg, int status); /** * @brief Aligns the runtime 'mysql_servers' table + checksums with the server state in MyHGM. * @@ -1103,11 +1142,11 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { */ bool drain_server_connections(const char *hostname, int port); /** - * @brief Whether any AWS RDS blue/green deployment is currently in a active switchover status. - * - * @details Derived from the runtime mysql_aws_rds_bgd_hostgroups 'status' column. + * @brief Flag/unflag every server in the writer and reader hostgroups of an AWS RDS blue/green + * deployment as "switchover in progress", so the read_only monitor (read_only_action_v2) takes + * no action on them while the BGD FSM is driving the switchover. */ - bool is_aws_rds_bgd_in_progress(); + void set_aws_rds_bgd_in_progress(unsigned int writer_hg, unsigned int reader_hg, bool in_progress); unsigned long long Get_Memory_Stats(); diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 849c51c877..bb5befc669 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -394,6 +394,14 @@ struct mon_metrics_map_idx { }; }; +/** + * @brief Server hostname and port. + */ +struct srv_addr_t { + std::string host; + int port = 0; +}; + /** * @brief A single node (row) of a 'SELECT * FROM mysql.rds_topology' result. */ @@ -450,9 +458,10 @@ struct AWS_RDS_BlueGreenPair { * - Replicas done: the table drains to empty (blue-reader DNS has propagated). * * The WRITER_SWITCHOVER_* values map 1:1 onto the mysql.rds_topology status strings. - * READER_SWITCHOVER_IN_PROGRESS is a ProxySQL inferred status with no topology-string mapping: we - * enter it after WRITER_SWITCHOVER_COMPLETED, deferring reader/DNS cleanup until the table drains - * to empty. + * READER_SWITCHOVER_IN_PROGRESS is a ProxySQL inferred status entered after + * WRITER_SWITCHOVER_COMPLETED; it defers reader/DNS cleanup until the topology table drains + * to empty. SWITCHOVER_COMPLETED is a short-lived status used for final cleanup before + * returning to NONE. */ enum class AWS_RDS_BGD_Status { NONE = 0, ///< no BGD topology / baseline @@ -461,14 +470,15 @@ enum class AWS_RDS_BGD_Status { WRITER_SWITCHOVER_IN_PROGRESS = 3, ///< "SWITCHOVER_IN_PROGRESS" WRITER_SWITCHOVER_POST_PROCESSING = 4, ///< "SWITCHOVER_IN_POST_PROCESSING" WRITER_SWITCHOVER_COMPLETED = 5, ///< "SWITCHOVER_COMPLETED" - READER_SWITCHOVER_IN_PROGRESS = 6, ///< ProxySQL inferred reader status; awaiting topology drain + deferred cleanup + READER_SWITCHOVER_IN_PROGRESS = 6, ///< ProxySQL inferred status; awaiting topology drain + deferred cleanup + SWITCHOVER_COMPLETED = 7, ///< short-lived status used for final cleanup before returning to NONE }; // Maps a switchover status enum to its stored/display string. const char* aws_rds_bgd_status_str(AWS_RDS_BGD_Status s); /** - * @brief Per-deployment switchover state carried by one RDS BGD worker thread. + * @brief Switchover state carried by RDS BGD worker thread. * * @details One worker (monitor_RDS_BGD_thread_HG) owns one writer hostgroup == * one blue/green deployment, so this struct lives on the worker's stack and is @@ -486,10 +496,11 @@ struct AWS_RDS_BGD_State { std::string last_topology_status; ///< raw mysql.rds_topology TARGET status from the previous poll (verbatim) std::vector bg_map; ///< [writer] always; [readers] only when green_reader_hg is configured - std::vector> shunned_readers; ///< (host,port) we shunned + std::vector shunned_readers; ///< readers we shunned AWS_RDS_BGD_Status bgd_status = AWS_RDS_BGD_Status::NONE; ///< drives the FSM and the deferred cleanup bool green_writer_added_in_hg = false; ///< green writer added to green_writer_hg + bool bgd_in_progress_set = false; ///< servers flagged as switchover-in-progress unsigned int next_check_interval_ms = 0; ///< FSM-controlled interval; 0 => baseline std::string next_check_host; ///< FSM-pinned probe host; when set (the green IP), the worker @@ -513,18 +524,9 @@ inline const char* const BGD_STATUS_IN_PROGRESS = "SWITCHOVER_IN_PROGRESS"; inline const char* const BGD_STATUS_POST_PROC = "SWITCHOVER_IN_POST_PROCESSING"; inline const char* const BGD_STATUS_COMPLETED = "SWITCHOVER_COMPLETED"; -// While any AWS RDS blue/green deployment is mid-switchover, the read_only monitor polls just that -// deployment's servers at this tightened interval (250ms) so it detects the writer's read_only flips -// quickly; matches the BGD FSM's own fast poll tiers. The full-fleet pass stays at -// mysql-monitor_read_only_interval. -#define READ_ONLY_BGD_LOOP_INTERVAL_US 250000 -#define READ_ONLY_NEXT_LOOP_INTERVAL_US 500000 - -// read_only monitor server-enumeration queries. -// Every server that belongs to a replication hostgroup and status NOT IN (2,3,5) +// read_only monitor server-enumeration query. +// Every server that belongs to a replication hostgroup and status NOT IN (OFFLINE_SOFT, OFFLINE_HARD, SHUNNED_AWS_BGD) #define SELECT_SERVERS_FOR_READ_ONLY "SELECT hostname, port, MAX(use_ssl) use_ssl, check_type, reader_hostgroup FROM mysql_servers JOIN mysql_replication_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE status NOT IN (2,3,5) GROUP BY hostname, port ORDER BY RANDOM()" -// Fast pass: only servers in an AWS RDS blue/green deployment -#define SELECT_RDS_BGD_SERVERS_FOR_READ_ONLY "SELECT hostname, port, MAX(use_ssl) use_ssl, 'read_only' check_type, reader_hostgroup FROM mysql_servers JOIN mysql_aws_rds_bgd_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE active=1 AND status NOT IN (2,3,5) GROUP BY hostname, port ORDER BY RANDOM()" // Defined in MySQL_HostGroups_Manager.h; forward-declared here because the include cycle // (Monitor.hpp -> HGM.h -> cpp.h -> Monitor.hpp) can leave them undefined at this point. Only @@ -659,11 +661,18 @@ class MySQL_Monitor { * connections, and shuns/enforces reader handling. State carried across cycles * lives in @p st. * - * @param st Per-deployment switchover state (worker-owned, mutated here). + * @param st BGD switchover state. * @param topology Parsed mysql.rds_topology result for this cycle. */ void handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_Result& topology); - // Deferred teardown: runs once mysql.rds_topology drains after SWITCHOVER_COMPLETED. + /** + * @brief Run deferred switchover teardown after mysql.rds_topology drains. + * + * @details Restores post-switchover reader handling, unshuns readers, drops DNS pins, + * drains green hostgroups, and clears BGD switchover state. + * + * @param st BGD switchover state. + */ void handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st); /** * @brief Evict stale DNS and drain connections for the deployment's green hostgroups after switchover. @@ -678,16 +687,33 @@ class MySQL_Monitor { * @param st Switchover state. */ void aws_rds_bgd_drain_green_hg(AWS_RDS_BGD_State& st); - // Called by the BGD worker when the topology table is absent/empty/vanished; routes to the - // deferred cleanup when bgd_status is READER_SWITCHOVER_IN_PROGRESS, else preserves the baseline release. + /** + * @brief Handle an absent, empty, or vanished mysql.rds_topology table. + * + * @details Routes to deferred cleanup when bgd_status is READER_SWITCHOVER_IN_PROGRESS; + * otherwise clears any in-progress switchover state for this deployment. + * + * @param st BGD switchover state. + */ void aws_rds_bgd_handle_topology_absent(AWS_RDS_BGD_State& st); - // Apply one switchover step's reader-HG mutations, with the action derived from bgd_status: - // POST_PROCESSING shuns the readers and (when writer_info is set) adds the writer as a reader; - // READER_SWITCHOVER_IN_PROGRESS unshuns the readers and (when writer_info is set) removes the writer. - void aws_rds_bgd_reconfigure_reader_hg( - AWS_RDS_BGD_Status bgd_status, unsigned int reader_hg, - std::vector>& unmapped_readers, - srv_info_t* writer_info, srv_opts_t* writer_opts); + /** + * @brief Apply BGD hostgroup changes for the current switchover status. + * + * @details POST_PROCESSING configures the writer placement and shuns unmapped readers. + * SWITCHOVER_COMPLETED unshuns readers and removes the writer from reader HG when + * writer_is_also_reader is false. Runtime mysql_servers and checksum are re-generated + * when server hostgroup membership changes. + * + * @param bgd_status Current BGD FSM status driving the action. + * @param writer Writer server to configure. + * @param writer_is_also_reader Whether the writer should also remain in reader_hg. + * @param reader_hg Reader hostgroup for reader shun/unshun and optional writer membership. + * @param readers Reader servers to shun or unshun. + */ + void aws_rds_bgd_hostgroup_action( + AWS_RDS_BGD_Status bgd_status, + srv_addr_t& writer, bool writer_is_also_reader, + unsigned int reader_hg, std::vector& readers); void * monitor_replication_lag(); void * monitor_dns_cache(); diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 13556afbb6..7940526ae7 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -3573,14 +3573,19 @@ SQLite3_result * MySQL_HostGroups_Manager::SQL3_Connection_Pool(bool _reset, int } /** - * @brief New implementation of the read_only_action method that does not depend on the admin table. - * The method checks each server in the provided list and adjusts the servers according to their corresponding read_only value. - * If any change has occured, checksum is calculated. + * @brief Reconcile writer/reader hostgroup placement from read_only monitor results. * - * @param mysql_servers List of servers having hostname, port and read only value. - * + * @details New implementation of the read_only_action that does not depend on the admin table. + * Checks each server in the provided list and adjusts writer/reader hostgroup placement + * according to the corresponding read_only value. If any change occurs, the runtime + * mysql_servers table and checksum are regenerated. When `force` is false, + * servers flagged as AWS RDS BGD switchover-in-progress are skipped; when true, the supplied + * state is applied even for those servers. + * + * @param mysql_servers Servers and their observed/read-only state. + * @param force Force state changes regardless of AWS RDS BGD switchover state. */ -void MySQL_HostGroups_Manager::read_only_action_v2(const std::list& mysql_servers) { +void MySQL_HostGroups_Manager::read_only_action_v2(const std::list& mysql_servers, bool force) { bool update_mysql_servers_table = false; @@ -3606,6 +3611,13 @@ void MySQL_HostGroups_Manager::read_only_action_v2(const std::listis_aws_rds_bgd_in_progress() && !force) { + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "Skipping read_only_action_v2() for server '%s:%d' because AWS RDS BGD switchover is in progress\n", + hostname.c_str(), port); + continue; + } + const std::vector& writer_map = host_server_mapping->get(HostGroup_Server_Mapping::Type::WRITER); is_writer = !writer_map.empty(); @@ -3900,7 +3912,59 @@ bool MySQL_HostGroups_Manager::aws_rds_bgd_set_shun_server(unsigned int hostgrou return changed; } -void MySQL_HostGroups_Manager::aws_rds_bgd_set_switchover_status(unsigned int writer_hg, int status) { +/** + * @brief Configure the AWS RDS BGD writer's writer/reader hostgroup membership. + * + * @details Ensures the writer is present in its writer hostgroup, with optional reader + * hostgroup membership controlled by writer_is_also_reader. + * + * @param hostname Server hostname to configure. + * @param port Server port to configure. + * @param writer_is_also_reader Whether the writer should also be present in reader hostgroup. + * + * @return true if hostgroup membership changed. + * + * @note Caller must hold wrlock(). + */ +bool MySQL_HostGroups_Manager::aws_rds_bgd_configure_writer(const char *hostname, int port, bool writer_is_also_reader) { + const std::string srv_id = std::string(hostname) + ":::" + std::to_string(port); + auto itr = hostgroup_server_mapping.find(srv_id); + + if (itr == hostgroup_server_mapping.end() || !itr->second) { + proxy_warning("AWS RDS BGD: server %s:%d not found in hostgroup_server_mapping\n", hostname, port); + return false; + } + + HostGroup_Server_Mapping* srv_map = itr->second.get(); + bool changed = false; + + if (srv_map->get(HostGroup_Server_Mapping::Type::WRITER).empty()) { + if (srv_map->get(HostGroup_Server_Mapping::Type::READER).empty()) { + proxy_warning("AWS RDS BGD: server %s:%d has no writer or reader hostgroup mapping\n", hostname, port); + return false; + } + + srv_map->copy_if_not_exists(HostGroup_Server_Mapping::Type::WRITER, HostGroup_Server_Mapping::Type::READER); + proxy_info("AWS RDS BGD: adding server %s:%d to writer hostgroup\n", hostname, port); + changed = true; + } + + if (writer_is_also_reader) { + if (srv_map->get(HostGroup_Server_Mapping::Type::READER).empty()) { + srv_map->copy_if_not_exists(HostGroup_Server_Mapping::Type::READER, HostGroup_Server_Mapping::Type::WRITER); + proxy_info("AWS RDS BGD: adding server %s:%d to reader hostgroup\n", hostname, port); + changed = true; + } + } else if (!srv_map->get(HostGroup_Server_Mapping::Type::READER).empty()) { + srv_map->clear(HostGroup_Server_Mapping::Type::READER); + proxy_info("AWS RDS BGD: removing server %s:%d from reader hostgroup\n", hostname, port); + changed = true; + } + + return changed; +} + +void MySQL_HostGroups_Manager::aws_rds_bgd_set_runtime_status(unsigned int writer_hg, int status) { char query[128]; snprintf(query, sizeof(query), "UPDATE mysql_aws_rds_bgd_hostgroups SET status=%d WHERE writer_hostgroup=%u", status, writer_hg); @@ -3909,24 +3973,30 @@ void MySQL_HostGroups_Manager::aws_rds_bgd_set_switchover_status(unsigned int wr wrunlock(); } -bool MySQL_HostGroups_Manager::is_aws_rds_bgd_in_progress() { - bool in_progress = false; - char *error = NULL; - int cols = 0; - int affected_rows = 0; - SQLite3_result *resultset = NULL; +void MySQL_HostGroups_Manager::set_aws_rds_bgd_in_progress(unsigned int writer_hg, unsigned int reader_hg, bool in_progress) { wrlock(); - mydb->execute_statement( - (char *)"SELECT EXISTS(SELECT 1 FROM mysql_aws_rds_bgd_hostgroups WHERE status!=0)", - &error, &cols, &affected_rows, &resultset); - wrunlock(); - if (resultset) { - if (resultset->rows_count && resultset->rows[0]->fields[0]) { - in_progress = (atoi(resultset->rows[0]->fields[0]) != 0); + + unsigned int hgs[2] = { writer_hg, reader_hg }; + for (unsigned int i = 0; i < 2; i++) { + MyHGC* myhgc = MyHGC_find(hgs[i]); + if (myhgc == nullptr || myhgc->mysrvs == nullptr) { + continue; + } + for (unsigned int j = 0; j < myhgc->mysrvs->cnt(); j++) { + MySrvC* s = myhgc->mysrvs->idx(j); + const std::string srv_id = std::string(s->address) + ":::" + std::to_string(s->port); + auto itr = hostgroup_server_mapping.find(srv_id); + if (itr != hostgroup_server_mapping.end() && itr->second) { + if (in_progress) { + itr->second->set_aws_rds_bgd_in_progress(); + } else { + itr->second->clear_aws_rds_bgd_in_progress(); + } + } } - delete resultset; } - return in_progress; + + wrunlock(); } /** diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 48566b23b8..875e7f4566 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -3691,25 +3691,18 @@ void * MySQL_Monitor::monitor_read_only() { unsigned long long t1; unsigned long long t2; - // next loop iteration time for regular read_only checks unsigned long long next_loop_at=0; - // next loop iteration time for read_only checks on RDS blue/green deployment servers - unsigned long long next_bgd_loop_at = 0; - int rds_topology_check_counter = 0; + int topology_loop = 0; while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true) { - // whether to run read_only checks on RDS blue/green deployment servers only - bool rds_bgd_only_loop = false; - - // whether to run mysql.rds_topology check for RDS servers - // in addition to regular read_only checks for all servers - bool rds_topology_check = false; - int rds_topology_check_interval = mysql_thread___monitor_aws_rds_topology_discovery_interval; + int topology_loop_max = mysql_thread___monitor_aws_rds_topology_discovery_interval; + bool do_discovery_check = false; unsigned int glover; char *error=NULL; SQLite3_result *resultset=NULL; - const char *query = NULL; + // add support for SSL + char *query=(char *)SELECT_SERVERS_FOR_READ_ONLY; t1=monotonic_time(); if (!GloMTH) return NULL; // quick exit during shutdown/restart @@ -3720,52 +3713,36 @@ void * MySQL_Monitor::monitor_read_only() { next_loop_at=0; } - bool bgd_active = MyHGM->is_aws_rds_bgd_in_progress(); - if (bgd_active) { - if (t1 < next_loop_at && t1 >= next_bgd_loop_at) { - rds_bgd_only_loop = true; - } - next_bgd_loop_at = t1 + READ_ONLY_BGD_LOOP_INTERVAL_US; - } else { - // BGD is not active && regular read_only interval time has not elapsed - if (t1 < next_loop_at) { - goto __sleep_monitor_read_only; - } - } - - if (rds_bgd_only_loop) { - query = SELECT_RDS_BGD_SERVERS_FOR_READ_ONLY; - } else { - query = SELECT_SERVERS_FOR_READ_ONLY; - next_loop_at = t1 + 1000ULL * (unsigned int) mysql_thread___monitor_read_only_interval; + if (t1 < next_loop_at) { + goto __sleep_monitor_read_only; } - + next_loop_at=t1+1000*mysql_thread___monitor_read_only_interval; proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); - resultset = MyHGM->execute_query((char*) query, &error); + resultset = MyHGM->execute_query(query, &error); assert(resultset); if (error) { proxy_error("Error on %s : %s\n", query, error); goto __end_monitor_read_only_loop; } - + if (resultset->rows_count == 0) { goto __end_monitor_read_only_loop; } - if (!rds_bgd_only_loop && rds_topology_check_interval > 0) { - if (rds_topology_check_counter >= rds_topology_check_interval) { - rds_topology_check = true; - rds_topology_check_counter = 0; - } - rds_topology_check_counter += 1; + if (topology_loop_max > 0) { // if the discovery interval is set to zero, do not query for the topology + if (topology_loop >= topology_loop_max) { + do_discovery_check = true; + topology_loop = 0; + } + topology_loop += 1; } // resultset must be initialized before calling monitor_read_only_async - monitor_read_only_async(resultset, rds_topology_check); + monitor_read_only_async(resultset, do_discovery_check); if (shutdown) return NULL; __end_monitor_read_only_loop: - if (!rds_bgd_only_loop && mysql_thread___monitor_enabled) { + if (mysql_thread___monitor_enabled==true) { char *query=NULL; query=(char *)"DELETE FROM mysql_server_read_only_log WHERE time_start_us < ?1"; auto [rc1, statement_unique] = monitordb->prepare_v2(query); @@ -3787,21 +3764,14 @@ void * MySQL_Monitor::monitor_read_only() { delete resultset; __sleep_monitor_read_only: - t2 = monotonic_time(); - unsigned long long st = 0; - if (bgd_active) { - if (t2 < next_bgd_loop_at) { - st = next_bgd_loop_at - t2; - usleep(st); - } - } else { - if (t2 < next_loop_at) { - st = next_loop_at - t2; - if (st > READ_ONLY_NEXT_LOOP_INTERVAL_US) { - st = READ_ONLY_NEXT_LOOP_INTERVAL_US; - } - usleep(st); + t2=monotonic_time(); + if (t2 500000) { + st = 500000; } + usleep(st); } } @@ -6698,6 +6668,36 @@ static int aws_rds_bgd_async_query(MySQL_Monitor_State_Data *mmsd, const char *q return 0; } +/** +* @brief Flag the servers as switchover-in-progress so the read_only monitor leaves them alone. +* +* @details During a switchover AWS makes the blue writer read-only; without this, read_only_action_v2 would +* demote/relocate it and fight the BGD FSM. Set once when status reaches INITIATED+ and held through +* SWITCHOVER_COMPLETED; cleared at NONE by aws_rds_bgd_clear_bgd_in_progress +*/ +static void aws_rds_bgd_set_bgd_in_progress(AWS_RDS_BGD_State& st) { + if (st.bgd_in_progress_set) { + return; + } + + MyHGM->set_aws_rds_bgd_in_progress(st.writer_hg, st.reader_hg, true); + st.bgd_in_progress_set = true; + proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: switchover in progress, suspending read_only monitor actions on these hostgroups until SWITCHOVER_COMPLETED\n", + st.writer_hg, st.reader_hg); +} + +/** +* @brief Re-enable read_only monitor action on the deployment's servers (undo aws_rds_bgd_set_bgd_in_progress). +*/ +static void aws_rds_bgd_clear_bgd_in_progress(AWS_RDS_BGD_State& st) { + if (!st.bgd_in_progress_set) { + return; + } + + MyHGM->set_aws_rds_bgd_in_progress(st.writer_hg, st.reader_hg, false); + st.bgd_in_progress_set = false; +} + /** * @brief Set the deployment's switchover status and persist it in runtime table. */ @@ -6711,7 +6711,11 @@ static void aws_rds_bgd_set_status(AWS_RDS_BGD_State& st, AWS_RDS_BGD_Status sta aws_rds_bgd_status_str(st.bgd_status), aws_rds_bgd_status_str(status)); st.bgd_status = status; - MyHGM->aws_rds_bgd_set_switchover_status(st.writer_hg, static_cast(status)); + MyHGM->aws_rds_bgd_set_runtime_status(st.writer_hg, static_cast(status)); + + if (status == AWS_RDS_BGD_Status::NONE) { + aws_rds_bgd_clear_bgd_in_progress(st); + } } void * monitor_RDS_BGD_thread_HG(void *arg) { @@ -6730,7 +6734,7 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { AWS_RDS_BGD_State st; st.writer_hg = wHG; - MyHGM->aws_rds_bgd_set_switchover_status(wHG, static_cast(AWS_RDS_BGD_Status::NONE)); + MyHGM->aws_rds_bgd_set_runtime_status(wHG, static_cast(AWS_RDS_BGD_Status::NONE)); unsigned int MySQL_Monitor__thread_MySQL_Thread_Variables_version; MySQL_Thread * mysql_thr = new MySQL_Thread(); @@ -7015,6 +7019,8 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { } __exit_monitor_RDS_BGD_thread_HG_now: + aws_rds_bgd_clear_bgd_in_progress(st); + if (mmsd) { delete mmsd; mmsd = NULL; @@ -7272,6 +7278,8 @@ const char* aws_rds_bgd_status_str(AWS_RDS_BGD_Status s) { return "WRITER_SWITCHOVER_COMPLETED"; case AWS_RDS_BGD_Status::READER_SWITCHOVER_IN_PROGRESS: return "READER_SWITCHOVER_IN_PROGRESS"; + case AWS_RDS_BGD_Status::SWITCHOVER_COMPLETED: + return "SWITCHOVER_COMPLETED"; } return "UNKNOWN"; } @@ -7279,13 +7287,15 @@ const char* aws_rds_bgd_status_str(AWS_RDS_BGD_Status s) { /** * @brief Run the status-driven blue/green switchover FSM for one deployment. * -* @details Dispatches on the deployment switchover status read from the TARGET (green) row -* of mysql.rds_topology (the TARGET row carries the status in every phase, including -* COMPLETED where the SOURCE row is absent). State carried across poll cycles (the -* blue<->green map, resolved green IPs, the pinned probe host, enforcement bookkeeping, -* and the next poll interval) lives in 'st', owned by the calling worker thread. +* @details Invoked each poll cycle by the BGD worker after it fetches the +* mysql.rds_topology result. Dispatches on the deployment's switchover status +* (AVAILABLE -> SWITCHOVER_INITIATED -> IN_PROGRESS -> IN_POST_PROCESSING -> +* COMPLETED): builds the blue<->green map, pre-resolves green IPs, repoints the +* blue hostnames onto the green IPs in the DNS cache, drains blue free +* connections, and shuns/enforces reader handling. State carried across cycles +* lives in @p st. * -* @param st Per-deployment switchover state (worker-owned, mutated here). +* @param st BGD switchover state (worker-owned, mutated here). * @param topology Parsed mysql.rds_topology result for this cycle. */ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_Result& topology) { @@ -7341,6 +7351,18 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo aws_rds_bgd_build_map(st, topology); aws_rds_bgd_resolve_green_ips(st); aws_rds_bgd_add_green_writer_in_hg(st); + aws_rds_bgd_set_bgd_in_progress(st); + + if (st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_IN_PROGRESS) { + // Demote the blue writer (RO=1) + for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { + if (p.is_writer) { + auto srv = read_only_server_t{ p.blue_host, (port_t)p.port, 1 }; + MyHGM->read_only_action_v2(std::list{srv}, true); + break; + } + } + } } else if (st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_POST_PROCESSING) { st.next_check_interval_ms = 100; @@ -7351,6 +7373,15 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo aws_rds_bgd_build_map(st, topology); aws_rds_bgd_resolve_green_ips(st); aws_rds_bgd_add_green_writer_in_hg(st); + aws_rds_bgd_set_bgd_in_progress(st); + + srv_addr_t writer; + for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { + if (p.is_writer) { + writer = srv_addr_t{ p.blue_host, p.port }; + break; + } + } // Repoint each mapped blue host onto its green IP and drain existing // connections so new backend work resolves to green. @@ -7371,7 +7402,8 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo } // Blue readers without a green counterpart must stop serving reads. - std::vector> blue_readers; + + std::vector blue_readers; MyHGM->wrlock(); MyHGC* rhgc = MyHGM->MyHGC_lookup(st.reader_hg); if (rhgc && rhgc->mysrvs) { @@ -7380,15 +7412,19 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { continue; } - blue_readers.push_back({ std::string(s->address), s->port }); + if (writer.host == s->address && writer.port == s->port) { + continue; + } + blue_readers.push_back(srv_addr_t{ std::string(s->address), s->port }); } } MyHGM->wrunlock(); - std::vector> unmapped_readers; - for (const std::pair& br : blue_readers) { + + std::vector unmapped_readers; + for (const srv_addr_t& br : blue_readers) { bool mapped = false; for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { - if (p.blue_host == br.first && p.port == br.second) { + if (p.blue_host == br.host && p.port == br.port) { mapped = true; break; } @@ -7398,27 +7434,15 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo } } - // If shunning the unmapped readers would leave the reader HG with no serving readers, - // add the writer into the reader HG so reads keep flowing during the switchover. - // This a no-ops when read_only monitor has added the writer to reader HG already. - std::unique_ptr writer_info; - std::unique_ptr writer_opts; + bool writer_is_also_reader = (st.writer_is_also_reader != 0); if (!unmapped_readers.empty() && unmapped_readers.size() == blue_readers.size()) { - for (AWS_RDS_BlueGreenPair& p : st.bg_map) { - if (p.is_writer) { - writer_info = std::make_unique(srv_info_t{ p.blue_host, (uint16_t)p.port, "AWS RDS BGD writer as reader" }); - writer_opts = std::make_unique(srv_opts_t{ p.blue_weight, p.blue_max_conns, p.blue_use_ssl }); - proxy_info( - "AWS RDS BGD [wHG=%u rHG=%u]: reader HG would be emptied by shun; adding writer '%s:%d' to reader HG\n", - st.writer_hg, st.reader_hg, p.blue_host.c_str(), p.port); - break; - } - } + // All blue readers would be transitioned to SHUNNED_AWS_BGD, leaving the reader HG empty. + // Temporarily enforce writer_is_also_reader until the reader switchover completes. + writer_is_also_reader = true; } - // Shun the unmapped blue readers. - aws_rds_bgd_reconfigure_reader_hg( - st.bgd_status, st.reader_hg, unmapped_readers, writer_info.get(), writer_opts.get()); + aws_rds_bgd_hostgroup_action(st.bgd_status, writer, writer_is_also_reader, st.reader_hg, unmapped_readers); + st.shunned_readers.insert(st.shunned_readers.end(), unmapped_readers.begin(), unmapped_readers.end()); } else if (st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_COMPLETED) { @@ -7440,47 +7464,49 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo st.next_check_interval_ms = 0; } else { - // NONE / unrecognized status: take no action, stay at baseline interval + // unrecognized status: take no action, stay at baseline interval st.next_check_interval_ms = 0; } } /** -* @brief Apply one switchover step's reader-HG mutations, with the action derived from bgd_status. +* @brief Apply BGD hostgroup changes for the current switchover status. +* +* @details POST_PROCESSING configures the writer placement and shuns unmapped readers. +* SWITCHOVER_COMPLETED unshuns readers and removes the writer from reader HG when +* writer_is_also_reader is false. Runtime mysql_servers and checksum are re-generated +* when server hostgroup membership changes. * -* @details POST_PROCESSING shuns the unmapped readers and, when writer_info is set, adds the writer as -* a reader (using writer_opts). READER_SWITCHOVER_IN_PROGRESS unshuns the readers and, when writer_info -* is set, removes the writer. +* @param bgd_status Current BGD FSM status driving the action. +* @param writer Writer server to configure. +* @param writer_is_also_reader Whether the writer should also remain in reader_hg. +* @param reader_hg Reader hostgroup for reader shun/unshun and optional writer membership. +* @param readers Reader servers to shun or unshun. */ -void MySQL_Monitor::aws_rds_bgd_reconfigure_reader_hg( - AWS_RDS_BGD_Status bgd_status, unsigned int reader_hg, - std::vector>& unmapped_readers, - srv_info_t* writer_info, srv_opts_t* writer_opts) +void MySQL_Monitor::aws_rds_bgd_hostgroup_action( + AWS_RDS_BGD_Status bgd_status, + srv_addr_t& writer, bool writer_is_also_reader, + unsigned int reader_hg, std::vector& readers) { - bool post_proc = (bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_POST_PROCESSING); - // WRITER_SWITCHOVER_POST_PROCESSING => shun; - // READER_SWITCHOVER_IN_PROGRESS => unshun - bool shun = post_proc; bool changed = false; + bool shun_readers = false; MyHGM->wrlock(); - if (writer_info) { - if (post_proc && writer_info && writer_opts) { - if (MyHGM->create_new_server_in_hg(reader_hg, *writer_info, *writer_opts) == 0) { - changed = true; - } - } else { - if (MyHGM->remove_server_in_hg(reader_hg, writer_info->addr, writer_info->port) == 0) { - changed = true; - } + if (bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_POST_PROCESSING) { + changed |= MyHGM->aws_rds_bgd_configure_writer(writer.host.c_str(), writer.port, writer_is_also_reader); + shun_readers = true; + } else if (bgd_status == AWS_RDS_BGD_Status::SWITCHOVER_COMPLETED) { + if (!writer_is_also_reader) { + changed |= (MyHGM->remove_server_in_hg(reader_hg, writer.host, writer.port) == 0); } + } else { + MyHGM->wrunlock(); + return; } - for (std::pair& s : unmapped_readers) { - if (MyHGM->aws_rds_bgd_set_shun_server(reader_hg, s.first.c_str(), s.second, shun)) { - changed = true; - } + for (srv_addr_t& s : readers) { + MyHGM->aws_rds_bgd_set_shun_server(reader_hg, s.host.c_str(), s.port, shun_readers); } if (changed) { @@ -7491,43 +7517,39 @@ void MySQL_Monitor::aws_rds_bgd_reconfigure_reader_hg( } /** -* @brief Deferred switchover teardown: run once mysql.rds_topology has drained after COMPLETED. +* @brief Run deferred switchover teardown after mysql.rds_topology drains. * -* @details Blue-reader DNS has propagated to the promoted instances by the time the topology -* table goes empty, so this: returns the writer to a writer-only role, restores the shunned -* blue readers to ONLINE immediately (no recovery delay), drops the blue->green DNS pins, and -* clears the per-worker switchover state so a future switchover starts from a clean FSM. +* @details Restores post-switchover reader handling, unshuns readers, drops DNS pins, +* drains green hostgroups, and clears BGD switchover state. +* +* @param st BGD switchover state. */ void MySQL_Monitor::handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st) { if (st.bgd_status != AWS_RDS_BGD_Status::READER_SWITCHOVER_IN_PROGRESS) { return; } - proxy_info( - "AWS RDS BGD [wHG=%u rHG=%u]: switchover status '%s' -> '%s'; running post-switchover cleanup\n", - st.writer_hg, st.reader_hg, aws_rds_bgd_status_str(st.bgd_status), "SWITCHOVER_COMPLETED" - ); + aws_rds_bgd_set_status(st, AWS_RDS_BGD_Status::SWITCHOVER_COMPLETED); + proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: running post-switchover cleanup\n", st.writer_hg, st.reader_hg); - // Restore the writer's original role in the reader HG based on writer_is_also_reader config and + // Restore the writer's original reader role based on writer_is_also_reader config and // unshun the previously shunned blue readers. - std::unique_ptr writer_info; - if (st.writer_is_also_reader == 0) { - for (AWS_RDS_BlueGreenPair& p : st.bg_map) { - if (p.is_writer) { - writer_info = std::make_unique(srv_info_t{ p.blue_host, (uint16_t)p.port, "AWS RDS BGD writer" }); - break; - } + srv_addr_t writer; + for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { + if (p.is_writer) { + writer = srv_addr_t{ p.blue_host, p.port }; + break; } } - - aws_rds_bgd_reconfigure_reader_hg(st.bgd_status, st.reader_hg, st.shunned_readers, writer_info.get(), NULL); + bool writer_is_also_reader = (st.writer_is_also_reader != 0); + aws_rds_bgd_hostgroup_action(st.bgd_status, writer, writer_is_also_reader, st.reader_hg, st.shunned_readers); // Drop DNS cache + purge connections for the previously shunned readers // so their blue names re-resolve to the promoted (green) instances. if (!st.shunned_readers.empty()) { - for (const std::pair& br : st.shunned_readers) { - dns_cache->remove(br.first); - My_Conn_Pool->purge_connections(br.first.c_str(), br.second); + for (const srv_addr_t& br : st.shunned_readers) { + dns_cache->remove(br.host); + My_Conn_Pool->purge_connections(br.host.c_str(), br.port); } st.shunned_readers.clear(); } @@ -7583,7 +7605,7 @@ void MySQL_Monitor::aws_rds_bgd_drain_green_hg(AWS_RDS_BGD_State& st) { } for (unsigned int hg : green_hgs) { - std::vector> servers; + std::vector servers; MyHGM->wrlock(); MyHGC* hgc = MyHGM->MyHGC_lookup(hg); @@ -7593,14 +7615,14 @@ void MySQL_Monitor::aws_rds_bgd_drain_green_hg(AWS_RDS_BGD_State& st) { if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { continue; } - servers.push_back({ std::string(s->address), s->port }); + servers.push_back(srv_addr_t{ std::string(s->address), s->port }); } } MyHGM->wrunlock(); - for (std::pair& srv : servers) { - std::string& host = srv.first; - int port = srv.second; + for (srv_addr_t& srv : servers) { + std::string& host = srv.host; + int port = srv.port; dns_cache->remove(host); MyHGM->drain_server_connections(host.c_str(), port); My_Conn_Pool->purge_connections(host.c_str(), port); @@ -7612,11 +7634,12 @@ void MySQL_Monitor::aws_rds_bgd_drain_green_hg(AWS_RDS_BGD_State& st) { } /** -* @brief Invoked by the BGD worker whenever mysql.rds_topology is absent/empty/vanished. +* @brief Handle an absent, empty, or vanished mysql.rds_topology table. +* +* @details Routes to deferred cleanup when bgd_status is READER_SWITCHOVER_IN_PROGRESS; +* otherwise clears any in-progress switchover state for this deployment. * -* @details When a post-switchover teardown is pending, an empty topology table means blue-reader -* DNS has propagated, so run the deferred cleanup. Otherwise keep the pre-existing behavior of -* releasing the read_only fast-poll engagement for this deployment. +* @param st BGD switchover state. */ void MySQL_Monitor::aws_rds_bgd_handle_topology_absent(AWS_RDS_BGD_State& st) { if (st.bgd_status == AWS_RDS_BGD_Status::READER_SWITCHOVER_IN_PROGRESS) { From 22ea5c49ea9abd8e30c78e6d67263b108095d45b Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Fri, 10 Jul 2026 06:59:36 +0000 Subject: [PATCH 28/81] fix: Move AWS RDS BGD switchover-in-progress flag from HGM to Monitor - Store per-server switchover flag in Monitor-owned map guarded by `aws_rds_bgd_mutex` - Skip flagged servers in `monitor_read_only_async` before dispatching read_only checks Signed-off-by: Wazir Ahmed --- include/Base_HostGroups_Manager.h | 2 +- include/MySQL_HostGroups_Manager.h | 29 +---------- include/MySQL_Monitor.hpp | 33 ++++++++++-- lib/MySQL_HostGroups_Manager.cpp | 40 +------------- lib/MySQL_Monitor.cpp | 83 ++++++++++++++++++++++++++---- 5 files changed, 108 insertions(+), 79 deletions(-) diff --git a/include/Base_HostGroups_Manager.h b/include/Base_HostGroups_Manager.h index 35d2a2e630..1d539dad9d 100644 --- a/include/Base_HostGroups_Manager.h +++ b/include/Base_HostGroups_Manager.h @@ -570,11 +570,11 @@ class Base_HostGroups_Manager { PtrArray *MyHostGroups; std::unordered_mapMyHostGroups_map; - HGC * MyHGC_find(unsigned int); HGC * MyHGC_create(unsigned int); public: Base_HostGroups_Manager(); + HGC * MyHGC_find(unsigned int); HGC * MyHGC_lookup(unsigned int); SQLite3_result * execute_query(char *query, char **error); diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 3119803cc3..83d2b42730 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -605,21 +605,6 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { return readonly_flag; } - inline - void set_aws_rds_bgd_in_progress() { - aws_rds_bgd_in_progress = true; - } - - inline - bool is_aws_rds_bgd_in_progress() { - return aws_rds_bgd_in_progress; - } - - inline - void clear_aws_rds_bgd_in_progress() { - aws_rds_bgd_in_progress = false; - } - private: unsigned int get_hostgroup_id(Type type, const Node& node) const; MySrvC* insert_HGM(unsigned int hostgroup_id, const MySrvC* srv); @@ -628,7 +613,6 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { std::array, TYPE_SIZE_> mapping; // index 0 contains reader and 1 contains writer hostgroups int readonly_flag; MySQL_HostGroups_Manager* myHGM; - bool aws_rds_bgd_in_progress = false; }; /** @@ -1068,14 +1052,11 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { * @details New implementation of the read_only_action that does not depend on the admin table. * Checks each server in the provided list and adjusts writer/reader hostgroup placement * according to the corresponding read_only value. If any change occurs, the runtime - * mysql_servers table and checksum are regenerated. When `force` is false, - * servers flagged as AWS RDS BGD switchover-in-progress are skipped; when true, the supplied - * state is applied even for those servers. + * mysql_servers table and checksum are regenerated. * * @param mysql_servers Servers and their observed/read-only state. - * @param force Force state changes regardless of AWS RDS BGD switchover state. */ - void read_only_action_v2(const std::list& mysql_servers, bool force = false); + void read_only_action_v2(const std::list& mysql_servers); unsigned int get_servers_table_version(); void wait_servers_table_version(unsigned, unsigned); bool shun_and_killall(char *hostname, int port); @@ -1141,12 +1122,6 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { * @return true if a matching server was found. */ bool drain_server_connections(const char *hostname, int port); - /** - * @brief Flag/unflag every server in the writer and reader hostgroups of an AWS RDS blue/green - * deployment as "switchover in progress", so the read_only monitor (read_only_action_v2) takes - * no action on them while the BGD FSM is driving the switchover. - */ - void set_aws_rds_bgd_in_progress(unsigned int writer_hg, unsigned int reader_hg, bool in_progress); unsigned long long Get_Memory_Stats(); diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index bb5befc669..a5bb0213b3 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -474,6 +474,11 @@ enum class AWS_RDS_BGD_Status { SWITCHOVER_COMPLETED = 7, ///< short-lived status used for final cleanup before returning to NONE }; +enum class AWS_RDS_BGD_Server_Status { + NONE = 0, + IN_PROGRESS = 1 +}; + // Maps a switchover status enum to its stored/display string. const char* aws_rds_bgd_status_str(AWS_RDS_BGD_Status s); @@ -500,7 +505,7 @@ struct AWS_RDS_BGD_State { AWS_RDS_BGD_Status bgd_status = AWS_RDS_BGD_Status::NONE; ///< drives the FSM and the deferred cleanup bool green_writer_added_in_hg = false; ///< green writer added to green_writer_hg - bool bgd_in_progress_set = false; ///< servers flagged as switchover-in-progress + bool bgd_in_progress_set = false; ///< deployment's servers flagged in aws_rds_bgd_server_status unsigned int next_check_interval_ms = 0; ///< FSM-controlled interval; 0 => baseline std::string next_check_host; ///< FSM-pinned probe host; when set (the green IP), the worker @@ -525,8 +530,8 @@ inline const char* const BGD_STATUS_POST_PROC = "SWITCHOVER_IN_POST_PROCESSIN inline const char* const BGD_STATUS_COMPLETED = "SWITCHOVER_COMPLETED"; // read_only monitor server-enumeration query. -// Every server that belongs to a replication hostgroup and status NOT IN (OFFLINE_SOFT, OFFLINE_HARD, SHUNNED_AWS_BGD) -#define SELECT_SERVERS_FOR_READ_ONLY "SELECT hostname, port, MAX(use_ssl) use_ssl, check_type, reader_hostgroup FROM mysql_servers JOIN mysql_replication_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE status NOT IN (2,3,5) GROUP BY hostname, port ORDER BY RANDOM()" +// Every server that belongs to a replication hostgroup and status NOT IN (OFFLINE_SOFT, OFFLINE_HARD) +#define SELECT_SERVERS_FOR_READ_ONLY "SELECT hostname, port, MAX(use_ssl) use_ssl, check_type, reader_hostgroup FROM mysql_servers JOIN mysql_replication_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE status NOT IN (2,3) GROUP BY hostname, port ORDER BY RANDOM()" // Defined in MySQL_HostGroups_Manager.h; forward-declared here because the include cycle // (Monitor.hpp -> HGM.h -> cpp.h -> Monitor.hpp) can leave them undefined at this point. Only @@ -593,6 +598,7 @@ class MySQL_Monitor { std::map AWS_Aurora_Hosts_Map; SQLite3_result *AWS_Aurora_Hosts_resultset; uint64_t AWS_Aurora_Hosts_resultset_checksum; + std::unordered_map aws_rds_bgd_server_status; SQLite3_result *AWS_RDS_BGD_Hosts_resultset; uint64_t AWS_RDS_BGD_Hosts_resultset_checksum; unsigned int num_threads; @@ -714,6 +720,27 @@ class MySQL_Monitor { AWS_RDS_BGD_Status bgd_status, srv_addr_t& writer, bool writer_is_also_reader, unsigned int reader_hg, std::vector& readers); + /** + * @brief Check whether a server is flagged as BGD switchover-in-progress. + * + * @param hostname Server hostname. + * @param port Server port. + * + * @return true if the server is flagged IN_PROGRESS. + */ + bool is_aws_rds_bgd_server_in_progress(const std::string& hostname, int port); + /** + * @brief Flag/unflag every server in BGD hostgroups as switchover-in-progress. + * + * @details Called by the BGD worker at switchover initiation (INITIATED / IN_PROGRESS / + * POST_PROCESSING) and cleared after SWITCHOVER_COMPLETED. Iterates the writer and reader + * hostgroups and marks all member servers in the shared aws_rds_bgd_server_status map. + * + * @param writer_hg Writer hostgroup for the deployment. + * @param reader_hg Reader hostgroup for the deployment. + * @param in_progress true to flag servers, false to clear. + */ + void set_aws_rds_bgd_server_in_progress(unsigned int writer_hg, unsigned int reader_hg, bool in_progress); void * monitor_replication_lag(); void * monitor_dns_cache(); diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 7940526ae7..e4a9cd1f3b 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -3578,14 +3578,11 @@ SQLite3_result * MySQL_HostGroups_Manager::SQL3_Connection_Pool(bool _reset, int * @details New implementation of the read_only_action that does not depend on the admin table. * Checks each server in the provided list and adjusts writer/reader hostgroup placement * according to the corresponding read_only value. If any change occurs, the runtime - * mysql_servers table and checksum are regenerated. When `force` is false, - * servers flagged as AWS RDS BGD switchover-in-progress are skipped; when true, the supplied - * state is applied even for those servers. + * mysql_servers table and checksum are regenerated. * * @param mysql_servers Servers and their observed/read-only state. - * @param force Force state changes regardless of AWS RDS BGD switchover state. */ -void MySQL_HostGroups_Manager::read_only_action_v2(const std::list& mysql_servers, bool force) { +void MySQL_HostGroups_Manager::read_only_action_v2(const std::list& mysql_servers) { bool update_mysql_servers_table = false; @@ -3611,13 +3608,6 @@ void MySQL_HostGroups_Manager::read_only_action_v2(const std::listis_aws_rds_bgd_in_progress() && !force) { - proxy_debug(PROXY_DEBUG_MONITOR, 5, - "Skipping read_only_action_v2() for server '%s:%d' because AWS RDS BGD switchover is in progress\n", - hostname.c_str(), port); - continue; - } - const std::vector& writer_map = host_server_mapping->get(HostGroup_Server_Mapping::Type::WRITER); is_writer = !writer_map.empty(); @@ -3973,32 +3963,6 @@ void MySQL_HostGroups_Manager::aws_rds_bgd_set_runtime_status(unsigned int write wrunlock(); } -void MySQL_HostGroups_Manager::set_aws_rds_bgd_in_progress(unsigned int writer_hg, unsigned int reader_hg, bool in_progress) { - wrlock(); - - unsigned int hgs[2] = { writer_hg, reader_hg }; - for (unsigned int i = 0; i < 2; i++) { - MyHGC* myhgc = MyHGC_find(hgs[i]); - if (myhgc == nullptr || myhgc->mysrvs == nullptr) { - continue; - } - for (unsigned int j = 0; j < myhgc->mysrvs->cnt(); j++) { - MySrvC* s = myhgc->mysrvs->idx(j); - const std::string srv_id = std::string(s->address) + ":::" + std::to_string(s->port); - auto itr = hostgroup_server_mapping.find(srv_id); - if (itr != hostgroup_server_mapping.end() && itr->second) { - if (in_progress) { - itr->second->set_aws_rds_bgd_in_progress(); - } else { - itr->second->clear_aws_rds_bgd_in_progress(); - } - } - } - } - - wrunlock(); -} - /** * @brief Aligns the runtime 'mysql_servers' table + checksums with the server state in MyHGM. * diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 875e7f4566..bdb75442f7 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6669,32 +6669,28 @@ static int aws_rds_bgd_async_query(MySQL_Monitor_State_Data *mmsd, const char *q } /** -* @brief Flag the servers as switchover-in-progress so the read_only monitor leaves them alone. -* -* @details During a switchover AWS makes the blue writer read-only; without this, read_only_action_v2 would -* demote/relocate it and fight the BGD FSM. Set once when status reaches INITIATED+ and held through -* SWITCHOVER_COMPLETED; cleared at NONE by aws_rds_bgd_clear_bgd_in_progress +* @brief Flag servers as switchover-in-progress so the read_only monitor skips them. */ static void aws_rds_bgd_set_bgd_in_progress(AWS_RDS_BGD_State& st) { if (st.bgd_in_progress_set) { return; } - MyHGM->set_aws_rds_bgd_in_progress(st.writer_hg, st.reader_hg, true); + GloMyMon->set_aws_rds_bgd_server_in_progress(st.writer_hg, st.reader_hg, true); st.bgd_in_progress_set = true; - proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: switchover in progress, suspending read_only monitor actions on these hostgroups until SWITCHOVER_COMPLETED\n", + proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: switchover in progress, suspending read_only monitor checks on writer/reader hostgroups until SWITCHOVER_COMPLETED\n", st.writer_hg, st.reader_hg); } /** -* @brief Re-enable read_only monitor action on the deployment's servers (undo aws_rds_bgd_set_bgd_in_progress). +* @brief Clear the switchover-in-progress flag from the aws_rds_bgd_server_status map */ static void aws_rds_bgd_clear_bgd_in_progress(AWS_RDS_BGD_State& st) { if (!st.bgd_in_progress_set) { return; } - MyHGM->set_aws_rds_bgd_in_progress(st.writer_hg, st.reader_hg, false); + GloMyMon->set_aws_rds_bgd_server_in_progress(st.writer_hg, st.reader_hg, false); st.bgd_in_progress_set = false; } @@ -7358,7 +7354,7 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { if (p.is_writer) { auto srv = read_only_server_t{ p.blue_host, (port_t)p.port, 1 }; - MyHGM->read_only_action_v2(std::list{srv}, true); + MyHGM->read_only_action_v2(std::list{srv}); break; } } @@ -7649,6 +7645,65 @@ void MySQL_Monitor::aws_rds_bgd_handle_topology_absent(AWS_RDS_BGD_State& st) { } } + +/** +* @brief Check whether a server is flagged as BGD switchover-in-progress. +* +* @param hostname Server hostname. +* @param port Server port. +* +* @return true if the server is flagged IN_PROGRESS. +*/ +bool MySQL_Monitor::is_aws_rds_bgd_server_in_progress(const std::string& hostname, int port) { + std::string key = hostname + ":::" + std::to_string(port); + pthread_mutex_lock(&aws_rds_bgd_mutex); + auto it = aws_rds_bgd_server_status.find(key); + bool r = (it != aws_rds_bgd_server_status.end() + && it->second == AWS_RDS_BGD_Server_Status::IN_PROGRESS); + pthread_mutex_unlock(&aws_rds_bgd_mutex); + return r; +} + +/** +* @brief Flag/unflag every server in BGD hostgroups as switchover-in-progress. +* +* @details Called by the BGD worker at switchover initiation (INITIATED / IN_PROGRESS / +* POST_PROCESSING) and cleared after SWITCHOVER_COMPLETED. Iterates the writer and reader +* hostgroups and marks all member servers in the shared aws_rds_bgd_server_status map. +* +* @param writer_hg Writer hostgroup for the deployment. +* @param reader_hg Reader hostgroup for the deployment. +* @param in_progress true to flag servers, false to clear. +*/ +void MySQL_Monitor::set_aws_rds_bgd_server_in_progress(unsigned int writer_hg, unsigned int reader_hg, bool in_progress) { + std::vector keys; + MyHGM->wrlock(); + unsigned int hgs[2] = { writer_hg, reader_hg }; + for (unsigned int i = 0; i < 2; i++) { + MyHGC* myhgc = MyHGM->MyHGC_find(hgs[i]); + if (myhgc == nullptr || myhgc->mysrvs == nullptr) { + continue; + } + for (unsigned int j = 0; j < myhgc->mysrvs->cnt(); j++) { + MySrvC* s = myhgc->mysrvs->idx(j); + keys.push_back(std::string(s->address) + ":::" + std::to_string(s->port)); + } + } + MyHGM->wrunlock(); + + pthread_mutex_lock(&aws_rds_bgd_mutex); + if (in_progress) { + for (const auto& k : keys) { + aws_rds_bgd_server_status[k] = AWS_RDS_BGD_Server_Status::IN_PROGRESS; + } + } else { + for (const auto& k : keys) { + aws_rds_bgd_server_status.erase(k); + } + } + pthread_mutex_unlock(&aws_rds_bgd_mutex); +} + /** * @brief AWS RDS BGD monitor thread entry point. * @@ -8896,6 +8951,14 @@ void MySQL_Monitor::monitor_read_only_async(SQLite3_result* resultset, bool do_d for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { const SQLite3_row* r = *it; + + if (is_aws_rds_bgd_server_in_progress(r->fields[0], atoi(r->fields[1]))) { + proxy_info( + "Skipping read_only check for '%s:%d' because AWS RDS BGD switchover is in progress\n", + r->fields[0], atoi(r->fields[1])); + continue; + } + bool rc_ping = server_responds_to_ping(r->fields[0], atoi(r->fields[1])); if (rc_ping) { // only if server is responding to pings MySQL_Monitor_State_Data_Task_Type task_type = MON_READ_ONLY; From ded81f857279738de1cda9eea69cff45e0548d1e Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sun, 12 Jul 2026 13:07:49 +0000 Subject: [PATCH 29/81] fix: Abort `async_connect` when server goes offline or unhealthy - `async_connect()` did not check `IsServerOffline()`, so in-flight connections to a server that was shunned, drained, or marked unhealthy after selection would block until connect_timeout_server. Signed-off-by: Wazir Ahmed --- lib/MySQL_Monitor.cpp | 4 +++- lib/mysql_connection.cpp | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index bdb75442f7..511f8e3eac 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6692,6 +6692,8 @@ static void aws_rds_bgd_clear_bgd_in_progress(AWS_RDS_BGD_State& st) { GloMyMon->set_aws_rds_bgd_server_in_progress(st.writer_hg, st.reader_hg, false); st.bgd_in_progress_set = false; + proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: switchover completed, resuming read_only monitor checks on writer/reader hostgroups\n", + st.writer_hg, st.reader_hg); } /** @@ -8953,7 +8955,7 @@ void MySQL_Monitor::monitor_read_only_async(SQLite3_result* resultset, bool do_d const SQLite3_row* r = *it; if (is_aws_rds_bgd_server_in_progress(r->fields[0], atoi(r->fields[1]))) { - proxy_info( + proxy_debug(PROXY_DEBUG_MONITOR, 5, "Skipping read_only check for '%s:%d' because AWS RDS BGD switchover is in progress\n", r->fields[0], atoi(r->fields[1])); continue; diff --git a/lib/mysql_connection.cpp b/lib/mysql_connection.cpp index 20470ec714..bc1e0b6238 100644 --- a/lib/mysql_connection.cpp +++ b/lib/mysql_connection.cpp @@ -2139,6 +2139,15 @@ int MySQL_Connection::async_connect(short event) { creation_time = monotonic_time(); return 0; } + + // Abort if the server went offline or was marked unhealthy while waiting to connect. + // The server status can change (shunned by monitor, AWS BGD switchover, manual OFFLINE) + // or the connection can be marked unhealthy (AWS BGD drain) between server selection and + // connection completion. + if (IsServerOffline()) { + return -1; + } + handler(event); switch (async_state_machine) { case ASYNC_CONNECT_SUCCESSFUL: From b1fd57b245d4b43246b879fd00e64ecdd50a02df Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Mon, 13 Jul 2026 05:47:07 +0000 Subject: [PATCH 30/81] feat: Handle AWS RDS BGD switchover rollbacks - Use `handle_aws_rds_bgd_post_switchover` as a common handler for both normal post-switchover cleanup and rollback. - Add backwards status transition detection in `handle_aws_rds_bgd` to catch switchover cancellations. - Call rollback on topology-absent and worker exit. - Pin topology probes to the green writer IP during switchover. Fall back to blue writer probing after 3 green probe failures. Signed-off-by: Wazir Ahmed --- include/MySQL_Monitor.hpp | 17 +++++- lib/MySQL_Monitor.cpp | 117 ++++++++++++++++++++++++++++++-------- 2 files changed, 106 insertions(+), 28 deletions(-) diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index a5bb0213b3..0b64084bbe 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -510,6 +510,7 @@ struct AWS_RDS_BGD_State { unsigned int next_check_interval_ms = 0; ///< FSM-controlled interval; 0 => baseline std::string next_check_host; ///< FSM-pinned probe host; when set (the green IP), the worker ///< polls it directly instead of selecting among the blue hosts + unsigned int next_check_host_failures = 0; ///< consecutive failures polling next_check_host; clears it after 3 }; /** @@ -672,14 +673,24 @@ class MySQL_Monitor { */ void handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_Result& topology); /** - * @brief Run deferred switchover teardown after mysql.rds_topology drains. + * @brief Run deferred switchover teardown or rollback cleanup. * * @details Restores post-switchover reader handling, unshuns readers, drops DNS pins, * drains green hostgroups, and clears BGD switchover state. * - * @param st BGD switchover state. + * When rollback is false (normal post-switchover), the caller must be in + * READER_SWITCHOVER_IN_PROGRESS; the function advances through + * SWITCHOVER_COMPLETED before clearing to NONE. + * + * When rollback is true (topology table disappeared or worker exit mid-switchover), + * the function accepts any non-NONE bgd_status, restores the blue writer to the + * writer hostgroup if it was demoted, then runs the same cleanup and resets + * switchover state. + * + * @param st BGD switchover state. + * @param rollback True if called due to a rollback/cancellation, false for normal completion. */ - void handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st); + void handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st, bool rollback = false); /** * @brief Evict stale DNS and drain connections for the deployment's green hostgroups after switchover. * diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 511f8e3eac..726f01d386 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6803,6 +6803,7 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true && exit_now==false) { unsigned int glover; t1 = monotonic_time(); + bool poll_success = false; if (!GloMTH) goto __exit_monitor_RDS_BGD_thread_HG_now; @@ -6973,12 +6974,14 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { // the BGD thread only monitors blue/green hostgroups; parse the topology // (shared with the read_only path) and hand the struct to the handler. if (mmsd->result && mysql_num_rows(mmsd->result) > 0) { + poll_success = true; AWS_RDS_Topology_Result topo = GloMyMon->parse_aws_rds_topology(mmsd->result); proxy_debug(PROXY_DEBUG_MONITOR, 5, "AWS RDS BGD [wHG=%u]: topology probe on %s:%d (blue_green=%d, nodes=%zu)\n", wHG, mmsd->hostname, mmsd->port, topo.blue_green ? 1 : 0, topo.nodes.size()); GloMyMon->handle_aws_rds_bgd(st, topo); } else { + poll_success = true; // Query succeeded with no rows: mysql.rds_topology has drained (blue-reader // DNS fully propagated). Run post-switchover cleanup. GloMyMon->aws_rds_bgd_handle_topology_absent(st); @@ -6991,6 +6994,24 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { } __end_of_loop: + if (!st.next_check_host.empty()) { + if (poll_success) { + st.next_check_host_failures = 0; + } else { + st.next_check_host_failures++; + if (st.next_check_host_failures >= 3) { + proxy_warning("AWS RDS BGD [wHG=%u rHG=%u]: green probe host %s unreachable after %u attempts, falling back to blue and clearing DNS pins\n", + wHG, st.reader_hg, st.next_check_host.c_str(), st.next_check_host_failures); + st.next_check_host.clear(); + st.next_check_host_failures = 0; + for (const auto& p : st.bg_map) { + GloMyMon->dns_cache->remove(p.blue_host); + GloMyMon->My_Conn_Pool->purge_connections(p.blue_host.c_str(), p.port); + } + } + } + } + mmsd->t2 = monotonic_time(); // the FSM tightens the interval to 100ms while a switchover is in flight // (st.next_check_interval_ms); otherwise fall back to the configured baseline. @@ -7017,7 +7038,9 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { } __exit_monitor_RDS_BGD_thread_HG_now: - aws_rds_bgd_clear_bgd_in_progress(st); + if (st.bgd_status != AWS_RDS_BGD_Status::NONE) { + GloMyMon->handle_aws_rds_bgd_post_switchover(st, true); + } if (mmsd) { delete mmsd; @@ -7185,8 +7208,8 @@ static void aws_rds_bgd_resolve_green_ips(AWS_RDS_BGD_State& st) { p.green_ip = ip; p.green_ip_ttl = 0; proxy_debug(PROXY_DEBUG_MONITOR, 7, - "AWS RDS BGD [wHG=%u]: green '%s' IP %s (DNS_Cache)\n", - st.writer_hg, p.green_host.c_str(), p.green_ip.c_str()); + "AWS RDS BGD [wHG=%u rHG=%u]: green '%s' IP %s (DNS_Cache)\n", + st.writer_hg, st.reader_hg, p.green_host.c_str(), p.green_ip.c_str()); continue; } // Cache miss (green is not a monitored server): resolve DNS now and track its TTL. @@ -7197,8 +7220,8 @@ static void aws_rds_bgd_resolve_green_ips(AWS_RDS_BGD_State& st) { p.green_ip_ttl = monotonic_time() + (1000ULL * (unsigned long long)mysql_thread___monitor_local_dns_cache_ttl); proxy_debug(PROXY_DEBUG_MONITOR, 7, - "AWS RDS BGD [wHG=%u]: green '%s' IP %s (resolved, ttl=%lus)\n", - st.writer_hg, p.green_host.c_str(), p.green_ip.c_str(), + "AWS RDS BGD [wHG=%u rHG=%u]: green '%s' IP %s (resolved, ttl=%lus)\n", + st.writer_hg, st.reader_hg, p.green_host.c_str(), p.green_ip.c_str(), (unsigned long)mysql_thread___monitor_local_dns_cache_ttl); } } @@ -7209,9 +7232,8 @@ static void aws_rds_bgd_resolve_green_ips(AWS_RDS_BGD_State& st) { if (p.is_writer && !p.green_ip.empty()) { if (st.next_check_host != p.green_ip) { st.next_check_host = p.green_ip; - proxy_debug(PROXY_DEBUG_MONITOR, 5, - "AWS RDS BGD [wHG=%u]: pinning topology probe to green IP %s\n", - st.writer_hg, p.green_ip.c_str()); + proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: pinning rds_topology probe to green IP %s\n", + st.writer_hg, st.reader_hg, p.green_ip.c_str()); } break; } @@ -7328,6 +7350,23 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo return; } + // Detect backwards transition: the topology status moved to an earlier + // phase than what we've already processed. This happens when a user + // cancels the switchover from the AWS side, reverting to AVAILABLE, + // or when AWS aborts the switchover due to an error. Roll back all + // accumulated side effects, then re-enter the target state's setup. + if (topology_status < st.bgd_status) { + handle_aws_rds_bgd_post_switchover(st, true); + if (topology_status == AWS_RDS_BGD_Status::AVAILABLE) { + aws_rds_bgd_set_status(st, topology_status); + st.next_check_interval_ms = 250; + aws_rds_bgd_build_map(st, topology); + aws_rds_bgd_resolve_green_ips(st); + aws_rds_bgd_add_green_writer_in_hg(st); + } + return; + } + if (topology_status == st.bgd_status) { // no phase change return; @@ -7515,20 +7554,52 @@ void MySQL_Monitor::aws_rds_bgd_hostgroup_action( } /** -* @brief Run deferred switchover teardown after mysql.rds_topology drains. +* @brief Run deferred switchover teardown or rollback cleanup. * * @details Restores post-switchover reader handling, unshuns readers, drops DNS pins, * drains green hostgroups, and clears BGD switchover state. * -* @param st BGD switchover state. +* When rollback is false (normal post-switchover), the caller must be in +* READER_SWITCHOVER_IN_PROGRESS; the function advances through +* SWITCHOVER_COMPLETED before clearing to NONE. +* +* When rollback is true (topology disappeared or worker exit mid-switchover), +* the function accepts any non-NONE bgd_status, restores the blue writer to the +* writer hostgroup if it was demoted, then runs the same cleanup and resets +* directly to NONE without the intermediate SWITCHOVER_COMPLETED state. +* +* @param st BGD switchover state. +* @param rollback True if called due to a rollback/cancellation, false for normal completion. */ -void MySQL_Monitor::handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st) { - if (st.bgd_status != AWS_RDS_BGD_Status::READER_SWITCHOVER_IN_PROGRESS) { +void MySQL_Monitor::handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st, bool rollback) { + if (st.bgd_status == AWS_RDS_BGD_Status::NONE) { + return; + } + + if (!rollback && st.bgd_status != AWS_RDS_BGD_Status::READER_SWITCHOVER_IN_PROGRESS) { return; } - aws_rds_bgd_set_status(st, AWS_RDS_BGD_Status::SWITCHOVER_COMPLETED); - proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: running post-switchover cleanup\n", st.writer_hg, st.reader_hg); + if (rollback) { + proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: rolling back from %s\n", + st.writer_hg, st.reader_hg, aws_rds_bgd_status_str(st.bgd_status)); + + // Restore the blue writer to the writer hostgroup. + // If writer exists in writer hostgroup, this is a no-op. + if (st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_IN_PROGRESS + || st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_POST_PROCESSING) { + for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { + if (p.is_writer) { + auto srv = read_only_server_t{ p.blue_host, (port_t)p.port, 0 }; + MyHGM->read_only_action_v2(std::list{srv}); + break; + } + } + } + } else { + aws_rds_bgd_set_status(st, AWS_RDS_BGD_Status::SWITCHOVER_COMPLETED); + proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: running post-switchover cleanup\n", st.writer_hg, st.reader_hg); + } // Restore the writer's original reader role based on writer_is_also_reader config and // unshun the previously shunned blue readers. @@ -7540,7 +7611,7 @@ void MySQL_Monitor::handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st) { } } bool writer_is_also_reader = (st.writer_is_also_reader != 0); - aws_rds_bgd_hostgroup_action(st.bgd_status, writer, writer_is_also_reader, st.reader_hg, st.shunned_readers); + aws_rds_bgd_hostgroup_action(AWS_RDS_BGD_Status::SWITCHOVER_COMPLETED, writer, writer_is_also_reader, st.reader_hg, st.shunned_readers); // Drop DNS cache + purge connections for the previously shunned readers // so their blue names re-resolve to the promoted (green) instances. @@ -7552,13 +7623,8 @@ void MySQL_Monitor::handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st) { st.shunned_readers.clear(); } - // Drop DNS pins for the mapped readers so their blue names resolve natively - // to the promoted (green) instances. The writer pin was already cleared at - // WRITER_SWITCHOVER_COMPLETED, so skip it here. + // Drop DNS pins for all mapped pairs for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { - if (p.is_writer) { - continue; - } dns_cache->remove(p.blue_host); My_Conn_Pool->purge_connections(p.blue_host.c_str(), p.port); } @@ -7574,7 +7640,7 @@ void MySQL_Monitor::handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st) { aws_rds_bgd_set_status(st, AWS_RDS_BGD_Status::NONE); proxy_info( - "AWS RDS BGD [wHG=%u rHG=%u]: post-switchover cleanup complete; state cleared\n", + "AWS RDS BGD [wHG=%u rHG=%u]: switchover cleanup complete; state cleared\n", st.writer_hg, st.reader_hg); } @@ -7635,15 +7701,16 @@ void MySQL_Monitor::aws_rds_bgd_drain_green_hg(AWS_RDS_BGD_State& st) { * @brief Handle an absent, empty, or vanished mysql.rds_topology table. * * @details Routes to deferred cleanup when bgd_status is READER_SWITCHOVER_IN_PROGRESS; -* otherwise clears any in-progress switchover state for this deployment. +* for any other non-NONE state, runs rollback cleanup to reverse accumulated side +* effects before resetting to NONE. * * @param st BGD switchover state. */ void MySQL_Monitor::aws_rds_bgd_handle_topology_absent(AWS_RDS_BGD_State& st) { if (st.bgd_status == AWS_RDS_BGD_Status::READER_SWITCHOVER_IN_PROGRESS) { handle_aws_rds_bgd_post_switchover(st); - } else { - aws_rds_bgd_set_status(st, AWS_RDS_BGD_Status::NONE); + } else if (st.bgd_status != AWS_RDS_BGD_Status::NONE) { + handle_aws_rds_bgd_post_switchover(st, true); } } From b574e3665690f8d0408e96996fa2139ef0509354 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Mon, 13 Jul 2026 08:21:32 +0000 Subject: [PATCH 31/81] fix: Normalize AWS BGD shunned status in cluster sync Signed-off-by: Wazir Ahmed --- include/Base_HostGroups_Manager.h | 3 ++- include/MySQL_HostGroups_Manager.h | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/include/Base_HostGroups_Manager.h b/include/Base_HostGroups_Manager.h index 1d539dad9d..6d5764dd4d 100644 --- a/include/Base_HostGroups_Manager.h +++ b/include/Base_HostGroups_Manager.h @@ -128,7 +128,7 @@ class MetricsCollector; * @brief Generates the 'mysql_servers_v2' resultset exposed to other ProxySQL cluster members. * @details The generated resultset is used for the checksum computation of the runtime ProxySQL config * ('mysql_servers_v2' checksum), and it's also forwarded to other cluster members when querying the Admin - * interface with 'CLUSTER_QUERY_MYSQL_SERVERS_V2'. It makes 'SHUNNED' state equivalent to 'ONLINE', and also + * interface with 'CLUSTER_QUERY_MYSQL_SERVERS_V2'. It makes 'SHUNNED' and 'SHUNNED_AWS_BGD' states equivalent to 'ONLINE', and also * filters out any 'OFFLINE_HARD' entries. This is done because none of the statuses are valid configuration * statuses, they are local, transient status that ProxySQL uses during operation. */ @@ -137,6 +137,7 @@ class MetricsCollector; "hostgroup_id, hostname, port, gtid_port, " \ "CASE" \ " WHEN status=\"SHUNNED\" THEN \"ONLINE\"" \ + " WHEN status=\"SHUNNED_AWS_BGD\" THEN \"ONLINE\"" \ " ELSE status " \ "END AS status, " \ "weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment " \ diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 83d2b42730..c25081447a 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -120,7 +120,7 @@ * @brief Generates the 'mysql_servers_v2' resultset exposed to other ProxySQL cluster members. * @details The generated resultset is used for the checksum computation of the runtime ProxySQL config * ('mysql_servers_v2' checksum), and it's also forwarded to other cluster members when querying the Admin - * interface with 'CLUSTER_QUERY_MYSQL_SERVERS_V2'. It makes 'SHUNNED' state equivalent to 'ONLINE', and also + * interface with 'CLUSTER_QUERY_MYSQL_SERVERS_V2'. It makes 'SHUNNED' and 'SHUNNED_AWS_BGD' states equivalent to 'ONLINE', and also * filters out any 'OFFLINE_HARD' entries. This is done because none of the statuses are valid configuration * statuses, they are local, transient status that ProxySQL uses during operation. */ @@ -129,6 +129,7 @@ "hostgroup_id, hostname, port, gtid_port, " \ "CASE" \ " WHEN status=\"SHUNNED\" THEN \"ONLINE\"" \ + " WHEN status=\"SHUNNED_AWS_BGD\" THEN \"ONLINE\"" \ " ELSE status " \ "END AS status, " \ "weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment " \ From 7d272074cde766fd2818ebc7539d204ae7106ed4 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Wed, 15 Jul 2026 04:32:16 +0000 Subject: [PATCH 32/81] fix: Track BGD green-writer ownership and skip green-HG drain on rollback - Only the auto-added green writer is removed during rollback; manually configured green rows are left untouched. - Green HG draining is now limited to successful switchover completion. Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 2 + include/MySQL_Monitor.hpp | 13 ++- lib/MySQL_HostGroups_Manager.cpp | 5 +- lib/MySQL_Monitor.cpp | 137 ++++++++++++++++++++--------- 4 files changed, 104 insertions(+), 53 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index c25081447a..f64ec7e9fb 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -1121,6 +1121,8 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { * @param hostname Address of the server to match. * @param port Port of the server to match. * @return true if a matching server was found. + * + * @note Caller must hold wrlock(). */ bool drain_server_connections(const char *hostname, int port); diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 0b64084bbe..40136463d5 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -692,18 +692,15 @@ class MySQL_Monitor { */ void handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st, bool rollback = false); /** - * @brief Evict stale DNS and drain connections for the deployment's green hostgroups after switchover. + * @brief Clean up the deployment's green hostgroups after switchover or rollback. * - * @details No-op unless the green writer/reader hostgroups are configured (the explicit green-HG path). - * For every non-OFFLINE_HARD member of each green hostgroup this drops the DNS cache entry, drains the - * server's backend connections and purges the monitor connection pool. - * - * The servers are left in place; the monitor shuns them on ping/connect errors once the retired green - * DNS names stop resolving to an IP. + * @details Successful cleanup drains all configured green-hostgroup members. Rollback drains and removes + * only the green writer auto-added by the BGD worker; user-configured rows are left unchanged. * * @param st Switchover state. + * @param rollback Whether cleanup is handling a rollback. */ - void aws_rds_bgd_drain_green_hg(AWS_RDS_BGD_State& st); + void aws_rds_bgd_drain_green_hg(AWS_RDS_BGD_State& st, bool rollback); /** * @brief Handle an absent, empty, or vanished mysql.rds_topology table. * diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 2b161130fa..c8032da62c 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -3999,12 +3999,12 @@ void MySQL_HostGroups_Manager::publish_mysql_servers_to_runtime() { * @param hostname Address of the server to match. * @param port Port of the server to match. * @return true if a matching server was found. + * + * @note Caller must hold wrlock(). */ bool MySQL_HostGroups_Manager::drain_server_connections(const char *hostname, int port) { bool found = false; - wrlock(); - for (unsigned int i = 0; i < MyHostGroups->len; i++) { MyHGC *myhgc = (MyHGC *)MyHostGroups->index(i); if (!myhgc || !myhgc->mysrvs) { @@ -4025,7 +4025,6 @@ bool MySQL_HostGroups_Manager::drain_server_connections(const char *hostname, in } } - wrunlock(); return found; } diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index b8872795bd..51cb58ac91 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -7244,11 +7244,12 @@ static void aws_rds_bgd_resolve_green_ips(AWS_RDS_BGD_State& st) { * @brief Add the green writer to green_writer_hostgroup, when that hostgroup is configured. * * @details Mirrors the blue writer's connection settings (weight/max_connections/use_ssl) onto -* the green writer. No-op when green_writer_hostgroup is NULL (the auto-discovery path) or when -* the green writer was already added on a prior poll. +* the green writer. Existing rows, including OFFLINE_HARD rows, are left unchanged. When +* green_writer_added_in_hg is true (BGD owns this row), an OFFLINE_HARD tombstone left by a +* prior rollback is re-enabled. */ static void aws_rds_bgd_add_green_writer_in_hg(AWS_RDS_BGD_State& st) { - if (st.green_writer_hg < 0 || st.green_writer_added_in_hg) { + if (st.green_writer_hg < 0) { return; } for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { @@ -7256,9 +7257,23 @@ static void aws_rds_bgd_add_green_writer_in_hg(AWS_RDS_BGD_State& st) { srv_info_t srv_info { p.green_host, (uint16_t)p.port, "AWS RDS BGD green writer" }; srv_opts_t srv_opts { p.blue_weight, p.blue_max_conns, p.blue_use_ssl }; MyHGM->wrlock(); - MyHGM->create_new_server_in_hg((uint32_t)st.green_writer_hg, srv_info, srv_opts); + MySrvC* existing = MyHGM->find_server_in_hg((uint32_t)st.green_writer_hg, p.green_host, p.port); + bool publish = false; + + if (existing == nullptr) { + st.green_writer_added_in_hg = + (MyHGM->create_new_server_in_hg((uint32_t)st.green_writer_hg, srv_info, srv_opts) == 0); + publish = st.green_writer_added_in_hg; + } else if (st.green_writer_added_in_hg + && existing->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + existing->set_status(MYSQL_SERVER_STATUS_ONLINE); + publish = true; + } + + if (publish) { + MyHGM->publish_mysql_servers_to_runtime(); + } MyHGM->wrunlock(); - st.green_writer_added_in_hg = true; break; } } @@ -7434,7 +7449,9 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo "AWS RDS BGD [wHG=%u rHG=%u]: repointed blue '%s' to green IP %s\n", st.writer_hg, st.reader_hg, p.blue_host.c_str(), p.green_ip.c_str()); + MyHGM->wrlock(); MyHGM->drain_server_connections(p.blue_host.c_str(), p.port); + MyHGM->wrunlock(); My_Conn_Pool->purge_connections(p.blue_host.c_str(), p.port); } @@ -7629,14 +7646,13 @@ void MySQL_Monitor::handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st, bo My_Conn_Pool->purge_connections(p.blue_host.c_str(), p.port); } - aws_rds_bgd_drain_green_hg(st); + aws_rds_bgd_drain_green_hg(st, rollback); // state cleanup st.bg_map.clear(); st.last_topology_status.clear(); st.next_check_host.clear(); st.next_check_interval_ms = 0; - st.green_writer_added_in_hg = false; aws_rds_bgd_set_status(st, AWS_RDS_BGD_Status::NONE); proxy_info( @@ -7645,54 +7661,91 @@ void MySQL_Monitor::handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st, bo } /** -* @brief Evict stale DNS and drain connections for the deployment's green hostgroups after switchover. -* -* @details No-op unless the green writer/reader hostgroups are configured (the explicit green-HG path). -* For every non-OFFLINE_HARD member of each green hostgroup this drops the DNS cache entry, drains the -* server's backend connections and purges the monitor connection pool. +* @brief Clean up the deployment's green hostgroups after switchover or rollback. * -* The servers are left in place; the monitor shuns them on ping/connect errors once the retired green -* DNS names stop resolving to an IP. +* @details Successful cleanup drains all configured green-hostgroup members. Rollback drains and removes +* only the green writer auto-added by the BGD worker; user-configured rows are left unchanged. * * @param st Switchover state. +* @param rollback Whether cleanup is handling a rollback. */ -void MySQL_Monitor::aws_rds_bgd_drain_green_hg(AWS_RDS_BGD_State& st) { - std::vector green_hgs; - if (st.green_writer_hg >= 0) { - green_hgs.push_back((unsigned int)st.green_writer_hg); - } - if (st.green_reader_hg >= 0) { - green_hgs.push_back((unsigned int)st.green_reader_hg); - } - if (green_hgs.empty()) { - return; - } +void MySQL_Monitor::aws_rds_bgd_drain_green_hg(AWS_RDS_BGD_State& st, bool rollback) { + struct hg_srv_t { + int hostgroup; + srv_addr_t server; + bool remove; + }; + std::vector targets; + bool servers_removed = false; - for (unsigned int hg : green_hgs) { - std::vector servers; + MyHGM->wrlock(); + + if (rollback) { + if (st.green_writer_added_in_hg && st.green_writer_hg >= 0) { + for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { + if (!p.is_writer) + continue; + + MySrvC* existing = MyHGM->find_server_in_hg( + (uint32_t)st.green_writer_hg, p.green_host, p.port); + if (existing) { + existing->ConnectionsUsed->mark_connections_unhealthy(); + servers_removed = + (MyHGM->remove_server_in_hg( + (uint32_t)st.green_writer_hg, p.green_host, p.port) == 0); + targets.push_back(hg_srv_t{ + st.green_writer_hg, srv_addr_t{ p.green_host, p.port }, true }); + } + break; + } + } + } else { + std::vector green_hgs; + if (st.green_writer_hg >= 0) + green_hgs.push_back(st.green_writer_hg); + if (st.green_reader_hg >= 0) + green_hgs.push_back(st.green_reader_hg); + + for (int hg : green_hgs) { + MyHGC* hgc = MyHGM->MyHGC_find(hg); + if (!hgc || !hgc->mysrvs) + continue; - MyHGM->wrlock(); - MyHGC* hgc = MyHGM->MyHGC_lookup(hg); - if (hgc && hgc->mysrvs) { for (unsigned int j = 0; j < hgc->mysrvs->cnt(); j++) { MySrvC* s = hgc->mysrvs->idx(j); - if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { - continue; + if (s->get_status() != MYSQL_SERVER_STATUS_OFFLINE_HARD) { + targets.push_back(hg_srv_t{ + hg, srv_addr_t{ std::string(s->address), s->port }, false }); } - servers.push_back(srv_addr_t{ std::string(s->address), s->port }); } } - MyHGM->wrunlock(); - for (srv_addr_t& srv : servers) { - std::string& host = srv.host; - int port = srv.port; - dns_cache->remove(host); - MyHGM->drain_server_connections(host.c_str(), port); - My_Conn_Pool->purge_connections(host.c_str(), port); + for (const hg_srv_t& target : targets) { + MyHGM->drain_server_connections( + target.server.host.c_str(), target.server.port); + } + } + + if (servers_removed) + MyHGM->publish_mysql_servers_to_runtime(); + + MyHGM->wrunlock(); + + for (const hg_srv_t& target : targets) { + dns_cache->remove(target.server.host); + My_Conn_Pool->purge_connections( + target.server.host.c_str(), target.server.port); + + if (target.remove) { + proxy_info( + "AWS RDS BGD [wHG=%u rHG=%u]: removed auto-added green writer '%s:%d' from HG %d\n", + st.writer_hg, st.reader_hg, target.server.host.c_str(), target.server.port, + target.hostgroup); + } else { proxy_info( - "AWS RDS BGD [wHG=%u rHG=%u]: connections drained from green HG %u server '%s:%d'\n", - st.writer_hg, st.reader_hg, hg, host.c_str(), port); + "AWS RDS BGD [wHG=%u rHG=%u]: connections drained from green HG %d server '%s:%d'\n", + st.writer_hg, st.reader_hg, target.hostgroup, target.server.host.c_str(), + target.server.port); } } } From 20247dcf0375222cb1b4100dc925adcba6ea8f8f Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Wed, 15 Jul 2026 07:13:48 +0000 Subject: [PATCH 33/81] fix: Use writer port and SSL for AWS BGD green probes - Track the configured green writer SSL setting - Select green probe parameters from the writer pair Signed-off-by: Wazir Ahmed --- include/MySQL_Monitor.hpp | 1 + lib/MySQL_Monitor.cpp | 47 +++++++++++++++++++++++++++++++++------ 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 40136463d5..1eb1576770 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -438,6 +438,7 @@ struct AWS_RDS_BlueGreenPair { int64_t blue_weight = 1; ///< Blue server weight mirrored onto the green server when it is added. int64_t blue_max_conns = 1000; ///< Blue server max_connections mirrored onto the green server when it is added. int32_t blue_use_ssl = 0; ///< Blue server SSL setting mirrored onto the green server when it is added. + int32_t green_use_ssl = -1; ///< Green server SSL; -1 means unset (use blue_use_ssl). std::string green_ip; ///< Green host IP resolved at SWITCHOVER_INITIATED and held warm. unsigned long long green_ip_ttl = 0; ///< Expiry for green_ip when resolved by the BGD thread; 0 means DNS_Cache-sourced. bool is_writer = false; ///< True when this pair maps the blue writer. diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 51cb58ac91..8390782c84 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6847,10 +6847,25 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { int poll_port; bool poll_use_ssl; if (!st.next_check_host.empty()) { - poll_host = st.next_check_host.c_str(); - poll_port = hpa[0].port; // port/use_ssl are uniform across the deployment - poll_use_ssl = hpa[0].use_ssl; - } else { + bool found_writer = false; + for (const auto& p : st.bg_map) { + if (p.is_writer) { + poll_host = st.next_check_host.c_str(); + poll_port = p.port; + poll_use_ssl = (p.green_use_ssl >= 0) ? p.green_use_ssl : p.blue_use_ssl; + found_writer = true; + break; + } + } + if (!found_writer) { + // Highly unlikely: next_check_host is set but bg_map has no writer pair. + // Clear the green pin and fall through to blue host selection. + st.next_check_host.clear(); + st.next_check_host_failures = 0; + } + } + + if (st.next_check_host.empty()) { found_pingable_host = false; rnd = (size_t) rand(); rnd %= num_hosts; @@ -7119,7 +7134,7 @@ static void aws_rds_bgd_build_map(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_ MyHGM->wrlock(); // blue writer: the writer_hostgroup member whose name matches the green TARGET. - MyHGC* whgc = MyHGM->MyHGC_lookup(st.writer_hg); + MyHGC* whgc = MyHGM->MyHGC_find(st.writer_hg); if (whgc && whgc->mysrvs) { for (unsigned int j = 0; j < whgc->mysrvs->cnt(); j++) { MySrvC* s = whgc->mysrvs->idx(j); @@ -7135,6 +7150,24 @@ static void aws_rds_bgd_build_map(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_ p.blue_max_conns = s->max_connections; p.blue_use_ssl = s->use_ssl; p.is_writer = true; + + // read the green writer's use_ssl config + if (st.green_writer_hg >= 0) { + MyHGC* gwhgc = MyHGM->MyHGC_find((unsigned int)st.green_writer_hg); + if (gwhgc && gwhgc->mysrvs) { + for (unsigned int k = 0; k < gwhgc->mysrvs->cnt(); k++) { + MySrvC* gs = gwhgc->mysrvs->idx(k); + if (gs->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + continue; + } + if (aws_rds_bgd_match_host(gs->address, green_writer_host)) { + p.green_use_ssl = gs->use_ssl; + break; + } + } + } + } + proxy_debug(PROXY_DEBUG_MONITOR, 7, "AWS RDS BGD [wHG=%u]: mapped blue writer '%s:%d' <-> green '%s'\n", st.writer_hg, p.blue_host.c_str(), p.port, p.green_host.c_str()); @@ -7147,7 +7180,7 @@ static void aws_rds_bgd_build_map(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_ // reader pairs: match blue readers to user-added green readers by name. if (st.green_reader_hg >= 0) { std::vector green_reader_hosts; - MyHGC* grhgc = MyHGM->MyHGC_lookup((unsigned int)st.green_reader_hg); + MyHGC* grhgc = MyHGM->MyHGC_find((unsigned int)st.green_reader_hg); if (grhgc && grhgc->mysrvs) { for (unsigned int j = 0; j < grhgc->mysrvs->cnt(); j++) { MySrvC* s = grhgc->mysrvs->idx(j); @@ -7158,7 +7191,7 @@ static void aws_rds_bgd_build_map(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_ } } - MyHGC* rhgc = MyHGM->MyHGC_lookup(st.reader_hg); + MyHGC* rhgc = MyHGM->MyHGC_find(st.reader_hg); if (rhgc && rhgc->mysrvs) { for (unsigned int j = 0; j < rhgc->mysrvs->cnt(); j++) { MySrvC* s = rhgc->mysrvs->idx(j); From 727b2166bf2128b3b58a37ec31ff654b5f725058 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Wed, 15 Jul 2026 13:41:08 +0000 Subject: [PATCH 34/81] fix: Track green hosts in AWS RDS BGD checksum - Compute host checksum based on both blue and green hosts - Skip `OFFLINE_SOFT` servers when building BGD pairs Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 6 +-- include/MySQL_Monitor.hpp | 4 +- lib/MySQL_HostGroups_Manager.cpp | 83 ++++++++++++++++++++++++------ lib/MySQL_Monitor.cpp | 48 +++++++++-------- 4 files changed, 99 insertions(+), 42 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index f64ec7e9fb..3f481b5ad3 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -1214,10 +1214,10 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { /** * @brief Rebuilds the AWS RDS BGD monitor's host resultset. * - * @details Rebuilds `GloMyMon->AWS_RDS_BGD_Hosts_resultset` (and its checksum) from the - * `mysql_servers` x `mysql_aws_rds_bgd_hostgroups` join used by the RDS BGD monitor thread. + * @details Rebuilds `GloMyMon->AWS_RDS_Blue_Hosts_resultset` and publishes a checksum combining + * the blue hosts with the green hosts. * - * @param lock When true, the monitor's `aws_rds_bgd_mutex` is taken internally. + * @param lock When true, the monitor's `aws_rds_bgd_mutex` is taken internally. */ void update_aws_rds_bgd_hosts_monitor_resultset(bool lock=false); /** diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 1eb1576770..cd340ef014 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -601,8 +601,8 @@ class MySQL_Monitor { SQLite3_result *AWS_Aurora_Hosts_resultset; uint64_t AWS_Aurora_Hosts_resultset_checksum; std::unordered_map aws_rds_bgd_server_status; - SQLite3_result *AWS_RDS_BGD_Hosts_resultset; - uint64_t AWS_RDS_BGD_Hosts_resultset_checksum; + SQLite3_result *AWS_RDS_Blue_Hosts_resultset; + uint64_t AWS_RDS_BGD_Hosts_checksum; unsigned int num_threads; unsigned int aux_threads; unsigned int started_threads; diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index c8032da62c..8f8bd61595 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -7133,7 +7133,7 @@ void MySQL_HostGroups_Manager::update_aws_aurora_hosts_monitor_resultset(bool lo } } -const char SELECT_AWS_RDS_BGD_SERVERS_FOR_MONITOR[] { +const char SELECT_AWS_RDS_BGD_BLUE_SERVERS_FOR_MONITOR[] { "SELECT writer_hostgroup, reader_hostgroup, hostname, port, MAX(use_ssl) use_ssl, green_writer_hostgroup," " green_reader_hostgroup, check_interval_ms, check_timeout_ms, writer_is_also_reader FROM mysql_servers" " JOIN mysql_aws_rds_bgd_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup" @@ -7141,11 +7141,19 @@ const char SELECT_AWS_RDS_BGD_SERVERS_FOR_MONITOR[] { " GROUP BY writer_hostgroup, hostname, port" }; +const char SELECT_AWS_RDS_BGD_GREEN_SERVERS_FOR_MONITOR[] { + "SELECT bgd.writer_hostgroup, srv.hostgroup_id, srv.hostname, srv.port, srv.use_ssl FROM mysql_servers AS srv" + " JOIN mysql_aws_rds_bgd_hostgroups AS bgd ON srv.hostgroup_id=bgd.green_writer_hostgroup" + " OR srv.hostgroup_id=bgd.green_reader_hostgroup" + " WHERE bgd.active=1 AND srv.status NOT IN (2,3)" + " ORDER BY bgd.writer_hostgroup, srv.hostgroup_id, srv.hostname, srv.port" +}; + /** * @brief Rebuilds the AWS RDS BGD monitor's host resultset. * - * @details Rebuilds `GloMyMon->AWS_RDS_BGD_Hosts_resultset` (and its checksum) from the - * `mysql_servers` x `mysql_aws_rds_bgd_hostgroups` join used by the RDS BGD monitor thread. + * @details Rebuilds `GloMyMon->AWS_RDS_Blue_Hosts_resultset` and publishes a checksum combining + * the blue hosts with the green hosts. * * @param lock When true, the monitor's `aws_rds_bgd_mutex` is taken internally. */ @@ -7154,21 +7162,66 @@ void MySQL_HostGroups_Manager::update_aws_rds_bgd_hosts_monitor_resultset(bool l pthread_mutex_lock(&GloMyMon->aws_rds_bgd_mutex); } - SQLite3_result* resultset = nullptr; - { - char* error = nullptr; - int cols = 0; - int affected_rows = 0; - mydb->execute_statement(SELECT_AWS_RDS_BGD_SERVERS_FOR_MONITOR, &error, &cols, &affected_rows, &resultset); - } + // Unlike other monitor resultset/checksum pairs, BGD intentionally tracks different data in each. + // + // AWS_RDS_Blue_Hosts_resultset contains only blue hosts. The BGD monitor dispatcher uses it to start + // workers, and each worker uses it as the list of servers eligible for mysql.rds_topology polling. + // + // AWS_RDS_BGD_Hosts_checksum combines the blue and green resultset checksums. Workers and the dispatcher + // use it as a generation signal: relevant changes in mysql_servers or mysql_aws_rds_bgd_hostgroups + // stop the old workers so replacements rebuild the blue/green map from the current runtime configuration. + + SQLite3_result* blue_resultset = nullptr; + SQLite3_result* green_resultset = nullptr; + char* blue_error = nullptr; + char* green_error = nullptr; + int blue_cols = 0; + int green_cols = 0; + int blue_affected_rows = 0; + int green_affected_rows = 0; - if (resultset) { - if (GloMyMon->AWS_RDS_BGD_Hosts_resultset) { - delete GloMyMon->AWS_RDS_BGD_Hosts_resultset; + mydb->execute_statement( + SELECT_AWS_RDS_BGD_BLUE_SERVERS_FOR_MONITOR, + &blue_error, &blue_cols, &blue_affected_rows, &blue_resultset); + mydb->execute_statement( + SELECT_AWS_RDS_BGD_GREEN_SERVERS_FOR_MONITOR, + &green_error, &green_cols, &green_affected_rows, &green_resultset); + + if (blue_error || green_error || !blue_resultset || !green_resultset) { + if (blue_error) { + proxy_error("Error refreshing AWS RDS BGD blue hosts: %s\n", blue_error); + } + if (green_error) { + proxy_error("Error refreshing AWS RDS BGD green hosts: %s\n", green_error); + } + free(blue_error); + free(green_error); + delete blue_resultset; + delete green_resultset; + + if (lock) { + pthread_mutex_unlock(&GloMyMon->aws_rds_bgd_mutex); } - GloMyMon->AWS_RDS_BGD_Hosts_resultset=resultset; - GloMyMon->AWS_RDS_BGD_Hosts_resultset_checksum=resultset->raw_checksum(); + return; + } + + const uint64_t blue_checksum = blue_resultset->raw_checksum(); + const uint64_t green_checksum = green_resultset->raw_checksum(); + SpookyHash hash; + hash.Init(19, 3); + hash.Update(&blue_checksum, sizeof(blue_checksum)); + hash.Update(&green_checksum, sizeof(green_checksum)); + + uint64_t combined_checksum = 0; + uint64_t ignored = 0; + hash.Final(&combined_checksum, &ignored); + + if (GloMyMon->AWS_RDS_Blue_Hosts_resultset) { + delete GloMyMon->AWS_RDS_Blue_Hosts_resultset; } + GloMyMon->AWS_RDS_Blue_Hosts_resultset = blue_resultset; + GloMyMon->AWS_RDS_BGD_Hosts_checksum = combined_checksum; + delete green_resultset; if (lock) { pthread_mutex_unlock(&GloMyMon->aws_rds_bgd_mutex); diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 8390782c84..0ca54823c6 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -1145,8 +1145,8 @@ MySQL_Monitor::MySQL_Monitor() { pthread_mutex_init(&proxysql_servers_mutex, NULL); AWS_Aurora_Hosts_resultset=NULL; AWS_Aurora_Hosts_resultset_checksum = 0; - AWS_RDS_BGD_Hosts_resultset=NULL; - AWS_RDS_BGD_Hosts_resultset_checksum = 0; + AWS_RDS_Blue_Hosts_resultset=NULL; + AWS_RDS_BGD_Hosts_checksum = 0; shutdown=false; monitor_enabled=true; // default // create new SQLite datatabase @@ -1247,9 +1247,9 @@ MySQL_Monitor::~MySQL_Monitor() { delete AWS_Aurora_Hosts_resultset; AWS_Aurora_Hosts_resultset=NULL; } - if (AWS_RDS_BGD_Hosts_resultset) { - delete AWS_RDS_BGD_Hosts_resultset; - AWS_RDS_BGD_Hosts_resultset=NULL; + if (AWS_RDS_Blue_Hosts_resultset) { + delete AWS_RDS_Blue_Hosts_resultset; + AWS_RDS_Blue_Hosts_resultset=NULL; } std::map::iterator it2; AWS_Aurora_monitor_node *node=NULL; @@ -6740,7 +6740,7 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { MySQL_Monitor__thread_MySQL_Thread_Variables_version = GloMTH->get_global_version(); mysql_thr->refresh_variables(); - uint64_t initial_raw_checksum = 0; + uint64_t initial_checksum = 0; // initial data load from the monitor resultset // Columns: @@ -6748,8 +6748,8 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { // 5 green_writer_hostgroup, 6 green_reader_hostgroup, 7 check_interval_ms, // 8 check_timeout_ms, 9 writer_is_also_reader pthread_mutex_lock(&GloMyMon->aws_rds_bgd_mutex); - initial_raw_checksum = GloMyMon->AWS_RDS_BGD_Hosts_resultset_checksum; - for (SQLite3_row *r : GloMyMon->AWS_RDS_BGD_Hosts_resultset->rows) { + initial_checksum = GloMyMon->AWS_RDS_BGD_Hosts_checksum; + for (SQLite3_row *r : GloMyMon->AWS_RDS_Blue_Hosts_resultset->rows) { if (atoi(r->fields[0]) == (int)wHG) { num_hosts++; if (st.reader_hg == 0) { @@ -6774,7 +6774,7 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { } host_def_t *hpa = (host_def_t *)malloc(sizeof(host_def_t)*(num_hosts ? num_hosts : 1)); - for (SQLite3_row *r : GloMyMon->AWS_RDS_BGD_Hosts_resultset->rows) { + for (SQLite3_row *r : GloMyMon->AWS_RDS_Blue_Hosts_resultset->rows) { if (atoi(r->fields[0]) == (int)wHG) { hpa[cur_host_idx].host = strdup(r->fields[2]); hpa[cur_host_idx].port = atoi(r->fields[3]); @@ -6791,7 +6791,7 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { unsigned long long t1 = 0; unsigned long long next_loop_at = 0; bool crc = false; - uint64_t current_raw_checksum = 0; + uint64_t current_checksum = 0; size_t rnd; bool found_pingable_host = false; bool rc_ping = false; @@ -6818,9 +6818,9 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { // if the host list/definition changed, terminate so the dispatcher respawns pthread_mutex_lock(&GloMyMon->aws_rds_bgd_mutex); - current_raw_checksum = GloMyMon->AWS_RDS_BGD_Hosts_resultset_checksum; + current_checksum = GloMyMon->AWS_RDS_BGD_Hosts_checksum; pthread_mutex_unlock(&GloMyMon->aws_rds_bgd_mutex); - if (current_raw_checksum != initial_raw_checksum) { + if (current_checksum != initial_checksum) { exit_now = true; break; } @@ -7138,7 +7138,8 @@ static void aws_rds_bgd_build_map(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_ if (whgc && whgc->mysrvs) { for (unsigned int j = 0; j < whgc->mysrvs->cnt(); j++) { MySrvC* s = whgc->mysrvs->idx(j); - if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD + || s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_SOFT) { continue; } if (aws_rds_bgd_match_host(s->address, green_writer_host)) { @@ -7157,7 +7158,8 @@ static void aws_rds_bgd_build_map(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_ if (gwhgc && gwhgc->mysrvs) { for (unsigned int k = 0; k < gwhgc->mysrvs->cnt(); k++) { MySrvC* gs = gwhgc->mysrvs->idx(k); - if (gs->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + if (gs->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD + || gs->get_status() == MYSQL_SERVER_STATUS_OFFLINE_SOFT) { continue; } if (aws_rds_bgd_match_host(gs->address, green_writer_host)) { @@ -7184,7 +7186,8 @@ static void aws_rds_bgd_build_map(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_ if (grhgc && grhgc->mysrvs) { for (unsigned int j = 0; j < grhgc->mysrvs->cnt(); j++) { MySrvC* s = grhgc->mysrvs->idx(j); - if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD + || s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_SOFT) { continue; } green_reader_hosts.push_back(s->address); @@ -7195,7 +7198,8 @@ static void aws_rds_bgd_build_map(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_ if (rhgc && rhgc->mysrvs) { for (unsigned int j = 0; j < rhgc->mysrvs->cnt(); j++) { MySrvC* s = rhgc->mysrvs->idx(j); - if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD + || s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_SOFT) { continue; } for (const std::string& green_reader_host : green_reader_hosts) { @@ -7877,7 +7881,7 @@ void * MySQL_Monitor::monitor_aws_rds_bgd() { MySQL_Monitor__thread_MySQL_Thread_Variables_version = GloMTH->get_global_version(); mysql_thr->refresh_variables(); - uint64_t last_raw_checksum = 0; + uint64_t last_checksum = 0; unsigned int *hgs_array = NULL; pthread_t *pthreads_array = NULL; unsigned int hgs_num = 0; @@ -7895,11 +7899,11 @@ void * MySQL_Monitor::monitor_aws_rds_bgd() { // respawn the per-writer-HG workers when the host list/definition changes pthread_mutex_lock(&aws_rds_bgd_mutex); - uint64_t new_raw_checksum = AWS_RDS_BGD_Hosts_resultset->raw_checksum(); + uint64_t new_checksum = AWS_RDS_BGD_Hosts_checksum; pthread_mutex_unlock(&aws_rds_bgd_mutex); - if (new_raw_checksum != last_raw_checksum) { + if (new_checksum != last_checksum) { proxy_info("Detected new/changed definition for AWS RDS monitoring\n"); - last_raw_checksum = new_raw_checksum; + last_checksum = new_checksum; if (pthreads_array) { for (unsigned int i=0; i < hgs_num; i++) { pthread_join(pthreads_array[i], NULL); @@ -7913,10 +7917,10 @@ void * MySQL_Monitor::monitor_aws_rds_bgd() { hgs_num = 0; pthread_mutex_lock(&aws_rds_bgd_mutex); - unsigned int num_rows = AWS_RDS_BGD_Hosts_resultset->rows_count; + unsigned int num_rows = AWS_RDS_Blue_Hosts_resultset->rows_count; if (num_rows) { unsigned int *tmp_hgs_array = (unsigned int *)malloc(sizeof(unsigned int)*num_rows); - for (SQLite3_row *r : AWS_RDS_BGD_Hosts_resultset->rows) { + for (SQLite3_row *r : AWS_RDS_Blue_Hosts_resultset->rows) { int wHG = atoi(r->fields[0]); bool found = false; for (unsigned int i=0; i < hgs_num; i++) { From d45c953d27ab19bd4d40482650064d4d86ab91ff Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 16 Jul 2026 04:31:53 +0000 Subject: [PATCH 35/81] fix: Refresh AWS RDS BGD checksum after mysql_servers commits Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 5 ++--- include/MySQL_Monitor.hpp | 2 +- lib/MySQL_HostGroups_Manager.cpp | 13 ++++++------- lib/MySQL_Monitor.cpp | 2 +- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 3f481b5ad3..148303b94f 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -722,9 +722,8 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { * `incoming_aws_rds_bgd_hostgroups` resultset. * * @details Inserts each staged row with `auto_generated=0` (config-loaded entries are - * user-defined) and NULL green hostgroups preserved, clears the staging resultset, then - * republishes the host list to the RDS BGD monitor thread via - * `update_aws_rds_bgd_hosts_monitor_resultset()`. No-op when nothing is staged. + * user-defined) and NULL green hostgroups preserved, then clears the staging resultset. + * No-op when nothing is staged. */ void generate_mysql_aws_rds_bgd_hostgroups_table(); SQLite3_result *incoming_aws_rds_bgd_hostgroups; diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index cd340ef014..c41c4f0945 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -655,7 +655,7 @@ class MySQL_Monitor { * * @details Spawns one worker (monitor_RDS_BGD_thread_HG) per writer hostgroup; each worker picks a pingable host, * probes 'mysql.rds_topology' and dispatches to a handler based on the detected topology shape. - * Workers are (re)spawned whenever the AWS_RDS_BGD_Hosts_resultset checksum changes. + * Workers are (re)spawned whenever the AWS_RDS_BGD_Hosts_checksum changes. */ void * monitor_aws_rds_bgd(); /** diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 8f8bd61595..f9414244ae 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1616,6 +1616,8 @@ bool MySQL_HostGroups_Manager::commit( // NOTE: In order to guarantee the latest generated version, this should be kept after all the // calls to 'generate_mysql_servers'. update_table_mysql_servers_for_monitor(false); + // Refresh BGD monitoring after all runtime server changes are applied. + update_aws_rds_bgd_hosts_monitor_resultset(true); wrunlock(); unsigned long long curtime2=monotonic_time(); @@ -6548,13 +6550,6 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_rds_bgd_hostgroups_table() { delete incoming_aws_rds_bgd_hostgroups; incoming_aws_rds_bgd_hostgroups=NULL; - - // publish the refreshed host list to the RDS monitor thread - if (GloMyMon) { - pthread_mutex_lock(&GloMyMon->aws_rds_bgd_mutex); - update_aws_rds_bgd_hosts_monitor_resultset(false); - pthread_mutex_unlock(&GloMyMon->aws_rds_bgd_mutex); - } } @@ -7158,6 +7153,10 @@ const char SELECT_AWS_RDS_BGD_GREEN_SERVERS_FOR_MONITOR[] { * @param lock When true, the monitor's `aws_rds_bgd_mutex` is taken internally. */ void MySQL_HostGroups_Manager::update_aws_rds_bgd_hosts_monitor_resultset(bool lock) { + if (!GloMyMon) { + return; + } + if (lock) { pthread_mutex_lock(&GloMyMon->aws_rds_bgd_mutex); } diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 0ca54823c6..28a6274dd9 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -7868,7 +7868,7 @@ void MySQL_Monitor::set_aws_rds_bgd_server_in_progress(unsigned int writer_hg, u * * @details Spawns one worker (monitor_RDS_BGD_thread_HG) per writer hostgroup; each worker picks a pingable host, * probes 'mysql.rds_topology' and dispatches to a handler based on the detected topology shape. -* Workers are (re)spawned whenever the AWS_RDS_BGD_Hosts_resultset checksum changes. +* Workers are (re)spawned whenever the AWS_RDS_BGD_Hosts_checksum changes. */ void * MySQL_Monitor::monitor_aws_rds_bgd() { // Wait for GloMTH to be initialized From cdffd77ee87f8e5cddbf6dbeece6ca1127fe2fa7 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 16 Jul 2026 06:33:07 +0000 Subject: [PATCH 36/81] fix: Revise AWS RDS BGD green drain policy - Preserve auto-added and user-configured green hosts during rollback and after a successful switchover. - Skip draining connections from green hosts during rollback. - After a successful switchover, drain connections from every non-OFFLINE_HARD green host without deleting rows or changing statuses. Signed-off-by: Wazir Ahmed --- include/MySQL_Monitor.hpp | 16 ++--- lib/MySQL_Monitor.cpp | 131 +++++++++++++------------------------- 2 files changed, 51 insertions(+), 96 deletions(-) diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index c41c4f0945..393eabd94b 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -505,7 +505,6 @@ struct AWS_RDS_BGD_State { std::vector shunned_readers; ///< readers we shunned AWS_RDS_BGD_Status bgd_status = AWS_RDS_BGD_Status::NONE; ///< drives the FSM and the deferred cleanup - bool green_writer_added_in_hg = false; ///< green writer added to green_writer_hg bool bgd_in_progress_set = false; ///< deployment's servers flagged in aws_rds_bgd_server_status unsigned int next_check_interval_ms = 0; ///< FSM-controlled interval; 0 => baseline @@ -677,7 +676,8 @@ class MySQL_Monitor { * @brief Run deferred switchover teardown or rollback cleanup. * * @details Restores post-switchover reader handling, unshuns readers, drops DNS pins, - * drains green hostgroups, and clears BGD switchover state. + * and clears BGD switchover state. Normal post-switchover cleanup also drains + * connections from green hosts; rollback leaves green rows and connections unchanged. * * When rollback is false (normal post-switchover), the caller must be in * READER_SWITCHOVER_IN_PROGRESS; the function advances through @@ -685,23 +685,21 @@ class MySQL_Monitor { * * When rollback is true (topology table disappeared or worker exit mid-switchover), * the function accepts any non-NONE bgd_status, restores the blue writer to the - * writer hostgroup if it was demoted, then runs the same cleanup and resets - * switchover state. + * writer hostgroup if it was demoted, then resets switchover state. * * @param st BGD switchover state. * @param rollback True if called due to a rollback/cancellation, false for normal completion. */ void handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st, bool rollback = false); /** - * @brief Clean up the deployment's green hostgroups after switchover or rollback. + * @brief Drain connections from green hosts after switchover. * - * @details Successful cleanup drains all configured green-hostgroup members. Rollback drains and removes - * only the green writer auto-added by the BGD worker; user-configured rows are left unchanged. + * @details Drains connections from every non-OFFLINE_HARD green host. Server rows + * and statuses are left unchanged. * * @param st Switchover state. - * @param rollback Whether cleanup is handling a rollback. */ - void aws_rds_bgd_drain_green_hg(AWS_RDS_BGD_State& st, bool rollback); + void aws_rds_bgd_drain_green_hg(AWS_RDS_BGD_State& st); /** * @brief Handle an absent, empty, or vanished mysql.rds_topology table. * diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 28a6274dd9..9dfd7a8c41 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -7281,38 +7281,30 @@ static void aws_rds_bgd_resolve_green_ips(AWS_RDS_BGD_State& st) { * @brief Add the green writer to green_writer_hostgroup, when that hostgroup is configured. * * @details Mirrors the blue writer's connection settings (weight/max_connections/use_ssl) onto -* the green writer. Existing rows, including OFFLINE_HARD rows, are left unchanged. When -* green_writer_added_in_hg is true (BGD owns this row), an OFFLINE_HARD tombstone left by a -* prior rollback is re-enabled. +* the green writer. Existing rows, including OFFLINE_HARD rows, are left unchanged. */ static void aws_rds_bgd_add_green_writer_in_hg(AWS_RDS_BGD_State& st) { if (st.green_writer_hg < 0) { return; } for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { - if (p.is_writer) { - srv_info_t srv_info { p.green_host, (uint16_t)p.port, "AWS RDS BGD green writer" }; - srv_opts_t srv_opts { p.blue_weight, p.blue_max_conns, p.blue_use_ssl }; - MyHGM->wrlock(); - MySrvC* existing = MyHGM->find_server_in_hg((uint32_t)st.green_writer_hg, p.green_host, p.port); - bool publish = false; - - if (existing == nullptr) { - st.green_writer_added_in_hg = - (MyHGM->create_new_server_in_hg((uint32_t)st.green_writer_hg, srv_info, srv_opts) == 0); - publish = st.green_writer_added_in_hg; - } else if (st.green_writer_added_in_hg - && existing->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { - existing->set_status(MYSQL_SERVER_STATUS_ONLINE); - publish = true; - } + if (!p.is_writer) { + continue; + } - if (publish) { + srv_info_t srv_info { p.green_host, (uint16_t)p.port, "AWS RDS BGD green writer" }; + srv_opts_t srv_opts { p.blue_weight, p.blue_max_conns, p.blue_use_ssl }; + MyHGM->wrlock(); + MySrvC* srvc = MyHGM->find_server_in_hg( + (uint32_t)st.green_writer_hg, p.green_host, p.port); + if (srvc == nullptr) { + int rc = MyHGM->create_new_server_in_hg((uint32_t)st.green_writer_hg, srv_info, srv_opts); + if (rc == 0) { MyHGM->publish_mysql_servers_to_runtime(); } - MyHGM->wrunlock(); - break; } + MyHGM->wrunlock(); + break; } } @@ -7611,7 +7603,8 @@ void MySQL_Monitor::aws_rds_bgd_hostgroup_action( * @brief Run deferred switchover teardown or rollback cleanup. * * @details Restores post-switchover reader handling, unshuns readers, drops DNS pins, -* drains green hostgroups, and clears BGD switchover state. +* and clears BGD switchover state. Normal post-switchover cleanup also drains +* connections from green hosts; rollback leaves green rows and connections unchanged. * * When rollback is false (normal post-switchover), the caller must be in * READER_SWITCHOVER_IN_PROGRESS; the function advances through @@ -7619,8 +7612,7 @@ void MySQL_Monitor::aws_rds_bgd_hostgroup_action( * * When rollback is true (topology disappeared or worker exit mid-switchover), * the function accepts any non-NONE bgd_status, restores the blue writer to the -* writer hostgroup if it was demoted, then runs the same cleanup and resets -* directly to NONE without the intermediate SWITCHOVER_COMPLETED state. +* writer hostgroup if it was demoted, then resets switchover state. * * @param st BGD switchover state. * @param rollback True if called due to a rollback/cancellation, false for normal completion. @@ -7683,7 +7675,9 @@ void MySQL_Monitor::handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st, bo My_Conn_Pool->purge_connections(p.blue_host.c_str(), p.port); } - aws_rds_bgd_drain_green_hg(st, rollback); + if (!rollback) { + aws_rds_bgd_drain_green_hg(st); + } // state cleanup st.bg_map.clear(); @@ -7698,73 +7692,43 @@ void MySQL_Monitor::handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st, bo } /** -* @brief Clean up the deployment's green hostgroups after switchover or rollback. +* @brief Drain connections from green hosts after switchover. * -* @details Successful cleanup drains all configured green-hostgroup members. Rollback drains and removes -* only the green writer auto-added by the BGD worker; user-configured rows are left unchanged. +* @details Drains connections from every non-OFFLINE_HARD green host. Server rows +* and statuses are left unchanged. * * @param st Switchover state. -* @param rollback Whether cleanup is handling a rollback. */ -void MySQL_Monitor::aws_rds_bgd_drain_green_hg(AWS_RDS_BGD_State& st, bool rollback) { +void MySQL_Monitor::aws_rds_bgd_drain_green_hg(AWS_RDS_BGD_State& st) { struct hg_srv_t { int hostgroup; srv_addr_t server; - bool remove; }; std::vector targets; - bool servers_removed = false; MyHGM->wrlock(); - if (rollback) { - if (st.green_writer_added_in_hg && st.green_writer_hg >= 0) { - for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { - if (!p.is_writer) - continue; - - MySrvC* existing = MyHGM->find_server_in_hg( - (uint32_t)st.green_writer_hg, p.green_host, p.port); - if (existing) { - existing->ConnectionsUsed->mark_connections_unhealthy(); - servers_removed = - (MyHGM->remove_server_in_hg( - (uint32_t)st.green_writer_hg, p.green_host, p.port) == 0); - targets.push_back(hg_srv_t{ - st.green_writer_hg, srv_addr_t{ p.green_host, p.port }, true }); - } - break; - } + for (int hg : { st.green_writer_hg, st.green_reader_hg }) { + if (hg < 0) { + continue; } - } else { - std::vector green_hgs; - if (st.green_writer_hg >= 0) - green_hgs.push_back(st.green_writer_hg); - if (st.green_reader_hg >= 0) - green_hgs.push_back(st.green_reader_hg); - - for (int hg : green_hgs) { - MyHGC* hgc = MyHGM->MyHGC_find(hg); - if (!hgc || !hgc->mysrvs) - continue; - - for (unsigned int j = 0; j < hgc->mysrvs->cnt(); j++) { - MySrvC* s = hgc->mysrvs->idx(j); - if (s->get_status() != MYSQL_SERVER_STATUS_OFFLINE_HARD) { - targets.push_back(hg_srv_t{ - hg, srv_addr_t{ std::string(s->address), s->port }, false }); - } - } + MyHGC* hgc = MyHGM->MyHGC_find(hg); + if (!hgc || !hgc->mysrvs) { + continue; } - - for (const hg_srv_t& target : targets) { - MyHGM->drain_server_connections( - target.server.host.c_str(), target.server.port); + for (unsigned int j = 0; j < hgc->mysrvs->cnt(); j++) { + MySrvC* s = hgc->mysrvs->idx(j); + if (s->get_status() != MYSQL_SERVER_STATUS_OFFLINE_HARD) { + targets.push_back(hg_srv_t{ + hg, srv_addr_t{ std::string(s->address), s->port } }); + } } } - if (servers_removed) - MyHGM->publish_mysql_servers_to_runtime(); + for (const hg_srv_t& target : targets) { + MyHGM->drain_server_connections( + target.server.host.c_str(), target.server.port); + } MyHGM->wrunlock(); @@ -7773,17 +7737,10 @@ void MySQL_Monitor::aws_rds_bgd_drain_green_hg(AWS_RDS_BGD_State& st, bool rollb My_Conn_Pool->purge_connections( target.server.host.c_str(), target.server.port); - if (target.remove) { - proxy_info( - "AWS RDS BGD [wHG=%u rHG=%u]: removed auto-added green writer '%s:%d' from HG %d\n", - st.writer_hg, st.reader_hg, target.server.host.c_str(), target.server.port, - target.hostgroup); - } else { - proxy_info( - "AWS RDS BGD [wHG=%u rHG=%u]: connections drained from green HG %d server '%s:%d'\n", - st.writer_hg, st.reader_hg, target.hostgroup, target.server.host.c_str(), - target.server.port); - } + proxy_info( + "AWS RDS BGD [wHG=%u rHG=%u]: connections drained from green HG %d server '%s:%d'\n", + st.writer_hg, st.reader_hg, target.hostgroup, target.server.host.c_str(), + target.server.port); } } From ac4167cd016ae50e1697d5ac4782eb9f61da3072 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 16 Jul 2026 07:04:33 +0000 Subject: [PATCH 37/81] fix: Exclude `OFFLINE_SOFT` servers from AWS RDS BGD action - Skip `OFFLINE_SOFT` blue readers during post-processing. - Leave `OFFLINE_SOFT` green hosts untouched during successful cleanup. Signed-off-by: Wazir Ahmed --- include/MySQL_Monitor.hpp | 4 ++-- lib/MySQL_Monitor.cpp | 16 ++++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 393eabd94b..2f233f1766 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -694,8 +694,8 @@ class MySQL_Monitor { /** * @brief Drain connections from green hosts after switchover. * - * @details Drains connections from every non-OFFLINE_HARD green host. Server rows - * and statuses are left unchanged. + * @details Drains connections from every green host that is neither OFFLINE_SOFT nor + * OFFLINE_HARD. Server rows and statuses are left unchanged. * * @param st Switchover state. */ diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 9dfd7a8c41..a82bb4b889 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -7492,7 +7492,8 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo if (rhgc && rhgc->mysrvs) { for (unsigned int j = 0; j < rhgc->mysrvs->cnt(); j++) { MySrvC* s = rhgc->mysrvs->idx(j); - if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_SOFT + || s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { continue; } if (writer.host == s->address && writer.port == s->port) { @@ -7694,8 +7695,8 @@ void MySQL_Monitor::handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st, bo /** * @brief Drain connections from green hosts after switchover. * -* @details Drains connections from every non-OFFLINE_HARD green host. Server rows -* and statuses are left unchanged. +* @details Drains connections from every green host that is neither OFFLINE_SOFT nor +* OFFLINE_HARD. Server rows and statuses are left unchanged. * * @param st Switchover state. */ @@ -7718,10 +7719,13 @@ void MySQL_Monitor::aws_rds_bgd_drain_green_hg(AWS_RDS_BGD_State& st) { } for (unsigned int j = 0; j < hgc->mysrvs->cnt(); j++) { MySrvC* s = hgc->mysrvs->idx(j); - if (s->get_status() != MYSQL_SERVER_STATUS_OFFLINE_HARD) { - targets.push_back(hg_srv_t{ - hg, srv_addr_t{ std::string(s->address), s->port } }); + if (s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_SOFT + || s->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + continue; } + + targets.push_back(hg_srv_t{ + hg, srv_addr_t{ std::string(s->address), s->port } }); } } From 0a1bc28df8e7d7876576dfe50a3800535ddc2c43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Thu, 16 Jul 2026 21:02:02 +0200 Subject: [PATCH 38/81] docs: Establish AWS RDS BGD validation contract Document current monitor mechanics separately from lifecycle claims pending author validation. Keep the PR documentation-only so validated policy can drive a later implementation PR. --- doc/AWS_RDS_BLUE_GREEN_MONITOR.md | 1032 +++++++++++++++++++++++++++++ 1 file changed, 1032 insertions(+) create mode 100644 doc/AWS_RDS_BLUE_GREEN_MONITOR.md diff --git a/doc/AWS_RDS_BLUE_GREEN_MONITOR.md b/doc/AWS_RDS_BLUE_GREEN_MONITOR.md new file mode 100644 index 0000000000..fc5f1ddbd4 --- /dev/null +++ b/doc/AWS_RDS_BLUE_GREEN_MONITOR.md @@ -0,0 +1,1032 @@ +# AWS RDS Blue/Green Monitor + +**Document status:** AUTHOR VALIDATION COMPLETE; FOLLOW-UP HANDOFF DEFINED; +IMPLEMENTATION CONFORMANCE OPEN + +**Applies to:** Amazon RDS Multi-AZ DB instance blue/green deployment monitoring + +**Primary monitor entry points:** `include/MySQL_Monitor.hpp`, +`lib/MySQL_Monitor.cpp` + +**Related implementation:** `include/DNS_Cache.hpp`, `lib/DNS_Cache.cpp`, +`include/MySQL_HostGroups_Manager.h`, `lib/MySQL_HostGroups_Manager.cpp`, +`include/mysql_connection.h`, `lib/mysql_connection.cpp`, +`lib/MySrvConnList.cpp`, `lib/ProxySQL_Admin.cpp`, `lib/ProxySQL_Config.cpp`, and +`include/ProxySQL_Admin_Tables_Definitions.h`. This is the non-exhaustive +side-effect/configuration surface referenced by later `SOURCE-CODE` sections. + +## Purpose + +This document defines the AWS observations, current ProxySQL behavior, and +safety requirements for the AWS RDS blue/green deployment monitor introduced +by [PR #5861](https://github.com/sysown/proxysql/pull/5861). + +The document is intentionally explicit about evidence. AWS behavior for which +this review has not recorded author evidence is not presented as an operational +guarantee. This evidence status does not imply that the author originally +inferred, assumed, or failed to observe the behavior. + +## Evidence Labels + +| Label | Dimension | Meaning | +|---|---|---| +| `SOURCE-CODE` | Provenance | Direct description of current implementation; it does not validate an external AWS claim. | +| `AUTHOR-VALIDATED` | External evidence | AWS behavior confirmed by the feature author. The statement must identify whether it is an AWS-provided contract or scoped observation. | +| `AUTHOR-ACCEPTED-POLICY` | Intent | ProxySQL behavior explicitly accepted by the feature author, including a deliberate policy choice made under an external uncertainty. | +| `REVIEW-VALIDATION-PENDING` | Review evidence | An external claim present in the PR, source comments, or implementation contract for which this review has not yet recorded the author's evidence or correction. It does not characterize how the author derived the claim. | +| `IMPLEMENTATION-CONFORMANCE-OPEN` | Review finding | The evidence or policy decision is resolved, but current source does not implement it or lacks verification. | +| `PROPOSED-POLICY` | Intent | Reviewer-proposed hardening that is not part of the current implementation or an author-accepted production contract unless separately promoted to `AUTHOR-ACCEPTED-POLICY`. | + +Labels may be combined. `SOURCE-CODE, REVIEW-VALIDATION-PENDING` means the +current code or comments encode an external claim whose supporting evidence has +not yet been recorded in this review. `SOURCE-CODE` alone must be used only for +internal mechanics and never promotes an external claim. + +A `REVIEW-VALIDATION-PENDING` claim must be promoted to `AUTHOR-VALIDATED`, +replaced by an explicit `AUTHOR-ACCEPTED-POLICY`, or corrected before the +author-validation gate closes. Closing that evidence gate does not imply that +the implementation conforms to the recorded decision or that a reviewer has +accepted the operational risk. + +## Author Evidence Record + +The feature author supplied the following evidence in the +[author-validation response](https://github.com/sysown/proxysql/pull/5934#issuecomment-4972444890): + +- An [AWS-provided RDS topology metadata document](https://github.com/user-attachments/files/30019110/RDS_Topology_metadata.md) + describing the `mysql.rds_topology` schema, roles, statuses, switchover + stages, traffic availability, and polling guidance. +- A [timestamped topology trace](https://github.com/user-attachments/files/30019175/aws-rds-topology-watch.txt) + from one complete switchover, polled at approximately 250 ms. +- Source-code references and additional author observations for cancellation, + reader behavior, and green-hostname retirement. + +The captured deployment used RDS MySQL 8.4.x, a Multi-AZ DB instance with two +read replicas, and `eu-north-1`. The trace covers one complete switchover; the +author separately observed one cancellation. A statement supported only by +that trace or an unrecorded author observation is scoped accordingly and is not +promoted to a universal AWS guarantee. + +The author then answered the eight remaining decisions in the +[counter-review response](https://github.com/sysown/proxysql/pull/5934#issuecomment-4989347968). +That response explicitly: + +- Defines the matched blue writer's configured port as the direct green-probe + port and accepts that a source/target pair using different ports is not + supported. +- Defines explicit-mode TLS from the matched green writer's `mysql_servers` + row and automatic-mode TLS from the matched blue writer's row. +- Makes green hostgroup membership persistent until an administrator removes + it, including membership created automatically at runtime. +- Accepts one-shot cleanup and loss of per-effect completion state rather than + a retained or durable cleanup ledger. +- Accepts the current same-phase DNS-resolution failure for this PR and commits + to a later per-pair reconciliation change. +- Accepts cleanup-on-worker-exit followed by fresh worker state, and a + no-persistence fresh start after a full ProxySQL process restart. + +The source changes reviewed with that response are commits `7d272074c` through +`ac4167cd0`, based on `0a37316c9`. A stated policy is recorded as resolved even +when implementation conformance is still open; those cases are called out +explicitly below. + +The author subsequently clarified two ProxySQL-internal contracts during the +review: + +- User configuration in the persistent `mysql_aws_rds_bgd_hostgroups` table + requires both green hostgroup values. Nullable green hostgroups belong only + to runtime rows generated by automatic discovery; those rows carry + `auto_generated=1` and are skipped when runtime state is saved back to the + persistent configuration table. +- Reads of connection state without an additional per-connection lock are an + accepted project-level risk. For BGD connection retirement, `healthy=false` + is to become a terminal marker: `MySQL_Connection::reset()` must not restore + it, and both local and global pool-return paths must destroy an unhealthy + connection instead of caching it. + +The author also selected the existing cluster simulator under +`test/deps/cluster_simulator` and its TAP group integration under +`test/tap/groups` as the test foundation. The simulator foundation and the BGD +scenario suite are deliberately separate follow-up PRs. Registration in +`groups.json` is not considered CI integration by itself; the BGD simulator +group must be executed by an automatic PR check. + +## Scope + +This document covers: + +- Detection of blue/green topology through `mysql.rds_topology`. +- Mapping of configured blue writer and reader servers to green servers. +- Green address resolution and direct topology probing. +- Writer and reader switchover handling. +- DNS pinning, hostgroup changes, and backend connection draining. +- Successful finalization, cancellation rollback, and worker replacement. +- Explicit and automatic green-hostgroup configuration. + +This document does not define Aurora monitoring, Group Replication monitoring, +Galera monitoring, or PostgreSQL behavior. + +## Terminology + +The table includes current implementation terms and proposed hardening +concepts; proposed concepts are explicitly labeled. + +| Term | Definition | +|---|---| +| Blue | The source deployment before switchover. | +| Green | The target deployment before switchover. | +| Observation | The result of one topology query or lifecycle event. | +| Controller state | `PROPOSED-POLICY`: ProxySQL's progress and policy across observations. | +| External effect | A DNS, hostgroup, monitor, or connection change visible outside controller bookkeeping. | +| Effect ledger | `PROPOSED-POLICY`: The identities and results of external effects requiring retry or cleanup. | +| Finalization | `PROPOSED-POLICY`: Cleanup on the proposed successful-completion path after observed writer completion and the accepted reader-completion signal. | +| Rollback | `PROPOSED-POLICY`: Restoration after cancellation or topology disappearance before observed writer completion. | + +## Topology Shape + +`SOURCE-CODE`: `parse_aws_rds_topology` sets `blue_green` once, from the first +fetched row, when the role and status columns exist and that row's cells for +both columns are non-NULL. Empty strings still meet this current non-NULL test. +Later rows do not re-evaluate or reverse the classification. + +`SOURCE-CODE, AUTHOR-VALIDATED (AWS-PROVIDED CONTRACT)`: Actual RDS blue/green +rows use the source and target role values and recognized target status values +listed below. The AWS-provided metadata document defines these values, and all +five statuses appeared in the supplied trace. + +Role values: + +```text +BLUE_GREEN_DEPLOYMENT_SOURCE +BLUE_GREEN_DEPLOYMENT_TARGET +``` + +Target status values: + +```text +AVAILABLE +SWITCHOVER_INITIATED +SWITCHOVER_IN_PROGRESS +SWITCHOVER_IN_POST_PROCESSING +SWITCHOVER_COMPLETED +``` + +`SOURCE-CODE`: If there is no row, either column is absent, or either first-row +cell is NULL, `blue_green` remains false. Later malformed rows do not change a +true first-row classification. Other RDS topology shapes may be processed by +the Multi-AZ Cluster discovery path. + +`AUTHOR-VALIDATED (SCOPED OBSERVATION)`: While both rows were present in the +supplied trace, the source and target rows carried the same status. The trace +does not by itself establish that equality as a universal contract. + +## Observation Model + +`PROPOSED-POLICY`: The hardened controller design distinguishes the following +observation and lifecycle-event vocabulary. `SOURCE-CODE`: The +`TOPOLOGY_ABSENT` inputs correspond to the current table-existence query and +`ER_NO_SUCH_TABLE` metadata-fetch paths, but these observations are not the +current implementation state enum. + +| Observation | Meaning | +|---|---| +| `TOPOLOGY_ABSENT` | The existence query returns zero rows, or a metadata fetch reports `ER_NO_SUCH_TABLE`. The table is not available; this is not a generic failure. | +| `TOPOLOGY_EMPTY` | A successful metadata query returns zero topology rows. | +| `AVAILABLE` | The target reports the recognized `AVAILABLE` status. | +| `WRITER_INITIATED` | The target reports the recognized `SWITCHOVER_INITIATED` status. | +| `WRITER_IN_PROGRESS` | The target reports the recognized `SWITCHOVER_IN_PROGRESS` status. | +| `WRITER_POST_PROCESSING` | The target reports the recognized `SWITCHOVER_IN_POST_PROCESSING` status. | +| `WRITER_COMPLETED` | The target reports the recognized `SWITCHOVER_COMPLETED` status. | +| `UNKNOWN_STATUS` | The target has a non-empty, unrecognized status. | +| `MALFORMED_TOPOLOGY` | The target, endpoint, role, status, or required identity is missing. | +| `QUERY_FAILED` | The query times out, the connection fails, or SQL reports an error other than documented absence. | +| `CONFIG_CHANGED` | The monitor result-set checksum or generation changes and may replace the worker without losing deployment state. | +| `CONFIG_DISABLED` | The deployment remains configured but is disabled; outstanding effects require phase-appropriate settlement. | +| `CONFIG_REMOVED` | The deployment configuration is removed; outstanding effects require phase-appropriate settlement before the context is removed. | +| `WORKER_RESTARTED` | A replacement worker attaches to and resumes the existing context. | + +`PROPOSED-POLICY`: `TOPOLOGY_ABSENT`, `TOPOLOGY_EMPTY`, and `QUERY_FAILED` are +not interchangeable. Query failure never proves cancellation or completion. + +## Validated Lifecycle Evidence + +`SOURCE-CODE, AUTHOR-VALIDATED (AWS-PROVIDED CONTRACT AND SCOPED OBSERVATION)`: +The AWS-provided metadata document defines the five forward phases. The supplied +trace observed the following row lifecycle: + +```text +Two rows: + SOURCE = blue + TARGET = green + status = AVAILABLE + +Two rows: + repeated observations of SWITCHOVER_INITIATED + -> repeated observations of SWITCHOVER_IN_PROGRESS + -> repeated observations of SWITCHOVER_IN_POST_PROCESSING + +One target row: + repeated observations of SWITCHOVER_COMPLETED + +Zero rows: + observed approximately 44 seconds after SWITCHOVER_COMPLETED in this trace +``` + +`AUTHOR-VALIDATED (AWS-PROVIDED CONTRACT)`: `SWITCHOVER_COMPLETED` means writer +DNS propagation completed and the original source endpoint points to the +target. The status sequence permits cancellation during `SWITCHOVER_INITIATED` +and `SWITCHOVER_IN_PROGRESS`; rollback is no longer allowed in +`SWITCHOVER_IN_POST_PROCESSING`. + +`AUTHOR-VALIDATED (SCOPED OBSERVATION)`: The trace observed monotonic forward +phase changes, with repeated identical observations while each phase remained +active. At `SWITCHOVER_COMPLETED`, the source row disappeared and the target +row remained for approximately 44 seconds before the table became empty. The +duration is not fixed. The table remained present in `information_schema`; an +`ER_NO_SUCH_TABLE` outcome was not observed. + +`AUTHOR-ACCEPTED-POLICY`: After writer completion has been observed, +`TOPOLOGY_EMPTY` is the accepted reader-cleanup signal. The author observed +reader errors before the table drained and normal reader behavior afterward. +The metadata table contains writer topology only, the trace did not measure +reader DNS timing, and AWS does not document table emptiness as proof of reader +DNS propagation. This policy therefore records an explicitly accepted +operational correlation, not an AWS guarantee. + +`AUTHOR-ACCEPTED-POLICY`: `TOPOLOGY_ABSENT` remains distinct from +`TOPOLOGY_EMPTY` in diagnostics but selects the same phase-specific policy: +rollback before observed writer completion and reader cleanup afterward. Only +the empty-table outcome was observed. + +## Current ProxySQL State Machine + +`SOURCE-CODE`: This is the nominal ordering encoded by the enum names and the +lifecycle currently described by the implementation. The arrows are not +enforced transition edges: + +```text +NONE + -> AVAILABLE + -> WRITER_SWITCHOVER_INITIATED + -> WRITER_SWITCHOVER_IN_PROGRESS + -> WRITER_SWITCHOVER_POST_PROCESSING + -> WRITER_SWITCHOVER_COMPLETED + -> READER_SWITCHOVER_IN_PROGRESS + -> SWITCHOVER_COMPLETED + -> NONE +``` + +`SOURCE-CODE`: The current handler converts the target status and compares enum +ordering with the stored status. A lower value is treated as a backward +transition: the handler runs the current rollback cleanup, and only an observed +`AVAILABLE` phase is then re-entered and initialized. Forward transitions do +not require their immediate predecessor, so a worker can still first observe +`WRITER_SWITCHOVER_POST_PROCESSING` and perform setup in that phase. + +`SOURCE-CODE`: An unknown non-empty target status converts to `NONE`. From an +active higher-valued state this is treated as a backward transition and invokes +rollback cleanup. An empty target status takes an earlier path that directly +sets `NONE` and does not invoke rollback cleanup. A special guard ignores a +repeated raw `WRITER_SWITCHOVER_COMPLETED` while local state is +`READER_SWITCHOVER_IN_PROGRESS`. A newly observed +`WRITER_SWITCHOVER_COMPLETED` is first stored and then immediately advanced to +`READER_SWITCHOVER_IN_PROGRESS` in the same handler call. + +`SOURCE-CODE`: `READER_SWITCHOVER_IN_PROGRESS` and `SWITCHOVER_COMPLETED` are +ProxySQL-inferred and cleanup states, not raw AWS status strings. + +`SOURCE-CODE`: Phase transitions currently perform these actions: + +| Phase or observation | Current action | +|---|---| +| `AVAILABLE` | Set the next-check interval to 250 ms; build the blue/green mapping; resolve green IPs; optionally add the green writer to its configured hostgroup. | +| `WRITER_SWITCHOVER_INITIATED` | On transition, set the next-check interval to 100 ms; invoke mapping, green-IP resolution, and optional green-writer setup; suppress read-only checks for the deployment hostgroups. | +| `WRITER_SWITCHOVER_IN_PROGRESS` | On transition, set the next-check interval to 100 ms; invoke the same setup; suppress read-only checks; demote the mapped blue writer to read-only. | +| `WRITER_SWITCHOVER_POST_PROCESSING` | On transition, set the next-check interval to 100 ms; invoke the same setup; enable or sustain read-only suppression; pin mapped blue names to resolved green IPs; drain matching connections; configure writer placement; shun unmapped readers. | +| `WRITER_SWITCHOVER_COMPLETED` | Enter the inferred reader phase; remove the writer DNS-cache entry; retain mapped reader pins until the topology drains; reset the next-check interval override to the baseline value of `0`. | +| `READER_SWITCHOVER_IN_PROGRESS` plus empty or absent topology | Run successful cleanup: reconcile configured writer membership in the reader hostgroup; unshun recorded readers; remove DNS-cache entries and purge monitor-pool connections for recorded shunned readers and all mapped pairs; drain configured green hostgroups; clear worker bookkeeping; transition through `SWITCHOVER_COMPLETED` to `NONE`. | +| Empty or absent topology from any other non-`NONE` state | Run rollback cleanup: conditionally restore a writer demoted during `WRITER_SWITCHOVER_IN_PROGRESS` or `WRITER_SWITCHOVER_POST_PROCESSING`; reconcile blue reader membership and shuns; remove blue DNS-cache entries; purge related blue monitor-pool connections; leave green rows, statuses, DNS entries, and connections unchanged; clear worker bookkeeping; then enter `NONE`. | +| Recognized backward status transition | Run rollback cleanup. If the new raw status is `AVAILABLE`, re-enter `AVAILABLE`, set the 250 ms interval, and rebuild mapping, resolution, and optional green-writer placement. Other backward statuses leave the worker in `NONE` after cleanup. | +| Worker exit with non-`NONE` state | Run rollback cleanup before destroying the worker-local state. A replacement worker starts with a new state instance. | + +### Current Probe Tuple + +`SOURCE-CODE, AUTHOR-ACCEPTED-POLICY`: When direct probing is active, the +worker takes the port from the writer pair in `bg_map`, not from an arbitrary +blue polling row. That pair records the configured blue writer's port. The +implementation does not consume or validate the TARGET topology row's port; +the author explicitly accepts that a matched source/target pair with different +ports is unsupported. + +`SOURCE-CODE, AUTHOR-ACCEPTED-POLICY`: Automatic mode has no independent green +`mysql_servers` row from which to read TLS configuration, so it intentionally +uses the matched blue writer's `use_ssl` value. + +`SOURCE-CODE, IMPLEMENTATION-CONFORMANCE-OPEN`: Explicit mode is intended to +use the matching green writer row's `use_ssl`. In commit `20247dcf0`, however, +the lookup calls `aws_rds_bgd_match_host(gs->address, green_writer_host)`. +That helper expects a blue hostname as its first argument and a green hostname +as its second argument. Passing the configured green hostname and TARGET green +hostname therefore does not match the ordinary +`-green-.` case. `green_use_ssl` remains unset and the +probe silently falls back to the blue writer's value. AWS-08's policy decision +is resolved, but the current source does not yet implement it. + +### Current Phase-Equality Behavior + +`SOURCE-CODE`: After status conversion, the handler returns immediately when +the converted status equals the stored status. Phase actions run on transition, +not on every observation. Consequently, a transient mapping or DNS failure is +not retried while the same phase continues. + +`AUTHOR-ACCEPTED-POLICY`: The author accepts this failure mode for the current +feature PR. In particular, a first DNS failure in +`WRITER_SWITCHOVER_POST_PROCESSING` can leave a pair unpinned and its old +connections undrained for the remainder of that phase. A subsequent PR is to +add worker-local, per-pair reconciliation that retries unresolved addresses +and applies pin/drain once, rather than rerunning the entire phase action. + +### Current Connection-Retirement Behavior + +`SOURCE-CODE`: `MySrvConnList::mark_connections_unhealthy()` marks every used +connection selected by a BGD drain with both `healthy=false` and +`reusable=false`. Free connections are deleted immediately. The intended used +connection lifecycle is therefore retirement after its current owner releases +it, not cancellation of an in-flight query solely because the drain began. + +`SOURCE-CODE`: `MySQL_Connection::reset()` currently assigns both +`healthy=true` and `reusable=true`. The author observes that the two backend +reset call sites, `handler_again___status_RESETTING_CONNECTION` and +`handler_again___status_CHANGING_USER_SERVER`, are surrounded by logic that +destroys rather than reuses the affected backend connection. That observation +reduces the known exposure, but the terminal nature of a BGD drain remains +implicit and distributed across callers. + +`AUTHOR-ACCEPTED-POLICY, IMPLEMENTATION-CONFORMANCE-OPEN`: The follow-up uses +the existing `healthy` field rather than introducing a second retirement flag: + +1. `MySQL_Connection::reset()` resets session state but does not change + `healthy` from false to true. +2. `MySQL_Thread::push_MyConn_local()` checks `healthy` before adding a + connection to the thread-local cache. An unhealthy connection is sent to + the global destruction path and cannot enter `cached_connections`. +3. `MySQL_HostGroups_Manager::push_MyConn_to_pool()` checks `healthy` after + removing the connection from `ConnectionsUsed` and destroys an unhealthy + connection before any optimization or insertion into `ConnectionsFree`. +4. `push_MyConn_to_pool_array()` remains covered because reusable entries + delegate to `push_MyConn_to_pool()`; the existing `reusable=false` branch + already destroys a drained connection directly. + +The accepted state flow is: + +```text +ACTIVE_BACKEND + healthy=true, reusable=true + | + | BGD drain marks a used connection + v +RETIRE_ON_RELEASE + healthy=false, reusable=false + | + | optional connection/session reset + | (healthy remains false) + v +POOL_RETURN + | + +--> push_MyConn_local: unhealthy -> global destruction path + | + `--> push_MyConn_to_pool: unhealthy -> delete + +No transition returns RETIRE_ON_RELEASE to a free or local pool. +``` + +`AUTHOR-ACCEPTED-POLICY`: Reads of `healthy` in these paths use the same +unlocked connection-field convention used elsewhere in ProxySQL. The author +accepts that race model for this focused fix. This decision does not assert +that a C++ data race is generally safe or introduce a broader locking policy. + +### Current Topology-Absence Behavior + +`SOURCE-CODE`: If local state is `READER_SWITCHOVER_IN_PROGRESS`, +`aws_rds_bgd_handle_topology_absent` runs the full current successful cleanup. +For every other non-`NONE` state, it calls the same cleanup helper with +`rollback=true`. + +`SOURCE-CODE`: Rollback conditionally moves a writer demoted during +`WRITER_SWITCHOVER_IN_PROGRESS` or `WRITER_SWITCHOVER_POST_PROCESSING` back to +the writer role. It then runs the common completion hostgroup action, unshuns +recorded readers, removes DNS-cache entries and purges monitor-pool connections +for recorded shunned readers and all mapped blue pairs, clears the worker +bookkeeping, and enters `NONE`, which clears read-only suppression. Rollback +does not drain green connections, remove green DNS entries, change green +statuses, or remove green rows. + +`SOURCE-CODE, AUTHOR-ACCEPTED-POLICY`: Successful cleanup drains connections +for every server in the configured green writer and reader hostgroups except +`OFFLINE_SOFT` and `OFFLINE_HARD` servers. It also removes those green +hostnames from the DNS and monitor connection caches. It leaves all green +server rows and statuses unchanged. The author assigns membership cleanup to +the administrator, including for a row automatically added to runtime by BGD. + +`SOURCE-CODE`: The current rollback is a one-shot best-effort procedure. Its +effect operations do not return an action result to this controller, there is no +owned effect ledger, and the worker state is cleared even when external state +has not been verified. + +`AUTHOR-ACCEPTED-POLICY`: This loss of per-effect completion and retry state is +intentional. The author accepts the possibility that process termination during +cleanup prevents the controller from proving that every intended postcondition +was reached. The retained-ledger design below remains a reviewer proposal, not +accepted follow-up work. + +`SOURCE-CODE`: The interval result depends on the caller path: + +- A metadata fetch reporting `ER_NO_SUCH_TABLE` sets + `next_check_interval_ms` to `0` before calling the helper. +- Successful or rollback cleanup resets the interval as part of state cleanup. +- If the helper is called while state is already `NONE`, it performs no cleanup; + an existence query or successful empty metadata query does not independently + reset an existing interval override in that case. + +`SOURCE-CODE`: PR 1 documents this current behavior. PR 1 does not change this. + +### Current Worker Lifetime + +`SOURCE-CODE`: State lives on the per-writer-hostgroup worker stack. The worker +and dispatcher compare a generation checksum that combines eligible blue and +green runtime rows. An Admin `mysql_servers` commit refreshes the checksum; +when its value changes, the old worker exits and the dispatcher creates a +replacement. If old state is non-`NONE`, the exit path runs one-shot rollback +cleanup before discarding it. The mapping, shunned-reader records, probe +target, and cleanup identities are not transferred; the replacement starts +with fresh state and rebuilds its map from current runtime configuration. + +`SOURCE-CODE`: The combined checksum fixes the earlier case in which an Admin +commit adding an eligible green row could leave a nonempty partial map alive. +It is a configuration-generation signal, not a per-effect result ledger and +not a retry trigger for DNS recovery. In-process BGD calls to +`publish_mysql_servers_to_runtime()` do not refresh this generation checksum; +that permits the current worker's own hostgroup actions to continue without +self-replacement. + +`AUTHOR-ACCEPTED-POLICY`: Cleanup-on-detach followed by fresh worker state is +the selected worker-replacement contract. A replacement whose first +observation is `SWITCHOVER_COMPLETED` does not reconstruct the prior worker's +map or effect ownership; it enters `READER_SWITCHOVER_IN_PROGRESS` and waits +for topology drain. + +### Current Source Anchors + +`SOURCE-CODE`: The source entry points for the current mechanics are +`parse_aws_rds_topology`, `handle_aws_rds_bgd`, +`aws_rds_bgd_handle_topology_absent`, +`handle_aws_rds_bgd_post_switchover`, and `monitor_RDS_BGD_thread_HG`. These +entry points should be reviewed with this document whenever behavior changes. + +## External Effects And Cleanup Ledger + +`PROPOSED-POLICY`: This section records the reviewer's stronger recovery model +for comparison and possible future reconsideration. The author explicitly +selected one-shot cleanup, worker-local state, and no durable BGD ledger. None +of the ledger states or invariants below is therefore an accepted requirement +for PR #5861 or the accepted same-phase reconciliation follow-up. + +`PROPOSED-POLICY`: Every externally visible effect must have a stable identity +and a cleanup record before the effect is considered applied. + +`PROPOSED-POLICY`: Every mutable effect uses the same stable ownership key: +deployment ID, deployment generation, action ID, and resource ID. The resource +ID identifies the effect-specific resource and does not change when its value +changes. Mutable result data is recorded separately and includes the before +value, intended or applied value, last observed value, and command result. In +particular, a resolved IP, its resolution source, and its expiry are result +data, not stable identity. + +`PROPOSED-POLICY`: An effect has one of these states: + +- `PENDING`: The intended value is not yet verified as applied. +- `APPLIED`: The intended value is verified, but cleanup or handoff remains. +- `REVERTED`: Compare-and-restore verified the owned effect was undone. +- `COMMITTED`: An irreversible effect was verified complete, or ownership of a + retained effect was explicitly handed off to desired runtime configuration. +- `CONFLICT`: The resource no longer has the value applied by this owner, so + automatic restoration would overwrite a newer value or owner. + +`PROPOSED-POLICY`: The active cleanup ledger contains only unsettled, +controller-owned effects in `PENDING`, `APPLIED`, or `CONFLICT`. `REVERTED` and +`COMMITTED` entries leave the active ledger; they may remain as audit +tombstones outside it. + +`PROPOSED-POLICY`: Cleanup uses compare-and-restore. An inverse action is +applied only when the resource still equals the applied value owned by the +ledger entry. Otherwise the effect becomes `CONFLICT`, remains in the active +ledger, and exposes `FAULTED`; cleanup does not overwrite newer configuration +or another owner. + +`PROPOSED-POLICY`: A transitional effect superseded by the accepted desired +post-success state becomes `COMMITTED` only after the controller verifies an +explicit handoff of the resource and its intended value into desired runtime +configuration. A conflict or permanent inability to complete that handoff +enters `FAULTED` with the active ledger retained. + +| Effect | Required identity | Successful postcondition | Required recovery | +|---|---|---|---| +| Blue/green mapping | Common owner key; resource identity is the blue endpoint, green endpoint, and role within the deployment. | All required endpoints are mapped; endpoint and role values are recorded as result data. | Clear the owned mapping or reconstruct it from a new complete observation. | +| Green resolution | Common owner key; resource identity is the green hostname and probe purpose. | A complete writer probe target is available; resolved IP, source, and expiry are recorded as result data. | Retry resolution or clear the incomplete owned result. | +| Direct probe | Common owner key; resource identity is the deployment probe slot, with host or IP, port, and SSL mode recorded as intended and applied result data. | Topology checks use the mapped writer endpoint. | Compare-and-restore the configured blue probe candidates. | +| Green writer placement | Common owner key; resource identity is the hostgroup and server. | The intended server is present with the mapped options, recorded as the applied value. | Compare-and-restore the prior placement, or mark `COMMITTED` only after explicit handoff of the retained placement into desired runtime configuration. | +| Monitor suppression | Common owner key; resource identity is the affected hostgroup and monitor scope. | Read-only monitoring skips only the intended servers. | Compare-and-restore the prior suppression value. | +| Blue writer demotion | Common owner key; resource identity is the server and affected placement; original and applied role and placement are result data. | The temporary role and placement are visible. | During rollback, compare-and-restore the original role and placement. After observed writer completion, successful finalization or `SAFE_TEARDOWN` reconciles and hands off the role and placement to accepted desired post-switchover runtime configuration, then marks the effect `COMMITTED`; it never restores obsolete blue solely because completion was observed or configuration was removed. A conflict or permanent inability to settle enters `FAULTED` with the active ledger retained. | +| DNS pin | Common owner key; resource identity is the hostname; pinned IP is applied result data. | Lookup returns the owned pinned IP. | Compare-and-restore only when the action owner still owns the pin. | +| Connection drain | Common owner key; resource identity is the server and drain generation. | Every connection predating the drain generation is verified retired and cannot be reused. | Never revert; mark `COMMITTED` only after every connection predating the generation is verified retired. | +| Reader shun | Common owner key; resource identity is the hostgroup, hostname, and port; previous status is before-value result data. | The intended reader is `SHUNNED_AWS_BGD`. | Compare-and-restore the recorded prior status. | +| Writer reader-hostgroup membership | Common owner key; resource identity is the writer server and reader hostgroup; original and applied membership are result data. | Membership matches the transition policy. | Compare-and-restore the configured membership. | + +`PROPOSED-POLICY`: The effect ledger is stored as keyed sets or maps. Repeated +observations cannot create duplicate records or ambiguous effect ownership. + +## Required Controller Invariants + +`PROPOSED-POLICY`: The hardened controller must maintain the following safety +and liveness invariants. They describe intended behavior, not the current +implementation. + +### Safety + +1. `IDLE` has no unsettled temporary effects in the active cleanup ledger; + settled audit tombstones may remain outside it. +2. Every visible controller-owned effect remains in the active cleanup ledger + until it is `REVERTED` or `COMMITTED`. +3. Worker termination cannot destroy the only cleanup record for an effect. +4. Repeated observations retry incomplete actions. +5. Completed actions are not reapplied without a new action generation. +6. An unknown or malformed observation, or a query failure, preserves the last + safe state and causes no destructive transition. +7. Topology disappearance before observed writer completion selects rollback. +8. Successful finalization requires observed writer completion. +9. A drained connection cannot become reusable. +10. A direct target includes the writer host or IP, port, and SSL mode. +11. Persistent configuration, runtime configuration, the hostgroup manager, + and exported configuration agree on nullability. +12. Rollback and successful finalization are idempotent. +13. A stale worker cannot apply a result to a newer deployment generation. +14. No deployment-registry lock is held during DNS, SQLite, hostgroup-manager, + connection-pool, or socket operations. + +### Liveness + +`PROPOSED-POLICY`: Liveness holds under fair scheduling, eventual recovery of +retryable dependencies, and unchanged effect ownership. A permanent failure or +ownership conflict converges to externally visible `FAULTED` with the active +ledger retained, rather than an unsafe overwrite or infinite silent retry. + +1. Transient DNS failure remains retryable. +2. Under these liveness conditions, a cancelled deployment eventually restores + the blue configuration; a permanent failure or ownership conflict instead + exposes `FAULTED` while retaining its active ledger. +3. Under these liveness conditions, successful completion eventually removes + temporary pins and shuns; a permanent failure or ownership conflict instead + exposes `FAULTED` while retaining its active ledger. +4. Worker replacement resumes the deployment or safely rolls it back. +5. Failure for one blue/green pair does not hide other pairs. +6. Fast polling is bounded and has an observable reason. +7. Configuration removal cannot abandon outstanding effects. + +## Proposed Controller Model + +`PROPOSED-POLICY`: Controller state is separate from the raw AWS status. The +following state diagram is a reviewer-proposed alternative; it is not the +current implementation enum or an author-accepted implementation contract. + +```text +IDLE + -> TRACKING + -> PREPARING + -> CUTOVER + -> REPOINTING + -> AWAITING_READER_DNS + -> FINALIZING_SUCCESS + -> IDLE only when the active cleanup ledger is settled and empty + +Any active state + topology disappears before observed writer completion + -> ROLLING_BACK + -> IDLE only when the active cleanup ledger is settled and empty + +Any state at or after observed writer completion + configuration removal + -> SAFE_TEARDOWN + -> IDLE only when the active cleanup ledger is settled and empty + +Any state + permanent failure or unrecoverable inconsistency + -> FAULTED with the active cleanup ledger and deployment context retained +``` + +### Observation-Driven Transitions + +`PROPOSED-POLICY`: The controller applies the following transitions from +observations and lifecycle events. The accepted reader-completion signal is a +symbolic policy input pending author validation; it may ultimately be defined +as empty or absent topology after observed writer completion. + +`PROPOSED-POLICY`: Each deployment context has a configuration-management mode +orthogonal to its raw AWS phase. `ACTIVE` means validated configuration still +manages the deployment. `CONFIG_DISABLED` is the explicit-disable form of the +existing `CONFIG_REMOVED` lifecycle event. Either event latches +`REMOVAL_REQUESTED` as a cleanup request. Raw topology observations never clear +`REMOVAL_REQUESTED`. Only an explicit, validated re-add or re-enable of the +same deployment under a new configuration generation may request a return to +`ACTIVE`, and only after ownership and configuration reconciliation succeeds. + +`PROPOSED-POLICY`: Permanent-failure and ownership-conflict rules have highest +precedence. The latched management mode and `ROLLING_BACK` rules are evaluated +next, before repeated, regressed, or generic forward-phase mappings. An +eligible forward-mapping state therefore excludes `ROLLING_BACK`, `FAULTED`, +`FINALIZING_SUCCESS`, and `SAFE_TEARDOWN`. + +`PROPOSED-POLICY`: The normal recognized-phase mappings below apply only to an +initial or forward observation. An observation equal to the recorded phase +uses the repeated-phase rule. An observation lower than the highest trusted +completion evidence uses the regression rule and never causes a reverse +transition. For a newly observed deployment, its initial `IDLE` context is a +pre-completion nonterminal context for these mappings. + +| Controller state and input | Required transition or action | +|---|---| +| Any + permanent failure or ownership conflict | Enter `FAULTED` with the active ledger and deployment context retained. | +| Any + `QUERY_FAILED` | Preserve state and effects, then retry. | +| Any + `UNKNOWN_STATUS` or `MALFORMED_TOPOLOGY` | Preserve state, expose the input, and make no destructive transition; action-result or error policy may enter `FAULTED` when the condition is classified permanent. | +| `ROLLING_BACK` + any pre-completion recognized phase | Remain in `ROLLING_BACK` and record the observation for diagnostics; do not resume cutover unless validated configuration explicitly re-enables the deployment and ownership reconciliation accepts it. | +| `ROLLING_BACK` + `WRITER_COMPLETED` | Stop or cancel pending rollback commands that would restore obsolete blue or remove the promoted target, latch writer-completion evidence, and enter `SAFE_TEARDOWN` when management mode is `REMOVAL_REQUESTED`; otherwise select an author-validated post-completion recovery or finalization path. | +| `ROLLING_BACK` + `TOPOLOGY_ABSENT` or `TOPOLOGY_EMPTY` | Remain in `ROLLING_BACK` and continue reconciling rollback effects. | +| `REMOVAL_REQUESTED` + any observation at or after writer completion | Enter or remain in `SAFE_TEARDOWN`; never apply a generic `AWAITING_READER_DNS` transition or restore blue solely from the raw phase. | +| Any + repeated observation of the same phase | Keep the controller state and reconcile incomplete effects. | +| Any + a regressed recognized phase | Preserve the highest trusted completion evidence and active effects, expose the regression, and make no reverse destructive transition until an author-validated policy decides how to handle it. | +| `IDLE` or `TRACKING` + `AVAILABLE` | Enter or remain in `TRACKING`; reconcile mapping, resolution, and preparation without applying cutover effects. | +| Any eligible forward-mapping state + `WRITER_INITIATED` | Enter `PREPARING`; record the latest observation and reconstruct or reconcile prerequisites. | +| Any eligible forward-mapping state + `WRITER_IN_PROGRESS` | Enter `CUTOVER`; record the latest observation and reconstruct or reconcile prerequisites and required cutover actions. | +| Any eligible forward-mapping state + `WRITER_POST_PROCESSING` | Enter `REPOINTING`; record the latest observation and reconstruct or reconcile all unmet prerequisites and repoint actions. | +| Any eligible forward-mapping state + `WRITER_COMPLETED` | Enter `AWAITING_READER_DNS`; record writer-completion evidence, then reconstruct and verify every unmet prerequisite effect or establish that it is obsolete under an author-validated policy; never infer that a skipped action succeeded. | +| Any pre-completion state + `TOPOLOGY_ABSENT` or `TOPOLOGY_EMPTY` | Enter `ROLLING_BACK`. | +| `ACTIVE` + `AWAITING_READER_DNS` + `ACCEPTED_READER_COMPLETION_SIGNAL` | Enter `FINALIZING_SUCCESS`. | +| `ACTIVE` + `CONFIG_CHANGED` | Preserve the deployment context and active ledger, then reconcile validated new configuration. | +| Any pre-completion state + `CONFIG_REMOVED` or `CONFIG_DISABLED` | Latch `REMOVAL_REQUESTED` and enter `ROLLING_BACK`. | +| Any state at or after observed writer completion + `CONFIG_REMOVED` or `CONFIG_DISABLED` | Latch `REMOVAL_REQUESTED` and enter `SAFE_TEARDOWN`; never restore blue solely because configuration was removed or disabled. | +| `REMOVAL_REQUESTED` + ordinary `CONFIG_CHANGED` or raw topology | Preserve `REMOVAL_REQUESTED`; do not resume generic forward processing. | +| `REMOVAL_REQUESTED` + explicit validated re-add or re-enable | Start a new configuration generation, reconcile ownership and configuration, and return to `ACTIVE` at the controller state appropriate to retained trusted evidence only after reconciliation accepts ownership. | +| Any + `WORKER_RESTARTED` | Attach the new worker generation to the same deployment context, state, and active ledger. | +| `ROLLING_BACK` + active ledger settled and empty | Enter `IDLE`. | +| `FINALIZING_SUCCESS` or `SAFE_TEARDOWN` + active ledger settled and empty | Enter `IDLE`. | + +`PROPOSED-POLICY`: Every action execution returns one classified result: + +- `SUCCEEDED`: The command changed the owned resource and verification observed + the intended value. +- `ALREADY_SATISFIED`: Verification found the owned intended value without + needing to repeat the command. +- `RETRYABLE_FAILURE`: The effect remains pending for a later reconciliation. +- `PERMANENT_FAILURE`: Policy cannot safely complete or retry the effect; enter + `FAULTED` with the active ledger retained. +- `OWNERSHIP_CONFLICT`: The resource does not equal the value applied by this + owner; retain the entry as `CONFLICT` and enter `FAULTED` without overwriting + it. +- `STALE_RESULT`: The result belongs to an older deployment or worker + generation and must not mutate current state or effects. + +`PROPOSED-POLICY`: Repetition count alone does not make a failure permanent. +The executor classification and author-validated error policy determine +whether a failure is retryable or permanent. + +### Reconciliation + +`PROPOSED-POLICY`: Phase changes update controller status and emit an +observable log record, but actions are reconciled on every poll. +Actions returning `RETRYABLE_FAILURE` remain pending. Completed actions return +`ALREADY_SATISFIED` instead of repeating the effect, and stale results are +discarded. Advancing an observation does not prove that its actions completed. + +### Successful Finalization + +`PROPOSED-POLICY`: Successful finalization is selected only when management +mode remains `ACTIVE` after observed writer completion and the controller has +the reader-completion signal accepted by policy after author validation. It +then removes temporary DNS pins, reconciles reader status and writer +reader-hostgroup membership, reconciles writer role and placement with the +accepted desired post-switchover runtime configuration, drains obsolete +green-hostgroup connections, clears the direct probe and monitor suppression, +and clears mapping and resolution records. Successfully restored reversible +entries become `REVERTED`. Connection drains become `COMMITTED` only after +every connection predating the drain generation is verified retired. Any +retained green placement becomes `COMMITTED` only after explicit handoff into +desired runtime configuration. The blue-writer-demotion record becomes +`COMMITTED` only after the controller verifies handoff of writer role and +placement into the accepted desired post-switchover runtime configuration; it +never restores obsolete blue solely after observed writer completion. A +conflict or permanent inability to complete that handoff enters `FAULTED` with +the active ledger retained. The controller enters `IDLE` only when the active +cleanup ledger is settled and empty; settled audit tombstones may remain +outside it. + +`PROPOSED-POLICY`: Successful finalization uses the ordered per-effect +settlement checklist in **Safe Teardown** wherever it applies. Its terminal +desired configuration is the accepted `ACTIVE` post-switchover configuration, +rather than removal intent, but it uses the same evidence gate, ownership +checks, dependency ordering, and `FAULTED` behavior. + +### Safe Teardown + +`PROPOSED-POLICY`: `SAFE_TEARDOWN` is selected when management mode is +`REMOVAL_REQUESTED` at or after observed writer completion. It settles effects +against the accepted terminal post-switchover removal intent and never restores +blue merely because configuration was removed or disabled. + +`PROPOSED-POLICY`: Reader-related DNS pins, reader shuns, writer +reader-hostgroup membership, and direct-probe protection cannot be cleared +before the accepted reader-completion evidence or an author-validated +equivalent is observed. The dispatcher-owned cleanup executor continues +observing through a retained complete probe target and deployment context even +after the configured worker is removed. If accepted evidence cannot be +obtained, or an effect cannot be settled safely because of permanent failure or +ownership conflict, the controller enters externally visible `FAULTED`, +retains the active ledger and context, and does not clear effects merely to +reach `IDLE`. + +`PROPOSED-POLICY`: After the evidence gate is satisfied, the controller settles +effects in this order: + +1. DNS pins: Remove each pin only when the action owner still owns its applied + value. +2. Reader shuns: Compare-and-restore each prior value or hand it off to the + accepted terminal desired configuration. A missing resource that removal + intent deliberately deleted may be `ALREADY_SATISFIED` only after ownership + validation. +3. Writer reader-hostgroup membership, blue writer demotion, and + green writer placement: Reconcile and hand them off to the accepted terminal + post-switchover configuration or removal intent, then mark them `COMMITTED`. + An ownership conflict or permanent settlement failure enters `FAULTED`. +4. Monitor suppression: Clear it only after DNS, reader, and routing-placement + protection are settled. +5. Direct probe: Clear it only after it is no longer needed to obtain evidence + or complete cleanup. +6. Connection drain: Mark it `COMMITTED` only after every connection predating + the drain generation is verified retired. +7. Blue/green mapping and green resolution records: Commit or clear them only + after every dependent effect is settled. + +`PROPOSED-POLICY`: The deployment context enters `IDLE` and becomes eligible +for removal only when the active cleanup ledger is empty. + +### Cancellation Rollback + +`PROPOSED-POLICY`: Topology absence before observed writer completion enters +rollback. Rollback removes owned DNS pins; restores the configured blue probe +candidates, recorded reader statuses, and recorded hostgroup placement; +compare-and-restores the original blue writer role and placement; clears +monitor suppression; removes temporary green placement according to the +validated placement policy; and clears mapping and resolution records. The +controller marks a reversible entry `REVERTED` only after compare-and-restore +verifies the inverse. An irreversible drain or explicitly handed-off retained +placement becomes `COMMITTED` under the ledger rules. The controller enters +`IDLE` only when the active cleanup ledger is settled and empty; a conflict +instead retains the active ledger in `FAULTED`. + +### Query And Data Errors + +`PROPOSED-POLICY`: Query and data errors select the following recovery +behavior; they do not imply successful completion. + +| Condition | Required controller behavior | +|---|---| +| Query timeout or connection failure | Preserve state and retry. | +| Unknown status | Preserve state, expose the unknown value, and make no destructive transition. | +| Malformed topology | Preserve state, expose the malformed input, and make no destructive transition. | +| DNS failure | Keep the resolution action pending. | +| Required mapping missing | Keep the mapping action pending. | +| One reader action fails | Retain completed reader actions and retry the failed reader action. | +| Permanent action failure | Enter `FAULTED` with the active cleanup ledger retained. | +| Ownership conflict | Enter `FAULTED` with the conflicting entry and active ledger retained; do not overwrite the resource. | +| Repeated rollback failure | Remain in `ROLLING_BACK` while classified retryable, or enter `FAULTED` when classified permanent; repetition count alone is not permanent, and the controller never enters `IDLE` with an unsettled entry. | + +## Worker And Configuration Lifetime + +`PROPOSED-POLICY`: Controller state and its effect ledger have deployment +lifetime, not worker-stack lifetime. + +```text +dispatcher creates or retrieves deployment context + -> worker generation N attaches + -> configuration checksum changes + -> generation N detaches; deployment context remains + -> worker generation N+1 attaches and resumes reconciliation +``` + +`PROPOSED-POLICY`: Configuration changes update the deployment context without +clearing it. Disabling or removing configuration before observed writer +completion latches `REMOVAL_REQUESTED` and requests rollback. At or after +observed writer completion, removal latches `REMOVAL_REQUESTED` and enters +`SAFE_TEARDOWN`, including when `FINALIZING_SUCCESS` had already begun; it never +restores blue solely because configuration was removed. + +`PROPOSED-POLICY`: Post-completion configuration removal follows the evidence +gate and ordered settlement checklist in **Safe Teardown**. The terminal desired +configuration reflects validated removal intent. The same checklist reconciles +writer role and placement and marks the demotion record `COMMITTED` only after +verified handoff; it never compare-and-restores obsolete blue after observed +writer completion. + +`PROPOSED-POLICY`: When a configuration worker is removed, the dispatcher +retains or starts an independent cleanup executor until the active ledger is +settled or an externally visible `FAULTED` state is reached. The dispatcher +removes the deployment context only after it reaches `IDLE` with a settled, +empty active ledger. It does not erase a context in `FAULTED`. + +`PROPOSED-POLICY`: Durable SQLite persistence is unnecessary only if every +controller-owned effect is proven to disappear, revert, or be reconstructable +after a full ProxySQL process restart. This includes DNS pins, runtime shuns, +monitor suppression, backend connections and their drain generations, +green writer placement, blue writer demotion, +writer reader-hostgroup membership, and the direct probe target. If any effect +does not meet that condition, the controller persists its ledger or provides +deterministic startup recovery. +The author instead accepts a no-persistence fresh start and the loss of +per-effect ownership across process restart. The condition above is therefore +a reviewer hardening criterion, not a pending author-validation question or a +guarantee of current behavior. + +## Configuration Model + +The feature has blue writer and reader hostgroups. Green hostgroup nullability +depends on row origin; it is not a user-selectable mixed-mode configuration. + +| Row origin and storage | Green writer hostgroup | Green reader hostgroup | Semantics | +|---|---|---|---| +| User row in persistent Admin configuration | Value required | Value required | Explicit green hostgroups. The persistent schema declares both columns `NOT NULL`; a user `NULL` insert is rejected by SQLite. | +| User row materialized into runtime/HGM | Value | Value | The values from persistent configuration are retained with `auto_generated=0`. | +| Runtime row created by automatic discovery | `NULL` | `NULL` | Automatic green handling. The row carries `auto_generated=1` and exists only in runtime/HGM state. | +| Any user mixed combination | Invalid | Invalid | A user row cannot select automatic handling for only one green role. | + +`SOURCE-CODE, AUTHOR-VALIDATED`: The runtime Admin and HGM schemas allow the +two green columns to be nullable because they must represent auto-generated +rows. That storage capability does not make `NULL` valid in the persistent +user table. Defensive `NULL` binding while materializing or dumping HGM rows +likewise does not expand the user configuration contract. + +`SOURCE-CODE, AUTHOR-VALIDATED`: Saving runtime BGD hostgroups to the persistent +Admin table skips every row whose runtime `auto_generated` field is nonzero. +Consequently, a runtime auto-generated row with two `NULL` green hostgroups is +not inserted into the persistent `NOT NULL` table. User rows have both values +and are saved normally. + +`PROPOSED-POLICY`: Configuration updates have generations. Configuration +removal latches `REMOVAL_REQUESTED`. Re-adding or re-enabling the deployment +creates a new validated generation and must reconcile ownership before +resuming controller processing. + +## Author Validation Checklist + +The author response is recorded below. `RESOLVED` means the external evidence +has been scoped correctly or the author explicitly accepted the policy or +limitation. An implementation can still fail to conform to a resolved policy; +that is tracked separately rather than reopening the evidence decision. + +| ID | Recorded author evidence or decision | Review disposition | +|---|---|---| +| AWS-01a | One trace and the AWS examples show the source row present through all pre-completion phases. | `RESOLVED`: This is scoped observation, not a universal prohibition. Missing source identity remains `MALFORMED_TOPOLOGY` and causes no destructive transition. | +| AWS-01b | The trace shows the source row disappearing at `SWITCHOVER_COMPLETED`, leaving one target row. | `RESOLVED`: Source-row absence after completion is expected in the observed lifecycle but is not independently the reader-completion signal. | +| AWS-01c | One trace and the AWS examples show the target row present in every nonempty result. | `RESOLVED`: This is scoped observation. A missing target remains `MALFORMED_TOPOLOGY`. | +| AWS-01d | `TOPOLOGY_EMPTY` was observed only after writer completion. | `RESOLVED`: The author accepts rollback before completion and reader cleanup afterward. The pre-completion impossibility is not stated as an AWS guarantee. | +| AWS-01e | `TOPOLOGY_ABSENT` was not observed; the table remained present and empty. | `RESOLVED AS POLICY`: Preserve the distinct observation but select the same phase boundary as `TOPOLOGY_EMPTY`. | +| AWS-02a | The AWS-provided contract permits rollback during initiated and in-progress; the author separately observed cancellation returning to `AVAILABLE`. | `RESOLVED`: Pre-completion cancellation selects the current one-shot rollback path. Retained settlement is a reviewer proposal, not accepted policy. | +| AWS-02b | The AWS-provided contract says rollback is no longer allowed during post-processing. | `RESOLVED`: At or after writer completion, never restore obsolete blue solely because of cancellation or removal. | +| AWS-03 | The author accepts the same phase-specific policy for empty and absent topology while retaining distinct diagnostics. | `RESOLVED AS POLICY`. | +| AWS-04 | The contract defines `SWITCHOVER_COMPLETED` as writer DNS completion; the trace observes source-row removal in that completed snapshot. | `RESOLVED`: Use the status, not row count alone, as writer-DNS evidence. | +| AWS-05a | The target existed through every phase and lingered about 44 seconds after completion in one trace. | `RESOLVED`: Record the duration only as variable, single-observation evidence. | +| AWS-05b | The author accepts `TOPOLOGY_EMPTY` after observed writer completion as the reader-cleanup signal despite no AWS guarantee or direct reader-DNS timestamp. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: The observational risk is explicit. | +| AWS-06 | The author observed the green hostname stop resolving after completion while the promoted IP survived. | `RESOLVED AS SCOPED OBSERVATION`: Retain a complete probe target while it is needed. | +| AWS-07 | The author agrees the evidence does not establish universal source/target port equality. Commit `20247dcf0` takes the probe port from the matched blue writer pair. Pair-specific port mismatch is explicitly unsupported; different pairs may use different ports. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: Use the matched blue writer's configured port and accept failure for a target using a different port. Do not present equality as an AWS guarantee. | +| AWS-08 | The author agrees `use_ssl` is ProxySQL configuration. Automatic mode uses the matched blue writer's value; explicit mode must use the matched green writer row's value. | `RESOLVED AS AUTHOR-ACCEPTED POLICY; IMPLEMENTATION OPEN`: The tuple sources are precise, but commit `20247dcf0` invokes the hostname matcher with two green names, so normal explicit rows do not set `green_use_ssl` and the probe falls back to blue TLS. | +| AWS-09 | The topology contains writer endpoints only; incomplete explicit reader mapping is expected and unmatched blue readers are shunned. | `RESOLVED AS POLICY`: Track and reconcile readers independently. | +| AWS-10 | Commits `cdffd77ee` and `ac4167cd0` retain auto-added and user-configured green rows on rollback and success. Rollback leaves green connections untouched; success drains eligible green connections but leaves rows and statuses unchanged. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: Green membership is persistent runtime configuration, not a temporary owned effect. Administrative cleanup is required even for an auto-added row. | +| AWS-11a | The author explicitly chooses one-shot worker-exit/configuration-change cleanup and no retained retry ledger. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: Loss of the cleanup context, including when a process terminates during cleanup, is accepted. The stronger retained rollback model is not PR2 scope. | +| AWS-11b | The author explicitly applies the same one-shot choice after completion and relies on the current phase-specific cleanup path. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: No retained `SAFE_TEARDOWN` executor or per-effect settlement record is required. This acceptance does not prove each one-shot operation succeeds. | +| AWS-12a | Commits `727b2166b` and `d45c953d2` combine eligible blue/green rows into the worker generation checksum and refresh it after Admin `mysql_servers` commits. The author accepts that DNS recovery alone does not retry a failed same-phase setup. | `RESOLVED AS AUTHOR-ACCEPTED POLICY AND FOLLOW-UP`: Current PR may leave a POST_PROCESSING pair unpinned and undrained after first-resolution failure. A subsequent PR must implement per-pair retry and exactly-once pin/drain behavior. | +| AWS-12b | The author selects cleanup-on-worker-exit and fresh replacement state. Persistent green membership and rollback-time green connections have no worker ownership under AWS-10. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: A replacement first observing COMPLETED may enter the inferred reader phase without reconstructing the prior map or effects. | +| AWS-13 | The author separates worker replacement from full restart. Replacement performs one-shot rollback then starts fresh. Full restart rebuilds DNS cache, pools, suppression, maps, probe target, and FSM; configured state reloads, while an unsynchronized auto-added runtime green row disappears. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: No durable BGD progress or ownership persistence is required. This is an accepted fresh-start contract, not a traced per-effect guarantee. | +| CFG-01a | User-configured rows require both green hostgroup values. The persistent Admin table declares both columns `NOT NULL`. | `RESOLVED AS AUTHOR-VALIDATED PROXYSQL CONTRACT`: A user `NULL` or mixed row is invalid; no configuration-nullability follow-up is required. | +| CFG-01b | Automatic discovery creates runtime/HGM rows with both green hostgroups `NULL` and `auto_generated=1`; runtime-to-persistent save skips those rows. | `RESOLVED AS AUTHOR-VALIDATED PROXYSQL CONTRACT`: Runtime nullability is intentional and does not conflict with persistent user constraints. | + +### Probe Target Validation Matrix + +| Mode | Host/IP source | Port source | SSL source | Author decision/evidence | +|---|---|---|---|---| +| Automatic | `AUTHOR-VALIDATED`: Resolved IP of the TARGET endpoint from `mysql.rds_topology`. | `AUTHOR-ACCEPTED-POLICY`: Matched blue writer's configured port. A different TARGET port is unsupported and is not forbidden by the recorded AWS evidence. | `AUTHOR-ACCEPTED-POLICY`: Matched blue writer's `use_ssl`, because no independent green row exists. | Policy resolved and implemented by writer-pair selection in `20247dcf0`. | +| Explicit | `AUTHOR-VALIDATED`: Resolved IP of the TARGET endpoint from `mysql.rds_topology`; the configured green writer must identify that target. | `AUTHOR-ACCEPTED-POLICY`: Matched blue writer's configured port. A different TARGET or explicit-green port is unsupported. | `AUTHOR-ACCEPTED-POLICY`: Exact matching green writer row's `use_ssl`. | Policy resolved; implementation is nonconforming because the current two-green-name matcher call does not select the explicit row. | + +`AUTHOR-ACCEPTED-POLICY`: A direct probe target is a complete host or IP, port, +and SSL tuple derived from the matched writer pair, never from an arbitrary +monitor row. The author accepts the blue-port constraint above. Explicit TLS +must be looked up by exact green writer identity, including the supported port, +rather than by applying a blue-to-green matcher to two green names. + +### Restart Validation Matrix + +`AUTHOR-ACCEPTED-POLICY`: Worker replacement and full process restart are +different fresh-start events. Neither recovers a durable BGD ledger. The table +records the selected behavior, not a claim that every one-shot operation has +been traced or verified under crash injection. + +| Effect | Worker replacement in the same process | Full ProxySQL process restart | +|---|---|---| +| Blue DNS pins | The exiting worker attempts to remove mapped blue DNS entries and purge their monitor-pool connections before discarding state. No result is retained for the replacement. | DNS cache and monitor connection pools are recreated; no BGD pin ownership is recovered. | +| Reader shuns | The exiting worker attempts to unshun only readers recorded in its local `shunned_readers` list. The replacement receives no list. | Runtime-only BGD shuns are discarded; server status is rebuilt from administrator configuration. | +| Monitor suppression | Entering `NONE` clears the worker's in-progress suppression entries for its current hostgroup members. | Suppression state is recreated empty. | +| Active connections and drains | Rollback purges mapped blue and recorded-reader monitor-pool connections. Green connections are intentionally untouched. No drain generation or completion result transfers. | Connection pools are recreated; no connection or drain-generation record survives. | +| Green placement | Auto-added and administrator-configured green rows remain in same-process runtime state. They are intentionally not worker-owned. | Administrator-configured rows reload. An auto-added runtime-only row disappears unless independently configured or synchronized into restart input. | +| Blue demotion | Exit cleanup attempts to restore a writer demoted in `WRITER_SWITCHOVER_IN_PROGRESS` or `WRITER_SWITCHOVER_POST_PROCESSING`. The replacement trusts current runtime placement. | Writer status and placement rebuild from administrator configuration. | +| Writer reader-hostgroup membership | Exit cleanup runs the current completion hostgroup action using local map and configuration values, then discards the map. | Membership rebuilds from administrator configuration. | +| Direct probe | The local direct-probe IP and failure counter are discarded. The replacement derives a new target from its first observation and current map, except that a first COMPLETED observation does not rebuild prior effects. | Probe state is recreated empty and derived from newly observed topology. | +| Mapping and resolution | Local pairs and resolved IPs are discarded after one-shot exit cleanup. The replacement builds a new map when its observed phase runs setup. | Pair map and resolution results are recreated from configuration and topology. | + +`AUTHOR-ACCEPTED-POLICY`: If the process terminates during cleanup, no durable +record proves which operations completed. The author accepts that uncertainty +because the relevant in-memory structures are expected to be rebuilt at full +restart. This explicitly rejects the stronger recovery requirement proposed in +**External Effects And Cleanup Ledger** for the current feature and accepted +follow-up scope. + +## Test Mapping For Later PRs + +The six response commits add no automated test. The following cases exercise +the code and policy changed by those commits without assuming the declined +durable-ledger design. + +| Requirement | Named unit/simulator case | Named Admin/TAP case | Observable postcondition | +|---|---|---|---| +| Matched writer probe tuple | `writer_tuple_not_first_poll_row` | `multiple_blue_ports_and_ssl` | With a reader first in the polling result and different ports across pairs, the direct probe uses the mapped blue writer's port and never `hpa[0]`. | +| Automatic TLS source | `auto_green_inherits_writer_ssl` | `automatic_green_tls` | With no explicit green row, the direct probe and auto-added green writer use the matched blue writer's `use_ssl`. | +| Explicit TLS source | `explicit_green_ssl_override` | `explicit_green_tls_differs_from_blue` | With the same supported pair port but blue `use_ssl=0` and explicit green `use_ssl=1`, the direct IP probe enables TLS. This catches the current two-green-name matcher defect. | +| Unsupported within-pair port mismatch | `target_port_mismatch_policy` | `target_port_mismatch_diagnostic` | The implementation's blue-port choice is explicit and observable; the test must not claim AWS guarantees equality. A future rejection diagnostic is preferable to silent probing of the wrong port. | +| Eligible green generation checksum | `green_checksum_matrix` | `admin_green_add_remove_ssl_status` | Add/remove, port, `use_ssl`, and transitions into or out of `OFFLINE_SOFT`/`OFFLINE_HARD` change the checksum and replace workers; irrelevant changes do not. | +| Admin commit during active phase | `config_change_exits_worker` | `load_mysql_servers_mid_switchover` | The old worker runs one-shot rollback, the dispatcher joins it, and the replacement builds a new map from the committed runtime rows. | +| Green membership persistence | `green_row_persists_cancel_and_success` | `green_row_lifecycle` | Auto-added and user rows remain after rollback and success; no existing status is changed. | +| Green drain policy | `green_drain_status_matrix` | `green_hg_cleanup` | Rollback drains no green connections. Success drains `ONLINE`, `SHUNNED`, and `SHUNNED_AWS_BGD` green servers while leaving `OFFLINE_SOFT` and `OFFLINE_HARD` untouched. Rows remain present. | +| Offline status exclusions | `offline_servers_not_acted_on` | `offline_soft_hard_servers` | Blue servers in either offline status do not participate in mapping or unmatched-reader shunning; green servers in either status are not drained. | +| Terminal connection retirement | `unhealthy_survives_reset` | `drained_used_connection_not_repooled` | After a drain marks a used connection unhealthy, reset does not revive it and neither local nor global pool return can place it in a free cache. | +| Persistent/user green hostgroups | `user_green_hostgroups_not_null` | `user_configuration_requires_both_green_hgs` | Persistent user inserts with either green hostgroup `NULL` fail; a row with both values loads with `auto_generated=0`. | +| Automatic runtime row persistence | `auto_generated_null_green_hgs` | `save_runtime_skips_auto_generated_bgd` | Auto-discovery creates a runtime row with both green hostgroups `NULL` and `auto_generated=1`; saving runtime to memory/disk does not persist that row. | +| First observation COMPLETED | `fresh_worker_first_completed` | `replace_worker_at_completed` | Fresh state advances to the inferred reader phase without reconstructing a prior map, then finishes on topology drain. | +| Full restart fresh start | `restart_discards_bgd_state` | `proxysql_restart_fixture` | DNS cache, pools, suppression, mapping, probe target, and FSM are recreated; configured rows reload; an unsynchronized auto-added runtime-only green row does not. | +| Same-phase DNS retry follow-up | `dns_retry_same_post_per_pair` | `first_resolution_fails_then_succeeds` | Accepted follow-up only: the unresolved pair retries while phase is unchanged; successful pairs are not redrained; the recovered pair is pinned and drained exactly once. | +| Partial pair progress follow-up | `one_pair_fails` | `multiple_reader_fixture` | Accepted follow-up only: successful pair state is retained worker-locally and only the failed pair retries. | + +`PROPOSED-POLICY`: The simulator cases previously proposed for durable effect +ownership, compare-and-restore, retained `FAULTED` state, cleanup across stale +worker generations, and a persistent restart ledger remain useful reviewer +hardening ideas. They are not author-accepted follow-up requirements after AWS-11a, +AWS-11b, AWS-12b, and AWS-13. Implementing them would require a new policy +decision rather than treating this document as approval. + +## Review Gate + +The author has answered all 23 validation IDs. No external +`REVIEW-VALIDATION-PENDING` claim remains. The evidence gate is therefore +closed, with observational scope and accepted operational risks preserved in +the checklist rather than promoted to AWS guarantees. + +The source review remains open on implementation and verification: + +1. Fix explicit green TLS selection. The current + `aws_rds_bgd_match_host(gs->address, green_writer_host)` call supplies two + green names to a blue-to-green matcher, so an explicit green `use_ssl` + differing from blue is not selected. +2. Make unhealthy connection retirement terminal across reset, local pool + return, and global pool return. The follow-up uses the existing `healthy` + field and does not add a second flag. +3. Track the author-accepted same-phase DNS failure as required follow-up work. + Until per-pair reconciliation exists, a transient first resolution failure + in POST_PROCESSING can leave traffic unpinned and old connections undrained. + Acceptance documents the risk; it does not make the failure safe. +4. Add focused simulator and TAP coverage for the response commits and these + follow-ups. Registration in `test/tap/groups/groups.json` is insufficient: + an automatic PR check must build the BGD test flavor and execute the BGD + simulator group. + +### Follow-up PR Sequence + +PR #5861 remains the live umbrella PR into `v3.0`. Every implementation and +test follow-up below targets `feature/aws-rds-monitor`, so each accepted change +becomes part of #5861 rather than replacing or closing it. The originally +proposed broad durable-ledger/controller PR is not part of this sequence. + +| Review PR | Scope | Dependency and completion signal | +|---|---|---| +| PR1: #5934 | This document only: evidence, accepted risks, current behavior, and follow-up contract. | Ready for author approval; merge into `feature/aws-rds-monitor` before implementation follow-ups so their scope is stable. | +| PR2: BGD cluster-simulator foundation and CI | Extend the existing `test/deps/cluster_simulator` architecture with a BGD mode capable of serving ordered `mysql.rds_topology` observations and probe outcomes. Add the matching TEST build mode, thin TAP wrapper/group, one smoke payload, and an automatic CI job that actually executes the group. | No production behavior change. Provides the reusable harness required by PR6. A successful compile-only `CI-maketest` job is not completion evidence. | +| PR3: probe target and explicit TLS | Correct AWS-08 by selecting the exact supported explicit green writer row and its `use_ssl`, while retaining the matched blue writer port and automatic-mode blue TLS fallback. | Depends only on the documented contract. Focused unit/TAP evidence must distinguish blue `use_ssl=0` from explicit green `use_ssl=1`. | +| PR4: terminal connection retirement | Preserve `healthy=false` across `MySQL_Connection::reset()` and destroy unhealthy connections in local and global pool-return paths. Do not introduce another flag or a new locking policy. | Focused tests prove a drained used connection cannot enter either free pool after reset or release. | +| PR5: same-phase per-pair reconciliation | Replace phase-equality no-op behavior with worker-local reconciliation for incomplete map/resolution/pin/drain work. Retry only incomplete pairs and never redrain a pair already completed in the current worker generation. | Depends on the accepted one-shot worker model; it must not introduce durable ownership or restart recovery. | +| PR6: simulator-driven BGD scenario suite | Use PR2's simulator to cover normal lifecycle, late entry, cancellation, topology drain, direct probe tuple/TLS, offline exclusions, worker replacement, terminal connection retirement where observable, and PR5 DNS failure/recovery. | Depends on PR2 and should normally follow PR3-PR5 so the suite validates final behavior rather than encoding known failures. All payloads run in the automatic BGD simulator CI group. | + +Any retained cleanup ledger, durable restart ownership, or alternative +controller state machine requires a new author policy decision. Detailed +implementation handoffs for PR2 through PR6 are maintained in the review +worktree root for transfer to the author; they are review artifacts and are +intentionally not part of this document-only PR. From 177a4bb2880c59431d2cec57765669db3c024f1f Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 16 Jul 2026 19:40:29 +0000 Subject: [PATCH 39/81] fix: Refresh `mysql_servers_to_monitor` after adding discovered green host in green HG Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 3 ++- lib/MySQL_HostGroups_Manager.cpp | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 148303b94f..1d33bae58c 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -1106,7 +1106,8 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { * @brief Aligns the runtime 'mysql_servers' table + checksums with the server state in MyHGM. * * @details One-way alignment (in-memory -> runtime): regenerates the runtime 'mysql_servers' table - * from the current in-memory MyHGM state and recomputes/republishes the global checksum. + * from the current in-memory MyHGM state, recomputes/republishes the global checksum, and refreshes + * 'mysql_servers_to_monitor' for the regular monitor threads. * * @note Caller must hold wrlock(). */ diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index f9414244ae..ba2971ba75 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -3969,12 +3969,14 @@ void MySQL_HostGroups_Manager::aws_rds_bgd_set_runtime_status(unsigned int write * @brief Aligns the runtime 'mysql_servers' table + checksums with the server state in MyHGM. * * @details One-way alignment (in-memory -> runtime): regenerates the runtime 'mysql_servers' table - * from the current in-memory 'MyHGC'/'MySrvC' structures and recomputes/republishes the global - * 'mysql_servers' checksum for cluster sync. + * from the current in-memory 'MyHGC'/'MySrvC' structures, recomputes/republishes the global + * 'mysql_servers' checksum for cluster sync, and refreshes 'mysql_servers_to_monitor' for the + * regular monitor threads. * * @note the caller MUST already hold 'wrlock()'. */ void MySQL_HostGroups_Manager::publish_mysql_servers_to_runtime() { + // update runtime table purge_mysql_servers_table(); proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers\n"); mydb->execute("DELETE FROM mysql_servers"); @@ -3990,6 +3992,9 @@ void MySQL_HostGroups_Manager::publish_mysql_servers_to_runtime() { pthread_mutex_lock(&GloVars.checksum_mutex); update_glovars_mysql_servers_checksum(mysrvs_checksum); pthread_mutex_unlock(&GloVars.checksum_mutex); + + // update monitor table + update_table_mysql_servers_for_monitor(false); } /** From b05b907e0ca4d0aa068467115a034c83215cb56c Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Fri, 17 Jul 2026 07:02:25 +0000 Subject: [PATCH 40/81] Fixes for AWS RDS BGD topology probe and auto-discovery - `mysql.rds_topology` table only exists in blue/green writer hosts, not in read replicas. Restricted the probe candidate list to writer. - Add auto-discovered green writer to `green_writer_hostgroup` with hostgroup's default attributes. This aligns BGD monitor with other modules such as group-replication and aurora. Signed-off-by: Wazir Ahmed --- include/MySQL_Monitor.hpp | 4 ++-- lib/MySQL_HostGroups_Manager.cpp | 6 ++++-- lib/MySQL_Monitor.cpp | 28 ++++++++++++---------------- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 2f233f1766..f57803a9f0 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -652,8 +652,8 @@ class MySQL_Monitor { /** * @brief AWS RDS BGD monitor thread entry point. * - * @details Spawns one worker (monitor_RDS_BGD_thread_HG) per writer hostgroup; each worker picks a pingable host, - * probes 'mysql.rds_topology' and dispatches to a handler based on the detected topology shape. + * @details Spawns one worker (monitor_RDS_BGD_thread_HG) per writer hostgroup; each worker picks a pingable writer, + * probes 'mysql.rds_topology' and dispatches based on the detected topology shape. * Workers are (re)spawned whenever the AWS_RDS_BGD_Hosts_checksum changes. */ void * monitor_aws_rds_bgd(); diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index ba2971ba75..17d27ea804 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -7135,7 +7135,9 @@ void MySQL_HostGroups_Manager::update_aws_aurora_hosts_monitor_resultset(bool lo const char SELECT_AWS_RDS_BGD_BLUE_SERVERS_FOR_MONITOR[] { "SELECT writer_hostgroup, reader_hostgroup, hostname, port, MAX(use_ssl) use_ssl, green_writer_hostgroup," - " green_reader_hostgroup, check_interval_ms, check_timeout_ms, writer_is_also_reader FROM mysql_servers" + " green_reader_hostgroup, check_interval_ms, check_timeout_ms, writer_is_also_reader," + " MAX(hostgroup_id=writer_hostgroup) is_writer" + " FROM mysql_servers" " JOIN mysql_aws_rds_bgd_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup" " WHERE active=1 AND mysql_servers.status NOT IN (2,3)" " GROUP BY writer_hostgroup, hostname, port" @@ -7169,7 +7171,7 @@ void MySQL_HostGroups_Manager::update_aws_rds_bgd_hosts_monitor_resultset(bool l // Unlike other monitor resultset/checksum pairs, BGD intentionally tracks different data in each. // // AWS_RDS_Blue_Hosts_resultset contains only blue hosts. The BGD monitor dispatcher uses it to start - // workers, and each worker uses it as the list of servers eligible for mysql.rds_topology polling. + // workers, and each worker uses it to select its `mysql.rds_topology` probe candidates. // // AWS_RDS_BGD_Hosts_checksum combines the blue and green resultset checksums. Workers and the dispatcher // use it as a generation signal: relevant changes in mysql_servers or mysql_aws_rds_bgd_hostgroups diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index a82bb4b889..e278408475 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6746,12 +6746,14 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { // Columns: // 0 writer_hostgroup, 1 reader_hostgroup, 2 hostname, 3 port, 4 use_ssl, // 5 green_writer_hostgroup, 6 green_reader_hostgroup, 7 check_interval_ms, - // 8 check_timeout_ms, 9 writer_is_also_reader + // 8 check_timeout_ms, 9 writer_is_also_reader, 10 is_writer pthread_mutex_lock(&GloMyMon->aws_rds_bgd_mutex); initial_checksum = GloMyMon->AWS_RDS_BGD_Hosts_checksum; for (SQLite3_row *r : GloMyMon->AWS_RDS_Blue_Hosts_resultset->rows) { if (atoi(r->fields[0]) == (int)wHG) { - num_hosts++; + if (atoi(r->fields[10]) != 0) { + num_hosts++; + } if (st.reader_hg == 0) { st.reader_hg = atoi(r->fields[1]); } @@ -6775,7 +6777,8 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { host_def_t *hpa = (host_def_t *)malloc(sizeof(host_def_t)*(num_hosts ? num_hosts : 1)); for (SQLite3_row *r : GloMyMon->AWS_RDS_Blue_Hosts_resultset->rows) { - if (atoi(r->fields[0]) == (int)wHG) { + // r->writer_hostgroup == wHG && r->is_writer != 0 + if (atoi(r->fields[0]) == (int)wHG && atoi(r->fields[10]) != 0) { hpa[cur_host_idx].host = strdup(r->fields[2]); hpa[cur_host_idx].port = atoi(r->fields[3]); hpa[cur_host_idx].use_ssl = atoi(r->fields[4]); @@ -7279,9 +7282,6 @@ static void aws_rds_bgd_resolve_green_ips(AWS_RDS_BGD_State& st) { /** * @brief Add the green writer to green_writer_hostgroup, when that hostgroup is configured. -* -* @details Mirrors the blue writer's connection settings (weight/max_connections/use_ssl) onto -* the green writer. Existing rows, including OFFLINE_HARD rows, are left unchanged. */ static void aws_rds_bgd_add_green_writer_in_hg(AWS_RDS_BGD_State& st) { if (st.green_writer_hg < 0) { @@ -7293,15 +7293,11 @@ static void aws_rds_bgd_add_green_writer_in_hg(AWS_RDS_BGD_State& st) { } srv_info_t srv_info { p.green_host, (uint16_t)p.port, "AWS RDS BGD green writer" }; - srv_opts_t srv_opts { p.blue_weight, p.blue_max_conns, p.blue_use_ssl }; + srv_opts_t srv_opts { -1, -1, -1 }; MyHGM->wrlock(); - MySrvC* srvc = MyHGM->find_server_in_hg( - (uint32_t)st.green_writer_hg, p.green_host, p.port); - if (srvc == nullptr) { - int rc = MyHGM->create_new_server_in_hg((uint32_t)st.green_writer_hg, srv_info, srv_opts); - if (rc == 0) { - MyHGM->publish_mysql_servers_to_runtime(); - } + int rc = MyHGM->create_new_server_in_hg((uint32_t)st.green_writer_hg, srv_info, srv_opts); + if (rc == 0) { + MyHGM->publish_mysql_servers_to_runtime(); } MyHGM->wrunlock(); break; @@ -7827,8 +7823,8 @@ void MySQL_Monitor::set_aws_rds_bgd_server_in_progress(unsigned int writer_hg, u /** * @brief AWS RDS BGD monitor thread entry point. * -* @details Spawns one worker (monitor_RDS_BGD_thread_HG) per writer hostgroup; each worker picks a pingable host, -* probes 'mysql.rds_topology' and dispatches to a handler based on the detected topology shape. +* @details Spawns one worker (monitor_RDS_BGD_thread_HG) per writer hostgroup; each worker picks a pingable +* writer, probes 'mysql.rds_topology' and dispatches based on the detected topology shape. * Workers are (re)spawned whenever the AWS_RDS_BGD_Hosts_checksum changes. */ void * MySQL_Monitor::monitor_aws_rds_bgd() { From e259f2c64b2c3541f29de3d21d8e8b075f985d95 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sun, 19 Jul 2026 21:27:23 +0000 Subject: [PATCH 41/81] docs: Define AWS RDS BGD simulator design - Document the SQLite3 server contract, TAP helper API, and CI integration. - Move the monitor contract under the AWS Blue/Green documentation directory. Signed-off-by: Wazir Ahmed --- .../RDS_BGD_Monitor.md} | 24 +- doc/AWS_Blue_Green/RDS_BGD_Simulator.md | 676 ++++++++++++++++++ 2 files changed, 688 insertions(+), 12 deletions(-) rename doc/{AWS_RDS_BLUE_GREEN_MONITOR.md => AWS_Blue_Green/RDS_BGD_Monitor.md} (97%) create mode 100644 doc/AWS_Blue_Green/RDS_BGD_Simulator.md diff --git a/doc/AWS_RDS_BLUE_GREEN_MONITOR.md b/doc/AWS_Blue_Green/RDS_BGD_Monitor.md similarity index 97% rename from doc/AWS_RDS_BLUE_GREEN_MONITOR.md rename to doc/AWS_Blue_Green/RDS_BGD_Monitor.md index fc5f1ddbd4..efc5ed02a4 100644 --- a/doc/AWS_RDS_BLUE_GREEN_MONITOR.md +++ b/doc/AWS_Blue_Green/RDS_BGD_Monitor.md @@ -8,6 +8,8 @@ IMPLEMENTATION CONFORMANCE OPEN **Primary monitor entry points:** `include/MySQL_Monitor.hpp`, `lib/MySQL_Monitor.cpp` +**Simulator design:** [RDS_BGD_Simulator.md](RDS_BGD_Simulator.md) + **Related implementation:** `include/DNS_Cache.hpp`, `lib/DNS_Cache.cpp`, `include/MySQL_HostGroups_Manager.h`, `lib/MySQL_HostGroups_Manager.cpp`, `include/mysql_connection.h`, `lib/mysql_connection.cpp`, @@ -104,12 +106,11 @@ review: it, and both local and global pool-return paths must destroy an unhealthy connection instead of caching it. -The author also selected the existing cluster simulator under -`test/deps/cluster_simulator` and its TAP group integration under -`test/tap/groups` as the test foundation. The simulator foundation and the BGD -scenario suite are deliberately separate follow-up PRs. Registration in -`groups.json` is not considered CI integration by itself; the BGD simulator -group must be executed by an automatic PR check. +The BGD test foundation uses ProxySQL's SQLite3 server, compiled under +`TEST_RDS_BGD` and controlled directly by each TAP test. The simulator +foundation and the BGD scenario suite are deliberately separate follow-up PRs. +Registration in `groups.json` is not considered CI integration by itself; the +BGD simulator group must be executed by an automatic PR check. ## Scope @@ -1019,14 +1020,13 @@ proposed broad durable-ledger/controller PR is not part of this sequence. | Review PR | Scope | Dependency and completion signal | |---|---|---| | PR1: #5934 | This document only: evidence, accepted risks, current behavior, and follow-up contract. | Ready for author approval; merge into `feature/aws-rds-monitor` before implementation follow-ups so their scope is stable. | -| PR2: BGD cluster-simulator foundation and CI | Extend the existing `test/deps/cluster_simulator` architecture with a BGD mode capable of serving ordered `mysql.rds_topology` observations and probe outcomes. Add the matching TEST build mode, thin TAP wrapper/group, one smoke payload, and an automatic CI job that actually executes the group. | No production behavior change. Provides the reusable harness required by PR6. A successful compile-only `CI-maketest` job is not completion evidence. | +| PR2: BGD simulator foundation and CI | Add the TAP-controlled SQLite3-server simulator defined in [RDS_BGD_Simulator.md](RDS_BGD_Simulator.md): the `TEST_RDS_BGD` build mode, IP-keyed topology responses, common and BGD TAP helpers, a simulator group, an end-to-end acceptance smoke test, and an automatic CI job that executes the group. | No production behavior change. Provides the reusable harness required by PR6. A successful compile-only `CI-maketest` job is not completion evidence. | | PR3: probe target and explicit TLS | Correct AWS-08 by selecting the exact supported explicit green writer row and its `use_ssl`, while retaining the matched blue writer port and automatic-mode blue TLS fallback. | Depends only on the documented contract. Focused unit/TAP evidence must distinguish blue `use_ssl=0` from explicit green `use_ssl=1`. | | PR4: terminal connection retirement | Preserve `healthy=false` across `MySQL_Connection::reset()` and destroy unhealthy connections in local and global pool-return paths. Do not introduce another flag or a new locking policy. | Focused tests prove a drained used connection cannot enter either free pool after reset or release. | | PR5: same-phase per-pair reconciliation | Replace phase-equality no-op behavior with worker-local reconciliation for incomplete map/resolution/pin/drain work. Retry only incomplete pairs and never redrain a pair already completed in the current worker generation. | Depends on the accepted one-shot worker model; it must not introduce durable ownership or restart recovery. | -| PR6: simulator-driven BGD scenario suite | Use PR2's simulator to cover normal lifecycle, late entry, cancellation, topology drain, direct probe tuple/TLS, offline exclusions, worker replacement, terminal connection retirement where observable, and PR5 DNS failure/recovery. | Depends on PR2 and should normally follow PR3-PR5 so the suite validates final behavior rather than encoding known failures. All payloads run in the automatic BGD simulator CI group. | +| PR6: simulator-driven BGD scenario suite | Use PR2's simulator to cover configuration and discovery order, automatic and explicit rows, worker replacement, normal lifecycle, late entry, cancellation and rollback, topology drain, direct probe tuple/TLS, offline exclusions, terminal connection retirement where observable, and PR5 DNS failure/recovery. | Depends on PR2 and should normally follow PR3-PR5 so the suite validates final behavior rather than encoding known failures. All payloads run in the automatic BGD simulator CI group. | Any retained cleanup ledger, durable restart ownership, or alternative -controller state machine requires a new author policy decision. Detailed -implementation handoffs for PR2 through PR6 are maintained in the review -worktree root for transfer to the author; they are review artifacts and are -intentionally not part of this document-only PR. +controller state machine requires a new author policy decision. The simulator +contract and integration design consumed by PR2 and PR6 are defined in +[RDS_BGD_Simulator.md](RDS_BGD_Simulator.md). diff --git a/doc/AWS_Blue_Green/RDS_BGD_Simulator.md b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md new file mode 100644 index 0000000000..93c1e26c2f --- /dev/null +++ b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md @@ -0,0 +1,676 @@ +# AWS RDS Blue/Green Deployment Simulator + +**Document status:** DESIGN APPROVED; IMPLEMENTATION NOT STARTED + +**Applies to:** `TEST_RDS_BGD`, the SQLite3-server simulation surface, BGD TAP +helpers, the local Docker runner, and the matching GitHub Actions job + +**Related monitor contract:** [RDS_BGD_Monitor.md](RDS_BGD_Monitor.md) + +## Purpose + +This document defines the simulator used to test ProxySQL's AWS RDS Blue/Green +Deployment monitor. It combines the behavioral contract, SQLite3-server +changes, TAP helper API, network fixture, local runner, CI job, and supported +coverage into one implementation specification. + +## Architecture + +The TAP test is the scenario controller. It configures ProxySQL with AWS-style +hostnames, writes IP-keyed backend state to ProxySQL's SQLite3 server, changes +that state to drive the BGD FSM, and verifies ProxySQL through runtime, +statistics, and simulator probe-log tables. + +No `test/deps/cluster_simulator` process or backend database container is +required. A common TAP helper owns reusable SQLite3-server operations, while a +BGD helper translates explicit test intent into topology state and probe-log +queries. Neither helper advances the FSM or owns scenario timing. + +## `TEST_RDS_BGD` Boundary + +Simulator tables, BGD response interception, listener changes, and supporting +members are compiled only under `TEST_RDS_BGD`. A production build contains no +BGD simulator surface and preserves the existing production monitor queries, +connection behavior, and DNS behavior. + +The flag reuses shared TEST-mode SQLite3-server infrastructure, including the +existing `READONLY_STATUS` mechanism, without changing the behavior of +`TEST_AURORA`, `TEST_GALERA`, `TEST_GROUPREP`, `TEST_READONLY`, or +`TEST_REPLICATIONLAG` builds. + +## Network Model + +The BGD TAP group injects a shared `/etc/hosts` map containing AWS-style blue +and green names. Every hostname resolves to a distinct loopback IP and uses +port 3306, preserving the address shape used by AWS while a single wildcard +SQLite3-server listener handles all simulated endpoints. + +Tests add servers to ProxySQL by hostname and configure simulator state using +the corresponding IP. Distinct destination IPs retain the blue/green split +when ProxySQL resolves a hostname or directly probes the resolved green IP. + +The map reserves multiple clusters and two green endpoint sets for cluster 1. +Tests configure only the endpoints they need: separate clusters support +simultaneous switchovers, and the alternate cluster-1 green set supports a +second switchover after the previous FSM resets on empty or absent topology. + +## Backend Identity and Topology Ownership + +ProxySQL sends the production BGD queries unchanged. The SQLite3 server calls +`getsockname()` on the accepted connection and uses the resulting +`backend_ip, backend_port` as the simulator key; no hostname, address, port, or +comment is appended to the query. + +Topology belongs only to backend keys explicitly updated by the TAP test. A +normal scenario publishes metadata to the blue and green writer IPs; readers +have no topology table unless a test intentionally configures one. The helper +does not copy state between deployment members implicitly. + +Under `TEST_RDS_BGD`, use `sockaddr_storage` for IPv4 and IPv6-safe local +address extraction. Failure to resolve the accepted local address is a +simulator error and must not fall back to an arbitrary backend row. + +## SQLite3-Server Storage + +Create the following tables in `SQLite3_Server::init()` and store them in the +existing persistent `GloVars.sqlite3serverdb` database: + +```sql +CREATE TABLE RDS_BGD_CONTROL ( + backend_ip TEXT NOT NULL, + backend_port INTEGER NOT NULL, + topology_present INTEGER NOT NULL DEFAULT 0 CHECK (topology_present IN (0,1)), + error_code INTEGER NOT NULL DEFAULT 0, + error_msg TEXT NOT NULL DEFAULT '', + PRIMARY KEY (backend_ip, backend_port) +); + +CREATE TABLE RDS_BGD_TOPOLOGY ( + backend_ip TEXT NOT NULL, + backend_port INTEGER NOT NULL, + row_order INTEGER NOT NULL, + id TEXT NOT NULL, + endpoint TEXT NOT NULL, + topology_port INTEGER NOT NULL, + role TEXT NOT NULL, + status TEXT NOT NULL, + PRIMARY KEY (backend_ip, backend_port, row_order) +); + +CREATE TABLE RDS_BGD_PROBE_LOG ( + sequence_id INTEGER PRIMARY KEY AUTOINCREMENT, + backend_ip TEXT NOT NULL, + backend_port INTEGER NOT NULL, + probe_kind TEXT NOT NULL CHECK (probe_kind IN ('table_check','metadata')), + encrypted INTEGER NOT NULL CHECK (encrypted IN (0,1)) +); +``` + +Each SQLite3-server session already opens this database in WAL/FULLMUTEX mode. +TAP writes and monitor reads therefore share persistent state without an +attached in-memory schema or a control connection that keeps data alive. + +## Query Dispatch + +Run the existing SQL normalization first: collapse whitespace, remove trailing +spaces or semicolons, and compare case-insensitively. Intercept only a complete +match for one of the production BGD constants: + +```sql +SELECT 1 FROM information_schema.TABLES + WHERE TABLE_SCHEMA='mysql' AND TABLE_NAME='rds_topology' + +SELECT * FROM mysql.rds_topology +``` + +For either match, resolve the accepted backend key, load its control row, +select the response described below, append a probe-log row, and send the +result. An address-extraction failure returns a simulator error without +selecting state or logging an invalid backend identity. + +All other statements continue through normal SQLite3-server handling. TAP +control and inspection statements against the simulator tables are not +rewritten or recorded as BGD monitor probes. + +## Control-State Meaning + +The TAP helper updates the control and topology tables in one transaction, so +a monitor query observes either the previous state or the complete new state. +The supported states are: + +| `RDS_BGD_CONTROL` state | Topology rows | Meaning | +|---|---|---| +| No backend row | None | Backend is unconfigured; topology is absent. | +| `topology_present=1`, `error_code=0` | One or more | Return the configured topology. | +| `topology_present=1`, `error_code=0` | Empty | Table exists but contains no topology. | +| `topology_present=1`, `error_code!=0` | Unchanged | Table exists, but its metadata query fails. | +| `topology_present=0`, `error_code=1146` | Empty | Table has been dropped. | + +Topology update and delete operations clear `error_code` and `error_msg`. +Configured errors other than 1146 mark the table present and retain its rows; +error 1146 marks it absent. Dropping topology also removes its rows. Other flag +combinations are invalid helper state. + +## Topology Responses + +The table check consults only `topology_present`. Metadata handling applies a +configured error before reading topology rows: + +| Query | Selected backend state | MySQL response | +|---|---|---| +| Table check | No control row or `topology_present=0` | Successful result with zero rows. | +| Table check | `topology_present=1` | One column named `1`, containing one row with value `1`. | +| Metadata | No control row | Error 1146: `Table 'mysql.rds_topology' doesn't exist`. | +| Metadata | `error_code!=0` | Stored `error_code` and `error_msg`. | +| Metadata | `error_code=0`, `topology_present=0` | Error 1146 as a defensive fallback. | +| Metadata | `error_code=0`, `topology_present=1` | Ordered backend rows; an empty set remains successful. | + +A successful metadata result exposes `id, endpoint, port, role, status`. +`topology_port` supplies the `port` result, and `row_order` determines row +order. Rows belonging to another backend key must never enter the result. + +## Error Packets and Probe Log + +The existing `send_MySQL_ERR()` always returns error 1045. Add an overload that +accepts an error code and message; error 1146 uses SQLSTATE `42S02`, while other +configured simulator errors may use `HY000` unless a test requires a specific +mapping. + +Every handled topology-table check or metadata query appends one row to +`RDS_BGD_PROBE_LOG`, including empty and error responses. `sequence_id` +preserves order, `probe_kind` identifies the query, `backend_ip, backend_port` +identify the destination, and `encrypted` records the accepted stream's TLS +state. + +The TAP test is the only probe-log consumer; ProxySQL never reads it. A test +captures a sequence watermark before changing state and then reads later rows +to verify the selected destination and TLS mode. A probe-log insertion failure +is a simulator failure and must not be silently reported as a normal backend +response. + +## `read_only` Reuse + +Build the existing `READONLY_STATUS` table for `TEST_RDS_BGD`, but do not add a +BGD-specific read-only table or call `enable_readonly_testing()`. The TAP test +owns ProxySQL hostgroup and server configuration. + +For the production `SELECT @@global.read_only ...` monitor query, resolve the +same backend key and select `READONLY_STATUS` using the backend IP as its +`hostname` value. Return the configured value as one `read_only` column; a +missing entry uses the existing safe default of `read_only=1`. + +This path does not consult `RDS_BGD_CONTROL` or write `RDS_BGD_PROBE_LOG`. The +legacy `TEST_READONLY` query-suffix behavior remains unchanged in its own +build. + +## TAP Helper API + +The API follows existing TAP conventions: write methods return `EXIT_SUCCESS` +or `EXIT_FAILURE`, and read methods return the existing `rc_t` type. The +signatures below are the initial API and may grow with reviewed test cases. + +### Common Endpoint + +```cpp +struct Simulator_Endpoint { + std::string host; + int port; +}; +``` + +Identifies one simulated backend. For BGD topology and probe-log operations, +`host` is the backend IP. + +### `Cluster_Simulator` + +```cpp +int connect( + const char* host, + int port, + const char* username, + const char* password, + bool use_ssl = false); + +int read_only_update(const Simulator_Endpoint& backend, bool read_only); +``` + +`connect()` opens the SQLite3-server control connection with the MySQL client +API; the helper closes it when destroyed. `read_only_update()` changes the +existing `READONLY_STATUS` row for one backend. + +### Topology and Host Types + +```cpp +struct RDS_BGD_Topology_Row { + std::string id; + std::string endpoint; + int port; + std::string role; + std::string status; +}; + +struct RDS_BGD_Host { + std::string hostname; + std::string ip; + int port; + + Simulator_Endpoint endpoint() const; +}; +``` + +`RDS_BGD_Topology_Row` represents one `mysql.rds_topology` row using +C++11-compatible field types. `RDS_BGD_Host` keeps the ProxySQL-facing +hostname and simulator-facing IP together. + +### Shared Cluster Fixture + +```cpp +class RDS_BGD_Cluster { +public: + const RDS_BGD_Host& blue_writer() const; + const RDS_BGD_Host& green_writer() const; + const std::vector& blue_readers() const; + const std::vector& green_readers() const; + std::vector get_writers() const; + std::vector get_topology( + const std::string& status) const; +}; + +const RDS_BGD_Cluster& rds_bgd_test_cluster(); +``` + +The fixture encapsulates the shared `/etc/hosts` mapping. `get_writers()` +returns the selected blue and green writer IPs; `get_topology(status)` returns +the standard two-row SOURCE/TARGET topology using the writer hostnames and the +provided status. + +### BGD Topology Operations + +```cpp +int topology_update( + const std::vector& backends, + const std::vector& rows); + +int topology_delete(const std::vector& backends); + +int topology_drop(const std::vector& backends); + +int topology_error( + const std::vector& backends, + unsigned int error_code, + const std::string& error_msg); +``` + +`topology_update()` marks the table present, clears any configured error, and +replaces rows on only the supplied backends. `topology_delete()` clears rows +and errors while leaving the table present. + +`topology_drop()` clears rows, marks the table absent, and records error 1146 +with `Table 'mysql.rds_topology' doesn't exist`. `topology_error()` requires a +nonzero code; 1146 marks topology absent, while any other code marks it present +and leaves existing rows unchanged. + +### Probe-Log Operations + +```cpp +enum class RDS_BGD_Probe_Kind { + table_check, + metadata, +}; + +struct RDS_BGD_Probe_Log { + uint64_t sequence_id; + Simulator_Endpoint backend; + RDS_BGD_Probe_Kind probe_kind; + bool encrypted; +}; + +rc_t probe_log_last_sequence(); + +rc_t> probe_log_since(uint64_t sequence_id); + +rc_t wait_for_probe_log( + uint64_t sequence_id, + const Simulator_Endpoint& backend, + RDS_BGD_Probe_Kind probe_kind, + uint32_t timeout_ms, + int encrypted = -1); +``` + +The watermark method returns zero for an empty log. `probe_log_since()` returns +rows after a watermark. `wait_for_probe_log()` waits for one matching row; +`encrypted` is `-1` for either mode, `0` for plaintext, and `1` for TLS. + +## Typical TAP Test + +```cpp +int main() { + plan(3); + + CommandLine cl {}; + if (cl.getEnv()) BAIL_OUT("failed to load TAP environment"); + + MYSQL* admin = init_mysql_conn( + cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (!admin) BAIL_OUT("failed to connect to ProxySQL Admin"); + + const RDS_BGD_Cluster& cluster = rds_bgd_test_cluster(); + if (configure_proxysql_for_bgd(admin, cluster) != EXIT_SUCCESS) + BAIL_OUT("failed to configure ProxySQL"); + + std::pair sqlite_server; + if (extract_sqlite3_host_port(admin, sqlite_server) != EXIT_SUCCESS) + BAIL_OUT("failed to find SQLite3-server address"); + + RDS_BGD_Simulator simulator; + if (simulator.connect( + sqlite_server.first.c_str(), sqlite_server.second, + cl.username, cl.password) != EXIT_SUCCESS) + BAIL_OUT("failed to connect to SQLite3 server"); + + const rc_t mark = simulator.probe_log_last_sequence(); + if (mark.first != EXIT_SUCCESS) + BAIL_OUT("failed to read probe-log watermark"); + + const int update_rc = simulator.topology_update( + cluster.get_writers(), cluster.get_topology("AVAILABLE")); + ok(update_rc == EXIT_SUCCESS, "publish topology to both writer IPs"); + if (update_rc != EXIT_SUCCESS) + BAIL_OUT("failed to publish topology"); + + ok(wait_for_cond(admin, + "SELECT status='AVAILABLE' FROM runtime_mysql_aws_rds_bgd_hostgroups " + "WHERE writer_hostgroup=10", 5) == EXIT_SUCCESS, + "ProxySQL enters AVAILABLE"); + + const rc_t green_log = simulator.wait_for_probe_log( + mark.second, + cluster.green_writer().endpoint(), + RDS_BGD_Probe_Kind::metadata, + 5000); + ok(green_log.first == EXIT_SUCCESS, + "ProxySQL probes the green writer IP directly"); + + mysql_close(admin); + return exit_status(); +} +``` + +ProxySQL configuration remains test-local. The simulator changes backend +responses and reads probe evidence; assertions against ProxySQL use Admin SQL. + +## Build Integration + +Add `build_lib_test_rds_bgd`, `build_src_test_rds_bgd`, and the top-level +`test_rds_bgd` target. The lib and src targets compile with +`-DDEBUG -DTEST_RDS_BGD`; none depends on `build_cluster_simulator`. + +`test_rds_bgd` depends on `build_src_test_rds_bgd` and then invokes `make +debug` in `test/tap`: + +```text +build_deps_debug -> build_lib_test_rds_bgd -> build_src_test_rds_bgd + -> TAP debug build +``` + +Use `test_rds_bgd` as the single entry point. Do not invoke +`build_tap_test_debug` afterward because its `build_src_debug` dependency +selects the normal debug daemon. Add `-DTEST_RDS_BGD` to `testall` as well. + +## Local CI Group + +Add `test/tap/groups/cluster_sim_rds_bgd/` and execute it as +`cluster_sim_rds_bgd-g1`. + +| File | BGD-specific content | +|---|---| +| `env.sh` | Set `CLUSTER_SIM_HOST_FILE` and `SKIP_CLUSTER_START=1`. | +| `add-hosts` | Define the fixed hostname/IP map below. | +| `pre-proxysql.bash` | Keep the existing short startup wait before Admin writes. | +| `pre-proxysql.sql` | Add the simulator user and move the SQLite3 server to port 3306. | + +The group has no `infras.lst`, `CLUSTER_SIM_BINARY_PATH`, or +`CLUSTER_SIM_TESTS_ROOT`. The TAP binary controls the simulator directly. + +```bash +export CLUSTER_SIM_HOST_FILE="${WORKSPACE}/test/tap/groups/cluster_sim_rds_bgd/add-hosts" +export SKIP_CLUSTER_START=1 +``` + +### Fixed Host Map + +All BGD TAP tests use this map. Green endpoints retain the blue endpoint's +first label and append `-green-` before the common domain. + +```text +# Cluster 1: blue endpoints +db-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.11 +db-1-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.12 +db-1-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.13 + +# Cluster 1: green deployment A +db-1-green-iqu47r.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.14 +db-1-reader-1-green-dlzky7.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.15 +db-1-reader-2-green-3fpjuu.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.16 + +# Cluster 1: green deployment B, for repeated switchovers +db-1-green-s7m2kx.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.17 +db-1-reader-1-green-v4n8qp.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.18 +db-1-reader-2-green-w6h3rz.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.19 + +# Cluster 2: reserved for multi-cluster tests +db-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.20 +db-2-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.21 +db-2-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.22 +db-2-green-iqu47r.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.23 +db-2-reader-1-green-dlzky7.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.24 +db-2-reader-2-green-3fpjuu.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.25 + +# Cluster 3: reserved for multi-cluster tests +db-3.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.26 +db-3-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.27 +db-3-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.28 +db-3-green-iqu47r.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.29 +db-3-reader-1-green-dlzky7.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.30 +db-3-reader-2-green-3fpjuu.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.31 +``` + +Every endpoint uses port 3306. Cluster 1 with green deployment A is sufficient +for normal FSM cases. A repeated-switchover test completes deployment A, waits +for empty or absent topology to reset the FSM, replaces A's green hostgroup +rows with deployment B, and publishes the next topology. Clusters 2 and 3 are +available for simultaneous switchovers. + +### SQLite3-Server Hook + +`pre-proxysql.sql` provisions the simulator credentials and changes the +SQLite3-server listener from the CI default to port 3306: + +```sql +INSERT OR REPLACE INTO mysql_users + (username, password, default_hostgroup, active) + VALUES ('testuser', 'testuser', 0, 1); +LOAD MYSQL USERS TO RUNTIME; +SAVE MYSQL USERS TO DISK; + +SET sqliteserver-mysql_ifaces='0.0.0.0:3306'; +LOAD SQLITESERVER VARIABLES TO RUNTIME; +SAVE SQLITESERVER VARIABLES TO DISK; +``` + +The hook does not populate `mysql_servers` or +`mysql_aws_rds_bgd_hostgroups`; each test owns its ProxySQL configuration and +simulator transitions. From the TAP container, the control connection uses +`proxysql:3306` on the existing isolated Docker network. + +### Group Registration and Local Run + +Register each BGD TAP binary in `test/tap/groups/groups.json`: + +```json +"test_rds_bgd-t": [ "cluster_sim_rds_bgd-g1" ] +``` + +Add the group and its `make test_rds_bgd` requirement to the simulator table in +`test/infra/README.md`. Clean when switching compile flavors because Make does +not track changed preprocessor flags: + +```bash +make clean +make -j"$(nproc)" test_rds_bgd + +export INFRA_ID="rds-bgd-$(date +%s)" +export TAP_GROUP="cluster_sim_rds_bgd-g1" + +./test/infra/control/ensure-infras.bash +./test/infra/control/run-tests-isolated.bash +./test/infra/control/destroy-infras.bash +``` + +The existing runner injects the host aliases, starts ProxySQL with +`--sqlite3-server`, executes the registered TAP binaries in the test container, +and collects logs. No BGD branch is required in `ensure-infras.bash`, +`start-proxysql-isolated.bash`, or `run-tests-isolated.bash`. + +## GitHub Actions + +Add `.github/workflows/CI-rds-bgd-simulator.yml`. It runs on +`workflow_dispatch` and after a successful `CI-trigger`, follows the repository's +existing concurrency/cancellation pattern, and checks out the exact triggering +SHA. + +The regular Ubuntu TAP cache contains a daemon built without `TEST_RDS_BGD` and +must not be used as the BGD executable. The workflow therefore has a BGD build +job and a dependent execution job. + +The build job checks out the triggering SHA, installs or reuses the normal +Ubuntu TAP build dependencies, and runs: + +```bash +make -j"$(nproc)" test_rds_bgd +``` + +After verifying `src/proxysql` and `test/tap/tests/test_rds_bgd-t`, it saves the +build output as two BGD-specific cache entries, following the existing CI +separation between daemon and test artifacts: + +```text +${SHA}_ubuntu22-tap-rds-bgd_src -> src/ +${SHA}_ubuntu22-tap-rds-bgd_test -> test/ +``` + +The cache keys are exact and include the BGD build flavor. Do not configure +`restore-keys`: falling back to the normal Ubuntu TAP cache could execute a +daemon compiled without `TEST_RDS_BGD`. The workflow must grant the cache-save +permission required by its `workflow_run` context. + +The execution job depends on the build job, checks out the same SHA, and +restores both entries with `fail-on-cache-miss: true`. It verifies the restored +executables before building the runner image and starting the test group. One +producer can therefore supply the same flagged artifacts to additional BGD +execution jobs without rebuilding ProxySQL. + +| Execution-job step | Required behavior | +|---|---| +| Checkout | Check out the triggering SHA, not the default branch tip. | +| Restore `src` | Restore the exact BGD `_src` key into `src/`; fail on a miss. | +| Restore `test` | Restore the exact BGD `_test` key into `test/`; fail on a miss. | +| Verify artifacts | Confirm `src/proxysql` and `test/tap/tests/test_rds_bgd-t` are executable. | +| Build runner image | Build `test/infra/docker-base` as `proxysql-ci-base:latest`. | +| Start | Export the shared variables below and run `ensure-infras.bash`. | +| Test | Run `run-tests-isolated.bash`; this execution, not compilation alone, is the required check. | +| Cleanup | With `if: always()`, stop ProxySQL and run `destroy-infras.bash`; cleanup failures must not hide the test result. | +| Logs | On failure, upload `ci_infra_logs/` with the workflow name, SHA, and run number in the artifact name. | + +The start, test, and cleanup steps use the same values: + +```bash +export WORKSPACE="${GITHUB_WORKSPACE}" +export INFRA_ID="rds-bgd-${GITHUB_RUN_ID}" +export TAP_GROUP="cluster_sim_rds_bgd-g1" +source test/infra/common/env.sh +``` + +The job starts no backend infrastructure and never invokes +`test/deps/cluster_simulator`. Its pass condition is: the flagged build +succeeds, the BGD TAP group executes, every TAP test exits successfully, and +the standard runner reports no infrastructure or test failure. + +## Supported Test Coverage + +### Simulator Acceptance + +The simulator implementation needs one end-to-end smoke test, not a separate +unit-test suite for every helper method. `test_rds_bgd-t` proves that the +`TEST_RDS_BGD` daemon accepts TAP-controlled topology, ProxySQL observes an +`AVAILABLE` deployment, the green-IP probe is logged, and the automatic CI job +executes the group without `test/deps/cluster_simulator`. + +The configuration and lifecycle tests below exercise the remaining helper and +SQLite3-server paths through BGD behavior. Before changing simulator state, +each test captures a probe watermark; failures report the configured backend +state, last ProxySQL runtime state, and later probe rows. + +### Configuration and Discovery + +Configuration tests keep topology at `AVAILABLE` until the expected runtime row +and worker generation are stable. A relevant case then continues through a +switchover, proving that the configuration adopted during setup is the one used +by the FSM. + +| Case | Configuration sequence | Expected observations | +|---|---|---| +| Available topology before blue writer | Publish `AVAILABLE`, enable automatic discovery, then add the blue writer and its replication-hostgroup mapping. | The read-only discovery path creates one runtime BGD row with derived blue hostgroups, NULL green hostgroups, and `auto_generated=1`; its worker begins probing. | +| Blue deployment before BGD exists | Add the blue writer and readers while topology is absent, then publish `AVAILABLE`. | No BGD row is created before discovery; topology appearance creates the auto-generated runtime row and starts its worker. | +| Blue readers added after discovery | Start from an auto-generated row with only the blue writer, then add one or more blue readers and load servers to runtime. | The host checksum changes, the worker generation is replaced, probing resumes, and later reader actions use the new reader set. | +| Explicit BGD row before servers | Disable automatic discovery, load an explicit BGD hostgroup row, then add the blue writer, blue readers, green writer, and green readers. | The row remains `auto_generated=0`; no worker runs without an eligible blue server, and each relevant server change is incorporated by the replacement worker. | +| Servers before explicit BGD row | Add blue and green servers first with automatic discovery disabled, then load the explicit BGD hostgroup row. | No BGD worker runs before the row exists; loading it starts a worker that uses the existing server membership. | +| Blue first, green later | Configure blue servers, publish `AVAILABLE`, then add explicit green writer and reader rows before starting switchover. | Green membership changes replace the worker and rebuild its mapping; existing rows are not duplicated and the explicit green writer supplies its configured TLS mode. | +| Green timing variants | With explicit green hostgroups, add green nodes before `AVAILABLE`, after discovery, or after the worker starts but before switchover. | Each ordering converges on the same runtime membership and blue/green mapping before the FSM advances. | +| Automatic to explicit configuration | Allow discovery to create an automatic row, then load a user row with explicit green hostgroups. | The runtime row becomes user-defined with `auto_generated=0`, explicit green hostgroups replace NULLs, and a replacement worker uses the new configuration. | +| Configuration mutation | Change `active`, hostgroup IDs, `writer_is_also_reader`, check interval/timeout, server status, or `use_ssl`; also cover row disablement and removal. | Relevant checksum changes stop the old worker, run phase-appropriate cleanup, and start or suppress a worker from the new active configuration. | +| Persistence and validation | Save automatic and explicit runtime state, and attempt invalid persistent rows. | Auto-generated rows are not persisted; explicit rows are retained; missing or mixed green hostgroups and other schema-invalid configurations are rejected. | + +Configuration assertions use `runtime_mysql_aws_rds_bgd_hostgroups`, +`runtime_mysql_servers`, probe-log destinations, and subsequent hostgroup +effects. They do not depend only on worker log messages. + +### Switchover, Rollback, and Cleanup + +| Case | Simulator/configuration transition | Expected observations | +|---|---|---| +| Normal lifecycle | Advance AVAILABLE → INITIATED → IN_PROGRESS → POST_PROCESSING → COMPLETED, then make topology empty or absent. | Runtime status follows every phase; writer/reader placement, server status, DNS effects, and connection-pool changes occur at their defined boundaries; final cleanup returns status to `NONE`. | +| Cancellation rollback | Move from INITIATED or IN_PROGRESS back to `AVAILABLE`. | Accumulated effects are rolled back, the blue writer and reader policy are restored, probe pinning is rebuilt for AVAILABLE, and the deployment remains monitorable. | +| Pre-completion topology loss | Delete or drop topology before writer completion. | Empty and absent observations remain distinguishable, but both select rollback rather than successful finalization. | +| Configuration change during switchover | Add/remove servers, disable/remove the BGD row, or change relevant configuration while a non-NONE phase is active. | The old worker performs one-shot phase-appropriate rollback before its replacement uses the new configuration; stale mappings do not drive later actions. | +| Rollback postconditions | Trigger rollback after writer demotion, reader shunning, or DNS pinning has occurred. | Blue writer service and configured reader membership are restored, BGD-shunned readers are unshunned, pins and direct-probe state are cleared or rebuilt, runtime status resets, green rows remain, and green connections are not drained. | +| Successful cleanup | Complete writer switchover, enter reader switchover, then drain topology. | Eligible green connections are drained while green rows and statuses remain; readers are reconciled and the worker returns to `NONE`. | +| Late entry | Start a fresh worker with INITIATED, IN_PROGRESS, POST_PROCESSING, or COMPLETED already published. | The worker reconstructs only the state supported by that observation and applies the defined phase actions without requiring earlier samples. | +| Direct-probe policy | Vary blue/green IP, supported port, and blue versus explicit-green `use_ssl`. | Probe-log rows identify the green IP, matched writer port, and correct automatic or explicit TLS source. | +| Reader and offline handling | Use matched, unmatched, and `OFFLINE_SOFT`/`OFFLINE_HARD` blue and green readers. | Only eligible pairs are mapped or drained; unmatched readers follow BGD shun policy and offline nodes are excluded. | +| Metadata failures | Return an empty result, error 1146, or another configured query error from selected writers. | Absence, empty metadata, and generic query failure remain distinct and never masquerade as a successful switchover. | +| Repeated switchover | Complete cluster-1 deployment A, reset on empty/absent topology, replace its green rows with deployment B, and run again. | The second lifecycle uses deployment B without stale mapping, probe, or simulator state from deployment A. | +| Concurrent switchovers | Publish independent topology for clusters 1, 2, or 3 and advance them independently. | Multiple workers make isolated progress; configuration or topology changes in one cluster do not alter another. | + +The simulator does not claim to validate application traffic, AWS control-plane +timing, mutable DNS propagation, packet loss, or exact post-switchover address +movement. Those require separate integration infrastructure when a test's +assertion depends on them. + +## Code Boundaries + +| Area | Required change | +|---|---| +| `Makefile` | Add the BGD build targets and include `TEST_RDS_BGD` in `testall`. | +| `include/SQLite3_Server.h` | Add BGD table definitions/helpers and the coded-error overload under the flag. | +| `src/SQLite3_Server.cpp` | Add listener setup, endpoint extraction, table creation, BGD/read-only interception, and probe logging. | +| `test/tap` helpers | Add the common simulator and BGD-specific API defined above. | +| `test/tap/groups/cluster_sim_rds_bgd` | Add the fixed host map and SQLite3-server group configuration. | +| `test/tap/groups/groups.json` | Register BGD TAP binaries in `cluster_sim_rds_bgd-g1`. | +| `test/infra/README.md` | Document the group and its required `test_rds_bgd` build target. | +| `.github/workflows/CI-rds-bgd-simulator.yml` | Build the flagged flavor and execute the BGD simulator group automatically. | +| BGD production monitor | Reuse existing query constants; add no simulator query decoration or test initializer. | + +Existing simulator builds retain their behavior. The scenario, not the helper, +owns topology publication, FSM timing, ProxySQL configuration, and expected +outcomes. From ffc07edce168e4e1c75abc84c6deb288b6aeab6d Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 21 Jul 2026 20:04:49 +0000 Subject: [PATCH 42/81] fix: Retire unhealthy AWS RDS BGD connections - Preserve terminal unhealthy state when resetting backend connections. - Reject unhealthy connections from local and shared connection pools. - Keep pool-return health checks behind HGM availability guards. Signed-off-by: Wazir Ahmed --- lib/MySQL_HostGroups_Manager.cpp | 8 +++++++- lib/MySQL_Thread.cpp | 5 +++++ lib/mysql_connection.cpp | 1 - 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 17d27ea804..682f02fad7 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -2350,8 +2350,14 @@ void MySQL_HostGroups_Manager::push_MyConn_to_pool(MySQL_Connection *c, bool _lo goto __exit_push_MyConn_to_pool; } + if (!c->healthy) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Destroying unhealthy MySQL_Connection %p, server %s:%d\n", c, mysrvc->address, mysrvc->port); + delete c; + goto __exit_push_MyConn_to_pool; + } + // If the largest query length exceeds the threshold, destroy the connection - if (GloMTH && c->largest_query_length > (unsigned int)GloMTH->variables.threshold_query_length) { + if (c->largest_query_length > (unsigned int)GloMTH->variables.threshold_query_length) { proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Destroying MySQL_Connection %p, server %s:%d with status %d . largest_query_length = %lu\n", c, mysrvc->address, mysrvc->port, (int)mysrvc->get_status(), c->largest_query_length); delete c; goto __exit_push_MyConn_to_pool; diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index b3b913779f..f3f07eae7a 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -6582,6 +6582,11 @@ MySQL_Connection * MySQL_Thread::get_MyConn_local(unsigned int _hid, MySQL_Sessi * @param c Pointer to the MySQL_Connection object to be pushed to the local connection pool. */ void MySQL_Thread::push_MyConn_local(MySQL_Connection *c) { + if (!c->healthy) { + MyHGM->push_MyConn_to_pool(c); + return; + } + // Bounded local cache: cache 1-in-N releases (N = mysql_threads), push the // rest to the shared HGM pool so peer workers can pick them up. // At N=1 always cache (no sibling to share with). diff --git a/lib/mysql_connection.cpp b/lib/mysql_connection.cpp index bc1e0b6238..d3fe351a80 100644 --- a/lib/mysql_connection.cpp +++ b/lib/mysql_connection.cpp @@ -3107,7 +3107,6 @@ void MySQL_Connection::reset() { bool old_no_multiplex_hg = get_status(STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); bool old_compress = get_status(STATUS_MYSQL_CONNECTION_COMPRESSION); status_flags=0; - healthy=true; // reconfigure STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG set_status(old_no_multiplex_hg,STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); // reconfigure STATUS_MYSQL_CONNECTION_COMPRESSION From 4147c73dbcd8f320970aeecc68b728127192716c Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Wed, 22 Jul 2026 07:46:08 +0000 Subject: [PATCH 43/81] test: Add unit test for unhealthy MySQL connection retirement --- doc/AWS_Blue_Green/RDS_BGD_Monitor.md | 10 +- test/tap/groups/groups.json | 1 + test/tap/tests/unit/Makefile | 3 +- test/tap/tests/unit/config_write_unit-t.cpp | 3 + .../unit/connection_unhealthy_unit-t.cpp | 147 ++++++++++++++++++ 5 files changed, 159 insertions(+), 5 deletions(-) create mode 100644 test/tap/tests/unit/connection_unhealthy_unit-t.cpp diff --git a/doc/AWS_Blue_Green/RDS_BGD_Monitor.md b/doc/AWS_Blue_Green/RDS_BGD_Monitor.md index efc5ed02a4..dfef602bd6 100644 --- a/doc/AWS_Blue_Green/RDS_BGD_Monitor.md +++ b/doc/AWS_Blue_Green/RDS_BGD_Monitor.md @@ -998,9 +998,11 @@ The source review remains open on implementation and verification: `aws_rds_bgd_match_host(gs->address, green_writer_host)` call supplies two green names to a blue-to-green matcher, so an explicit green `use_ssl` differing from blue is not selected. -2. Make unhealthy connection retirement terminal across reset, local pool - return, and global pool return. The follow-up uses the existing `healthy` - field and does not add a second flag. +2. **COMPLETED:** Make unhealthy connection retirement terminal across reset, + local pool return, and global pool return. The follow-up uses the existing + `healthy` field and does not add a second flag. `connection_unhealthy_unit-t` + verifies that unhealthy connections remain terminal across reset and cannot + enter either free pool. 3. Track the author-accepted same-phase DNS failure as required follow-up work. Until per-pair reconciliation exists, a transient first resolution failure in POST_PROCESSING can leave traffic unpinned and old connections undrained. @@ -1022,7 +1024,7 @@ proposed broad durable-ledger/controller PR is not part of this sequence. | PR1: #5934 | This document only: evidence, accepted risks, current behavior, and follow-up contract. | Ready for author approval; merge into `feature/aws-rds-monitor` before implementation follow-ups so their scope is stable. | | PR2: BGD simulator foundation and CI | Add the TAP-controlled SQLite3-server simulator defined in [RDS_BGD_Simulator.md](RDS_BGD_Simulator.md): the `TEST_RDS_BGD` build mode, IP-keyed topology responses, common and BGD TAP helpers, a simulator group, an end-to-end acceptance smoke test, and an automatic CI job that executes the group. | No production behavior change. Provides the reusable harness required by PR6. A successful compile-only `CI-maketest` job is not completion evidence. | | PR3: probe target and explicit TLS | Correct AWS-08 by selecting the exact supported explicit green writer row and its `use_ssl`, while retaining the matched blue writer port and automatic-mode blue TLS fallback. | Depends only on the documented contract. Focused unit/TAP evidence must distinguish blue `use_ssl=0` from explicit green `use_ssl=1`. | -| PR4: terminal connection retirement | Preserve `healthy=false` across `MySQL_Connection::reset()` and destroy unhealthy connections in local and global pool-return paths. Do not introduce another flag or a new locking policy. | Focused tests prove a drained used connection cannot enter either free pool after reset or release. | +| PR4: terminal connection retirement (**complete**) | Preserve `healthy=false` across `MySQL_Connection::reset()` and destroy unhealthy connections in local and global pool-return paths. Do not introduce another flag or a new locking policy. | **Completed:** `connection_unhealthy_unit-t` proves a drained used connection cannot enter either free pool after reset or release. | | PR5: same-phase per-pair reconciliation | Replace phase-equality no-op behavior with worker-local reconciliation for incomplete map/resolution/pin/drain work. Retry only incomplete pairs and never redrain a pair already completed in the current worker generation. | Depends on the accepted one-shot worker model; it must not introduce durable ownership or restart recovery. | | PR6: simulator-driven BGD scenario suite | Use PR2's simulator to cover configuration and discovery order, automatic and explicit rows, worker replacement, normal lifecycle, late entry, cancellation and rollback, topology drain, direct probe tuple/TLS, offline exclusions, terminal connection retirement where observable, and PR5 DNS failure/recovery. | Depends on PR2 and should normally follow PR3-PR5 so the suite validates final behavior rather than encoding known failures. All payloads run in the automatic BGD simulator CI group. | diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index 05b9d31a1e..9b62161280 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -21,6 +21,7 @@ "config_validation_unit-t" : [ "unit-tests-g1" ], "config_write_unit-t" : [ "unit-tests-g1" ], "connection_pool_unit-t" : [ "unit-tests-g1" ], + "connection_unhealthy_unit-t" : [ "unit-tests-g1" ], "deprecate_eof_cache-t" : [ "legacy-g4","mariadb10-galera-g4","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g4","mysql84-gr-g4","mysql90-g4","mysql95-g4" ], "envvars-t" : [ "legacy-g1","mariadb10-galera-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql84-gr-g1","mysql90-g1","mysql90-gr-g1","mysql93-g1","mysql93-gr-g1","mysql95-g1","mysql95-gr-g1" ], "eof_cache_mixed_flags-t" : [ "legacy-g4","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g4","mysql90-g4","mysql95-g4" ], diff --git a/test/tap/tests/unit/Makefile b/test/tap/tests/unit/Makefile index d8ca44c12c..06071e962b 100644 --- a/test/tap/tests/unit/Makefile +++ b/test/tap/tests/unit/Makefile @@ -420,7 +420,8 @@ UNIT_TESTS := smoke_test-t query_cache_unit-t query_processor_unit-t \ gtid_server_data_unit-t \ admin_disk_upgrade_unit-t \ glovars_unit-t \ - pgsql_servers_ssl_params_unit-t + pgsql_servers_ssl_params_unit-t \ + connection_unhealthy_unit-t # Plugin-chassis + mysqlx-plugin unit tests — built only when # libproxysql.a was compiled with -DPROXYSQL40 (autodetected higher up diff --git a/test/tap/tests/unit/config_write_unit-t.cpp b/test/tap/tests/unit/config_write_unit-t.cpp index be0d19f151..1d72b74524 100644 --- a/test/tap/tests/unit/config_write_unit-t.cpp +++ b/test/tap/tests/unit/config_write_unit-t.cpp @@ -322,6 +322,7 @@ static void test_write_mysql_servers_empty() { "domain_name VARCHAR, max_lag_ms INT, check_interval_ms INT, check_timeout_ms INT, " "writer_is_also_reader INT, new_reader_weight INT, add_lag_ms INT, min_lag_ms INT, " "lag_num_checks INT, comment VARCHAR)"); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS); db->execute("CREATE TABLE mysql_hostgroup_attributes " "(hostgroup_id INT PRIMARY KEY, max_num_online_servers INT, autocommit INT, " "free_connections_pct INT, init_connect VARCHAR, multiplex INT, connection_warming INT, " @@ -372,6 +373,7 @@ static void test_write_mysql_servers_with_data() { "domain_name VARCHAR, max_lag_ms INT, check_interval_ms INT, check_timeout_ms INT, " "writer_is_also_reader INT, new_reader_weight INT, add_lag_ms INT, min_lag_ms INT, " "lag_num_checks INT, comment VARCHAR)"); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS); db->execute("CREATE TABLE mysql_hostgroup_attributes " "(hostgroup_id INT PRIMARY KEY, max_num_online_servers INT, autocommit INT, " "free_connections_pct INT, init_connect VARCHAR, multiplex INT, connection_warming INT, " @@ -420,6 +422,7 @@ static void test_write_mysql_servers_replication_hostgroups() { "domain_name VARCHAR, max_lag_ms INT, check_interval_ms INT, check_timeout_ms INT, " "writer_is_also_reader INT, new_reader_weight INT, add_lag_ms INT, min_lag_ms INT, " "lag_num_checks INT, comment VARCHAR)"); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS); db->execute("CREATE TABLE mysql_hostgroup_attributes " "(hostgroup_id INT PRIMARY KEY, max_num_online_servers INT, autocommit INT, " "free_connections_pct INT, init_connect VARCHAR, multiplex INT, connection_warming INT, " diff --git a/test/tap/tests/unit/connection_unhealthy_unit-t.cpp b/test/tap/tests/unit/connection_unhealthy_unit-t.cpp new file mode 100644 index 0000000000..599adcbb81 --- /dev/null +++ b/test/tap/tests/unit/connection_unhealthy_unit-t.cpp @@ -0,0 +1,147 @@ +/** + * @file connection_unhealthy_unit-t.cpp + * @brief Verify unhealthy MySQL connections cannot re-enter connection pools. + * + * Exercises the real MySQL connection, thread-local cache, and HostGroups + * Manager boundaries without opening a backend network connection. + */ + +#include "tap.h" +#include "test_globals.h" +#include "test_init.h" + +#include "proxysql.h" +#include "cpp.h" +#include "MySQL_Logger.hpp" + +extern MySQL_HostGroups_Manager *MyHGM; +extern MySQL_Threads_Handler *GloMTH; +extern MySQL_Logger *GloMyLogger; + +static MySrvC *create_server(unsigned int hostgroup_id, const char *address) { + srv_info_t info; + info.addr = address; + info.port = 3306; + info.kind = "connection-unhealthy-unit"; + + srv_opts_t opts; + opts.weigth = 1; + opts.max_conns = 100; + opts.use_ssl = 0; + + MyHGM->wrlock(); + int rc = MyHGM->create_new_server_in_hg(hostgroup_id, info, opts); + MyHGC *hostgroup = MyHGM->MyHGC_find(hostgroup_id); + MyHGM->wrunlock(); + + if (rc != 0 || hostgroup == nullptr || hostgroup->mysrvs->cnt() != 1) { + BAIL_OUT("failed to create server for hostgroup %u", hostgroup_id); + } + + return hostgroup->mysrvs->idx(0); +} + +static MySQL_Connection *create_used_connection(MySrvC *server, bool healthy) { + MySQL_Connection *connection = new MySQL_Connection(); + connection->mysql = mysql_init(nullptr); + if (connection->mysql == nullptr) { + delete connection; + BAIL_OUT("mysql_init() failed for unit-test connection"); + } + + connection->parent = server; + connection->healthy = healthy; + connection->reusable = healthy; + connection->async_state_machine = ASYNC_IDLE; + connection->largest_query_length = 0; + server->ConnectionsUsed->add(connection); + return connection; +} + +static void check_pool_state(MySrvC *server, unsigned int exp_used, unsigned int exp_free, const char *msg) { + unsigned int used = server->ConnectionsUsed->conns_length(); + unsigned int free = server->ConnectionsFree->conns_length(); + ok(used == exp_used && free == exp_free, "%s (used=%u, free=%u)", msg, used, free); +} + +static void test_unhealthy_global_pool() { + MySrvC *server = create_server(101, "unhealthy-global"); + MySQL_Connection *connection = create_used_connection(server, false); + + connection->reset(); + MyHGM->push_MyConn_to_pool(connection); + + check_pool_state(server, 0, 0, "reset unhealthy connection is destroyed at the global pool boundary"); +} + +static void test_unhealthy_local_pool(MySQL_Thread &worker) { + MySrvC *server = create_server(102, "unhealthy-local"); + MySQL_Connection *connection = create_used_connection(server, false); + + connection->reset(); + worker.push_MyConn_local(connection); + + check_pool_state(server, 0, 0, "reset unhealthy connection is destroyed at the local pool boundary"); + + // If the assertion failed because the connection entered the local cache, + // return it before continuing so later cases remain isolated. + worker.return_local_connections(); +} + +static void test_healthy_global_pool() { + MySrvC *server = create_server(103, "healthy-global"); + MySQL_Connection *connection = create_used_connection(server, true); + + MyHGM->push_MyConn_to_pool(connection); + + check_pool_state(server, 0, 1, "healthy connection enters the global free pool"); +} + +static void test_healthy_local_pool(MySQL_Thread &worker) { + MySrvC *server = create_server(104, "healthy-local"); + MySQL_Connection *connection = create_used_connection(server, true); + + worker.push_MyConn_local(connection); + check_pool_state(server, 1, 0, "healthy connection remains used while cached locally"); + + worker.return_local_connections(); + check_pool_state(server, 0, 1, "healthy local connection enters the global free pool when returned"); +} + +int main() { + plan(5); + + if (test_init_minimal() != 0) { + BAIL_OUT("test_init_minimal() failed"); + } + if (test_init_query_processor() != 0) { + BAIL_OUT("test_init_query_processor() failed"); + } + GloMyLogger = new MySQL_Logger(); + if (test_init_hostgroups() != 0) { + BAIL_OUT("test_init_hostgroups() failed"); + } + + // Make the local-cache decision deterministic: with one worker, every + // otherwise eligible connection is cached locally. + GloMTH->num_threads = 1; + { + MySQL_Thread worker; + if (!worker.init()) { + BAIL_OUT("MySQL_Thread::init() failed"); + } + + test_unhealthy_global_pool(); + test_unhealthy_local_pool(worker); + test_healthy_global_pool(); + test_healthy_local_pool(worker); + } + + test_cleanup_hostgroups(); + delete GloMyLogger; + GloMyLogger = nullptr; + test_cleanup_query_processor(); + test_cleanup_minimal(); + + return exit_status(); +} From 7204726f5a4e8ae41a9995bf274ba0a782318c6c Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Wed, 22 Jul 2026 06:49:00 +0000 Subject: [PATCH 44/81] fix: Select explicit green writer TLS by probe tuple - Match the configured TARGET by exact hostname and writer port - In `bgd_add_green_writer_in_hg()`, copy green writer's `use_ssl` into `st.bg_map` Signed-off-by: Wazir Ahmed --- doc/AWS_Blue_Green/RDS_BGD_Monitor.md | 34 ++++++++++++------------- doc/AWS_Blue_Green/RDS_BGD_Simulator.md | 2 ++ lib/MySQL_Monitor.cpp | 8 ++++-- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/doc/AWS_Blue_Green/RDS_BGD_Monitor.md b/doc/AWS_Blue_Green/RDS_BGD_Monitor.md index dfef602bd6..38e8f5e839 100644 --- a/doc/AWS_Blue_Green/RDS_BGD_Monitor.md +++ b/doc/AWS_Blue_Green/RDS_BGD_Monitor.md @@ -323,15 +323,14 @@ ports is unsupported. `mysql_servers` row from which to read TLS configuration, so it intentionally uses the matched blue writer's `use_ssl` value. -`SOURCE-CODE, IMPLEMENTATION-CONFORMANCE-OPEN`: Explicit mode is intended to -use the matching green writer row's `use_ssl`. In commit `20247dcf0`, however, -the lookup calls `aws_rds_bgd_match_host(gs->address, green_writer_host)`. -That helper expects a blue hostname as its first argument and a green hostname -as its second argument. Passing the configured green hostname and TARGET green -hostname therefore does not match the ordinary -`-green-.` case. `green_use_ssl` remains unset and the -probe silently falls back to the blue writer's value. AWS-08's policy decision -is resolved, but the current source does not yet implement it. +`SOURCE-CODE, AUTHOR-ACCEPTED-POLICY`: Explicit mode selects an eligible green +writer row by exact TARGET hostname and the matched blue writer's port. Map +construction copies `use_ssl` from an existing row. When discovery creates a +missing row or restores an `OFFLINE_HARD` row, the successful add path copies +the exact row's resolved `use_ssl` after hostgroup defaults are applied. A +valid initially empty configured green writer hostgroup produces no warning. +An `OFFLINE_SOFT` row remains ineligible and retains the matched-blue TLS +fallback. Simulator coverage for both row paths is assigned to PR6. ### Current Phase-Equality Behavior @@ -903,7 +902,7 @@ that is tracked separately rather than reopening the evidence decision. | AWS-05b | The author accepts `TOPOLOGY_EMPTY` after observed writer completion as the reader-cleanup signal despite no AWS guarantee or direct reader-DNS timestamp. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: The observational risk is explicit. | | AWS-06 | The author observed the green hostname stop resolving after completion while the promoted IP survived. | `RESOLVED AS SCOPED OBSERVATION`: Retain a complete probe target while it is needed. | | AWS-07 | The author agrees the evidence does not establish universal source/target port equality. Commit `20247dcf0` takes the probe port from the matched blue writer pair. Pair-specific port mismatch is explicitly unsupported; different pairs may use different ports. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: Use the matched blue writer's configured port and accept failure for a target using a different port. Do not present equality as an AWS guarantee. | -| AWS-08 | The author agrees `use_ssl` is ProxySQL configuration. Automatic mode uses the matched blue writer's value; explicit mode must use the matched green writer row's value. | `RESOLVED AS AUTHOR-ACCEPTED POLICY; IMPLEMENTATION OPEN`: The tuple sources are precise, but commit `20247dcf0` invokes the hostname matcher with two green names, so normal explicit rows do not set `green_use_ssl` and the probe falls back to blue TLS. | +| AWS-08 | The author agrees `use_ssl` is ProxySQL configuration. Automatic mode uses the matched blue writer's value; explicit mode must use the matched green writer row's value. | `RESOLVED AS AUTHOR-ACCEPTED POLICY; IMPLEMENTED`: Existing rows are selected by exact TARGET hostname and matched-blue port; successfully created or restored rows supply their resolved `use_ssl`. Simulator coverage remains assigned to PR6. | | AWS-09 | The topology contains writer endpoints only; incomplete explicit reader mapping is expected and unmatched blue readers are shunned. | `RESOLVED AS POLICY`: Track and reconcile readers independently. | | AWS-10 | Commits `cdffd77ee` and `ac4167cd0` retain auto-added and user-configured green rows on rollback and success. Rollback leaves green connections untouched; success drains eligible green connections but leaves rows and statuses unchanged. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: Green membership is persistent runtime configuration, not a temporary owned effect. Administrative cleanup is required even for an auto-added row. | | AWS-11a | The author explicitly chooses one-shot worker-exit/configuration-change cleanup and no retained retry ledger. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: Loss of the cleanup context, including when a process terminates during cleanup, is accepted. The stronger retained rollback model is not PR2 scope. | @@ -919,7 +918,7 @@ that is tracked separately rather than reopening the evidence decision. | Mode | Host/IP source | Port source | SSL source | Author decision/evidence | |---|---|---|---|---| | Automatic | `AUTHOR-VALIDATED`: Resolved IP of the TARGET endpoint from `mysql.rds_topology`. | `AUTHOR-ACCEPTED-POLICY`: Matched blue writer's configured port. A different TARGET port is unsupported and is not forbidden by the recorded AWS evidence. | `AUTHOR-ACCEPTED-POLICY`: Matched blue writer's `use_ssl`, because no independent green row exists. | Policy resolved and implemented by writer-pair selection in `20247dcf0`. | -| Explicit | `AUTHOR-VALIDATED`: Resolved IP of the TARGET endpoint from `mysql.rds_topology`; the configured green writer must identify that target. | `AUTHOR-ACCEPTED-POLICY`: Matched blue writer's configured port. A different TARGET or explicit-green port is unsupported. | `AUTHOR-ACCEPTED-POLICY`: Exact matching green writer row's `use_ssl`. | Policy resolved; implementation is nonconforming because the current two-green-name matcher call does not select the explicit row. | +| Explicit | `AUTHOR-VALIDATED`: Resolved IP of the TARGET endpoint from `mysql.rds_topology`; the configured green writer must identify that target. | `AUTHOR-ACCEPTED-POLICY`: Matched blue writer's configured port. A different TARGET or explicit-green port is unsupported. | `AUTHOR-ACCEPTED-POLICY`: Exact matching green writer row's resolved `use_ssl`. | Policy resolved and implemented; PR6 owns simulator coverage. | `AUTHOR-ACCEPTED-POLICY`: A direct probe target is a complete host or IP, port, and SSL tuple derived from the matched writer pair, never from an arbitrary @@ -963,7 +962,7 @@ durable-ledger design. |---|---|---|---| | Matched writer probe tuple | `writer_tuple_not_first_poll_row` | `multiple_blue_ports_and_ssl` | With a reader first in the polling result and different ports across pairs, the direct probe uses the mapped blue writer's port and never `hpa[0]`. | | Automatic TLS source | `auto_green_inherits_writer_ssl` | `automatic_green_tls` | With no explicit green row, the direct probe and auto-added green writer use the matched blue writer's `use_ssl`. | -| Explicit TLS source | `explicit_green_ssl_override` | `explicit_green_tls_differs_from_blue` | With the same supported pair port but blue `use_ssl=0` and explicit green `use_ssl=1`, the direct IP probe enables TLS. This catches the current two-green-name matcher defect. | +| Explicit TLS source | `explicit_green_ssl_override` | `explicit_green_tls_differs_from_blue` | With the same supported pair port but blue `use_ssl=0` and explicit green `use_ssl=1`, the direct IP probe enables TLS. This guards the exact explicit-green TLS selection. | | Unsupported within-pair port mismatch | `target_port_mismatch_policy` | `target_port_mismatch_diagnostic` | The implementation's blue-port choice is explicit and observable; the test must not claim AWS guarantees equality. A future rejection diagnostic is preferable to silent probing of the wrong port. | | Eligible green generation checksum | `green_checksum_matrix` | `admin_green_add_remove_ssl_status` | Add/remove, port, `use_ssl`, and transitions into or out of `OFFLINE_SOFT`/`OFFLINE_HARD` change the checksum and replace workers; irrelevant changes do not. | | Admin commit during active phase | `config_change_exits_worker` | `load_mysql_servers_mid_switchover` | The old worker runs one-shot rollback, the dispatcher joins it, and the replacement builds a new map from the committed runtime rows. | @@ -994,10 +993,9 @@ the checklist rather than promoted to AWS guarantees. The source review remains open on implementation and verification: -1. Fix explicit green TLS selection. The current - `aws_rds_bgd_match_host(gs->address, green_writer_host)` call supplies two - green names to a blue-to-green matcher, so an explicit green `use_ssl` - differing from blue is not selected. +1. **COMPLETED:** Select explicit green TLS from the exact supported writer row, + including a row created or restored during discovery. Simulator coverage + for the existing-row and discovered-row paths remains part of PR6. 2. **COMPLETED:** Make unhealthy connection retirement terminal across reset, local pool return, and global pool return. The follow-up uses the existing `healthy` field and does not add a second flag. `connection_unhealthy_unit-t` @@ -1023,10 +1021,10 @@ proposed broad durable-ledger/controller PR is not part of this sequence. |---|---|---| | PR1: #5934 | This document only: evidence, accepted risks, current behavior, and follow-up contract. | Ready for author approval; merge into `feature/aws-rds-monitor` before implementation follow-ups so their scope is stable. | | PR2: BGD simulator foundation and CI | Add the TAP-controlled SQLite3-server simulator defined in [RDS_BGD_Simulator.md](RDS_BGD_Simulator.md): the `TEST_RDS_BGD` build mode, IP-keyed topology responses, common and BGD TAP helpers, a simulator group, an end-to-end acceptance smoke test, and an automatic CI job that executes the group. | No production behavior change. Provides the reusable harness required by PR6. A successful compile-only `CI-maketest` job is not completion evidence. | -| PR3: probe target and explicit TLS | Correct AWS-08 by selecting the exact supported explicit green writer row and its `use_ssl`, while retaining the matched blue writer port and automatic-mode blue TLS fallback. | Depends only on the documented contract. Focused unit/TAP evidence must distinguish blue `use_ssl=0` from explicit green `use_ssl=1`. | +| PR3: probe target and explicit TLS (**complete**) | Correct AWS-08 by selecting the exact supported explicit green writer row and its resolved `use_ssl`, including a row created or restored during discovery, while retaining the matched blue writer port and automatic-mode blue TLS fallback. | **Completed:** production behavior conforms to AWS-08. Existing-row and discovered-row simulator coverage remains part of PR6. | | PR4: terminal connection retirement (**complete**) | Preserve `healthy=false` across `MySQL_Connection::reset()` and destroy unhealthy connections in local and global pool-return paths. Do not introduce another flag or a new locking policy. | **Completed:** `connection_unhealthy_unit-t` proves a drained used connection cannot enter either free pool after reset or release. | | PR5: same-phase per-pair reconciliation | Replace phase-equality no-op behavior with worker-local reconciliation for incomplete map/resolution/pin/drain work. Retry only incomplete pairs and never redrain a pair already completed in the current worker generation. | Depends on the accepted one-shot worker model; it must not introduce durable ownership or restart recovery. | -| PR6: simulator-driven BGD scenario suite | Use PR2's simulator to cover configuration and discovery order, automatic and explicit rows, worker replacement, normal lifecycle, late entry, cancellation and rollback, topology drain, direct probe tuple/TLS, offline exclusions, terminal connection retirement where observable, and PR5 DNS failure/recovery. | Depends on PR2 and should normally follow PR3-PR5 so the suite validates final behavior rather than encoding known failures. All payloads run in the automatic BGD simulator CI group. | +| PR6: simulator-driven BGD scenario suite | Use PR2's simulator to cover configuration and discovery order, automatic and explicit rows, worker replacement, normal lifecycle, late entry, cancellation and rollback, topology drain, direct probe tuple/TLS for existing and discovered explicit green rows, offline exclusions, terminal connection retirement where observable, and PR5 DNS failure/recovery. | Depends on PR2 and should normally follow PR3-PR5 so the suite validates final behavior rather than encoding known failures. All payloads run in the automatic BGD simulator CI group. | Any retained cleanup ledger, durable restart ownership, or alternative controller state machine requires a new author policy decision. The simulator diff --git a/doc/AWS_Blue_Green/RDS_BGD_Simulator.md b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md index 93c1e26c2f..54bcd06a4d 100644 --- a/doc/AWS_Blue_Green/RDS_BGD_Simulator.md +++ b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md @@ -626,6 +626,8 @@ by the FSM. | Explicit BGD row before servers | Disable automatic discovery, load an explicit BGD hostgroup row, then add the blue writer, blue readers, green writer, and green readers. | The row remains `auto_generated=0`; no worker runs without an eligible blue server, and each relevant server change is incorporated by the replacement worker. | | Servers before explicit BGD row | Add blue and green servers first with automatic discovery disabled, then load the explicit BGD hostgroup row. | No BGD worker runs before the row exists; loading it starts a worker that uses the existing server membership. | | Blue first, green later | Configure blue servers, publish `AVAILABLE`, then add explicit green writer and reader rows before starting switchover. | Green membership changes replace the worker and rebuild its mapping; existing rows are not duplicated and the explicit green writer supplies its configured TLS mode. | +| Existing explicit green TLS | Configure the blue writer with `use_ssl=0` and the exact green TARGET row at the supported port with `use_ssl=1`, then publish `AVAILABLE`. | The direct metadata probe targets the resolved green writer IP with `encrypted=1`. | +| Discovered green TLS defaults | Configure the blue writer with `use_ssl=0`, leave the configured green writer hostgroup empty, and set its `servers_defaults.use_ssl=1`, then publish `AVAILABLE`. | Discovery adds the exact TARGET row with runtime `use_ssl=1`, and the subsequent green-IP metadata probe has `encrypted=1`. | | Green timing variants | With explicit green hostgroups, add green nodes before `AVAILABLE`, after discovery, or after the worker starts but before switchover. | Each ordering converges on the same runtime membership and blue/green mapping before the FSM advances. | | Automatic to explicit configuration | Allow discovery to create an automatic row, then load a user row with explicit green hostgroups. | The runtime row becomes user-defined with `auto_generated=0`, explicit green hostgroups replace NULLs, and a replacement worker uses the new configuration. | | Configuration mutation | Change `active`, hostgroup IDs, `writer_is_also_reader`, check interval/timeout, server status, or `use_ssl`; also cover row disablement and removal. | Relevant checksum changes stop the old worker, run phase-appropriate cleanup, and start or suppress a worker from the new active configuration. | diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index e278408475..b82082ddfb 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -7165,7 +7165,7 @@ static void aws_rds_bgd_build_map(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_ || gs->get_status() == MYSQL_SERVER_STATUS_OFFLINE_SOFT) { continue; } - if (aws_rds_bgd_match_host(gs->address, green_writer_host)) { + if (strcasecmp(gs->address, green_writer_host.c_str()) == 0 && gs->port == p.port) { p.green_use_ssl = gs->use_ssl; break; } @@ -7287,7 +7287,7 @@ static void aws_rds_bgd_add_green_writer_in_hg(AWS_RDS_BGD_State& st) { if (st.green_writer_hg < 0) { return; } - for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { + for (AWS_RDS_BlueGreenPair& p : st.bg_map) { if (!p.is_writer) { continue; } @@ -7297,6 +7297,10 @@ static void aws_rds_bgd_add_green_writer_in_hg(AWS_RDS_BGD_State& st) { MyHGM->wrlock(); int rc = MyHGM->create_new_server_in_hg((uint32_t)st.green_writer_hg, srv_info, srv_opts); if (rc == 0) { + MySrvC* s = MyHGM->find_server_in_hg((unsigned int)st.green_writer_hg, p.green_host, p.port); + if (s) { + p.green_use_ssl = s->use_ssl; + } MyHGM->publish_mysql_servers_to_runtime(); } MyHGM->wrunlock(); From d5581c0b24196e801bea4a07382ed520d0d0db8a Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sun, 19 Jul 2026 21:45:54 +0000 Subject: [PATCH 45/81] docs: Refine AWS RDS BGD simulator coverage - Keep simulator probe coverage on the fixed port 3306 topology. - Move DNS failure recovery out of TAP integration coverage. Signed-off-by: Wazir Ahmed --- doc/AWS_Blue_Green/RDS_BGD_Monitor.md | 14 +++++++------- doc/AWS_Blue_Green/RDS_BGD_Simulator.md | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/AWS_Blue_Green/RDS_BGD_Monitor.md b/doc/AWS_Blue_Green/RDS_BGD_Monitor.md index 38e8f5e839..b741b79f02 100644 --- a/doc/AWS_Blue_Green/RDS_BGD_Monitor.md +++ b/doc/AWS_Blue_Green/RDS_BGD_Monitor.md @@ -960,11 +960,10 @@ durable-ledger design. | Requirement | Named unit/simulator case | Named Admin/TAP case | Observable postcondition | |---|---|---|---| -| Matched writer probe tuple | `writer_tuple_not_first_poll_row` | `multiple_blue_ports_and_ssl` | With a reader first in the polling result and different ports across pairs, the direct probe uses the mapped blue writer's port and never `hpa[0]`. | +| Matched writer probe destination | `writer_tuple_not_first_poll_row` | `matched_writer_destination` | With a reader first in the polling result and every endpoint at port 3306, the direct probe uses the mapped green writer destination and never `hpa[0]`. | | Automatic TLS source | `auto_green_inherits_writer_ssl` | `automatic_green_tls` | With no explicit green row, the direct probe and auto-added green writer use the matched blue writer's `use_ssl`. | -| Explicit TLS source | `explicit_green_ssl_override` | `explicit_green_tls_differs_from_blue` | With the same supported pair port but blue `use_ssl=0` and explicit green `use_ssl=1`, the direct IP probe enables TLS. This guards the exact explicit-green TLS selection. | -| Unsupported within-pair port mismatch | `target_port_mismatch_policy` | `target_port_mismatch_diagnostic` | The implementation's blue-port choice is explicit and observable; the test must not claim AWS guarantees equality. A future rejection diagnostic is preferable to silent probing of the wrong port. | -| Eligible green generation checksum | `green_checksum_matrix` | `admin_green_add_remove_ssl_status` | Add/remove, port, `use_ssl`, and transitions into or out of `OFFLINE_SOFT`/`OFFLINE_HARD` change the checksum and replace workers; irrelevant changes do not. | +| Explicit TLS source | `explicit_green_ssl_override` | `explicit_green_tls_differs_from_blue` | With every endpoint at port 3306, blue `use_ssl=0`, and explicit green `use_ssl=1`, the direct IP probe enables TLS. This guards the exact explicit-green TLS selection. | +| Eligible green generation checksum | `green_checksum_matrix` | `admin_green_add_remove_ssl_status` | Add/remove, `use_ssl`, and transitions into or out of `OFFLINE_SOFT`/`OFFLINE_HARD` change the checksum and replace workers; irrelevant changes do not. | | Admin commit during active phase | `config_change_exits_worker` | `load_mysql_servers_mid_switchover` | The old worker runs one-shot rollback, the dispatcher joins it, and the replacement builds a new map from the committed runtime rows. | | Green membership persistence | `green_row_persists_cancel_and_success` | `green_row_lifecycle` | Auto-added and user rows remain after rollback and success; no existing status is changed. | | Green drain policy | `green_drain_status_matrix` | `green_hg_cleanup` | Rollback drains no green connections. Success drains `ONLINE`, `SHUNNED`, and `SHUNNED_AWS_BGD` green servers while leaving `OFFLINE_SOFT` and `OFFLINE_HARD` untouched. Rows remain present. | @@ -974,7 +973,7 @@ durable-ledger design. | Automatic runtime row persistence | `auto_generated_null_green_hgs` | `save_runtime_skips_auto_generated_bgd` | Auto-discovery creates a runtime row with both green hostgroups `NULL` and `auto_generated=1`; saving runtime to memory/disk does not persist that row. | | First observation COMPLETED | `fresh_worker_first_completed` | `replace_worker_at_completed` | Fresh state advances to the inferred reader phase without reconstructing a prior map, then finishes on topology drain. | | Full restart fresh start | `restart_discards_bgd_state` | `proxysql_restart_fixture` | DNS cache, pools, suppression, mapping, probe target, and FSM are recreated; configured rows reload; an unsynchronized auto-added runtime-only green row does not. | -| Same-phase DNS retry follow-up | `dns_retry_same_post_per_pair` | `first_resolution_fails_then_succeeds` | Accepted follow-up only: the unresolved pair retries while phase is unchanged; successful pairs are not redrained; the recovered pair is pinned and drained exactly once. | +| Same-phase DNS retry follow-up | `dns_retry_same_post_per_pair` | — | Future resolver/unit coverage only: the unresolved pair retries while phase is unchanged; successful pairs are not redrained; the recovered pair is pinned and drained exactly once. Mutable DNS is outside simulator/TAP scope. | | Partial pair progress follow-up | `one_pair_fails` | `multiple_reader_fixture` | Accepted follow-up only: successful pair state is retained worker-locally and only the failed pair retries. | `PROPOSED-POLICY`: The simulator cases previously proposed for durable effect @@ -1001,7 +1000,8 @@ The source review remains open on implementation and verification: `healthy` field and does not add a second flag. `connection_unhealthy_unit-t` verifies that unhealthy connections remain terminal across reset and cannot enter either free pool. -3. Track the author-accepted same-phase DNS failure as required follow-up work. +3. Track the author-accepted same-phase DNS failure as required follow-up work + with focused resolver/unit coverage rather than simulator/TAP integration. Until per-pair reconciliation exists, a transient first resolution failure in POST_PROCESSING can leave traffic unpinned and old connections undrained. Acceptance documents the risk; it does not make the failure safe. @@ -1024,7 +1024,7 @@ proposed broad durable-ledger/controller PR is not part of this sequence. | PR3: probe target and explicit TLS (**complete**) | Correct AWS-08 by selecting the exact supported explicit green writer row and its resolved `use_ssl`, including a row created or restored during discovery, while retaining the matched blue writer port and automatic-mode blue TLS fallback. | **Completed:** production behavior conforms to AWS-08. Existing-row and discovered-row simulator coverage remains part of PR6. | | PR4: terminal connection retirement (**complete**) | Preserve `healthy=false` across `MySQL_Connection::reset()` and destroy unhealthy connections in local and global pool-return paths. Do not introduce another flag or a new locking policy. | **Completed:** `connection_unhealthy_unit-t` proves a drained used connection cannot enter either free pool after reset or release. | | PR5: same-phase per-pair reconciliation | Replace phase-equality no-op behavior with worker-local reconciliation for incomplete map/resolution/pin/drain work. Retry only incomplete pairs and never redrain a pair already completed in the current worker generation. | Depends on the accepted one-shot worker model; it must not introduce durable ownership or restart recovery. | -| PR6: simulator-driven BGD scenario suite | Use PR2's simulator to cover configuration and discovery order, automatic and explicit rows, worker replacement, normal lifecycle, late entry, cancellation and rollback, topology drain, direct probe tuple/TLS for existing and discovered explicit green rows, offline exclusions, terminal connection retirement where observable, and PR5 DNS failure/recovery. | Depends on PR2 and should normally follow PR3-PR5 so the suite validates final behavior rather than encoding known failures. All payloads run in the automatic BGD simulator CI group. | +| PR6: simulator-driven BGD scenario suite | Use PR2's simulator to cover configuration and discovery order, automatic and explicit rows, worker replacement, normal lifecycle, late entry, cancellation and rollback, topology drain, direct probe destination/TLS for existing and discovered explicit green rows, offline exclusions, and terminal connection retirement where observable. | Depends on PR2 and should normally follow PR3-PR4 so the suite validates final behavior rather than encoding known failures. All payloads run in the automatic BGD simulator CI group. | Any retained cleanup ledger, durable restart ownership, or alternative controller state machine requires a new author policy decision. The simulator diff --git a/doc/AWS_Blue_Green/RDS_BGD_Simulator.md b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md index 54bcd06a4d..d0befc1cd1 100644 --- a/doc/AWS_Blue_Green/RDS_BGD_Simulator.md +++ b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md @@ -648,7 +648,7 @@ effects. They do not depend only on worker log messages. | Rollback postconditions | Trigger rollback after writer demotion, reader shunning, or DNS pinning has occurred. | Blue writer service and configured reader membership are restored, BGD-shunned readers are unshunned, pins and direct-probe state are cleared or rebuilt, runtime status resets, green rows remain, and green connections are not drained. | | Successful cleanup | Complete writer switchover, enter reader switchover, then drain topology. | Eligible green connections are drained while green rows and statuses remain; readers are reconciled and the worker returns to `NONE`. | | Late entry | Start a fresh worker with INITIATED, IN_PROGRESS, POST_PROCESSING, or COMPLETED already published. | The worker reconstructs only the state supported by that observation and applies the defined phase actions without requiring earlier samples. | -| Direct-probe policy | Vary blue/green IP, supported port, and blue versus explicit-green `use_ssl`. | Probe-log rows identify the green IP, matched writer port, and correct automatic or explicit TLS source. | +| Direct-probe policy | Vary blue/green IP and blue versus explicit-green `use_ssl` while every endpoint uses port 3306. | Probe-log rows identify the correct green writer destination at port 3306 and the correct automatic or explicit TLS source. | | Reader and offline handling | Use matched, unmatched, and `OFFLINE_SOFT`/`OFFLINE_HARD` blue and green readers. | Only eligible pairs are mapped or drained; unmatched readers follow BGD shun policy and offline nodes are excluded. | | Metadata failures | Return an empty result, error 1146, or another configured query error from selected writers. | Absence, empty metadata, and generic query failure remain distinct and never masquerade as a successful switchover. | | Repeated switchover | Complete cluster-1 deployment A, reset on empty/absent topology, replace its green rows with deployment B, and run again. | The second lifecycle uses deployment B without stale mapping, probe, or simulator state from deployment A. | From 3ff57f40fb14a70cb9a280538346e7e14d5417fd Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sun, 19 Jul 2026 21:58:51 +0000 Subject: [PATCH 46/81] test: Add TAP-controlled RDS BGD simulator helpers Signed-off-by: Wazir Ahmed --- test/tap/tap/Makefile | 10 +- test/tap/tap/cluster_simulator.cpp | 94 ++++++++++ test/tap/tap/cluster_simulator.h | 39 ++++ test/tap/tap/rds_bgd_simulator.cpp | 287 +++++++++++++++++++++++++++++ test/tap/tap/rds_bgd_simulator.h | 87 +++++++++ 5 files changed, 515 insertions(+), 2 deletions(-) create mode 100644 test/tap/tap/cluster_simulator.cpp create mode 100644 test/tap/tap/cluster_simulator.h create mode 100644 test/tap/tap/rds_bgd_simulator.cpp create mode 100644 test/tap/tap/rds_bgd_simulator.h diff --git a/test/tap/tap/Makefile b/test/tap/tap/Makefile index d7245f406e..ead866e075 100644 --- a/test/tap/tap/Makefile +++ b/test/tap/tap/Makefile @@ -80,8 +80,14 @@ noise_utils_mysql8.o: noise_utils.cpp noise_utils.h utils.h command_line.h cpp-d mcp_client.o: mcp_client.cpp mcp_client.h libcurl$(SHLIB_EXT) $(CXX) -fPIC -c mcp_client.cpp $(IDIRS) $(OPT) -libtap_mariadb.a: tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a - $(AR) rcs libtap_mariadb.a tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo +cluster_simulator.o: cluster_simulator.cpp cluster_simulator.h + $(CXX) -fPIC -c cluster_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) + +rds_bgd_simulator.o: rds_bgd_simulator.cpp rds_bgd_simulator.h cluster_simulator.h utils.h + $(CXX) -fPIC -c rds_bgd_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) + +libtap_mariadb.a: tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o rds_bgd_simulator.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a + $(AR) rcs libtap_mariadb.a tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o rds_bgd_simulator.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo libtap_mysql57.a: tap.o command_line.o utils_mysql57.o noise_utils_mysql57.o mcp_client.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a $(AR) rcs libtap_mysql57.a tap.o command_line.o utils_mysql57.o noise_utils_mysql57.o mcp_client.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo diff --git a/test/tap/tap/cluster_simulator.cpp b/test/tap/tap/cluster_simulator.cpp new file mode 100644 index 0000000000..10b13de968 --- /dev/null +++ b/test/tap/tap/cluster_simulator.cpp @@ -0,0 +1,94 @@ +#include "cluster_simulator.h" + +#include + +#include "tap.h" + +Cluster_Simulator::Cluster_Simulator() : mysql_(nullptr) {} + +Cluster_Simulator::~Cluster_Simulator() { + if (mysql_ != nullptr) { + mysql_close(mysql_); + mysql_ = nullptr; + } +} + +int Cluster_Simulator::connect( + const char* host, + int port, + const char* username, + const char* password, + bool use_ssl) +{ + if (mysql_ != nullptr) { + mysql_close(mysql_); + mysql_ = nullptr; + } + + mysql_ = mysql_init(nullptr); + if (mysql_ == nullptr) { + diag("Failed to initialize the cluster simulator connection"); + return EXIT_FAILURE; + } + + unsigned long client_flags = 0; + if (use_ssl) { + mysql_ssl_set(mysql_, nullptr, nullptr, nullptr, nullptr, nullptr); + client_flags |= CLIENT_SSL; + } + + if (mysql_real_connect( + mysql_, host, username, password, nullptr, port, nullptr, client_flags) == nullptr) { + diag( + "Failed to connect to cluster simulator at %s:%d: %s", + host, port, mysql_error(mysql_)); + mysql_close(mysql_); + mysql_ = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int Cluster_Simulator::read_only_update( + const Simulator_Endpoint& backend, bool read_only) +{ + const std::string query { + "INSERT OR REPLACE INTO READONLY_STATUS(hostname,port,read_only) VALUES (" + + sql_quote(backend.host) + "," + std::to_string(backend.port) + "," + + (read_only ? "1" : "0") + ")" + }; + return execute(query); +} + +MYSQL* Cluster_Simulator::connection() const { + return mysql_; +} + +int Cluster_Simulator::execute(const std::string& query) { + if (mysql_ == nullptr) { + diag("Cluster simulator connection is not open"); + return EXIT_FAILURE; + } + + if (mysql_query(mysql_, query.c_str()) != 0) { + diag( + "Cluster simulator query failed (%u): %s; query: %s", + mysql_errno(mysql_), mysql_error(mysql_), query.c_str()); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +std::string Cluster_Simulator::sql_quote(const std::string& value) { + std::string quoted { "'" }; + for (char c : value) { + quoted += c; + if (c == '\'') { + quoted += '\''; + } + } + quoted += '\''; + return quoted; +} diff --git a/test/tap/tap/cluster_simulator.h b/test/tap/tap/cluster_simulator.h new file mode 100644 index 0000000000..2f25c06062 --- /dev/null +++ b/test/tap/tap/cluster_simulator.h @@ -0,0 +1,39 @@ +#ifndef TAP_CLUSTER_SIMULATOR_H +#define TAP_CLUSTER_SIMULATOR_H + +#include + +#include "mysql.h" + +struct Simulator_Endpoint { + std::string host; + int port; +}; + +class Cluster_Simulator { +public: + Cluster_Simulator(); + virtual ~Cluster_Simulator(); + + Cluster_Simulator(const Cluster_Simulator&) = delete; + Cluster_Simulator& operator=(const Cluster_Simulator&) = delete; + + int connect( + const char* host, + int port, + const char* username, + const char* password, + bool use_ssl = false); + + int read_only_update(const Simulator_Endpoint& backend, bool read_only); + +protected: + MYSQL* connection() const; + int execute(const std::string& query); + static std::string sql_quote(const std::string& value); + +private: + MYSQL* mysql_; +}; + +#endif // TAP_CLUSTER_SIMULATOR_H diff --git a/test/tap/tap/rds_bgd_simulator.cpp b/test/tap/tap/rds_bgd_simulator.cpp new file mode 100644 index 0000000000..10872f7336 --- /dev/null +++ b/test/tap/tap/rds_bgd_simulator.cpp @@ -0,0 +1,287 @@ +#include "rds_bgd_simulator.h" + +#include +#include +#include +#include + +#include "tap.h" + +namespace { + +const char* probe_kind_string(RDS_BGD_Probe_Kind kind) { + return kind == RDS_BGD_Probe_Kind::table_check ? "table_check" : "metadata"; +} + +rc_t parse_probe_kind(const std::string& value) { + if (value == "table_check") { + return { EXIT_SUCCESS, RDS_BGD_Probe_Kind::table_check }; + } + if (value == "metadata") { + return { EXIT_SUCCESS, RDS_BGD_Probe_Kind::metadata }; + } + return { EXIT_FAILURE, RDS_BGD_Probe_Kind::table_check }; +} + +} // namespace + +Simulator_Endpoint RDS_BGD_Host::endpoint() const { + return { ip, port }; +} + +RDS_BGD_Cluster::RDS_BGD_Cluster() + : blue_writer_ { + "db-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.11", 3306 }, + green_writer_ { + "db-1-green-iqu47r.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.14", 3306 }, + blue_readers_ { + { "db-1-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.12", 3306 }, + { "db-1-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.13", 3306 }, + }, + green_readers_ { + { "db-1-reader-1-green-dlzky7.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.15", 3306 }, + { "db-1-reader-2-green-3fpjuu.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.16", 3306 }, + } +{} + +const RDS_BGD_Host& RDS_BGD_Cluster::blue_writer() const { + return blue_writer_; +} + +const RDS_BGD_Host& RDS_BGD_Cluster::green_writer() const { + return green_writer_; +} + +const std::vector& RDS_BGD_Cluster::blue_readers() const { + return blue_readers_; +} + +const std::vector& RDS_BGD_Cluster::green_readers() const { + return green_readers_; +} + +std::vector RDS_BGD_Cluster::get_writers() const { + return { blue_writer_.endpoint(), green_writer_.endpoint() }; +} + +std::vector RDS_BGD_Cluster::get_topology( + const std::string& status) const +{ + return { + { blue_writer_.hostname, blue_writer_.hostname, blue_writer_.port, "SOURCE", status }, + { green_writer_.hostname, green_writer_.hostname, green_writer_.port, "TARGET", status }, + }; +} + +const RDS_BGD_Cluster& rds_bgd_test_cluster() { + static const RDS_BGD_Cluster cluster {}; + return cluster; +} + +int RDS_BGD_Simulator::topology_update( + const std::vector& backends, + const std::vector& rows) +{ + if (backends.empty()) { + return EXIT_FAILURE; + } + + std::vector statements {}; + for (const Simulator_Endpoint& backend : backends) { + const std::string predicate { backend_predicate(backend) }; + statements.push_back("DELETE FROM RDS_BGD_TOPOLOGY WHERE " + predicate); + statements.push_back( + "INSERT OR REPLACE INTO RDS_BGD_CONTROL" + "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + + sql_quote(backend.host) + "," + std::to_string(backend.port) + ",1,0,'')"); + + for (std::size_t row_order = 0; row_order < rows.size(); ++row_order) { + const RDS_BGD_Topology_Row& row = rows[row_order]; + statements.push_back( + "INSERT INTO RDS_BGD_TOPOLOGY" + "(backend_ip,backend_port,row_order,id,endpoint,topology_port,role,status) VALUES (" + + sql_quote(backend.host) + "," + std::to_string(backend.port) + "," + + std::to_string(row_order) + "," + sql_quote(row.id) + "," + + sql_quote(row.endpoint) + "," + std::to_string(row.port) + "," + + sql_quote(row.role) + "," + sql_quote(row.status) + ")"); + } + } + + return execute_transaction(statements); +} + +int RDS_BGD_Simulator::topology_delete( + const std::vector& backends) +{ + if (backends.empty()) { + return EXIT_FAILURE; + } + + std::vector statements {}; + for (const Simulator_Endpoint& backend : backends) { + statements.push_back( + "DELETE FROM RDS_BGD_TOPOLOGY WHERE " + backend_predicate(backend)); + statements.push_back( + "INSERT OR REPLACE INTO RDS_BGD_CONTROL" + "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + + sql_quote(backend.host) + "," + std::to_string(backend.port) + ",1,0,'')"); + } + return execute_transaction(statements); +} + +int RDS_BGD_Simulator::topology_drop( + const std::vector& backends) +{ + return topology_error( + backends, 1146, "Table 'mysql.rds_topology' doesn't exist"); +} + +int RDS_BGD_Simulator::topology_error( + const std::vector& backends, + unsigned int error_code, + const std::string& error_msg) +{ + if (backends.empty() || error_code == 0) { + return EXIT_FAILURE; + } + + const bool topology_present = error_code != 1146; + std::vector statements {}; + for (const Simulator_Endpoint& backend : backends) { + if (!topology_present) { + statements.push_back( + "DELETE FROM RDS_BGD_TOPOLOGY WHERE " + backend_predicate(backend)); + } + statements.push_back( + "INSERT OR REPLACE INTO RDS_BGD_CONTROL" + "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + + sql_quote(backend.host) + "," + std::to_string(backend.port) + "," + + (topology_present ? "1" : "0") + "," + std::to_string(error_code) + "," + + sql_quote(error_msg) + ")"); + } + return execute_transaction(statements); +} + +rc_t RDS_BGD_Simulator::probe_log_last_sequence() { + if (connection() == nullptr) { + return { EXIT_FAILURE, 0 }; + } + + const rc_t> result { + mysql_query_ext_rows( + connection(), "SELECT COALESCE(MAX(sequence_id),0) FROM RDS_BGD_PROBE_LOG") + }; + if (result.first != EXIT_SUCCESS || result.second.size() != 1 || + result.second.front().size() != 1) { + return { EXIT_FAILURE, 0 }; + } + + return { + EXIT_SUCCESS, + static_cast(std::strtoull(result.second.front().front().c_str(), nullptr, 10)) + }; +} + +rc_t> RDS_BGD_Simulator::probe_log_since( + uint64_t sequence_id) +{ + if (connection() == nullptr) { + return { EXIT_FAILURE, {} }; + } + + const std::string query { + "SELECT sequence_id,backend_ip,backend_port,probe_kind,encrypted " + "FROM RDS_BGD_PROBE_LOG WHERE sequence_id>" + std::to_string(sequence_id) + + " ORDER BY sequence_id" + }; + const rc_t> result { + mysql_query_ext_rows(connection(), query) + }; + if (result.first != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + + std::vector logs {}; + for (const mysql_res_row& row : result.second) { + if (row.size() != 5) { + return { EXIT_FAILURE, {} }; + } + const rc_t kind { parse_probe_kind(row[3]) }; + if (kind.first != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + logs.push_back({ + static_cast(std::strtoull(row[0].c_str(), nullptr, 10)), + { row[1], std::atoi(row[2].c_str()) }, + kind.second, + std::atoi(row[4].c_str()) != 0, + }); + } + + return { EXIT_SUCCESS, std::move(logs) }; +} + +rc_t RDS_BGD_Simulator::wait_for_probe_log( + uint64_t sequence_id, + const Simulator_Endpoint& backend, + RDS_BGD_Probe_Kind probe_kind, + uint32_t timeout_ms, + int encrypted) +{ + const uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; + do { + const rc_t> logs { probe_log_since(sequence_id) }; + if (logs.first != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + for (const RDS_BGD_Probe_Log& log : logs.second) { + if (log.backend.host == backend.host && log.backend.port == backend.port && + log.probe_kind == probe_kind && + (encrypted < 0 || log.encrypted == (encrypted != 0))) { + return { EXIT_SUCCESS, log }; + } + } + usleep(50000); + } while (monotonic_time() < deadline); + + const rc_t> logs { probe_log_since(sequence_id) }; + if (logs.first == EXIT_SUCCESS) { + for (const RDS_BGD_Probe_Log& log : logs.second) { + diag( + "Observed BGD probe sequence=%llu backend=%s:%d kind=%s encrypted=%d", + static_cast(log.sequence_id), + log.backend.host.c_str(), log.backend.port, + probe_kind_string(log.probe_kind), log.encrypted ? 1 : 0); + } + } + diag( + "Timed out waiting for BGD probe backend=%s:%d kind=%s encrypted=%d", + backend.host.c_str(), backend.port, probe_kind_string(probe_kind), encrypted); + return { ETIMEDOUT, {} }; +} + +int RDS_BGD_Simulator::execute_transaction( + const std::vector& statements) +{ + if (execute("START TRANSACTION") != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + for (const std::string& statement : statements) { + if (execute(statement) != EXIT_SUCCESS) { + (void)execute("ROLLBACK"); + return EXIT_FAILURE; + } + } + if (execute("COMMIT") != EXIT_SUCCESS) { + (void)execute("ROLLBACK"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +std::string RDS_BGD_Simulator::backend_predicate( + const Simulator_Endpoint& backend) +{ + return "backend_ip=" + sql_quote(backend.host) + + " AND backend_port=" + std::to_string(backend.port); +} diff --git a/test/tap/tap/rds_bgd_simulator.h b/test/tap/tap/rds_bgd_simulator.h new file mode 100644 index 0000000000..8edc492d6c --- /dev/null +++ b/test/tap/tap/rds_bgd_simulator.h @@ -0,0 +1,87 @@ +#ifndef TAP_RDS_BGD_SIMULATOR_H +#define TAP_RDS_BGD_SIMULATOR_H + +#include +#include +#include + +#include "cluster_simulator.h" +#include "utils.h" + +struct RDS_BGD_Topology_Row { + std::string id; + std::string endpoint; + int port; + std::string role; + std::string status; +}; + +struct RDS_BGD_Host { + std::string hostname; + std::string ip; + int port; + + Simulator_Endpoint endpoint() const; +}; + +class RDS_BGD_Cluster { +public: + const RDS_BGD_Host& blue_writer() const; + const RDS_BGD_Host& green_writer() const; + const std::vector& blue_readers() const; + const std::vector& green_readers() const; + std::vector get_writers() const; + std::vector get_topology( + const std::string& status) const; + +private: + friend const RDS_BGD_Cluster& rds_bgd_test_cluster(); + RDS_BGD_Cluster(); + + RDS_BGD_Host blue_writer_; + RDS_BGD_Host green_writer_; + std::vector blue_readers_; + std::vector green_readers_; +}; + +const RDS_BGD_Cluster& rds_bgd_test_cluster(); + +enum class RDS_BGD_Probe_Kind { + table_check, + metadata, +}; + +struct RDS_BGD_Probe_Log { + uint64_t sequence_id; + Simulator_Endpoint backend; + RDS_BGD_Probe_Kind probe_kind; + bool encrypted; +}; + +class RDS_BGD_Simulator : public Cluster_Simulator { +public: + int topology_update( + const std::vector& backends, + const std::vector& rows); + int topology_delete(const std::vector& backends); + int topology_drop(const std::vector& backends); + int topology_error( + const std::vector& backends, + unsigned int error_code, + const std::string& error_msg); + + rc_t probe_log_last_sequence(); + rc_t> probe_log_since(uint64_t sequence_id); + rc_t wait_for_probe_log( + uint64_t sequence_id, + const Simulator_Endpoint& backend, + RDS_BGD_Probe_Kind probe_kind, + uint32_t timeout_ms, + int encrypted = -1); + +private: + static std::string backend_predicate(const Simulator_Endpoint& backend); + int execute_transaction(const std::vector& statements); +}; + +#endif // TAP_RDS_BGD_SIMULATOR_H From 11e03685c2262f5da0815f57f69541ac56c2293d Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sun, 19 Jul 2026 22:01:54 +0000 Subject: [PATCH 47/81] test: Add TEST_RDS_BGD SQLite simulator Signed-off-by: Wazir Ahmed --- Makefile | 16 ++- include/SQLite3_Server.h | 12 ++- src/SQLite3_Server.cpp | 212 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 225 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index b7aa0c4d92..dcd4997c01 100644 --- a/Makefile +++ b/Makefile @@ -220,6 +220,10 @@ testreadonly: build_src_testreadonly build_cluster_simulator .PHONY: testreplicationlag testreplicationlag: build_src_testreplicationlag build_cluster_simulator +.PHONY: test_rds_bgd +test_rds_bgd: build_src_test_rds_bgd + cd test/tap && OPTZ="${O0} -ggdb -DDEBUG" CC=${CC} CXX=${CXX} ${MAKE} debug + .PHONY: testall testall: build_src_testall @@ -337,13 +341,21 @@ build_src_testreplicationlag: build_lib_testreplicationlag build_lib_testreplicationlag: build_deps_debug cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_REPLICATIONLAG" CC=${CC} CXX=${CXX} ${MAKE} +.PHONY: build_src_test_rds_bgd +build_src_test_rds_bgd: build_lib_test_rds_bgd + cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_RDS_BGD" CC=${CC} CXX=${CXX} ${MAKE} + +.PHONY: build_lib_test_rds_bgd +build_lib_test_rds_bgd: build_deps_debug + cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_RDS_BGD" CC=${CC} CXX=${CXX} ${MAKE} + .PHONY: build_src_testall build_src_testall: build_lib_testall - cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA -DTEST_GALERA -DTEST_GROUPREP -DTEST_READONLY -DTEST_REPLICATIONLAG" CC=${CC} CXX=${CXX} ${MAKE} + cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA -DTEST_GALERA -DTEST_GROUPREP -DTEST_READONLY -DTEST_REPLICATIONLAG -DTEST_RDS_BGD" CC=${CC} CXX=${CXX} ${MAKE} .PHONY: build_lib_testall build_lib_testall: build_deps_debug - cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA -DTEST_GALERA -DTEST_GROUPREP -DTEST_READONLY -DTEST_REPLICATIONLAG" CC=${CC} CXX=${CXX} ${MAKE} + cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA -DTEST_GALERA -DTEST_GROUPREP -DTEST_READONLY -DTEST_REPLICATIONLAG -DTEST_RDS_BGD" CC=${CC} CXX=${CXX} ${MAKE} .PHONY: build_tap_test build_tap_test: build_tap_tests diff --git a/include/SQLite3_Server.h b/include/SQLite3_Server.h index 09fe3f9bc0..26904b7368 100644 --- a/include/SQLite3_Server.h +++ b/include/SQLite3_Server.h @@ -51,15 +51,20 @@ class SQLite3_Server { std::unordered_map grouprep_map; std::vector *tables_defs_grouprep; #endif // TEST_GROUPREP +#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) + std::vector *tables_defs_readonly; +#endif #ifdef TEST_READONLY std::unordered_map readonly_map; - std::vector *tables_defs_readonly; #endif // TEST_READONLY +#ifdef TEST_RDS_BGD + std::vector *tables_defs_rds_bgd; +#endif // TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG std::unordered_map> replicationlag_map; std::vector* tables_defs_replicationlag; #endif // TEST_REPLICATIONLAG -#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) +#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) || defined(TEST_RDS_BGD) void insert_into_tables_defs(std::vector *, const char *table_name, const char *table_def); void drop_tables_defs(std::vector *tables_defs); void check_and_build_standard_tables(SQLite3DB *db, std::vector *tables_defs); @@ -122,5 +127,8 @@ class SQLite3_Server { void wrunlock(); void send_MySQL_OK(MySQL_Protocol *myprot, char *msg, int rows=0, uint16_t status=2); void send_MySQL_ERR(MySQL_Protocol *myprot, char *msg); +#ifdef TEST_RDS_BGD + void send_MySQL_ERR(MySQL_Protocol *myprot, uint16_t error_code, const char *msg); +#endif // TEST_RDS_BGD }; #endif // CLASS_PROXYSQL_SQLITE3_SERVER_H diff --git a/src/SQLite3_Server.cpp b/src/SQLite3_Server.cpp index 25ad6b6128..afe4463fcd 100644 --- a/src/SQLite3_Server.cpp +++ b/src/SQLite3_Server.cpp @@ -11,6 +11,9 @@ #include "proxysql_utils.h" #include "MySQL_Query_Processor.h" #include "SQLite3_Server.h" +#ifdef TEST_RDS_BGD +#include "MySQL_Monitor.hpp" +#endif #include #include @@ -373,8 +376,29 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p memcpy(query,(char *)pkt->ptr+sizeof(mysql_hdr)+1,query_length-1); query[query_length-1]=0; -#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) +#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) || defined(TEST_RDS_BGD) if (sess->client_myds->proxy_addr.addr == NULL) { +#ifdef TEST_RDS_BGD + struct sockaddr_storage addr; + socklen_t addr_len=sizeof(addr); + memset(&addr,0,addr_len); + if (getsockname(sess->client_myds->fd, (struct sockaddr *)&addr, &addr_len)==0) { + char buf[INET6_ADDRSTRLEN]; + const void *src=NULL; + if (addr.ss_family == AF_INET) { + struct sockaddr_in *ipv4 = (struct sockaddr_in *)&addr; + src = &ipv4->sin_addr; + sess->client_myds->proxy_addr.port = ntohs(ipv4->sin_port); + } else if (addr.ss_family == AF_INET6) { + struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)&addr; + src = &ipv6->sin6_addr; + sess->client_myds->proxy_addr.port = ntohs(ipv6->sin6_port); + } + if (src && inet_ntop(addr.ss_family, src, buf, sizeof(buf))) { + sess->client_myds->proxy_addr.addr = strdup(buf); + } + } +#else struct sockaddr addr; socklen_t addr_len=sizeof(struct sockaddr); memset(&addr,0,addr_len); @@ -402,8 +426,9 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } else { sess->client_myds->proxy_addr.addr = strdup("unknown"); } +#endif // TEST_RDS_BGD } -#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG +#endif // TEST simulation char *query_no_space=(char *)l_alloc(query_length); memcpy(query_no_space,query,query_length); @@ -574,13 +599,13 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p if (query_no_space_length==SELECT_VERSION_COMMENT_LEN) { if (!strncasecmp(SELECT_VERSION_COMMENT, query_no_space, query_no_space_length)) { l_free(query_length,query); -#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) +#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) || defined(TEST_RDS_BGD) char *a = (char *)"SELECT '(ProxySQL Automated Test Server) - %s'"; query = (char *)malloc(strlen(a)+strlen(sess->client_myds->proxy_addr.addr)); sprintf(query,a,sess->client_myds->proxy_addr.addr); #else query=l_strdup("SELECT '(ProxySQL SQLite3 Server)'"); -#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG +#endif // TEST simulation query_length=strlen(query)+1; goto __run_query; } @@ -778,8 +803,116 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p __run_query: if (run_query) { -#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) +#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) || defined(TEST_RDS_BGD) if (strncasecmp("SELECT",query_no_space,6)==0) { +#ifdef TEST_RDS_BGD + const bool rds_bgd_table_check = + strcasecmp(query_no_space, QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK) == 0; + const bool rds_bgd_metadata = + strcasecmp(query_no_space, QUERY_AWS_RDS_TOPOLOGY_DISCOVERY) == 0; + if (rds_bgd_table_check || rds_bgd_metadata) { + if (sess->client_myds->proxy_addr.addr == NULL || + sess->client_myds->proxy_addr.port <= 0) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1105, + "RDS BGD simulator could not identify the accepted backend address"); + run_query=false; + } else { + SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; + const std::string backend_ip { sess->client_myds->proxy_addr.addr }; + const int backend_port = sess->client_myds->proxy_addr.port; + const std::string predicate { + "backend_ip='" + backend_ip + "' AND backend_port=" + + std::to_string(backend_port) + }; + char *control_error=NULL; + int control_cols=0; + int control_affected_rows=0; + SQLite3_result *control_result=NULL; + const std::string control_query { + "SELECT topology_present,error_code,error_msg FROM RDS_BGD_CONTROL WHERE " + + predicate + }; + sqlite_sess->sessdb->execute_statement( + control_query.c_str(), &control_error, &control_cols, + &control_affected_rows, &control_result); + + if (control_error != NULL) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1105, control_error); + free(control_error); + run_query=false; + } else { + bool topology_present=false; + unsigned int configured_error=0; + std::string configured_error_msg {}; + if (control_result && control_result->rows_count == 1) { + SQLite3_row *row=control_result->rows.front(); + topology_present=atoi(row->fields[0]) != 0; + configured_error=static_cast(atoi(row->fields[1])); + configured_error_msg=row->fields[2] ? row->fields[2] : ""; + } + + const std::string log_query { + "INSERT INTO RDS_BGD_PROBE_LOG" + "(backend_ip,backend_port,probe_kind,encrypted) VALUES ('" + + backend_ip + "'," + std::to_string(backend_port) + ",'" + + (rds_bgd_table_check ? "table_check" : "metadata") + "'," + + (sess->client_myds->encrypted ? "1" : "0") + ")" + }; + if (!sqlite_sess->sessdb->execute(log_query.c_str())) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1105, + "RDS BGD simulator failed to record the topology probe"); + run_query=false; + } else if (rds_bgd_table_check) { + l_free(query_length,query); + query=l_strdup(topology_present ? "SELECT 1" : "SELECT 1 WHERE 0"); + query_length=strlen(query)+1; + } else if (configured_error != 0 || !topology_present) { + const uint16_t error_code = configured_error + ? static_cast(configured_error) : 1146; + const char *error_msg = configured_error + ? configured_error_msg.c_str() + : "Table 'mysql.rds_topology' doesn't exist"; + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, error_code, error_msg); + run_query=false; + } else { + const std::string topology_query { + "SELECT id,endpoint,topology_port AS port,role,status " + "FROM RDS_BGD_TOPOLOGY WHERE " + predicate + + " ORDER BY row_order" + }; + l_free(query_length,query); + query=l_strdup(topology_query.c_str()); + query_length=strlen(query)+1; + } + } + delete control_result; + } + } + + if (run_query && !rds_bgd_table_check && !rds_bgd_metadata && + strcasecmp(query_no_space, "SELECT @@global.read_only read_only") == 0) { + if (sess->client_myds->proxy_addr.addr == NULL || + sess->client_myds->proxy_addr.port <= 0) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1105, + "RDS BGD simulator could not identify the accepted backend address"); + run_query=false; + } else { + const std::string read_only_query { + "SELECT COALESCE((SELECT read_only FROM READONLY_STATUS WHERE hostname='" + + std::string(sess->client_myds->proxy_addr.addr) + "' AND port=" + + std::to_string(sess->client_myds->proxy_addr.port) + "),1) AS read_only" + }; + l_free(query_length,query); + query=l_strdup(read_only_query.c_str()); + query_length=strlen(query)+1; + } + } +#endif // TEST_RDS_BGD #ifdef TEST_AURORA if (strstr(query_no_space,(char *)"REPLICA_HOST_STATUS")) { pthread_mutex_lock(&GloSQLite3Server->aurora_mutex); @@ -913,7 +1046,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p sprintf(query,a,rand()%30+10); } } -#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG +#endif // TEST simulation SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; if (sess->autocommit==false) { sqlite3 *db = sqlite_sess->sessdb->get_db(); @@ -1295,6 +1428,16 @@ SQLite3_Server::~SQLite3_Server() { drop_tables_defs(tables_defs_grouprep); delete tables_defs_grouprep; #endif // TEST_GROUPREP + +#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) + drop_tables_defs(tables_defs_readonly); + delete tables_defs_readonly; +#endif + +#ifdef TEST_RDS_BGD + drop_tables_defs(tables_defs_rds_bgd); + delete tables_defs_rds_bgd; +#endif // TEST_RDS_BGD }; #ifdef TEST_AURORA @@ -1382,7 +1525,7 @@ SQLite3_Server::SQLite3_Server() { variables.read_only=false; -#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) +#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) || defined(TEST_RDS_BGD) string s = ""; #ifdef TEST_AURORA @@ -1410,12 +1553,19 @@ SQLite3_Server::SQLite3_Server() { s += "0.0.0.0:3306"; pthread_mutex_init(&test_replicationlag_mutex, NULL); #endif //TEST_REPLICATIONLAG +#ifdef TEST_RDS_BGD +#if !defined(TEST_READONLY) && !defined(TEST_REPLICATIONLAG) + if (!s.empty()) + s += ";"; + s += "0.0.0.0:3306"; +#endif +#endif // TEST_RDS_BGD variables.mysql_ifaces=strdup(s.c_str()); #else variables.mysql_ifaces=strdup("127.0.0.1:6030"); -#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG +#endif // TEST simulation }; @@ -1778,7 +1928,7 @@ void SQLite3_Server::populate_grouprep_table(MySQL_Session *sess, int txs_behind #endif // TEST_GALERA -#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) +#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) || defined(TEST_RDS_BGD) void SQLite3_Server::insert_into_tables_defs(std::vector *tables_defs, const char *table_name, const char *table_def) { table_def_t *td = new table_def_t; td->table_name=strdup(table_name); @@ -1808,7 +1958,7 @@ void SQLite3_Server::drop_tables_defs(std::vector *tables_defs) { delete td; } }; -#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG +#endif // TEST simulation void SQLite3_Server::wrlock() { pthread_rwlock_wrlock(&rwlock); @@ -1859,14 +2009,41 @@ bool SQLite3_Server::init() { check_and_build_standard_tables(sessdb, tables_defs_grouprep); GloAdmin->enable_grouprep_testing(); #endif // TEST_GALERA -#ifdef TEST_READONLY +#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) tables_defs_readonly = new std::vector; insert_into_tables_defs(tables_defs_readonly, (const char *)"READONLY_STATUS", (const char*)"CREATE TABLE READONLY_STATUS (hostname VARCHAR NOT NULL , port INT NOT NULL , read_only INT NOT NULL CHECK (read_only IN (0, 1)) DEFAULT 1 , PRIMARY KEY (hostname, port))"); check_and_build_standard_tables(sessdb, tables_defs_readonly); +#ifdef TEST_READONLY GloAdmin->enable_readonly_testing(); #endif // TEST_READONLY +#endif // TEST_READONLY || TEST_RDS_BGD +#ifdef TEST_RDS_BGD + tables_defs_rds_bgd = new std::vector; + insert_into_tables_defs(tables_defs_rds_bgd, + (const char *)"RDS_BGD_CONTROL", + (const char *)"CREATE TABLE RDS_BGD_CONTROL (" + "backend_ip TEXT NOT NULL, backend_port INTEGER NOT NULL, " + "topology_present INTEGER NOT NULL DEFAULT 0 CHECK (topology_present IN (0,1)), " + "error_code INTEGER NOT NULL DEFAULT 0, error_msg TEXT NOT NULL DEFAULT '', " + "PRIMARY KEY (backend_ip, backend_port))"); + insert_into_tables_defs(tables_defs_rds_bgd, + (const char *)"RDS_BGD_TOPOLOGY", + (const char *)"CREATE TABLE RDS_BGD_TOPOLOGY (" + "backend_ip TEXT NOT NULL, backend_port INTEGER NOT NULL, row_order INTEGER NOT NULL, " + "id TEXT NOT NULL, endpoint TEXT NOT NULL, topology_port INTEGER NOT NULL, " + "role TEXT NOT NULL, status TEXT NOT NULL, " + "PRIMARY KEY (backend_ip, backend_port, row_order))"); + insert_into_tables_defs(tables_defs_rds_bgd, + (const char *)"RDS_BGD_PROBE_LOG", + (const char *)"CREATE TABLE RDS_BGD_PROBE_LOG (" + "sequence_id INTEGER PRIMARY KEY AUTOINCREMENT, backend_ip TEXT NOT NULL, " + "backend_port INTEGER NOT NULL, probe_kind TEXT NOT NULL " + "CHECK (probe_kind IN ('table_check','metadata')), encrypted INTEGER NOT NULL " + "CHECK (encrypted IN (0,1)))"); + check_and_build_standard_tables(sessdb, tables_defs_rds_bgd); +#endif // TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG tables_defs_replicationlag = new std::vector; insert_into_tables_defs(tables_defs_replicationlag, @@ -1987,6 +2164,19 @@ void SQLite3_Server::send_MySQL_ERR(MySQL_Protocol *myprot, char *msg) { myds->DSS=STATE_SLEEP; } +#ifdef TEST_RDS_BGD +void SQLite3_Server::send_MySQL_ERR( + MySQL_Protocol *myprot, uint16_t error_code, const char *msg) +{ + assert(myprot); + MySQL_Data_Stream *myds=myprot->get_myds(); + myds->DSS=STATE_QUERY_SENT_DS; + char *sqlstate = error_code == 1146 ? (char *)"42S02" : (char *)"HY000"; + myprot->generate_pkt_ERR(true,NULL,NULL,1,error_code,sqlstate,msg); + myds->DSS=STATE_SLEEP; +} +#endif // TEST_RDS_BGD + #ifdef TEST_READONLY void SQLite3_Server::load_readonly_table(MySQL_Session *sess) { // this function needs to be called with lock on mutex readonly_mutex already acquired From a04c4fc51d7a6756b555a513db68ad5ce4408b58 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sun, 19 Jul 2026 22:03:45 +0000 Subject: [PATCH 48/81] fix: Stop dispatch after simulated BGD errors Signed-off-by: Wazir Ahmed --- src/SQLite3_Server.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/SQLite3_Server.cpp b/src/SQLite3_Server.cpp index afe4463fcd..60fd326d73 100644 --- a/src/SQLite3_Server.cpp +++ b/src/SQLite3_Server.cpp @@ -1047,6 +1047,11 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } } #endif // TEST simulation + if (!run_query) { + l_free(pkt->size-sizeof(mysql_hdr),query_no_space); + l_free(query_length,query); + return; + } SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; if (sess->autocommit==false) { sqlite3 *db = sqlite_sess->sessdb->get_db(); From 8ef0e9b2ab1caae5158b248d13ab646a5fe0a37f Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sun, 19 Jul 2026 22:04:38 +0000 Subject: [PATCH 49/81] test: Add RDS BGD simulator smoke coverage Signed-off-by: Wazir Ahmed --- test/infra/README.md | 3 +- test/tap/groups/cluster_sim_rds_bgd/add-hosts | 30 +++++ test/tap/groups/cluster_sim_rds_bgd/env.sh | 4 + .../cluster_sim_rds_bgd/pre-proxysql.bash | 5 + .../cluster_sim_rds_bgd/pre-proxysql.sql | 8 ++ test/tap/groups/groups.json | 1 + test/tap/tests/test_rds_bgd-t.cpp | 117 ++++++++++++++++++ 7 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 test/tap/groups/cluster_sim_rds_bgd/add-hosts create mode 100644 test/tap/groups/cluster_sim_rds_bgd/env.sh create mode 100755 test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.bash create mode 100644 test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.sql create mode 100644 test/tap/tests/test_rds_bgd-t.cpp diff --git a/test/infra/README.md b/test/infra/README.md index 194ad5b005..d2dd9c2f76 100644 --- a/test/infra/README.md +++ b/test/infra/README.md @@ -48,8 +48,9 @@ Groups whose name starts with `cluster_sim_` (e.g. `cluster_sim_aurora-g1`, `clu | `cluster_sim_group_repl-g` | `make testgrouprep` | | `cluster_sim_read_only-g` | `make testreadonly` | | `cluster_sim_repl_lag-g` | `make testreplicationlag` | +| `cluster_sim_rds_bgd-g` | `make test_rds_bgd` | -Each target sets the corresponding `-DTEST_` flag on the ProxySQL src and lib build and triggers the simulator binary build. A plain `make` is **not sufficient** for these groups. +Each target sets the corresponding `-DTEST_` flag on the ProxySQL src and lib build. The BGD target builds its TAP-controlled SQLite3-server simulator directly; the other targets also build `test/deps/cluster_simulator`. A plain `make` is **not sufficient** for these groups. --- ## 1. Core Concepts diff --git a/test/tap/groups/cluster_sim_rds_bgd/add-hosts b/test/tap/groups/cluster_sim_rds_bgd/add-hosts new file mode 100644 index 0000000000..4fec081138 --- /dev/null +++ b/test/tap/groups/cluster_sim_rds_bgd/add-hosts @@ -0,0 +1,30 @@ +# Cluster 1: blue endpoints +db-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.11 +db-1-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.12 +db-1-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.13 + +# Cluster 1: green deployment A +db-1-green-iqu47r.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.14 +db-1-reader-1-green-dlzky7.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.15 +db-1-reader-2-green-3fpjuu.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.16 + +# Cluster 1: green deployment B +db-1-green-s7m2kx.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.17 +db-1-reader-1-green-v4n8qp.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.18 +db-1-reader-2-green-w6h3rz.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.19 + +# Cluster 2 +db-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.20 +db-2-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.21 +db-2-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.22 +db-2-green-iqu47r.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.23 +db-2-reader-1-green-dlzky7.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.24 +db-2-reader-2-green-3fpjuu.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.25 + +# Cluster 3 +db-3.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.26 +db-3-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.27 +db-3-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.28 +db-3-green-iqu47r.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.29 +db-3-reader-1-green-dlzky7.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.30 +db-3-reader-2-green-3fpjuu.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.31 diff --git a/test/tap/groups/cluster_sim_rds_bgd/env.sh b/test/tap/groups/cluster_sim_rds_bgd/env.sh new file mode 100644 index 0000000000..482577530e --- /dev/null +++ b/test/tap/groups/cluster_sim_rds_bgd/env.sh @@ -0,0 +1,4 @@ +# shellcheck shell=bash + +export CLUSTER_SIM_HOST_FILE="${WORKSPACE}/test/tap/groups/cluster_sim_rds_bgd/add-hosts" +export SKIP_CLUSTER_START=1 diff --git a/test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.bash b/test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.bash new file mode 100755 index 0000000000..97e8edeb18 --- /dev/null +++ b/test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.bash @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -e + +# ProxySQL's Admin port is available before module startup has fully settled. +sleep 5 diff --git a/test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.sql b/test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.sql new file mode 100644 index 0000000000..ac40c96ec3 --- /dev/null +++ b/test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.sql @@ -0,0 +1,8 @@ +INSERT OR REPLACE INTO mysql_users (username, password, default_hostgroup, active) + VALUES ('testuser', 'testuser', 0, 1); +LOAD MYSQL USERS TO RUNTIME; +SAVE MYSQL USERS TO DISK; + +SET sqliteserver-mysql_ifaces='0.0.0.0:3306'; +LOAD SQLITESERVER VARIABLES TO RUNTIME; +SAVE SQLITESERVER VARIABLES TO DISK; diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index 9b62161280..5854dd00cf 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -334,6 +334,7 @@ "test_cluster_sim_group_repl-t" : [ "cluster_sim_group_repl-g1" ], "test_cluster_sim_read_only-t" : [ "cluster_sim_read_only-g1" ], "test_cluster_sim_repl_lag-t" : [ "cluster_sim_repl_lag-g1" ], + "test_rds_bgd-t" : [ "cluster_sim_rds_bgd-g1" ], "test_cluster_sync-t" : [ "legacy-g5","mysql-auto_increment_delay_multiplex=0-g3","mysql-multiplexing=false-g3","mysql-query_digests=0-g3","mysql-query_digests_keep_comment=1-g3","mysql84-g5","mysql90-g5","mysql95-g5" ], "test_cluster_sync_mysql_servers-t" : [ "legacy-g5","mysql-auto_increment_delay_multiplex=0-g3","mysql-multiplexing=false-g3","mysql-query_digests=0-g3","mysql-query_digests_keep_comment=1-g3","mysql84-g5","mysql90-g5","mysql95-g5" ], "test_cluster_sync_pgsql-t" : [ "legacy-g5","mysql-auto_increment_delay_multiplex=0-g3","mysql-multiplexing=false-g3","mysql-query_digests=0-g3","mysql-query_digests_keep_comment=1-g3" ], diff --git a/test/tap/tests/test_rds_bgd-t.cpp b/test/tap/tests/test_rds_bgd-t.cpp new file mode 100644 index 0000000000..94832af5bf --- /dev/null +++ b/test/tap/tests/test_rds_bgd-t.cpp @@ -0,0 +1,117 @@ +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_simulator.h" +#include "tap.h" +#include "utils.h" + +namespace { + +int execute_all(MYSQL* admin, const std::vector& queries) { + for (const std::string& query : queries) { + if (mysql_query(admin, query.c_str()) != 0) { + diag( + "Admin query failed (%u): %s; query: %s", + mysql_errno(admin), mysql_error(admin), query.c_str()); + return EXIT_FAILURE; + } + } + return EXIT_SUCCESS; +} + +int configure_proxysql_for_bgd( + MYSQL* admin, const RDS_BGD_Cluster& cluster) +{ + const RDS_BGD_Host& writer = cluster.blue_writer(); + return execute_all(admin, { + "DELETE FROM mysql_servers", + "DELETE FROM mysql_replication_hostgroups", + "DELETE FROM mysql_aws_rds_bgd_hostgroups", + "INSERT INTO mysql_replication_hostgroups(writer_hostgroup,reader_hostgroup) " + "VALUES (10,20)", + "INSERT INTO mysql_aws_rds_bgd_hostgroups(" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment) " + "VALUES (10,20,30,40,1,0,100,800,'BGD simulator smoke test')", + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,use_ssl,comment) VALUES (10,'" + + writer.hostname + "'," + std::to_string(writer.port) + ",0,'blue writer')", + "SET mysql-monitor_username='testuser'", + "SET mysql-monitor_password='testuser'", + "SET mysql-monitor_enabled='true'", + "LOAD MYSQL VARIABLES TO RUNTIME", + "LOAD MYSQL SERVERS TO RUNTIME", + }); +} + +} // namespace + +int main() { + plan(3); + + CommandLine cl {}; + if (cl.getEnv()) { + BAIL_OUT("failed to load TAP environment"); + } + + MYSQL* admin = init_mysql_conn( + cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + BAIL_OUT("failed to connect to ProxySQL Admin"); + } + + RDS_BGD_Simulator simulator {}; + if (simulator.connect( + cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + mysql_close(admin); + BAIL_OUT("failed to connect to the SQLite3-server simulator"); + } + + const RDS_BGD_Cluster& cluster = rds_bgd_test_cluster(); + for (const Simulator_Endpoint& writer : cluster.get_writers()) { + if (simulator.read_only_update(writer, false) != EXIT_SUCCESS) { + mysql_close(admin); + BAIL_OUT("failed to configure writer read_only state"); + } + } + + const rc_t mark = simulator.probe_log_last_sequence(); + if (mark.first != EXIT_SUCCESS) { + mysql_close(admin); + BAIL_OUT("failed to read the BGD probe-log watermark"); + } + + const int update_rc = simulator.topology_update( + cluster.get_writers(), cluster.get_topology("AVAILABLE")); + ok(update_rc == EXIT_SUCCESS, "publish AVAILABLE topology to both writer IPs"); + if (update_rc != EXIT_SUCCESS) { + mysql_close(admin); + BAIL_OUT("failed to publish BGD topology"); + } + + if (configure_proxysql_for_bgd(admin, cluster) != EXIT_SUCCESS) { + mysql_close(admin); + BAIL_OUT("failed to configure ProxySQL for BGD monitoring"); + } + + const int status_rc = wait_for_cond( + admin, + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_rds_bgd_hostgroups " + "WHERE writer_hostgroup=10 AND status='AVAILABLE'", + 10); + ok(status_rc == EXIT_SUCCESS, "ProxySQL enters the AVAILABLE BGD state"); + + const rc_t green_probe = simulator.wait_for_probe_log( + mark.second, + cluster.green_writer().endpoint(), + RDS_BGD_Probe_Kind::metadata, + 10000, + 0); + ok( + green_probe.first == EXIT_SUCCESS, + "ProxySQL probes topology directly on the green writer IP over plaintext"); + + mysql_close(admin); + return exit_status(); +} From 46592fede74227e6e86642135584a53ddfc62e3d Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sun, 19 Jul 2026 22:06:56 +0000 Subject: [PATCH 50/81] fix: Use AWS BGD topology role values Signed-off-by: Wazir Ahmed --- test/tap/tap/rds_bgd_simulator.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/tap/tap/rds_bgd_simulator.cpp b/test/tap/tap/rds_bgd_simulator.cpp index 10872f7336..7c23332b6f 100644 --- a/test/tap/tap/rds_bgd_simulator.cpp +++ b/test/tap/tap/rds_bgd_simulator.cpp @@ -68,8 +68,10 @@ std::vector RDS_BGD_Cluster::get_topology( const std::string& status) const { return { - { blue_writer_.hostname, blue_writer_.hostname, blue_writer_.port, "SOURCE", status }, - { green_writer_.hostname, green_writer_.hostname, green_writer_.port, "TARGET", status }, + { blue_writer_.hostname, blue_writer_.hostname, blue_writer_.port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", status }, + { green_writer_.hostname, green_writer_.hostname, green_writer_.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", status }, }; } From 9acdf653ecadb84d7c7c7fb5e8311a9e937fdea2 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sun, 19 Jul 2026 22:08:33 +0000 Subject: [PATCH 51/81] docs: Mark AWS RDS BGD simulator complete Signed-off-by: Wazir Ahmed --- doc/AWS_Blue_Green/RDS_BGD_Monitor.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/AWS_Blue_Green/RDS_BGD_Monitor.md b/doc/AWS_Blue_Green/RDS_BGD_Monitor.md index b741b79f02..f434c96fd8 100644 --- a/doc/AWS_Blue_Green/RDS_BGD_Monitor.md +++ b/doc/AWS_Blue_Green/RDS_BGD_Monitor.md @@ -1,7 +1,7 @@ # AWS RDS Blue/Green Monitor -**Document status:** AUTHOR VALIDATION COMPLETE; FOLLOW-UP HANDOFF DEFINED; -IMPLEMENTATION CONFORMANCE OPEN +**Document status:** AUTHOR VALIDATION COMPLETE; PR2 SIMULATOR FOUNDATION +COMPLETE; IMPLEMENTATION CONFORMANCE OPEN **Applies to:** Amazon RDS Multi-AZ DB instance blue/green deployment monitoring @@ -1020,7 +1020,7 @@ proposed broad durable-ledger/controller PR is not part of this sequence. | Review PR | Scope | Dependency and completion signal | |---|---|---| | PR1: #5934 | This document only: evidence, accepted risks, current behavior, and follow-up contract. | Ready for author approval; merge into `feature/aws-rds-monitor` before implementation follow-ups so their scope is stable. | -| PR2: BGD simulator foundation and CI | Add the TAP-controlled SQLite3-server simulator defined in [RDS_BGD_Simulator.md](RDS_BGD_Simulator.md): the `TEST_RDS_BGD` build mode, IP-keyed topology responses, common and BGD TAP helpers, a simulator group, an end-to-end acceptance smoke test, and an automatic CI job that executes the group. | No production behavior change. Provides the reusable harness required by PR6. A successful compile-only `CI-maketest` job is not completion evidence. | +| PR2: BGD simulator foundation — COMPLETED | Add the TAP-controlled SQLite3-server simulator defined in [RDS_BGD_Simulator.md](RDS_BGD_Simulator.md): the `TEST_RDS_BGD` build mode, IP-keyed topology responses, common and BGD TAP helpers, a simulator group, and an end-to-end acceptance smoke test. | Completed after the isolated local Docker group passed. Provides the reusable harness required by PR6; automatic GitHub Actions execution remains separate follow-up work under the review gate above. | | PR3: probe target and explicit TLS (**complete**) | Correct AWS-08 by selecting the exact supported explicit green writer row and its resolved `use_ssl`, including a row created or restored during discovery, while retaining the matched blue writer port and automatic-mode blue TLS fallback. | **Completed:** production behavior conforms to AWS-08. Existing-row and discovered-row simulator coverage remains part of PR6. | | PR4: terminal connection retirement (**complete**) | Preserve `healthy=false` across `MySQL_Connection::reset()` and destroy unhealthy connections in local and global pool-return paths. Do not introduce another flag or a new locking policy. | **Completed:** `connection_unhealthy_unit-t` proves a drained used connection cannot enter either free pool after reset or release. | | PR5: same-phase per-pair reconciliation | Replace phase-equality no-op behavior with worker-local reconciliation for incomplete map/resolution/pin/drain work. Retry only incomplete pairs and never redrain a pair already completed in the current worker generation. | Depends on the accepted one-shot worker model; it must not introduce durable ownership or restart recovery. | From 66e81887911930808d558c26335bde74aef118e3 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sun, 19 Jul 2026 23:49:15 +0000 Subject: [PATCH 52/81] fix: Ensure simulated backend address is initialized Signed-off-by: Wazir Ahmed --- src/SQLite3_Server.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/SQLite3_Server.cpp b/src/SQLite3_Server.cpp index 60fd326d73..1da52af4dd 100644 --- a/src/SQLite3_Server.cpp +++ b/src/SQLite3_Server.cpp @@ -398,6 +398,9 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p sess->client_myds->proxy_addr.addr = strdup(buf); } } + if (sess->client_myds->proxy_addr.addr == NULL) { + sess->client_myds->proxy_addr.addr = strdup("unknown"); + } #else struct sockaddr addr; socklen_t addr_len=sizeof(struct sockaddr); From 8f1dec3a26d1b144ccb47015ea6c64a43de70aa3 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 21 Jul 2026 20:01:45 +0000 Subject: [PATCH 53/81] fix: Reuse read-only simulation for AWS RDS BGD - Reuse the existing monitor query suffix and read-only cache instead of adding a BGD-specific lookup path. - Populate BGD read-only state through shared READONLY_STATUS hostname and port entries. Signed-off-by: Wazir Ahmed --- doc/AWS_Blue_Green/RDS_BGD_Simulator.md | 52 ++++++++------ include/SQLite3_Server.h | 10 ++- lib/MySQL_Monitor.cpp | 42 ++++++----- src/SQLite3_Server.cpp | 92 ++++--------------------- test/tap/tests/test_rds_bgd-t.cpp | 9 ++- 5 files changed, 81 insertions(+), 124 deletions(-) diff --git a/doc/AWS_Blue_Green/RDS_BGD_Simulator.md b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md index d0befc1cd1..cef6a78e65 100644 --- a/doc/AWS_Blue_Green/RDS_BGD_Simulator.md +++ b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md @@ -17,9 +17,10 @@ coverage into one implementation specification. ## Architecture The TAP test is the scenario controller. It configures ProxySQL with AWS-style -hostnames, writes IP-keyed backend state to ProxySQL's SQLite3 server, changes +hostnames, writes simulated backend state to ProxySQL's SQLite3 server, changes that state to drive the BGD FSM, and verifies ProxySQL through runtime, -statistics, and simulator probe-log tables. +statistics, and simulator probe-log tables. Topology state is keyed by backend +IP, while read-only state is keyed by the configured hostname. No `test/deps/cluster_simulator` process or backend database container is required. A common TAP helper owns reusable SQLite3-server operations, while a @@ -45,9 +46,10 @@ and green names. Every hostname resolves to a distinct loopback IP and uses port 3306, preserving the address shape used by AWS while a single wildcard SQLite3-server listener handles all simulated endpoints. -Tests add servers to ProxySQL by hostname and configure simulator state using -the corresponding IP. Distinct destination IPs retain the blue/green split -when ProxySQL resolves a hostname or directly probes the resolved green IP. +Tests add servers to ProxySQL by hostname. They configure topology state using +the corresponding IP and read-only state using the configured hostname and +port. Distinct destination IPs retain the blue/green split when ProxySQL +resolves a hostname or directly probes the resolved green IP. The map reserves multiple clusters and two green endpoint sets for cluster 1. Tests configure only the endpoints they need: separate clusters support @@ -128,9 +130,10 @@ select the response described below, append a probe-log row, and send the result. An address-extraction failure returns a simulator error without selecting state or logging an invalid backend identity. -All other statements continue through normal SQLite3-server handling. TAP -control and inspection statements against the simulator tables are not -rewritten or recorded as BGD monitor probes. +Simulated read-only checks follow the handling described below. All remaining +statements continue through normal SQLite3-server handling. TAP control and +inspection statements against the simulator tables are not rewritten or +recorded as BGD monitor probes. ## Control-State Meaning @@ -188,20 +191,21 @@ to verify the selected destination and TLS mode. A probe-log insertion failure is a simulator failure and must not be silently reported as a normal backend response. -## `read_only` Reuse +## Read-Only Simulation -Build the existing `READONLY_STATUS` table for `TEST_RDS_BGD`, but do not add a -BGD-specific read-only table or call `enable_readonly_testing()`. The TAP test -owns ProxySQL hostgroup and server configuration. +`TEST_RDS_BGD` builds the shared `READONLY_STATUS(hostname, port, read_only)` +table and the read-only cache, without calling `enable_readonly_testing()`. +The TAP test owns ProxySQL hostgroup and server configuration and writes each +read-only value using the AWS hostname configured in `mysql_servers`. -For the production `SELECT @@global.read_only ...` monitor query, resolve the -same backend key and select `READONLY_STATUS` using the backend IP as its -`hostname` value. Return the configured value as one `read_only` column; a -missing entry uses the existing safe default of `read_only=1`. +Read-only monitor tasks send the simulation query +`SELECT @@global.read_only read_only :`. The SQLite3 server +uses the suffix to read the cached value populated from `READONLY_STATUS` and +returns one `read_only` column. Table writes refresh the cache, and a missing +entry returns the safe default `read_only=1`. -This path does not consult `RDS_BGD_CONTROL` or write `RDS_BGD_PROBE_LOG`. The -legacy `TEST_READONLY` query-suffix behavior remains unchanged in its own -build. +BGD topology tasks send the production topology queries unchanged. Read-only +handling does not consult `RDS_BGD_CONTROL` or write `RDS_BGD_PROBE_LOG`. ## TAP Helper API @@ -219,7 +223,8 @@ struct Simulator_Endpoint { ``` Identifies one simulated backend. For BGD topology and probe-log operations, -`host` is the backend IP. +`host` is the backend IP. For `read_only_update()`, `host` is the AWS hostname +configured in ProxySQL. ### `Cluster_Simulator` @@ -236,7 +241,7 @@ int read_only_update(const Simulator_Endpoint& backend, bool read_only); `connect()` opens the SQLite3-server control connection with the MySQL client API; the helper closes it when destroyed. `read_only_update()` changes the -existing `READONLY_STATUS` row for one backend. +`READONLY_STATUS` row identified by configured hostname and port. ### Topology and Host Types @@ -367,6 +372,11 @@ int main() { sqlite_server.first.c_str(), sqlite_server.second, cl.username, cl.password) != EXIT_SUCCESS) BAIL_OUT("failed to connect to SQLite3 server"); + if (simulator.read_only_update( + { cluster.blue_writer().hostname, cluster.blue_writer().port }, false) != EXIT_SUCCESS || + simulator.read_only_update( + { cluster.green_writer().hostname, cluster.green_writer().port }, false) != EXIT_SUCCESS) + BAIL_OUT("failed to configure writer read_only state"); const rc_t mark = simulator.probe_log_last_sequence(); if (mark.first != EXIT_SUCCESS) diff --git a/include/SQLite3_Server.h b/include/SQLite3_Server.h index 26904b7368..3be2596de2 100644 --- a/include/SQLite3_Server.h +++ b/include/SQLite3_Server.h @@ -54,9 +54,9 @@ class SQLite3_Server { #if defined(TEST_READONLY) || defined(TEST_RDS_BGD) std::vector *tables_defs_readonly; #endif -#ifdef TEST_READONLY +#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) std::unordered_map readonly_map; -#endif // TEST_READONLY +#endif // TEST_READONLY || TEST_RDS_BGD #ifdef TEST_RDS_BGD std::vector *tables_defs_rds_bgd; #endif // TEST_RDS_BGD @@ -99,14 +99,14 @@ class SQLite3_Server { void init_grouprep_ifaces_string(std::string& s); group_rep_status grouprep_test_value(const std::string& srv_addr); #endif // TEST_GROUPREP -#ifdef TEST_READONLY +#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) pthread_mutex_t test_readonly_mutex; void load_readonly_table(MySQL_Session *sess); int readonly_test_value(char *p); int readonly_map_size() { return readonly_map.size(); } -#endif // TEST_READONLY +#endif // TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG pthread_mutex_t test_replicationlag_mutex; void load_replicationlag_table(MySQL_Session* sess); @@ -127,8 +127,6 @@ class SQLite3_Server { void wrunlock(); void send_MySQL_OK(MySQL_Protocol *myprot, char *msg, int rows=0, uint16_t status=2); void send_MySQL_ERR(MySQL_Protocol *myprot, char *msg); -#ifdef TEST_RDS_BGD void send_MySQL_ERR(MySQL_Protocol *myprot, uint16_t error_code, const char *msg); -#endif // TEST_RDS_BGD }; #endif // CLASS_PROXYSQL_SQLITE3_SERVER_H diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index b82082ddfb..1c46733700 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -682,7 +682,7 @@ void MySQL_Monitor_State_Data::init_async() { task_timeout_ = mysql_thread___monitor_ping_timeout; task_handler_ = &MySQL_Monitor_State_Data::ping_handler; break; -#ifndef TEST_READONLY +#if !defined(TEST_READONLY) && !defined(TEST_RDS_BGD) case MON_READ_ONLY: query_ = "SELECT @@global.read_only read_only"; async_state_machine_ = ASYNC_QUERY_START; @@ -713,13 +713,7 @@ void MySQL_Monitor_State_Data::init_async() { task_timeout_ = mysql_thread___monitor_read_only_timeout; task_handler_ = &MySQL_Monitor_State_Data::read_only_handler; break; - case MON_AWS_RDS_TOPOLOGY_DISCOVERY: - query_ = QUERY_AWS_RDS_TOPOLOGY_DISCOVERY; - async_state_machine_ = ASYNC_QUERY_START; - task_timeout_ = mysql_thread___monitor_read_only_timeout; - task_handler_ = &MySQL_Monitor_State_Data::read_only_handler; - break; -#else // TEST_READONLY +#else // TEST_READONLY || TEST_RDS_BGD case MON_READ_ONLY: case MON_INNODB_READ_ONLY: case MON_SUPER_READ_ONLY: @@ -731,7 +725,15 @@ void MySQL_Monitor_State_Data::init_async() { task_timeout_ = mysql_thread___monitor_read_only_timeout; task_handler_ = &MySQL_Monitor_State_Data::read_only_handler; break; -#endif // TEST_READONLY +#endif // TEST_READONLY || TEST_RDS_BGD +#if !defined(TEST_READONLY) || defined(TEST_RDS_BGD) + case MON_AWS_RDS_TOPOLOGY_DISCOVERY: + query_ = QUERY_AWS_RDS_TOPOLOGY_DISCOVERY; + async_state_machine_ = ASYNC_QUERY_START; + task_timeout_ = mysql_thread___monitor_read_only_timeout; + task_handler_ = &MySQL_Monitor_State_Data::read_only_handler; + break; +#endif // !TEST_READONLY || TEST_RDS_BGD case MON_GROUP_REPLICATION: async_state_machine_ = ASYNC_QUERY_START; #ifdef TEST_GROUPREP @@ -1758,7 +1760,17 @@ void * monitor_read_only_thread(const std::vector& da mmsd->t1=monotonic_time(); mmsd->interr=0; // reset the value -#ifndef TEST_READONLY +#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#ifdef TEST_RDS_BGD + if (mmsd->get_task_type() == MON_AWS_RDS_TOPOLOGY_DISCOVERY) { + monitor_query = QUERY_AWS_RDS_TOPOLOGY_DISCOVERY; + } else +#endif // TEST_RDS_BGD + { + monitor_query = "SELECT @@global.read_only read_only"; + monitor_query += " " + std::string(mmsd->hostname) + ":" + std::to_string(mmsd->port); + } +#else if (mmsd->get_task_type() == MON_INNODB_READ_ONLY) { monitor_query = "SELECT @@global.innodb_read_only read_only"; } else if (mmsd->get_task_type() == MON_SUPER_READ_ONLY) { @@ -1772,14 +1784,8 @@ void * monitor_read_only_thread(const std::vector& da } else { // default monitor_query = "SELECT @@global.read_only read_only"; } - mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql, monitor_query.c_str()); -#else // TEST_READONLY - { - monitor_query = "SELECT @@global.read_only read_only"; - monitor_query += " " + std::string(mmsd->hostname) + ":" + std::to_string(mmsd->port); - mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,monitor_query.c_str()); - } -#endif // TEST_READONLY +#endif // TEST_READONLY || TEST_RDS_BGD + mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,monitor_query.c_str()); while (mmsd->async_exit_status) { mmsd->async_exit_status=wait_for_mysql(mmsd->mysql, mmsd->async_exit_status); #ifdef DEBUG diff --git a/src/SQLite3_Server.cpp b/src/SQLite3_Server.cpp index 1da52af4dd..185c6ae4d5 100644 --- a/src/SQLite3_Server.cpp +++ b/src/SQLite3_Server.cpp @@ -361,7 +361,6 @@ vector get_hgs_info(SQLite3DB* db) { #endif void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *pkt) { - char *error=NULL; int cols; int affected_rows; @@ -378,7 +377,6 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p #if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) || defined(TEST_RDS_BGD) if (sess->client_myds->proxy_addr.addr == NULL) { -#ifdef TEST_RDS_BGD struct sockaddr_storage addr; socklen_t addr_len=sizeof(addr); memset(&addr,0,addr_len); @@ -401,37 +399,8 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p if (sess->client_myds->proxy_addr.addr == NULL) { sess->client_myds->proxy_addr.addr = strdup("unknown"); } -#else - struct sockaddr addr; - socklen_t addr_len=sizeof(struct sockaddr); - memset(&addr,0,addr_len); - int rc; - rc=getsockname(sess->client_myds->fd, &addr, &addr_len); - if (rc==0) { - char buf[512]; - switch (addr.sa_family) { - case AF_INET: { - struct sockaddr_in *ipv4 = (struct sockaddr_in *)&addr; - inet_ntop(addr.sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); - sess->client_myds->proxy_addr.addr = strdup(buf); - } - break; - case AF_INET6: { - struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)&addr; - inet_ntop(addr.sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); - sess->client_myds->proxy_addr.addr = strdup(buf); - } - break; - default: - sess->client_myds->proxy_addr.addr = strdup("unknown"); - break; - } - } else { - sess->client_myds->proxy_addr.addr = strdup("unknown"); - } -#endif // TEST_RDS_BGD } -#endif // TEST simulation +#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG || TEST_RDS_BGD char *query_no_space=(char *)l_alloc(query_length); memcpy(query_no_space,query,query_length); @@ -608,7 +577,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p sprintf(query,a,sess->client_myds->proxy_addr.addr); #else query=l_strdup("SELECT '(ProxySQL SQLite3 Server)'"); -#endif // TEST simulation +#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG || TEST_RDS_BGD query_length=strlen(query)+1; goto __run_query; } @@ -896,25 +865,6 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } } - if (run_query && !rds_bgd_table_check && !rds_bgd_metadata && - strcasecmp(query_no_space, "SELECT @@global.read_only read_only") == 0) { - if (sess->client_myds->proxy_addr.addr == NULL || - sess->client_myds->proxy_addr.port <= 0) { - GloSQLite3Server->send_MySQL_ERR( - &sess->client_myds->myprot, 1105, - "RDS BGD simulator could not identify the accepted backend address"); - run_query=false; - } else { - const std::string read_only_query { - "SELECT COALESCE((SELECT read_only FROM READONLY_STATUS WHERE hostname='" + - std::string(sess->client_myds->proxy_addr.addr) + "' AND port=" + - std::to_string(sess->client_myds->proxy_addr.port) + "),1) AS read_only" - }; - l_free(query_length,query); - query=l_strdup(read_only_query.c_str()); - query_length=strlen(query)+1; - } - } #endif // TEST_RDS_BGD #ifdef TEST_AURORA if (strstr(query_no_space,(char *)"REPLICA_HOST_STATUS")) { @@ -996,7 +946,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } } #endif // TEST_GROUPREP -#ifdef TEST_READONLY +#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) if (strncasecmp("SELECT @@global.read_only read_only ",query_no_space, strlen("SELECT @@global.read_only read_only "))==0) { if (strlen(query_no_space) > strlen("SELECT @@global.read_only read_only ")+5) { pthread_mutex_lock(&GloSQLite3Server->test_readonly_mutex); @@ -1013,7 +963,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p pthread_mutex_unlock(&GloSQLite3Server->test_readonly_mutex); } } -#endif // TEST_READONLY +#endif // TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG if ( strncasecmp("SELECT SLAVE STATUS ", query_no_space, strlen("SELECT SLAVE STATUS ")) == 0 @@ -1049,7 +999,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p sprintf(query,a,rand()%30+10); } } -#endif // TEST simulation +#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG || TEST_RDS_BGD if (!run_query) { l_free(pkt->size-sizeof(mysql_hdr),query_no_space); l_free(query_length,query); @@ -1118,7 +1068,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p bool deprecate_eof = sess->client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot, in_trans, deprecate_eof); delete resultset; -#ifdef TEST_READONLY +#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) if (strncasecmp("SELECT",query_no_space,6)) { if (strstr(query_no_space,(char *)"READONLY_STATUS")) { // the table is writable @@ -1127,7 +1077,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p pthread_mutex_unlock(&GloSQLite3Server->test_readonly_mutex); } } -#endif // TEST_READONLY +#endif // TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG if (strncasecmp("SELECT", query_no_space, 6)) { if (strstr(query_no_space, (char*)"REPLICATIONLAG_HOST_STATUS")) { @@ -1547,13 +1497,13 @@ SQLite3_Server::SQLite3_Server() { #ifdef TEST_GROUPREP init_grouprep_ifaces_string(s); #endif // TEST_GROUPREP -#ifdef TEST_READONLY - // for readonly test we listen on all IPs because we simulate a lot of clusters +#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) + // Read-only simulation listens on all IPs because it can simulate many clusters. if (!s.empty()) s += ";"; s += "0.0.0.0:3306"; pthread_mutex_init(&test_readonly_mutex, NULL); -#endif //TEST_READONLY +#endif // TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG // for replication test we listen on all IPs if (!s.empty()) @@ -1561,19 +1511,11 @@ SQLite3_Server::SQLite3_Server() { s += "0.0.0.0:3306"; pthread_mutex_init(&test_replicationlag_mutex, NULL); #endif //TEST_REPLICATIONLAG -#ifdef TEST_RDS_BGD -#if !defined(TEST_READONLY) && !defined(TEST_REPLICATIONLAG) - if (!s.empty()) - s += ";"; - s += "0.0.0.0:3306"; -#endif -#endif // TEST_RDS_BGD - variables.mysql_ifaces=strdup(s.c_str()); #else variables.mysql_ifaces=strdup("127.0.0.1:6030"); -#endif // TEST simulation +#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG || TEST_RDS_BGD }; @@ -1966,7 +1908,7 @@ void SQLite3_Server::drop_tables_defs(std::vector *tables_defs) { delete td; } }; -#endif // TEST simulation +#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG || TEST_RDS_BGD void SQLite3_Server::wrlock() { pthread_rwlock_wrlock(&rwlock); @@ -2172,10 +2114,7 @@ void SQLite3_Server::send_MySQL_ERR(MySQL_Protocol *myprot, char *msg) { myds->DSS=STATE_SLEEP; } -#ifdef TEST_RDS_BGD -void SQLite3_Server::send_MySQL_ERR( - MySQL_Protocol *myprot, uint16_t error_code, const char *msg) -{ +void SQLite3_Server::send_MySQL_ERR(MySQL_Protocol *myprot, uint16_t error_code, const char *msg) { assert(myprot); MySQL_Data_Stream *myds=myprot->get_myds(); myds->DSS=STATE_QUERY_SENT_DS; @@ -2183,9 +2122,8 @@ void SQLite3_Server::send_MySQL_ERR( myprot->generate_pkt_ERR(true,NULL,NULL,1,error_code,sqlstate,msg); myds->DSS=STATE_SLEEP; } -#endif // TEST_RDS_BGD -#ifdef TEST_READONLY +#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) void SQLite3_Server::load_readonly_table(MySQL_Session *sess) { // this function needs to be called with lock on mutex readonly_mutex already acquired GloAdmin->mysql_servers_wrlock(); @@ -2226,7 +2164,7 @@ int SQLite3_Server::readonly_test_value(char *p) { } return rc; } -#endif // TEST_READONLY +#endif // TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG void SQLite3_Server::load_replicationlag_table(MySQL_Session* sess) { diff --git a/test/tap/tests/test_rds_bgd-t.cpp b/test/tap/tests/test_rds_bgd-t.cpp index 94832af5bf..68f710d1de 100644 --- a/test/tap/tests/test_rds_bgd-t.cpp +++ b/test/tap/tests/test_rds_bgd-t.cpp @@ -69,8 +69,13 @@ int main() { } const RDS_BGD_Cluster& cluster = rds_bgd_test_cluster(); - for (const Simulator_Endpoint& writer : cluster.get_writers()) { - if (simulator.read_only_update(writer, false) != EXIT_SUCCESS) { + const RDS_BGD_Host* writers[] = { + &cluster.blue_writer(), + &cluster.green_writer(), + }; + for (const RDS_BGD_Host* writer : writers) { + if (simulator.read_only_update( + { writer->hostname, writer->port }, false) != EXIT_SUCCESS) { mysql_close(admin); BAIL_OUT("failed to configure writer read_only state"); } From f59abc7324fceeb9df6c850cd0f00ed53ea5272f Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Wed, 22 Jul 2026 17:13:57 +0000 Subject: [PATCH 54/81] test: Simplify AWS RDS BGD simulator helpers - Make cluster fixtures test-owned and reusable across TAP tests. - Simplify and document simulator APIs and probe-log handling. - Rename the smoke test and use the common topology-discovery path. Signed-off-by: Wazir Ahmed --- doc/AWS_Blue_Green/RDS_BGD_Simulator.md | 91 ++++--- include/SQLite3_Server.h | 2 - lib/MySQL_Monitor.cpp | 35 ++- test/infra/README.md | 2 +- test/tap/groups/cluster_sim_rds_bgd/add-hosts | 6 + test/tap/groups/cluster_sim_rds_bgd/env.sh | 8 + .../cluster_sim_rds_bgd/pre-proxysql.bash | 6 +- .../cluster_sim_rds_bgd/pre-proxysql.sql | 4 + test/tap/groups/groups.json | 2 +- test/tap/tap/cluster_simulator.cpp | 46 +--- test/tap/tap/cluster_simulator.h | 83 ++++-- test/tap/tap/rds_bgd_simulator.cpp | 181 +++++--------- test/tap/tap/rds_bgd_simulator.h | 236 ++++++++++++++---- test/tap/tap/rds_bgd_tap.h | 38 +++ test/tap/tests/test_rds_bgd-t.cpp | 122 --------- test/tap/tests/test_rds_bgd_smoke-t.cpp | 108 ++++++++ 16 files changed, 562 insertions(+), 408 deletions(-) create mode 100644 test/tap/tap/rds_bgd_tap.h delete mode 100644 test/tap/tests/test_rds_bgd-t.cpp create mode 100644 test/tap/tests/test_rds_bgd_smoke-t.cpp diff --git a/doc/AWS_Blue_Green/RDS_BGD_Simulator.md b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md index cef6a78e65..59cfa53a5c 100644 --- a/doc/AWS_Blue_Green/RDS_BGD_Simulator.md +++ b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md @@ -150,7 +150,7 @@ The supported states are: | `topology_present=0`, `error_code=1146` | Empty | Table has been dropped. | Topology update and delete operations clear `error_code` and `error_msg`. -Configured errors other than 1146 mark the table present and retain its rows; +Configured errors other than 1146 treat the table as present and retain its rows; error 1146 marks it absent. Dropping topology also removes its rows. Other flag combinations are invalid helper state. @@ -186,7 +186,7 @@ identify the destination, and `encrypted` records the accepted stream's TLS state. The TAP test is the only probe-log consumer; ProxySQL never reads it. A test -captures a sequence watermark before changing state and then reads later rows +reads the last sequence before changing state and then reads later rows to verify the selected destination and TLS mode. A probe-log insertion failure is a simulator failure and must not be silently reported as a normal backend response. @@ -230,13 +230,13 @@ configured in ProxySQL. ```cpp int connect( - const char* host, + char* host, int port, - const char* username, - const char* password, + char* username, + char* password, bool use_ssl = false); -int read_only_update(const Simulator_Endpoint& backend, bool read_only); +int read_only_update(Simulator_Endpoint backend, bool read_only); ``` `connect()` opens the SQLite3-server control connection with the MySQL client @@ -259,7 +259,7 @@ struct RDS_BGD_Host { std::string ip; int port; - Simulator_Endpoint endpoint() const; + Simulator_Endpoint endpoint(); }; ``` @@ -267,24 +267,23 @@ struct RDS_BGD_Host { C++11-compatible field types. `RDS_BGD_Host` keeps the ProxySQL-facing hostname and simulator-facing IP together. -### Shared Cluster Fixture +### Cluster Fixture ```cpp class RDS_BGD_Cluster { public: - const RDS_BGD_Host& blue_writer() const; - const RDS_BGD_Host& green_writer() const; - const std::vector& blue_readers() const; - const std::vector& green_readers() const; - std::vector get_writers() const; - std::vector get_topology( - const std::string& status) const; -}; + RDS_BGD_Host blue_writer; + RDS_BGD_Host green_writer; + std::vector blue_readers; + std::vector green_readers; -const RDS_BGD_Cluster& rds_bgd_test_cluster(); + std::vector get_writers(); + std::vector get_topology(std::string status); +}; ``` -The fixture encapsulates the shared `/etc/hosts` mapping. `get_writers()` +Each TAP test owns and initializes the cluster fixtures it uses. A fixture +keeps the selected `/etc/hosts` mapping together. `get_writers()` returns the selected blue and green writer IPs; `get_topology(status)` returns the standard two-row SOURCE/TARGET topology using the writer hostnames and the provided status. @@ -293,17 +292,17 @@ provided status. ```cpp int topology_update( - const std::vector& backends, - const std::vector& rows); + std::vector backends, + std::vector rows); -int topology_delete(const std::vector& backends); +int topology_delete(std::vector backends); -int topology_drop(const std::vector& backends); +int topology_drop(std::vector backends); int topology_error( - const std::vector& backends, + std::vector backends, unsigned int error_code, - const std::string& error_msg); + std::string error_msg); ``` `topology_update()` marks the table present, clears any configured error, and @@ -336,14 +335,14 @@ rc_t> probe_log_since(uint64_t sequence_id); rc_t wait_for_probe_log( uint64_t sequence_id, - const Simulator_Endpoint& backend, + Simulator_Endpoint backend, RDS_BGD_Probe_Kind probe_kind, uint32_t timeout_ms, int encrypted = -1); ``` -The watermark method returns zero for an empty log. `probe_log_since()` returns -rows after a watermark. `wait_for_probe_log()` waits for one matching row; +`probe_log_last_sequence()` returns zero for an empty log. `probe_log_since()` +returns rows after the supplied sequence. `wait_for_probe_log()` waits for one matching row; `encrypted` is `-1` for either mode, `0` for plaintext, and `1` for TLS. ## Typical TAP Test @@ -359,28 +358,22 @@ int main() { cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); if (!admin) BAIL_OUT("failed to connect to ProxySQL Admin"); - const RDS_BGD_Cluster& cluster = rds_bgd_test_cluster(); + RDS_BGD_Cluster cluster = bgd_cluster_init(); if (configure_proxysql_for_bgd(admin, cluster) != EXIT_SUCCESS) BAIL_OUT("failed to configure ProxySQL"); - std::pair sqlite_server; - if (extract_sqlite3_host_port(admin, sqlite_server) != EXIT_SUCCESS) - BAIL_OUT("failed to find SQLite3-server address"); - RDS_BGD_Simulator simulator; - if (simulator.connect( - sqlite_server.first.c_str(), sqlite_server.second, - cl.username, cl.password) != EXIT_SUCCESS) + if (simulator.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) BAIL_OUT("failed to connect to SQLite3 server"); if (simulator.read_only_update( - { cluster.blue_writer().hostname, cluster.blue_writer().port }, false) != EXIT_SUCCESS || + { cluster.blue_writer.hostname, cluster.blue_writer.port }, false) != EXIT_SUCCESS || simulator.read_only_update( - { cluster.green_writer().hostname, cluster.green_writer().port }, false) != EXIT_SUCCESS) + { cluster.green_writer.hostname, cluster.green_writer.port }, false) != EXIT_SUCCESS) BAIL_OUT("failed to configure writer read_only state"); - const rc_t mark = simulator.probe_log_last_sequence(); - if (mark.first != EXIT_SUCCESS) - BAIL_OUT("failed to read probe-log watermark"); + auto [seq_rc, last_seq] = simulator.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) + BAIL_OUT("failed to read the last probe-log sequence"); const int update_rc = simulator.topology_update( cluster.get_writers(), cluster.get_topology("AVAILABLE")); @@ -393,12 +386,12 @@ int main() { "WHERE writer_hostgroup=10", 5) == EXIT_SUCCESS, "ProxySQL enters AVAILABLE"); - const rc_t green_log = simulator.wait_for_probe_log( - mark.second, - cluster.green_writer().endpoint(), + auto [probe_rc, green_log] = simulator.wait_for_probe_log( + last_seq, + cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, 5000); - ok(green_log.first == EXIT_SUCCESS, + ok(probe_rc == EXIT_SUCCESS, "ProxySQL probes the green writer IP directly"); mysql_close(admin); @@ -518,7 +511,7 @@ simulator transitions. From the TAP container, the control connection uses Register each BGD TAP binary in `test/tap/groups/groups.json`: ```json -"test_rds_bgd-t": [ "cluster_sim_rds_bgd-g1" ] +"test_rds_bgd_smoke-t": [ "cluster_sim_rds_bgd-g1" ] ``` Add the group and its `make test_rds_bgd` requirement to the simulator table in @@ -560,7 +553,7 @@ Ubuntu TAP build dependencies, and runs: make -j"$(nproc)" test_rds_bgd ``` -After verifying `src/proxysql` and `test/tap/tests/test_rds_bgd-t`, it saves the +After verifying `src/proxysql` and `test/tap/tests/test_rds_bgd_smoke-t`, it saves the build output as two BGD-specific cache entries, following the existing CI separation between daemon and test artifacts: @@ -585,7 +578,7 @@ execution jobs without rebuilding ProxySQL. | Checkout | Check out the triggering SHA, not the default branch tip. | | Restore `src` | Restore the exact BGD `_src` key into `src/`; fail on a miss. | | Restore `test` | Restore the exact BGD `_test` key into `test/`; fail on a miss. | -| Verify artifacts | Confirm `src/proxysql` and `test/tap/tests/test_rds_bgd-t` are executable. | +| Verify artifacts | Confirm `src/proxysql` and `test/tap/tests/test_rds_bgd_smoke-t` are executable. | | Build runner image | Build `test/infra/docker-base` as `proxysql-ci-base:latest`. | | Start | Export the shared variables below and run `ensure-infras.bash`. | | Test | Run `run-tests-isolated.bash`; this execution, not compilation alone, is the required check. | @@ -611,14 +604,14 @@ the standard runner reports no infrastructure or test failure. ### Simulator Acceptance The simulator implementation needs one end-to-end smoke test, not a separate -unit-test suite for every helper method. `test_rds_bgd-t` proves that the +unit-test suite for every helper method. `test_rds_bgd_smoke-t` proves that the `TEST_RDS_BGD` daemon accepts TAP-controlled topology, ProxySQL observes an `AVAILABLE` deployment, the green-IP probe is logged, and the automatic CI job executes the group without `test/deps/cluster_simulator`. The configuration and lifecycle tests below exercise the remaining helper and SQLite3-server paths through BGD behavior. Before changing simulator state, -each test captures a probe watermark; failures report the configured backend +each test reads the last probe-log sequence; failures report the configured backend state, last ProxySQL runtime state, and later probe rows. ### Configuration and Discovery diff --git a/include/SQLite3_Server.h b/include/SQLite3_Server.h index 3be2596de2..a2ceaabe2a 100644 --- a/include/SQLite3_Server.h +++ b/include/SQLite3_Server.h @@ -53,8 +53,6 @@ class SQLite3_Server { #endif // TEST_GROUPREP #if defined(TEST_READONLY) || defined(TEST_RDS_BGD) std::vector *tables_defs_readonly; -#endif -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) std::unordered_map readonly_map; #endif // TEST_READONLY || TEST_RDS_BGD #ifdef TEST_RDS_BGD diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 1c46733700..589d05d1de 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -726,14 +726,12 @@ void MySQL_Monitor_State_Data::init_async() { task_handler_ = &MySQL_Monitor_State_Data::read_only_handler; break; #endif // TEST_READONLY || TEST_RDS_BGD -#if !defined(TEST_READONLY) || defined(TEST_RDS_BGD) case MON_AWS_RDS_TOPOLOGY_DISCOVERY: query_ = QUERY_AWS_RDS_TOPOLOGY_DISCOVERY; async_state_machine_ = ASYNC_QUERY_START; task_timeout_ = mysql_thread___monitor_read_only_timeout; task_handler_ = &MySQL_Monitor_State_Data::read_only_handler; break; -#endif // !TEST_READONLY || TEST_RDS_BGD case MON_GROUP_REPLICATION: async_state_machine_ = ASYNC_QUERY_START; #ifdef TEST_GROUPREP @@ -1760,31 +1758,26 @@ void * monitor_read_only_thread(const std::vector& da mmsd->t1=monotonic_time(); mmsd->interr=0; // reset the value -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) -#ifdef TEST_RDS_BGD if (mmsd->get_task_type() == MON_AWS_RDS_TOPOLOGY_DISCOVERY) { monitor_query = QUERY_AWS_RDS_TOPOLOGY_DISCOVERY; - } else -#endif // TEST_RDS_BGD - { + } else { +#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) monitor_query = "SELECT @@global.read_only read_only"; monitor_query += " " + std::string(mmsd->hostname) + ":" + std::to_string(mmsd->port); - } #else - if (mmsd->get_task_type() == MON_INNODB_READ_ONLY) { - monitor_query = "SELECT @@global.innodb_read_only read_only"; - } else if (mmsd->get_task_type() == MON_SUPER_READ_ONLY) { - monitor_query = "SELECT @@global.super_read_only read_only"; - } else if (mmsd->get_task_type() == MON_READ_ONLY__AND__INNODB_READ_ONLY) { - monitor_query = "SELECT @@global.read_only&@@global.innodb_read_only read_only"; - } else if (mmsd->get_task_type() == MON_READ_ONLY__OR__INNODB_READ_ONLY) { - monitor_query = "SELECT @@global.read_only|@@global.innodb_read_only read_only"; - } else if (mmsd->get_task_type() == MON_AWS_RDS_TOPOLOGY_DISCOVERY) { - monitor_query = QUERY_AWS_RDS_TOPOLOGY_DISCOVERY; - } else { // default - monitor_query = "SELECT @@global.read_only read_only"; - } + if (mmsd->get_task_type() == MON_INNODB_READ_ONLY) { + monitor_query = "SELECT @@global.innodb_read_only read_only"; + } else if (mmsd->get_task_type() == MON_SUPER_READ_ONLY) { + monitor_query = "SELECT @@global.super_read_only read_only"; + } else if (mmsd->get_task_type() == MON_READ_ONLY__AND__INNODB_READ_ONLY) { + monitor_query = "SELECT @@global.read_only&@@global.innodb_read_only read_only"; + } else if (mmsd->get_task_type() == MON_READ_ONLY__OR__INNODB_READ_ONLY) { + monitor_query = "SELECT @@global.read_only|@@global.innodb_read_only read_only"; + } else { // default + monitor_query = "SELECT @@global.read_only read_only"; + } #endif // TEST_READONLY || TEST_RDS_BGD + } mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,monitor_query.c_str()); while (mmsd->async_exit_status) { mmsd->async_exit_status=wait_for_mysql(mmsd->mysql, mmsd->async_exit_status); diff --git a/test/infra/README.md b/test/infra/README.md index d2dd9c2f76..7896a94108 100644 --- a/test/infra/README.md +++ b/test/infra/README.md @@ -50,7 +50,7 @@ Groups whose name starts with `cluster_sim_` (e.g. `cluster_sim_aurora-g1`, `clu | `cluster_sim_repl_lag-g` | `make testreplicationlag` | | `cluster_sim_rds_bgd-g` | `make test_rds_bgd` | -Each target sets the corresponding `-DTEST_` flag on the ProxySQL src and lib build. The BGD target builds its TAP-controlled SQLite3-server simulator directly; the other targets also build `test/deps/cluster_simulator`. A plain `make` is **not sufficient** for these groups. +Each target sets the corresponding `-DTEST_` flag on the ProxySQL src and lib builds and triggers the required simulator build. A plain `make` is **not sufficient** for these groups. --- ## 1. Core Concepts diff --git a/test/tap/groups/cluster_sim_rds_bgd/add-hosts b/test/tap/groups/cluster_sim_rds_bgd/add-hosts index 4fec081138..7147ba38e2 100644 --- a/test/tap/groups/cluster_sim_rds_bgd/add-hosts +++ b/test/tap/groups/cluster_sim_rds_bgd/add-hosts @@ -1,3 +1,9 @@ +# AWS RDS BGD simulator endpoint aliases. +# Format: " " per line; '#' comments allowed. +# These are injected into the ProxySQL container's /etc/hosts via Docker +# --add-host by test/infra/control/start-proxysql-isolated.bash when +# CLUSTER_SIM_HOST_FILE points at this file (see this group's env.sh). + # Cluster 1: blue endpoints db-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.11 db-1-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com 127.10.0.12 diff --git a/test/tap/groups/cluster_sim_rds_bgd/env.sh b/test/tap/groups/cluster_sim_rds_bgd/env.sh index 482577530e..5071a61929 100644 --- a/test/tap/groups/cluster_sim_rds_bgd/env.sh +++ b/test/tap/groups/cluster_sim_rds_bgd/env.sh @@ -1,4 +1,12 @@ # shellcheck shell=bash +# AWS RDS BGD simulator TAP group environment +# Inject AWS-style endpoint aliases into the ProxySQL container. export CLUSTER_SIM_HOST_FILE="${WORKSPACE}/test/tap/groups/cluster_sim_rds_bgd/add-hosts" + +# Skip background cluster nodes: the TAP test drives the primary ProxySQL's +# built-in SQLite3-server simulator directly. export SKIP_CLUSTER_START=1 + +# No backend infra: the TAP test controls simulated backend state through the +# SQLite3 server. Intentionally NOT setting DEFAULT_MYSQL_INFRA / DEFAULT_PGSQL_INFRA. diff --git a/test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.bash b/test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.bash index 97e8edeb18..d195d60ff0 100755 --- a/test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.bash +++ b/test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.bash @@ -1,5 +1,7 @@ #!/usr/bin/env bash set -e - -# ProxySQL's Admin port is available before module startup has fully settled. +# ProxySQL's admin port goes live before its startup is done. Wait for all +# init__variables() to complete before running pre-proxysql.sql; +# concurrent writes return SQLITE_LOCKED, which flush-variables functions +# treat as fatal (assert on rc != 0). sleep 5 diff --git a/test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.sql b/test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.sql index ac40c96ec3..40399d7dbd 100644 --- a/test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.sql +++ b/test/tap/groups/cluster_sim_rds_bgd/pre-proxysql.sql @@ -1,8 +1,12 @@ +-- Create and persist the client account used by the BGD TAP tests. INSERT OR REPLACE INTO mysql_users (username, password, default_hostgroup, active) VALUES ('testuser', 'testuser', 0, 1); LOAD MYSQL USERS TO RUNTIME; SAVE MYSQL USERS TO DISK; +-- When compiled with TEST_RDS_BGD, ProxySQL's monitor reaches its own SQLite3 +-- server on :3306. The default proxysql-ci.cnf pins the server to :6030, so +-- rebind it to :3306 here. SET sqliteserver-mysql_ifaces='0.0.0.0:3306'; LOAD SQLITESERVER VARIABLES TO RUNTIME; SAVE SQLITESERVER VARIABLES TO DISK; diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index 5854dd00cf..164e2088ad 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -334,7 +334,6 @@ "test_cluster_sim_group_repl-t" : [ "cluster_sim_group_repl-g1" ], "test_cluster_sim_read_only-t" : [ "cluster_sim_read_only-g1" ], "test_cluster_sim_repl_lag-t" : [ "cluster_sim_repl_lag-g1" ], - "test_rds_bgd-t" : [ "cluster_sim_rds_bgd-g1" ], "test_cluster_sync-t" : [ "legacy-g5","mysql-auto_increment_delay_multiplex=0-g3","mysql-multiplexing=false-g3","mysql-query_digests=0-g3","mysql-query_digests_keep_comment=1-g3","mysql84-g5","mysql90-g5","mysql95-g5" ], "test_cluster_sync_mysql_servers-t" : [ "legacy-g5","mysql-auto_increment_delay_multiplex=0-g3","mysql-multiplexing=false-g3","mysql-query_digests=0-g3","mysql-query_digests_keep_comment=1-g3","mysql84-g5","mysql90-g5","mysql95-g5" ], "test_cluster_sync_pgsql-t" : [ "legacy-g5","mysql-auto_increment_delay_multiplex=0-g3","mysql-multiplexing=false-g3","mysql-query_digests=0-g3","mysql-query_digests_keep_comment=1-g3" ], @@ -428,6 +427,7 @@ "test_query_rules_fast_routing_algorithm-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g4","mysql90-g4","mysql95-g4" ], "test_query_rules_routing-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g4","mysql90-g4","mysql95-g4" ], "test_query_timeout-t" : [ "legacy-g9","mariadb10-galera-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql84-gr-g9","mysql90-g4","mysql95-g4" ], + "test_rds_bgd_smoke-t" : [ "cluster_sim_rds_bgd-g1" ], "test_read_only_actions_offline_hard_servers-t" : [ "legacy-g5","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g5","mysql84-g9","mysql90-g4","mysql90-g5","mysql95-g4","mysql95-g5" ], "test_rw_binary_data-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql90-g4","mysql95-g4" ], "test_server_sess_status-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql90-g4","mysql95-g4" ], diff --git a/test/tap/tap/cluster_simulator.cpp b/test/tap/tap/cluster_simulator.cpp index 10b13de968..666d897ee0 100644 --- a/test/tap/tap/cluster_simulator.cpp +++ b/test/tap/tap/cluster_simulator.cpp @@ -4,22 +4,9 @@ #include "tap.h" -Cluster_Simulator::Cluster_Simulator() : mysql_(nullptr) {} +using namespace std; -Cluster_Simulator::~Cluster_Simulator() { - if (mysql_ != nullptr) { - mysql_close(mysql_); - mysql_ = nullptr; - } -} - -int Cluster_Simulator::connect( - const char* host, - int port, - const char* username, - const char* password, - bool use_ssl) -{ +int Cluster_Simulator::connect(char* host, int port, char* username, char* password, bool use_ssl) { if (mysql_ != nullptr) { mysql_close(mysql_); mysql_ = nullptr; @@ -37,11 +24,9 @@ int Cluster_Simulator::connect( client_flags |= CLIENT_SSL; } - if (mysql_real_connect( - mysql_, host, username, password, nullptr, port, nullptr, client_flags) == nullptr) { - diag( - "Failed to connect to cluster simulator at %s:%d: %s", - host, port, mysql_error(mysql_)); + auto ret = mysql_real_connect(mysql_, host, username, password, nullptr, port, nullptr, client_flags); + if (ret == nullptr) { + diag("Failed to connect to cluster simulator at %s:%d: %s", host, port, mysql_error(mysql_)); mysql_close(mysql_); mysql_ = nullptr; return EXIT_FAILURE; @@ -50,22 +35,16 @@ int Cluster_Simulator::connect( return EXIT_SUCCESS; } -int Cluster_Simulator::read_only_update( - const Simulator_Endpoint& backend, bool read_only) -{ - const std::string query { +int Cluster_Simulator::read_only_update(Endpoint backend, bool read_only) { + string query { "INSERT OR REPLACE INTO READONLY_STATUS(hostname,port,read_only) VALUES (" + - sql_quote(backend.host) + "," + std::to_string(backend.port) + "," + + sql_quote(backend.host) + "," + to_string(backend.port) + "," + (read_only ? "1" : "0") + ")" }; return execute(query); } -MYSQL* Cluster_Simulator::connection() const { - return mysql_; -} - -int Cluster_Simulator::execute(const std::string& query) { +int Cluster_Simulator::execute(string query) { if (mysql_ == nullptr) { diag("Cluster simulator connection is not open"); return EXIT_FAILURE; @@ -74,15 +53,16 @@ int Cluster_Simulator::execute(const std::string& query) { if (mysql_query(mysql_, query.c_str()) != 0) { diag( "Cluster simulator query failed (%u): %s; query: %s", - mysql_errno(mysql_), mysql_error(mysql_), query.c_str()); + mysql_errno(mysql_), mysql_error(mysql_), query.c_str() + ); return EXIT_FAILURE; } return EXIT_SUCCESS; } -std::string Cluster_Simulator::sql_quote(const std::string& value) { - std::string quoted { "'" }; +string Cluster_Simulator::sql_quote(string value) { + string quoted { "'" }; for (char c : value) { quoted += c; if (c == '\'') { diff --git a/test/tap/tap/cluster_simulator.h b/test/tap/tap/cluster_simulator.h index 2f25c06062..f769c254f0 100644 --- a/test/tap/tap/cluster_simulator.h +++ b/test/tap/tap/cluster_simulator.h @@ -5,32 +5,85 @@ #include "mysql.h" -struct Simulator_Endpoint { - std::string host; - int port; +using namespace std; + +/** + * @brief Identifies a simulated backend by address and listener port. + */ +struct Endpoint { + string host; ///< Hostname or IP address used to identify the simulated backend. + int port; ///< MySQL listener port of the simulated backend. }; +/** + * @brief Provides common control operations for TAP-driven cluster simulators. + * + * @details Owns the MySQL control connection to the SQLite3-server simulator and exposes + * backend state updates shared by technology-specific simulators. + */ class Cluster_Simulator { public: - Cluster_Simulator(); - virtual ~Cluster_Simulator(); + Cluster_Simulator() : mysql_(nullptr) {} + virtual ~Cluster_Simulator() { + if (mysql_ != nullptr) { + mysql_close(mysql_); + mysql_ = nullptr; + } + } Cluster_Simulator(const Cluster_Simulator&) = delete; Cluster_Simulator& operator=(const Cluster_Simulator&) = delete; - int connect( - const char* host, - int port, - const char* username, - const char* password, - bool use_ssl = false); + /** + * @brief Opens the simulator control connection. + * + * @details Replaces any existing control connection, optionally enables MySQL client + * TLS, and reports connection failures through TAP diagnostics. + * + * @param host Simulator hostname or IP address. + * @param port Simulator MySQL listener port. + * @param username MySQL username used by the control connection. + * @param password MySQL password used by the control connection. + * @param use_ssl Whether the control connection must use TLS. + * + * @return EXIT_SUCCESS when the connection is established; EXIT_FAILURE otherwise. + */ + int connect(char* host, int port, char* username, char* password, bool use_ssl = false); - int read_only_update(const Simulator_Endpoint& backend, bool read_only); + /** + * @brief Sets the simulated read-only state for a backend. + * + * @details Upserts `READONLY_STATUS` using the endpoint as its key. The state is + * consumed through the hostname-suffixed monitor-query path shared with + * `TEST_READONLY`. + * + * @param backend Hostname and port identifying the backend. + * @param read_only Whether the backend must report itself as read-only. + * + * @return EXIT_SUCCESS when the state is updated; EXIT_FAILURE otherwise. + */ + int read_only_update(Endpoint backend, bool read_only); protected: - MYSQL* connection() const; - int execute(const std::string& query); - static std::string sql_quote(const std::string& value); + MYSQL* connection() { return mysql_; } + + /** + * @brief Executes a query on the simulator control connection. + * + * @param query SQL statement to execute. + * + * @return EXIT_SUCCESS when the query succeeds; EXIT_FAILURE otherwise. + */ + int execute(string query); + + /** + * @brief Quotes a string value for use in simulator control SQL. + * + * @param value String value to quote. + * + * @return Single-quoted SQL literal with embedded quotes escaped. + */ + static string sql_quote(string value); private: MYSQL* mysql_; diff --git a/test/tap/tap/rds_bgd_simulator.cpp b/test/tap/tap/rds_bgd_simulator.cpp index 7c23332b6f..be18689b6b 100644 --- a/test/tap/tap/rds_bgd_simulator.cpp +++ b/test/tap/tap/rds_bgd_simulator.cpp @@ -7,13 +7,13 @@ #include "tap.h" -namespace { +using namespace std; const char* probe_kind_string(RDS_BGD_Probe_Kind kind) { return kind == RDS_BGD_Probe_Kind::table_check ? "table_check" : "metadata"; } -rc_t parse_probe_kind(const std::string& value) { +rc_t parse_probe_kind(string value) { if (value == "table_check") { return { EXIT_SUCCESS, RDS_BGD_Probe_Kind::table_check }; } @@ -23,88 +23,53 @@ rc_t parse_probe_kind(const std::string& value) { return { EXIT_FAILURE, RDS_BGD_Probe_Kind::table_check }; } -} // namespace - -Simulator_Endpoint RDS_BGD_Host::endpoint() const { +Endpoint RDS_BGD_Host::endpoint() { return { ip, port }; } -RDS_BGD_Cluster::RDS_BGD_Cluster() - : blue_writer_ { - "db-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.11", 3306 }, - green_writer_ { - "db-1-green-iqu47r.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.14", 3306 }, - blue_readers_ { - { "db-1-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.12", 3306 }, - { "db-1-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.13", 3306 }, - }, - green_readers_ { - { "db-1-reader-1-green-dlzky7.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.15", 3306 }, - { "db-1-reader-2-green-3fpjuu.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.16", 3306 }, - } -{} - -const RDS_BGD_Host& RDS_BGD_Cluster::blue_writer() const { - return blue_writer_; +Endpoint RDS_BGD_Host::host_endpoint() { + return { hostname, port }; } -const RDS_BGD_Host& RDS_BGD_Cluster::green_writer() const { - return green_writer_; +vector RDS_BGD_Cluster::get_writers() { + return { blue_writer.endpoint(), green_writer.endpoint() }; } -const std::vector& RDS_BGD_Cluster::blue_readers() const { - return blue_readers_; +vector RDS_BGD_Cluster::get_writer_hosts() { + return { blue_writer.host_endpoint(), green_writer.host_endpoint() }; } -const std::vector& RDS_BGD_Cluster::green_readers() const { - return green_readers_; -} - -std::vector RDS_BGD_Cluster::get_writers() const { - return { blue_writer_.endpoint(), green_writer_.endpoint() }; -} - -std::vector RDS_BGD_Cluster::get_topology( - const std::string& status) const -{ +vector RDS_BGD_Cluster::get_topology(string status) { return { - { blue_writer_.hostname, blue_writer_.hostname, blue_writer_.port, + { blue_writer.hostname, blue_writer.hostname, blue_writer.port, "BLUE_GREEN_DEPLOYMENT_SOURCE", status }, - { green_writer_.hostname, green_writer_.hostname, green_writer_.port, + { green_writer.hostname, green_writer.hostname, green_writer.port, "BLUE_GREEN_DEPLOYMENT_TARGET", status }, }; } -const RDS_BGD_Cluster& rds_bgd_test_cluster() { - static const RDS_BGD_Cluster cluster {}; - return cluster; -} - -int RDS_BGD_Simulator::topology_update( - const std::vector& backends, - const std::vector& rows) -{ +int RDS_BGD_Simulator::topology_update(vector backends, vector rows) { if (backends.empty()) { return EXIT_FAILURE; } - std::vector statements {}; - for (const Simulator_Endpoint& backend : backends) { - const std::string predicate { backend_predicate(backend) }; + vector statements {}; + for (Endpoint& backend : backends) { + string predicate { backend_predicate(backend) }; statements.push_back("DELETE FROM RDS_BGD_TOPOLOGY WHERE " + predicate); statements.push_back( "INSERT OR REPLACE INTO RDS_BGD_CONTROL" "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + - sql_quote(backend.host) + "," + std::to_string(backend.port) + ",1,0,'')"); + sql_quote(backend.host) + "," + to_string(backend.port) + ",1,0,'')"); - for (std::size_t row_order = 0; row_order < rows.size(); ++row_order) { - const RDS_BGD_Topology_Row& row = rows[row_order]; + for (size_t row_order = 0; row_order < rows.size(); ++row_order) { + RDS_BGD_Topology_Row& row = rows[row_order]; statements.push_back( "INSERT INTO RDS_BGD_TOPOLOGY" "(backend_ip,backend_port,row_order,id,endpoint,topology_port,role,status) VALUES (" + - sql_quote(backend.host) + "," + std::to_string(backend.port) + "," + - std::to_string(row_order) + "," + sql_quote(row.id) + "," + - sql_quote(row.endpoint) + "," + std::to_string(row.port) + "," + + sql_quote(backend.host) + "," + to_string(backend.port) + "," + + to_string(row_order) + "," + sql_quote(row.id) + "," + + sql_quote(row.endpoint) + "," + to_string(row.port) + "," + sql_quote(row.role) + "," + sql_quote(row.status) + ")"); } } @@ -112,44 +77,35 @@ int RDS_BGD_Simulator::topology_update( return execute_transaction(statements); } -int RDS_BGD_Simulator::topology_delete( - const std::vector& backends) -{ +int RDS_BGD_Simulator::topology_delete(vector backends) { if (backends.empty()) { return EXIT_FAILURE; } - std::vector statements {}; - for (const Simulator_Endpoint& backend : backends) { + vector statements {}; + for (Endpoint& backend : backends) { statements.push_back( "DELETE FROM RDS_BGD_TOPOLOGY WHERE " + backend_predicate(backend)); statements.push_back( "INSERT OR REPLACE INTO RDS_BGD_CONTROL" "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + - sql_quote(backend.host) + "," + std::to_string(backend.port) + ",1,0,'')"); + sql_quote(backend.host) + "," + to_string(backend.port) + ",1,0,'')"); } return execute_transaction(statements); } -int RDS_BGD_Simulator::topology_drop( - const std::vector& backends) -{ - return topology_error( - backends, 1146, "Table 'mysql.rds_topology' doesn't exist"); +int RDS_BGD_Simulator::topology_drop(vector backends) { + return topology_error(backends, 1146, "Table 'mysql.rds_topology' doesn't exist"); } -int RDS_BGD_Simulator::topology_error( - const std::vector& backends, - unsigned int error_code, - const std::string& error_msg) -{ +int RDS_BGD_Simulator::topology_error(vector backends, int error_code, string error_msg) { if (backends.empty() || error_code == 0) { return EXIT_FAILURE; } - const bool topology_present = error_code != 1146; - std::vector statements {}; - for (const Simulator_Endpoint& backend : backends) { + bool topology_present = error_code != 1146; + vector statements {}; + for (Endpoint& backend : backends) { if (!topology_present) { statements.push_back( "DELETE FROM RDS_BGD_TOPOLOGY WHERE " + backend_predicate(backend)); @@ -157,8 +113,8 @@ int RDS_BGD_Simulator::topology_error( statements.push_back( "INSERT OR REPLACE INTO RDS_BGD_CONTROL" "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + - sql_quote(backend.host) + "," + std::to_string(backend.port) + "," + - (topology_present ? "1" : "0") + "," + std::to_string(error_code) + "," + + sql_quote(backend.host) + "," + to_string(backend.port) + "," + + (topology_present ? "1" : "0") + "," + to_string(error_code) + "," + sql_quote(error_msg) + ")"); } return execute_transaction(statements); @@ -169,74 +125,69 @@ rc_t RDS_BGD_Simulator::probe_log_last_sequence() { return { EXIT_FAILURE, 0 }; } - const rc_t> result { - mysql_query_ext_rows( - connection(), "SELECT COALESCE(MAX(sequence_id),0) FROM RDS_BGD_PROBE_LOG") - }; - if (result.first != EXIT_SUCCESS || result.second.size() != 1 || - result.second.front().size() != 1) { + auto [rc, rows] = mysql_query_ext_rows( + connection(), "SELECT COALESCE(MAX(sequence_id),0) FROM RDS_BGD_PROBE_LOG"); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows.front().size() != 1) { return { EXIT_FAILURE, 0 }; } return { EXIT_SUCCESS, - static_cast(std::strtoull(result.second.front().front().c_str(), nullptr, 10)) + static_cast(strtoull(rows.front().front().c_str(), nullptr, 10)) }; } -rc_t> RDS_BGD_Simulator::probe_log_since( +rc_t> RDS_BGD_Simulator::probe_log_since( uint64_t sequence_id) { if (connection() == nullptr) { return { EXIT_FAILURE, {} }; } - const std::string query { + string query { "SELECT sequence_id,backend_ip,backend_port,probe_kind,encrypted " - "FROM RDS_BGD_PROBE_LOG WHERE sequence_id>" + std::to_string(sequence_id) + + "FROM RDS_BGD_PROBE_LOG WHERE sequence_id>" + to_string(sequence_id) + " ORDER BY sequence_id" }; - const rc_t> result { - mysql_query_ext_rows(connection(), query) - }; - if (result.first != EXIT_SUCCESS) { + auto [rc, rows] = mysql_query_ext_rows(connection(), query); + if (rc != EXIT_SUCCESS) { return { EXIT_FAILURE, {} }; } - std::vector logs {}; - for (const mysql_res_row& row : result.second) { + vector logs {}; + for (mysql_res_row& row : rows) { if (row.size() != 5) { return { EXIT_FAILURE, {} }; } - const rc_t kind { parse_probe_kind(row[3]) }; - if (kind.first != EXIT_SUCCESS) { + auto [kind_rc, kind] = parse_probe_kind(row[3]); + if (kind_rc != EXIT_SUCCESS) { return { EXIT_FAILURE, {} }; } logs.push_back({ - static_cast(std::strtoull(row[0].c_str(), nullptr, 10)), - { row[1], std::atoi(row[2].c_str()) }, - kind.second, - std::atoi(row[4].c_str()) != 0, + static_cast(strtoull(row[0].c_str(), nullptr, 10)), + { row[1], atoi(row[2].c_str()) }, + kind, + atoi(row[4].c_str()) != 0, }); } - return { EXIT_SUCCESS, std::move(logs) }; + return { EXIT_SUCCESS, move(logs) }; } rc_t RDS_BGD_Simulator::wait_for_probe_log( uint64_t sequence_id, - const Simulator_Endpoint& backend, + Endpoint backend, RDS_BGD_Probe_Kind probe_kind, uint32_t timeout_ms, int encrypted) { - const uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; + uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; do { - const rc_t> logs { probe_log_since(sequence_id) }; - if (logs.first != EXIT_SUCCESS) { + auto [rc, logs] = probe_log_since(sequence_id); + if (rc != EXIT_SUCCESS) { return { EXIT_FAILURE, {} }; } - for (const RDS_BGD_Probe_Log& log : logs.second) { + for (RDS_BGD_Probe_Log& log : logs) { if (log.backend.host == backend.host && log.backend.port == backend.port && log.probe_kind == probe_kind && (encrypted < 0 || log.encrypted == (encrypted != 0))) { @@ -246,9 +197,9 @@ rc_t RDS_BGD_Simulator::wait_for_probe_log( usleep(50000); } while (monotonic_time() < deadline); - const rc_t> logs { probe_log_since(sequence_id) }; - if (logs.first == EXIT_SUCCESS) { - for (const RDS_BGD_Probe_Log& log : logs.second) { + auto [rc, logs] = probe_log_since(sequence_id); + if (rc == EXIT_SUCCESS) { + for (RDS_BGD_Probe_Log& log : logs) { diag( "Observed BGD probe sequence=%llu backend=%s:%d kind=%s encrypted=%d", static_cast(log.sequence_id), @@ -262,13 +213,11 @@ rc_t RDS_BGD_Simulator::wait_for_probe_log( return { ETIMEDOUT, {} }; } -int RDS_BGD_Simulator::execute_transaction( - const std::vector& statements) -{ +int RDS_BGD_Simulator::execute_transaction(vector& statements) { if (execute("START TRANSACTION") != EXIT_SUCCESS) { return EXIT_FAILURE; } - for (const std::string& statement : statements) { + for (string& statement : statements) { if (execute(statement) != EXIT_SUCCESS) { (void)execute("ROLLBACK"); return EXIT_FAILURE; @@ -281,9 +230,7 @@ int RDS_BGD_Simulator::execute_transaction( return EXIT_SUCCESS; } -std::string RDS_BGD_Simulator::backend_predicate( - const Simulator_Endpoint& backend) -{ +string RDS_BGD_Simulator::backend_predicate(Endpoint backend) { return "backend_ip=" + sql_quote(backend.host) + - " AND backend_port=" + std::to_string(backend.port); + " AND backend_port=" + to_string(backend.port); } diff --git a/test/tap/tap/rds_bgd_simulator.h b/test/tap/tap/rds_bgd_simulator.h index 8edc492d6c..cd3233df5c 100644 --- a/test/tap/tap/rds_bgd_simulator.h +++ b/test/tap/tap/rds_bgd_simulator.h @@ -8,80 +8,226 @@ #include "cluster_simulator.h" #include "utils.h" +using namespace std; + +/** + * @brief Represents one row returned by the simulated `mysql.rds_topology` table. + */ struct RDS_BGD_Topology_Row { - std::string id; - std::string endpoint; - int port; - std::string role; - std::string status; + string id; ///< RDS topology node identifier. + string endpoint; ///< RDS hostname exposed by the topology row. + int port; ///< MySQL port exposed by the topology row. + string role; ///< Blue/green deployment role reported by RDS. + string status; ///< Blue/green deployment status reported by RDS. }; +/** + * @brief Describes one RDS BGD host and its fixed simulator address. + */ struct RDS_BGD_Host { - std::string hostname; - std::string ip; - int port; + string hostname; ///< AWS-style RDS hostname configured in ProxySQL. + string ip; ///< Fixed loopback address used by the simulator. + int port; ///< MySQL listener port shared by the hostname and IP. + + /** + * @brief Returns the IP/port endpoint used for topology simulation. + * + * @return Simulator endpoint containing this host's IP address and port. + */ + Endpoint endpoint(); - Simulator_Endpoint endpoint() const; + /** + * @brief Returns the hostname/port endpoint used for read-only simulation. + * + * @return Simulator endpoint containing this host's RDS hostname and port. + */ + Endpoint host_endpoint(); }; +/** + * @brief Holds the blue and green hosts participating in one simulated RDS BGD cluster. + * + * @details TAP tests populate the cluster with the deployment topology required by each + * scenario. Helper methods derive writer endpoints and AWS topology rows from + * the configured hosts. + */ class RDS_BGD_Cluster { public: - const RDS_BGD_Host& blue_writer() const; - const RDS_BGD_Host& green_writer() const; - const std::vector& blue_readers() const; - const std::vector& green_readers() const; - std::vector get_writers() const; - std::vector get_topology( - const std::string& status) const; + RDS_BGD_Host blue_writer; ///< Source writer configured in ProxySQL. + RDS_BGD_Host green_writer; ///< Target writer discovered from the topology. + vector blue_readers; ///< Source readers configured in ProxySQL. + vector green_readers; ///< Target readers discovered from the topology. -private: - friend const RDS_BGD_Cluster& rds_bgd_test_cluster(); - RDS_BGD_Cluster(); + /** + * @brief Returns both writer IP/port endpoints for topology simulation. + * + * @return Blue and green writer endpoints keyed by simulator IP address. + */ + vector get_writers(); - RDS_BGD_Host blue_writer_; - RDS_BGD_Host green_writer_; - std::vector blue_readers_; - std::vector green_readers_; -}; + /** + * @brief Returns both writer hostname/port endpoints for read-only simulation. + * + * @return Blue and green writer endpoints keyed by RDS hostname. + */ + vector get_writer_hosts(); -const RDS_BGD_Cluster& rds_bgd_test_cluster(); + /** + * @brief Builds the topology rows published by the simulated writers. + * + * @details Creates one source row for the blue writer and one target row for the green + * writer. The supplied deployment status is applied to both rows. + * + * @param status RDS blue/green deployment status to publish. + * + * @return Source and target rows for the simulated topology table. + */ + vector get_topology(string status); +}; +/** + * @brief Identifies the RDS BGD monitor query recorded in the simulator probe log. + */ enum class RDS_BGD_Probe_Kind { - table_check, - metadata, + table_check, ///< Query checking whether `mysql.rds_topology` exists. + metadata, ///< Query fetching rows from `mysql.rds_topology`. }; +/** + * @brief Describes one RDS BGD monitor query observed by the simulator. + */ struct RDS_BGD_Probe_Log { - uint64_t sequence_id; - Simulator_Endpoint backend; - RDS_BGD_Probe_Kind probe_kind; - bool encrypted; + uint64_t sequence_id; ///< Monotonically increasing probe-log sequence. + Endpoint backend; ///< Accepted backend IP address and port. + RDS_BGD_Probe_Kind probe_kind; ///< Type of topology query observed. + bool encrypted; ///< Whether the monitor connection used TLS. }; +/** + * @brief Controls RDS BGD topology responses and inspects monitor probes from TAP tests. + * + * @details Publishes per-backend topology rows, empty results, missing tables, or MySQL + * errors. It also reads the ordered probe log generated by the SQLite3-server + * simulator and reuses `Cluster_Simulator` for shared control operations. + */ class RDS_BGD_Simulator : public Cluster_Simulator { public: - int topology_update( - const std::vector& backends, - const std::vector& rows); - int topology_delete(const std::vector& backends); - int topology_drop(const std::vector& backends); - int topology_error( - const std::vector& backends, - unsigned int error_code, - const std::string& error_msg); + /** + * @brief Replaces the simulated topology returned by each backend. + * + * @details Deletes existing topology rows before inserting the supplied rows. The + * topology table is marked present, configured errors are cleared, and the + * complete update is applied atomically across all supplied backends. + * + * @param backends Backend IP/port endpoints that must return the topology. + * @param rows Topology rows to publish on each backend. + * + * @return EXIT_SUCCESS when every backend is updated; EXIT_FAILURE otherwise. + */ + int topology_update(vector backends, vector rows); + + /** + * @brief Configures each backend to return an empty topology result. + * + * @details Deletes all topology rows associated with the supplied backends while keeping + * the topology table present and clearing any configured metadata error. + * + * @param backends Backend IP/port endpoints that must return an empty result. + * + * @return EXIT_SUCCESS when every backend is updated; EXIT_FAILURE otherwise. + */ + int topology_delete(vector backends); + /** + * @brief Configures each backend to report that the topology table does not exist. + * + * @param backends Backend IP/port endpoints that must return MySQL error 1146. + * + * @return EXIT_SUCCESS when every backend is updated; EXIT_FAILURE otherwise. + */ + int topology_drop(vector backends); + + /** + * @brief Configures a MySQL error for topology queries on each backend. + * + * @details Stores the nonzero error code and message returned by subsequent metadata + * probes. Error 1146 marks the topology table absent and removes its existing + * rows; other error codes leave the table marked present. + * + * @param backends Backend IP/port endpoints that must return the error. + * @param error_code Nonzero MySQL error code to return. + * @param error_msg MySQL error message to return. + * + * @return EXIT_SUCCESS when every backend is updated; EXIT_FAILURE otherwise. + */ + int topology_error(vector backends, int error_code, string error_msg); + + /** + * @brief Reads the latest sequence from the RDS BGD probe log. + * + * @return EXIT_SUCCESS and the latest sequence, or zero when the log is empty; + * EXIT_FAILURE and zero when the query fails. + */ rc_t probe_log_last_sequence(); - rc_t> probe_log_since(uint64_t sequence_id); + + /** + * @brief Returns probe-log records newer than a sequence. + * + * @details Selects records with `sequence_id` strictly greater than the supplied value + * and preserves database sequence order in the returned vector. + * + * @param sequence_id Last probe-log sequence already observed by the TAP test. + * + * @return EXIT_SUCCESS and the matching records; EXIT_FAILURE and an empty vector + * when the query or record parsing fails. + */ + rc_t> probe_log_since(uint64_t sequence_id); + + /** + * @brief Waits for a matching RDS BGD probe-log record. + * + * @details Matches records newer than `sequence_id` by backend and probe kind. TLS state + * is matched when `encrypted` is zero or one; `-1` accepts either state. Observed + * probes are emitted through TAP diagnostics when the wait expires. + * + * @param sequence_id Last probe-log sequence observed before the expected probe. + * @param backend Backend IP/port endpoint expected to receive the probe. + * @param probe_kind Type of topology query expected. + * @param timeout_ms Maximum time to wait in milliseconds. + * @param encrypted Expected TLS state, or -1 to accept either state. + * + * @return EXIT_SUCCESS and the matching record; ETIMEDOUT and an empty record when + * the deadline expires; EXIT_FAILURE and an empty record when log retrieval fails. + */ rc_t wait_for_probe_log( uint64_t sequence_id, - const Simulator_Endpoint& backend, + Endpoint backend, RDS_BGD_Probe_Kind probe_kind, uint32_t timeout_ms, - int encrypted = -1); + int encrypted = -1 + ); private: - static std::string backend_predicate(const Simulator_Endpoint& backend); - int execute_transaction(const std::vector& statements); + /** + * @brief Builds the SQL predicate identifying one simulated backend. + * + * @param backend Backend IP/port endpoint to match. + * + * @return SQL predicate matching the backend control-table key. + */ + static string backend_predicate(Endpoint backend); + + /** + * @brief Executes simulator control statements in one transaction. + * + * @details Executes the supplied statements in order and commits only after every + * statement succeeds. A statement or commit failure triggers a rollback. + * + * @param statements SQL statements to execute atomically. + * + * @return EXIT_SUCCESS when the transaction commits; EXIT_FAILURE otherwise. + */ + int execute_transaction(vector& statements); }; #endif // TAP_RDS_BGD_SIMULATOR_H diff --git a/test/tap/tap/rds_bgd_tap.h b/test/tap/tap/rds_bgd_tap.h new file mode 100644 index 0000000000..abfe2118eb --- /dev/null +++ b/test/tap/tap/rds_bgd_tap.h @@ -0,0 +1,38 @@ +#ifndef TAP_TESTS_RDS_BGD_TAP_H +#define TAP_TESTS_RDS_BGD_TAP_H + +#include +#include +#include + +#include "rds_bgd_simulator.h" +#include "tap.h" + +using namespace std; + +inline RDS_BGD_Cluster bgd_cluster_init() { + return { + { "db-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.11", 3306 }, + { "db-1-green-iqu47r.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.14", 3306 }, + { + { "db-1-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.12", 3306 }, + { "db-1-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.13", 3306 }, + }, + { + { "db-1-reader-1-green-dlzky7.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.15", 3306 }, + { "db-1-reader-2-green-3fpjuu.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.16", 3306 }, + }, + }; +} + +inline int execute_all(MYSQL* admin, vector queries) { + for (string& query : queries) { + if (mysql_query(admin, query.c_str()) != 0) { + diag("Admin query failed (%u): %s; query: %s", mysql_errno(admin), mysql_error(admin), query.c_str()); + return EXIT_FAILURE; + } + } + return EXIT_SUCCESS; +} + +#endif // TAP_TESTS_RDS_BGD_TAP_H diff --git a/test/tap/tests/test_rds_bgd-t.cpp b/test/tap/tests/test_rds_bgd-t.cpp deleted file mode 100644 index 68f710d1de..0000000000 --- a/test/tap/tests/test_rds_bgd-t.cpp +++ /dev/null @@ -1,122 +0,0 @@ -#include -#include -#include - -#include "command_line.h" -#include "rds_bgd_simulator.h" -#include "tap.h" -#include "utils.h" - -namespace { - -int execute_all(MYSQL* admin, const std::vector& queries) { - for (const std::string& query : queries) { - if (mysql_query(admin, query.c_str()) != 0) { - diag( - "Admin query failed (%u): %s; query: %s", - mysql_errno(admin), mysql_error(admin), query.c_str()); - return EXIT_FAILURE; - } - } - return EXIT_SUCCESS; -} - -int configure_proxysql_for_bgd( - MYSQL* admin, const RDS_BGD_Cluster& cluster) -{ - const RDS_BGD_Host& writer = cluster.blue_writer(); - return execute_all(admin, { - "DELETE FROM mysql_servers", - "DELETE FROM mysql_replication_hostgroups", - "DELETE FROM mysql_aws_rds_bgd_hostgroups", - "INSERT INTO mysql_replication_hostgroups(writer_hostgroup,reader_hostgroup) " - "VALUES (10,20)", - "INSERT INTO mysql_aws_rds_bgd_hostgroups(" - "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," - "active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment) " - "VALUES (10,20,30,40,1,0,100,800,'BGD simulator smoke test')", - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,use_ssl,comment) VALUES (10,'" + - writer.hostname + "'," + std::to_string(writer.port) + ",0,'blue writer')", - "SET mysql-monitor_username='testuser'", - "SET mysql-monitor_password='testuser'", - "SET mysql-monitor_enabled='true'", - "LOAD MYSQL VARIABLES TO RUNTIME", - "LOAD MYSQL SERVERS TO RUNTIME", - }); -} - -} // namespace - -int main() { - plan(3); - - CommandLine cl {}; - if (cl.getEnv()) { - BAIL_OUT("failed to load TAP environment"); - } - - MYSQL* admin = init_mysql_conn( - cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); - if (admin == nullptr) { - BAIL_OUT("failed to connect to ProxySQL Admin"); - } - - RDS_BGD_Simulator simulator {}; - if (simulator.connect( - cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { - mysql_close(admin); - BAIL_OUT("failed to connect to the SQLite3-server simulator"); - } - - const RDS_BGD_Cluster& cluster = rds_bgd_test_cluster(); - const RDS_BGD_Host* writers[] = { - &cluster.blue_writer(), - &cluster.green_writer(), - }; - for (const RDS_BGD_Host* writer : writers) { - if (simulator.read_only_update( - { writer->hostname, writer->port }, false) != EXIT_SUCCESS) { - mysql_close(admin); - BAIL_OUT("failed to configure writer read_only state"); - } - } - - const rc_t mark = simulator.probe_log_last_sequence(); - if (mark.first != EXIT_SUCCESS) { - mysql_close(admin); - BAIL_OUT("failed to read the BGD probe-log watermark"); - } - - const int update_rc = simulator.topology_update( - cluster.get_writers(), cluster.get_topology("AVAILABLE")); - ok(update_rc == EXIT_SUCCESS, "publish AVAILABLE topology to both writer IPs"); - if (update_rc != EXIT_SUCCESS) { - mysql_close(admin); - BAIL_OUT("failed to publish BGD topology"); - } - - if (configure_proxysql_for_bgd(admin, cluster) != EXIT_SUCCESS) { - mysql_close(admin); - BAIL_OUT("failed to configure ProxySQL for BGD monitoring"); - } - - const int status_rc = wait_for_cond( - admin, - "SELECT COUNT(*)=1 FROM runtime_mysql_aws_rds_bgd_hostgroups " - "WHERE writer_hostgroup=10 AND status='AVAILABLE'", - 10); - ok(status_rc == EXIT_SUCCESS, "ProxySQL enters the AVAILABLE BGD state"); - - const rc_t green_probe = simulator.wait_for_probe_log( - mark.second, - cluster.green_writer().endpoint(), - RDS_BGD_Probe_Kind::metadata, - 10000, - 0); - ok( - green_probe.first == EXIT_SUCCESS, - "ProxySQL probes topology directly on the green writer IP over plaintext"); - - mysql_close(admin); - return exit_status(); -} diff --git a/test/tap/tests/test_rds_bgd_smoke-t.cpp b/test/tap/tests/test_rds_bgd_smoke-t.cpp new file mode 100644 index 0000000000..eb7ce5f971 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_smoke-t.cpp @@ -0,0 +1,108 @@ +/** + * @file test_rds_bgd_smoke-t.cpp + * @brief Smoke test for TAP-controlled AWS RDS BGD simulation. + * + * Test steps: + * 1. Connect to ProxySQL Admin and the SQLite3-server simulator. + * 2. Configure both simulated writers as writable. + * 3. Publish an AVAILABLE topology on the blue and green writer IPs. + * 4. Configure ProxySQL with the blue writer and BGD hostgroups. + * 5. Verify that ProxySQL reaches AVAILABLE and probes the green writer IP. + */ + +#include +#include +#include + +#include "rds_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +int configure_proxysql_for_bgd(MYSQL* admin, RDS_BGD_Cluster& cluster) { + RDS_BGD_Host& writer = cluster.blue_writer; + return execute_all(admin, { + "DELETE FROM mysql_servers", + "DELETE FROM mysql_replication_hostgroups", + "DELETE FROM mysql_aws_rds_bgd_hostgroups", + "INSERT INTO mysql_replication_hostgroups(writer_hostgroup,reader_hostgroup) " + "VALUES (10,20)", + "INSERT INTO mysql_aws_rds_bgd_hostgroups(" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment) " + "VALUES (10,20,30,40,1,0,100,800,'BGD simulator smoke test')", + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,use_ssl,comment) VALUES (10,'" + + writer.hostname + "'," + std::to_string(writer.port) + ",0,'blue writer')", + "SET mysql-monitor_username='testuser'", + "SET mysql-monitor_password='testuser'", + "SET mysql-monitor_enabled='true'", + "LOAD MYSQL VARIABLES TO RUNTIME", + "LOAD MYSQL SERVERS TO RUNTIME", + }); +} + +int main() { + plan(3); + + CommandLine cl {}; + if (cl.getEnv()) { + BAIL_OUT("failed to load TAP environment"); + } + + MYSQL* admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + BAIL_OUT("failed to connect to ProxySQL Admin"); + } + + RDS_BGD_Simulator sim {}; + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + mysql_close(admin); + BAIL_OUT("failed to connect to the SQLite3-server simulator"); + } + + // Initialize the test cluster and make both simulated writers writable. + RDS_BGD_Cluster cluster = bgd_cluster_init(); + for (Endpoint& writer : cluster.get_writer_hosts()) { + if (sim.read_only_update(writer, false) != EXIT_SUCCESS) { + mysql_close(admin); + BAIL_OUT("failed to configure writer read_only state"); + } + } + + // Record the last probe sequence before enabling BGD monitoring. + auto [rc, last_seq] = sim.probe_log_last_sequence(); + if (rc != EXIT_SUCCESS) { + mysql_close(admin); + BAIL_OUT("failed to read the last BGD probe-log sequence"); + } + + // Publish the AVAILABLE topology on both simulated writer IPs. + rc = sim.topology_update(cluster.get_writers(), cluster.get_topology("AVAILABLE")); + ok(rc == EXIT_SUCCESS, "publish AVAILABLE topology to both writer IPs"); + if (rc != EXIT_SUCCESS) { + mysql_close(admin); + BAIL_OUT("failed to publish BGD topology"); + } + + // Configure ProxySQL with the blue writer and BGD hostgroups. + if (configure_proxysql_for_bgd(admin, cluster) != EXIT_SUCCESS) { + mysql_close(admin); + BAIL_OUT("failed to configure ProxySQL for BGD monitoring"); + } + + // Wait for topology discovery to place the BGD hostgroups in AVAILABLE. + rc = wait_for_cond( + admin, + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_rds_bgd_hostgroups " + "WHERE writer_hostgroup=10 AND status='AVAILABLE'", + 3); + ok(rc == EXIT_SUCCESS, "ProxySQL enters the AVAILABLE BGD state"); + + // Verify that ProxySQL probes metadata directly on the green writer IP. + auto [probe_rc, green_probe] = sim.wait_for_probe_log( + last_seq, cluster.green_writer.endpoint(), + RDS_BGD_Probe_Kind::metadata, 3000, 0); + ok(probe_rc == EXIT_SUCCESS, "ProxySQL probes topology directly on the green writer IP over plaintext"); + + mysql_close(admin); + return exit_status(); +} From 0eca84a0ff0f4f96a3a753f643a2e9dbfff412c2 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 23 Jul 2026 05:42:31 +0000 Subject: [PATCH 55/81] fix: Align AWS RDS BGD simulator probe handling - Record topology probes before resolving the simulated response. - Align simulator documentation and local CI guidance with the implementation. Signed-off-by: Wazir Ahmed --- doc/AWS_Blue_Green/RDS_BGD_Monitor.md | 4 +- doc/AWS_Blue_Green/RDS_BGD_Simulator.md | 135 ++++++++++++------------ src/SQLite3_Server.cpp | 64 +++++------ test/infra/README.md | 13 ++- 4 files changed, 114 insertions(+), 102 deletions(-) diff --git a/doc/AWS_Blue_Green/RDS_BGD_Monitor.md b/doc/AWS_Blue_Green/RDS_BGD_Monitor.md index f434c96fd8..057accc594 100644 --- a/doc/AWS_Blue_Green/RDS_BGD_Monitor.md +++ b/doc/AWS_Blue_Green/RDS_BGD_Monitor.md @@ -1020,11 +1020,11 @@ proposed broad durable-ledger/controller PR is not part of this sequence. | Review PR | Scope | Dependency and completion signal | |---|---|---| | PR1: #5934 | This document only: evidence, accepted risks, current behavior, and follow-up contract. | Ready for author approval; merge into `feature/aws-rds-monitor` before implementation follow-ups so their scope is stable. | -| PR2: BGD simulator foundation — COMPLETED | Add the TAP-controlled SQLite3-server simulator defined in [RDS_BGD_Simulator.md](RDS_BGD_Simulator.md): the `TEST_RDS_BGD` build mode, IP-keyed topology responses, common and BGD TAP helpers, a simulator group, and an end-to-end acceptance smoke test. | Completed after the isolated local Docker group passed. Provides the reusable harness required by PR6; automatic GitHub Actions execution remains separate follow-up work under the review gate above. | +| PR2: BGD simulator foundation — COMPLETED | Add the TAP-controlled SQLite3-server simulator defined in [RDS_BGD_Simulator.md](RDS_BGD_Simulator.md): the `TEST_RDS_BGD` build mode, IP-keyed topology responses, common and BGD TAP helpers, a simulator group, and an end-to-end acceptance smoke test. | Completed after the isolated local Docker group passed. Provides the reusable harness required by PR6; GitHub workflow execution remains separate follow-up work under the review gate above. | | PR3: probe target and explicit TLS (**complete**) | Correct AWS-08 by selecting the exact supported explicit green writer row and its resolved `use_ssl`, including a row created or restored during discovery, while retaining the matched blue writer port and automatic-mode blue TLS fallback. | **Completed:** production behavior conforms to AWS-08. Existing-row and discovered-row simulator coverage remains part of PR6. | | PR4: terminal connection retirement (**complete**) | Preserve `healthy=false` across `MySQL_Connection::reset()` and destroy unhealthy connections in local and global pool-return paths. Do not introduce another flag or a new locking policy. | **Completed:** `connection_unhealthy_unit-t` proves a drained used connection cannot enter either free pool after reset or release. | | PR5: same-phase per-pair reconciliation | Replace phase-equality no-op behavior with worker-local reconciliation for incomplete map/resolution/pin/drain work. Retry only incomplete pairs and never redrain a pair already completed in the current worker generation. | Depends on the accepted one-shot worker model; it must not introduce durable ownership or restart recovery. | -| PR6: simulator-driven BGD scenario suite | Use PR2's simulator to cover configuration and discovery order, automatic and explicit rows, worker replacement, normal lifecycle, late entry, cancellation and rollback, topology drain, direct probe destination/TLS for existing and discovered explicit green rows, offline exclusions, and terminal connection retirement where observable. | Depends on PR2 and should normally follow PR3-PR4 so the suite validates final behavior rather than encoding known failures. All payloads run in the automatic BGD simulator CI group. | +| PR6: simulator-driven BGD scenario suite | Use PR2's simulator to cover configuration and discovery order, automatic and explicit rows, worker replacement, normal lifecycle, late entry, cancellation and rollback, topology drain, direct probe destination/TLS for existing and discovered explicit green rows, offline exclusions, and terminal connection retirement where observable. | Depends on PR2 and should normally follow PR3-PR4 so the suite validates final behavior rather than encoding known failures. All payloads run in the BGD simulator GitHub workflow. | Any retained cleanup ledger, durable restart ownership, or alternative controller state machine requires a new author policy decision. The simulator diff --git a/doc/AWS_Blue_Green/RDS_BGD_Simulator.md b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md index 59cfa53a5c..283df64949 100644 --- a/doc/AWS_Blue_Green/RDS_BGD_Simulator.md +++ b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md @@ -1,9 +1,9 @@ # AWS RDS Blue/Green Deployment Simulator -**Document status:** DESIGN APPROVED; IMPLEMENTATION NOT STARTED +**Document status:** SIMULATOR IMPLEMENTED; GITHUB WORKFLOW FOLLOW-UP **Applies to:** `TEST_RDS_BGD`, the SQLite3-server simulation surface, BGD TAP -helpers, the local Docker runner, and the matching GitHub Actions job +helpers, the local Docker runner, and supported simulator coverage **Related monitor contract:** [RDS_BGD_Monitor.md](RDS_BGD_Monitor.md) @@ -11,8 +11,9 @@ helpers, the local Docker runner, and the matching GitHub Actions job This document defines the simulator used to test ProxySQL's AWS RDS Blue/Green Deployment monitor. It combines the behavioral contract, SQLite3-server -changes, TAP helper API, network fixture, local runner, CI job, and supported -coverage into one implementation specification. +changes, TAP helper API, network fixture, local runner, and supported coverage +into one implementation specification. GitHub workflow execution is defined as +follow-up work. ## Architecture @@ -20,7 +21,7 @@ The TAP test is the scenario controller. It configures ProxySQL with AWS-style hostnames, writes simulated backend state to ProxySQL's SQLite3 server, changes that state to drive the BGD FSM, and verifies ProxySQL through runtime, statistics, and simulator probe-log tables. Topology state is keyed by backend -IP, while read-only state is keyed by the configured hostname. +IP and port, while read-only state is keyed by the configured hostname and port. No `test/deps/cluster_simulator` process or backend database container is required. A common TAP helper owns reusable SQLite3-server operations, while a @@ -125,10 +126,10 @@ SELECT 1 FROM information_schema.TABLES SELECT * FROM mysql.rds_topology ``` -For either match, resolve the accepted backend key, load its control row, -select the response described below, append a probe-log row, and send the -result. An address-extraction failure returns a simulator error without -selecting state or logging an invalid backend identity. +For either match, resolve the accepted backend key, append a probe-log row, +load its control row, and select the response described below. An +address-extraction failure returns a simulator error without selecting state or +logging an invalid backend identity. Simulated read-only checks follow the handling described below. All remaining statements continue through normal SQLite3-server handling. TAP control and @@ -137,9 +138,9 @@ recorded as BGD monitor probes. ## Control-State Meaning -The TAP helper updates the control and topology tables in one transaction, so -a monitor query observes either the previous state or the complete new state. -The supported states are: +The TAP helper publishes control and topology changes atomically. Monitor probes +read committed simulator state without holding a cross-query snapshot. The +supported states are: | `RDS_BGD_CONTROL` state | Topology rows | Meaning | |---|---|---| @@ -216,7 +217,7 @@ signatures below are the initial API and may grow with reviewed test cases. ### Common Endpoint ```cpp -struct Simulator_Endpoint { +struct Endpoint { std::string host; int port; }; @@ -236,7 +237,7 @@ int connect( char* password, bool use_ssl = false); -int read_only_update(Simulator_Endpoint backend, bool read_only); +int read_only_update(Endpoint backend, bool read_only); ``` `connect()` opens the SQLite3-server control connection with the MySQL client @@ -259,7 +260,8 @@ struct RDS_BGD_Host { std::string ip; int port; - Simulator_Endpoint endpoint(); + Endpoint endpoint(); + Endpoint host_endpoint(); }; ``` @@ -277,31 +279,32 @@ public: std::vector blue_readers; std::vector green_readers; - std::vector get_writers(); + std::vector get_writers(); + std::vector get_writer_hosts(); std::vector get_topology(std::string status); }; ``` Each TAP test owns and initializes the cluster fixtures it uses. A fixture -keeps the selected `/etc/hosts` mapping together. `get_writers()` -returns the selected blue and green writer IPs; `get_topology(status)` returns -the standard two-row SOURCE/TARGET topology using the writer hostnames and the -provided status. +keeps the selected `/etc/hosts` mapping together. `get_writers()` returns the +selected blue and green writer IPs, `get_writer_hosts()` returns their configured +hostnames, and `get_topology(status)` returns the standard two-row SOURCE/TARGET +topology using the writer hostnames and the provided status. ### BGD Topology Operations ```cpp int topology_update( - std::vector backends, + std::vector backends, std::vector rows); -int topology_delete(std::vector backends); +int topology_delete(std::vector backends); -int topology_drop(std::vector backends); +int topology_drop(std::vector backends); int topology_error( - std::vector backends, - unsigned int error_code, + std::vector backends, + int error_code, std::string error_msg); ``` @@ -324,7 +327,7 @@ enum class RDS_BGD_Probe_Kind { struct RDS_BGD_Probe_Log { uint64_t sequence_id; - Simulator_Endpoint backend; + Endpoint backend; RDS_BGD_Probe_Kind probe_kind; bool encrypted; }; @@ -335,7 +338,7 @@ rc_t> probe_log_since(uint64_t sequence_id); rc_t wait_for_probe_log( uint64_t sequence_id, - Simulator_Endpoint backend, + Endpoint backend, RDS_BGD_Probe_Kind probe_kind, uint32_t timeout_ms, int encrypted = -1); @@ -359,17 +362,13 @@ int main() { if (!admin) BAIL_OUT("failed to connect to ProxySQL Admin"); RDS_BGD_Cluster cluster = bgd_cluster_init(); - if (configure_proxysql_for_bgd(admin, cluster) != EXIT_SUCCESS) - BAIL_OUT("failed to configure ProxySQL"); - - RDS_BGD_Simulator simulator; + RDS_BGD_Simulator simulator {}; if (simulator.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) BAIL_OUT("failed to connect to SQLite3 server"); - if (simulator.read_only_update( - { cluster.blue_writer.hostname, cluster.blue_writer.port }, false) != EXIT_SUCCESS || - simulator.read_only_update( - { cluster.green_writer.hostname, cluster.green_writer.port }, false) != EXIT_SUCCESS) - BAIL_OUT("failed to configure writer read_only state"); + for (Endpoint& writer : cluster.get_writer_hosts()) { + if (simulator.read_only_update(writer, false) != EXIT_SUCCESS) + BAIL_OUT("failed to configure writer read_only state"); + } auto [seq_rc, last_seq] = simulator.probe_log_last_sequence(); if (seq_rc != EXIT_SUCCESS) @@ -381,6 +380,9 @@ int main() { if (update_rc != EXIT_SUCCESS) BAIL_OUT("failed to publish topology"); + if (configure_proxysql_for_bgd(admin, cluster) != EXIT_SUCCESS) + BAIL_OUT("failed to configure ProxySQL"); + ok(wait_for_cond(admin, "SELECT status='AVAILABLE' FROM runtime_mysql_aws_rds_bgd_hostgroups " "WHERE writer_hostgroup=10", 5) == EXIT_SUCCESS, @@ -404,8 +406,8 @@ responses and reads probe evidence; assertions against ProxySQL use Admin SQL. ## Build Integration -Add `build_lib_test_rds_bgd`, `build_src_test_rds_bgd`, and the top-level -`test_rds_bgd` target. The lib and src targets compile with +The build provides `build_lib_test_rds_bgd`, `build_src_test_rds_bgd`, and the +top-level `test_rds_bgd` target. The lib and src targets compile with `-DDEBUG -DTEST_RDS_BGD`; none depends on `build_cluster_simulator`. `test_rds_bgd` depends on `build_src_test_rds_bgd` and then invokes `make @@ -416,13 +418,13 @@ build_deps_debug -> build_lib_test_rds_bgd -> build_src_test_rds_bgd -> TAP debug build ``` -Use `test_rds_bgd` as the single entry point. Do not invoke +`test_rds_bgd` is the single entry point. Do not invoke `build_tap_test_debug` afterward because its `build_src_debug` dependency -selects the normal debug daemon. Add `-DTEST_RDS_BGD` to `testall` as well. +selects the normal debug daemon. `testall` includes `-DTEST_RDS_BGD` as well. ## Local CI Group -Add `test/tap/groups/cluster_sim_rds_bgd/` and execute it as +The `test/tap/groups/cluster_sim_rds_bgd/` group executes as `cluster_sim_rds_bgd-g1`. | File | BGD-specific content | @@ -508,19 +510,19 @@ simulator transitions. From the TAP container, the control connection uses ### Group Registration and Local Run -Register each BGD TAP binary in `test/tap/groups/groups.json`: +Each BGD TAP binary is registered in `test/tap/groups/groups.json`: ```json "test_rds_bgd_smoke-t": [ "cluster_sim_rds_bgd-g1" ] ``` -Add the group and its `make test_rds_bgd` requirement to the simulator table in -`test/infra/README.md`. Clean when switching compile flavors because Make does -not track changed preprocessor flags: +The simulator table in `test/infra/README.md` records the group and its +`make test_rds_bgd` requirement. Clean when switching compile flavors because +Make does not track changed preprocessor flags: ```bash make clean -make -j"$(nproc)" test_rds_bgd +PROXYSQL40=1 make -j"$(nproc)" test_rds_bgd export INFRA_ID="rds-bgd-$(date +%s)" export TAP_GROUP="cluster_sim_rds_bgd-g1" @@ -535,9 +537,10 @@ The existing runner injects the host aliases, starts ProxySQL with and collects logs. No BGD branch is required in `ensure-infras.bash`, `start-proxysql-isolated.bash`, or `run-tests-isolated.bash`. -## GitHub Actions +## GitHub Workflow Follow-up -Add `.github/workflows/CI-rds-bgd-simulator.yml`. It runs on +A separate follow-up adds `.github/workflows/CI-rds-bgd-simulator.yml`; the +workflow is not part of the simulator implementation described above. It runs on `workflow_dispatch` and after a successful `CI-trigger`, follows the repository's existing concurrency/cancellation pattern, and checks out the exact triggering SHA. @@ -550,7 +553,7 @@ The build job checks out the triggering SHA, installs or reuses the normal Ubuntu TAP build dependencies, and runs: ```bash -make -j"$(nproc)" test_rds_bgd +PROXYSQL40=1 make -j"$(nproc)" test_rds_bgd ``` After verifying `src/proxysql` and `test/tap/tests/test_rds_bgd_smoke-t`, it saves the @@ -603,16 +606,16 @@ the standard runner reports no infrastructure or test failure. ### Simulator Acceptance -The simulator implementation needs one end-to-end smoke test, not a separate +The simulator implementation includes one end-to-end smoke test, not a separate unit-test suite for every helper method. `test_rds_bgd_smoke-t` proves that the `TEST_RDS_BGD` daemon accepts TAP-controlled topology, ProxySQL observes an -`AVAILABLE` deployment, the green-IP probe is logged, and the automatic CI job -executes the group without `test/deps/cluster_simulator`. +`AVAILABLE` deployment, and the green-IP probe is logged. The isolated local +runner executes the test without `test/deps/cluster_simulator`. -The configuration and lifecycle tests below exercise the remaining helper and -SQLite3-server paths through BGD behavior. Before changing simulator state, -each test reads the last probe-log sequence; failures report the configured backend -state, last ProxySQL runtime state, and later probe rows. +The follow-up configuration and lifecycle tests below exercise the remaining +helper and SQLite3-server paths through BGD behavior. Before changing simulator +state, each test reads the last probe-log sequence; failures report the +configured backend state, last ProxySQL runtime state, and later probe rows. ### Configuration and Discovery @@ -664,17 +667,17 @@ assertion depends on them. ## Code Boundaries -| Area | Required change | +| Area | Current boundary | |---|---| -| `Makefile` | Add the BGD build targets and include `TEST_RDS_BGD` in `testall`. | -| `include/SQLite3_Server.h` | Add BGD table definitions/helpers and the coded-error overload under the flag. | -| `src/SQLite3_Server.cpp` | Add listener setup, endpoint extraction, table creation, BGD/read-only interception, and probe logging. | -| `test/tap` helpers | Add the common simulator and BGD-specific API defined above. | -| `test/tap/groups/cluster_sim_rds_bgd` | Add the fixed host map and SQLite3-server group configuration. | -| `test/tap/groups/groups.json` | Register BGD TAP binaries in `cluster_sim_rds_bgd-g1`. | -| `test/infra/README.md` | Document the group and its required `test_rds_bgd` build target. | -| `.github/workflows/CI-rds-bgd-simulator.yml` | Build the flagged flavor and execute the BGD simulator group automatically. | -| BGD production monitor | Reuse existing query constants; add no simulator query decoration or test initializer. | +| `Makefile` | Provides the BGD build targets and includes `TEST_RDS_BGD` in `testall`. | +| `include/SQLite3_Server.h` | Defines the BGD tables and shared simulator helpers. | +| `src/SQLite3_Server.cpp` | Handles endpoint extraction, table creation, BGD/read-only interception, and probe logging. | +| `test/tap` helpers | Provide the common simulator and BGD-specific API defined above. | +| `test/tap/groups/cluster_sim_rds_bgd` | Defines the fixed host map and SQLite3-server group configuration. | +| `test/tap/groups/groups.json` | Registers BGD TAP binaries in `cluster_sim_rds_bgd-g1`. | +| `test/infra/README.md` | Documents the group and its required `test_rds_bgd` build target. | +| GitHub workflow | Follow-up work builds the flagged flavor and executes the BGD simulator group. | +| BGD production monitor | Reuses existing query constants without simulator query decoration or a test initializer. | Existing simulator builds retain their behavior. The scenario, not the helper, owns topology publication, FSM timing, ProxySQL configuration, and expected diff --git a/src/SQLite3_Server.cpp b/src/SQLite3_Server.cpp index 185c6ae4d5..ee3ebb4085 100644 --- a/src/SQLite3_Server.cpp +++ b/src/SQLite3_Server.cpp @@ -797,51 +797,54 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p "backend_ip='" + backend_ip + "' AND backend_port=" + std::to_string(backend_port) }; - char *control_error=NULL; - int control_cols=0; - int control_affected_rows=0; - SQLite3_result *control_result=NULL; - const std::string control_query { - "SELECT topology_present,error_code,error_msg FROM RDS_BGD_CONTROL WHERE " + - predicate + const std::string log_query { + "INSERT INTO RDS_BGD_PROBE_LOG" + "(backend_ip,backend_port,probe_kind,encrypted) VALUES ('" + + backend_ip + "'," + std::to_string(backend_port) + ",'" + + (rds_bgd_table_check ? "table_check" : "metadata") + "'," + + (sess->client_myds->encrypted ? "1" : "0") + ")" }; - sqlite_sess->sessdb->execute_statement( - control_query.c_str(), &control_error, &control_cols, - &control_affected_rows, &control_result); - - if (control_error != NULL) { + if (!sqlite_sess->sessdb->execute(log_query.c_str())) { GloSQLite3Server->send_MySQL_ERR( - &sess->client_myds->myprot, 1105, control_error); - free(control_error); + &sess->client_myds->myprot, 1105, + "RDS BGD simulator failed to record the topology probe"); run_query=false; } else { + char *control_error=NULL; + int control_cols=0; + int control_affected_rows=0; + SQLite3_result *control_result=NULL; + const std::string control_query { + "SELECT topology_present,error_code,error_msg FROM RDS_BGD_CONTROL WHERE " + + predicate + }; + sqlite_sess->sessdb->execute_statement( + control_query.c_str(), &control_error, &control_cols, + &control_affected_rows, &control_result); + + if (control_error != NULL) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1105, control_error); + free(control_error); + run_query=false; + } + bool topology_present=false; unsigned int configured_error=0; std::string configured_error_msg {}; - if (control_result && control_result->rows_count == 1) { + if (run_query && control_result && control_result->rows_count == 1) { SQLite3_row *row=control_result->rows.front(); topology_present=atoi(row->fields[0]) != 0; configured_error=static_cast(atoi(row->fields[1])); configured_error_msg=row->fields[2] ? row->fields[2] : ""; } + delete control_result; - const std::string log_query { - "INSERT INTO RDS_BGD_PROBE_LOG" - "(backend_ip,backend_port,probe_kind,encrypted) VALUES ('" + - backend_ip + "'," + std::to_string(backend_port) + ",'" + - (rds_bgd_table_check ? "table_check" : "metadata") + "'," + - (sess->client_myds->encrypted ? "1" : "0") + ")" - }; - if (!sqlite_sess->sessdb->execute(log_query.c_str())) { - GloSQLite3Server->send_MySQL_ERR( - &sess->client_myds->myprot, 1105, - "RDS BGD simulator failed to record the topology probe"); - run_query=false; - } else if (rds_bgd_table_check) { + if (run_query && rds_bgd_table_check) { l_free(query_length,query); query=l_strdup(topology_present ? "SELECT 1" : "SELECT 1 WHERE 0"); query_length=strlen(query)+1; - } else if (configured_error != 0 || !topology_present) { + } else if (run_query && (configured_error != 0 || !topology_present)) { const uint16_t error_code = configured_error ? static_cast(configured_error) : 1146; const char *error_msg = configured_error @@ -850,7 +853,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p GloSQLite3Server->send_MySQL_ERR( &sess->client_myds->myprot, error_code, error_msg); run_query=false; - } else { + } else if (run_query) { const std::string topology_query { "SELECT id,endpoint,topology_port AS port,role,status " "FROM RDS_BGD_TOPOLOGY WHERE " + predicate + @@ -861,7 +864,6 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p query_length=strlen(query)+1; } } - delete control_result; } } diff --git a/test/infra/README.md b/test/infra/README.md index 7896a94108..44efef3ce5 100644 --- a/test/infra/README.md +++ b/test/infra/README.md @@ -39,7 +39,11 @@ This will: ## 0.2. Simulator-backed TAP groups -Groups whose name starts with `cluster_sim_` (e.g. `cluster_sim_aurora-g1`, `cluster_sim_galera-g1`) drive ProxySQL via the in-repo `cluster_simulator` under `test/deps/cluster_simulator/`. The simulator mutates ProxySQL's internal cluster state through code paths gated by compile-time `#ifdef` flags, so the ProxySQL binary **must** be built with the matching flag or state mutations become no-ops and tests fail silently. +Groups whose name starts with `cluster_sim_` exercise simulator-specific +ProxySQL paths gated by compile-time `#ifdef` flags. State is driven either by +the in-repo `test/deps/cluster_simulator` process or directly by TAP helpers +through SQLite3-server, as in RDS BGD. The ProxySQL binary **must** be built with +the matching flag or state mutations become no-ops and tests fail silently. | Simulator group | Required build target | |-----------------------------|---------------------------| @@ -48,9 +52,12 @@ Groups whose name starts with `cluster_sim_` (e.g. `cluster_sim_aurora-g1`, `clu | `cluster_sim_group_repl-g` | `make testgrouprep` | | `cluster_sim_read_only-g` | `make testreadonly` | | `cluster_sim_repl_lag-g` | `make testreplicationlag` | -| `cluster_sim_rds_bgd-g` | `make test_rds_bgd` | +| `cluster_sim_rds_bgd-g` | `make test_rds_bgd` | -Each target sets the corresponding `-DTEST_` flag on the ProxySQL src and lib builds and triggers the required simulator build. A plain `make` is **not sufficient** for these groups. +Each target sets the corresponding `-DTEST_` flag on the ProxySQL src +and lib builds and builds the TAP artifacts. Targets backed by +`test/deps/cluster_simulator` build that process as well. A plain `make` is +**not sufficient** for these groups. --- ## 1. Core Concepts From e8bbc5c38516122bad94f7cbd21a6c97c1708377 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 23 Jul 2026 10:29:11 +0000 Subject: [PATCH 56/81] fix: Retry AWS RDS BGD DNS pinning in same phase - Retry green IP resolution while topology remains in an eligible phase. - Track per-pair pin completion to avoid repeated drain and purge actions. Signed-off-by: Wazir Ahmed --- doc/AWS_Blue_Green/RDS_BGD_Monitor.md | 46 +++++++++-------- include/MySQL_Monitor.hpp | 7 +++ lib/MySQL_Monitor.cpp | 71 ++++++++++++++++++--------- 3 files changed, 77 insertions(+), 47 deletions(-) diff --git a/doc/AWS_Blue_Green/RDS_BGD_Monitor.md b/doc/AWS_Blue_Green/RDS_BGD_Monitor.md index 057accc594..eb12f2e651 100644 --- a/doc/AWS_Blue_Green/RDS_BGD_Monitor.md +++ b/doc/AWS_Blue_Green/RDS_BGD_Monitor.md @@ -1,7 +1,6 @@ # AWS RDS Blue/Green Monitor -**Document status:** AUTHOR VALIDATION COMPLETE; PR2 SIMULATOR FOUNDATION -COMPLETE; IMPLEMENTATION CONFORMANCE OPEN +**Document status:** AUTHOR VALIDATION COMPLETE; IMPLEMENTATION CONFORMANCE OPEN **Applies to:** Amazon RDS Multi-AZ DB instance blue/green deployment monitoring @@ -82,8 +81,8 @@ That response explicitly: it, including membership created automatically at runtime. - Accepts one-shot cleanup and loss of per-effect completion state rather than a retained or durable cleanup ledger. -- Accepts the current same-phase DNS-resolution failure for this PR and commits - to a later per-pair reconciliation change. +- Records the same-phase DNS-resolution failure for later per-pair + reconciliation. - Accepts cleanup-on-worker-exit followed by fresh worker state, and a no-persistence fresh start after a full ProxySQL process restart. @@ -334,17 +333,16 @@ fallback. Simulator coverage for both row paths is assigned to PR6. ### Current Phase-Equality Behavior -`SOURCE-CODE`: After status conversion, the handler returns immediately when -the converted status equals the stored status. Phase actions run on transition, -not on every observation. Consequently, a transient mapping or DNS failure is -not retried while the same phase continues. +`SOURCE-CODE`: After status conversion, when the converted status equals the +stored status, the handler retries green-IP resolution from `AVAILABLE` through +`WRITER_SWITCHOVER_POST_PROCESSING`. It then returns without rebuilding the +configuration-derived pair map or rerunning the full phase action. -`AUTHOR-ACCEPTED-POLICY`: The author accepts this failure mode for the current -feature PR. In particular, a first DNS failure in -`WRITER_SWITCHOVER_POST_PROCESSING` can leave a pair unpinned and its old -connections undrained for the remainder of that phase. A subsequent PR is to -add worker-local, per-pair reconciliation that retries unresolved addresses -and applies pin/drain once, rather than rerunning the entire phase action. +`SOURCE-CODE`: During an equal `WRITER_SWITCHOVER_POST_PROCESSING` observation, +the handler pins and drains only pairs whose green IP is available and whose +worker-local `green_ip_pinned` flag is false. Unresolved pairs remain eligible +for the next observation, while completed pairs are skipped for the remainder +of that worker generation. ### Current Connection-Retirement Behavior @@ -489,7 +487,7 @@ entry points should be reviewed with this document whenever behavior changes. for comparison and possible future reconsideration. The author explicitly selected one-shot cleanup, worker-local state, and no durable BGD ledger. None of the ledger states or invariants below is therefore an accepted requirement -for PR #5861 or the accepted same-phase reconciliation follow-up. +for PR #5861 or the same-phase reconciliation follow-up. `PROPOSED-POLICY`: Every externally visible effect must have a stable identity and a cleanup record before the effect is considered applied. @@ -907,7 +905,7 @@ that is tracked separately rather than reopening the evidence decision. | AWS-10 | Commits `cdffd77ee` and `ac4167cd0` retain auto-added and user-configured green rows on rollback and success. Rollback leaves green connections untouched; success drains eligible green connections but leaves rows and statuses unchanged. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: Green membership is persistent runtime configuration, not a temporary owned effect. Administrative cleanup is required even for an auto-added row. | | AWS-11a | The author explicitly chooses one-shot worker-exit/configuration-change cleanup and no retained retry ledger. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: Loss of the cleanup context, including when a process terminates during cleanup, is accepted. The stronger retained rollback model is not PR2 scope. | | AWS-11b | The author explicitly applies the same one-shot choice after completion and relies on the current phase-specific cleanup path. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: No retained `SAFE_TEARDOWN` executor or per-effect settlement record is required. This acceptance does not prove each one-shot operation succeeds. | -| AWS-12a | Commits `727b2166b` and `d45c953d2` combine eligible blue/green rows into the worker generation checksum and refresh it after Admin `mysql_servers` commits. The author accepts that DNS recovery alone does not retry a failed same-phase setup. | `RESOLVED AS AUTHOR-ACCEPTED POLICY AND FOLLOW-UP`: Current PR may leave a POST_PROCESSING pair unpinned and undrained after first-resolution failure. A subsequent PR must implement per-pair retry and exactly-once pin/drain behavior. | +| AWS-12a | Commits `727b2166b` and `d45c953d2` combine eligible blue/green rows into the worker generation checksum and refresh it after Admin `mysql_servers` commits. The author assigned failed same-phase DNS setup to a per-pair follow-up. | `RESOLVED; IMPLEMENTED`: Eligible same-phase observations retry green-IP resolution, and worker-local `green_ip_pinned` state prevents repeated pin/drain work for completed pairs. The existing simulator cannot reproduce mutable DNS recovery, so simulator verification is unavailable for this case. | | AWS-12b | The author selects cleanup-on-worker-exit and fresh replacement state. Persistent green membership and rollback-time green connections have no worker ownership under AWS-10. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: A replacement first observing COMPLETED may enter the inferred reader phase without reconstructing the prior map or effects. | | AWS-13 | The author separates worker replacement from full restart. Replacement performs one-shot rollback then starts fresh. Full restart rebuilds DNS cache, pools, suppression, maps, probe target, and FSM; configured state reloads, while an unsynchronized auto-added runtime green row disappears. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: No durable BGD progress or ownership persistence is required. This is an accepted fresh-start contract, not a traced per-effect guarantee. | | CFG-01a | User-configured rows require both green hostgroup values. The persistent Admin table declares both columns `NOT NULL`. | `RESOLVED AS AUTHOR-VALIDATED PROXYSQL CONTRACT`: A user `NULL` or mixed row is invalid; no configuration-nullability follow-up is required. | @@ -973,8 +971,8 @@ durable-ledger design. | Automatic runtime row persistence | `auto_generated_null_green_hgs` | `save_runtime_skips_auto_generated_bgd` | Auto-discovery creates a runtime row with both green hostgroups `NULL` and `auto_generated=1`; saving runtime to memory/disk does not persist that row. | | First observation COMPLETED | `fresh_worker_first_completed` | `replace_worker_at_completed` | Fresh state advances to the inferred reader phase without reconstructing a prior map, then finishes on topology drain. | | Full restart fresh start | `restart_discards_bgd_state` | `proxysql_restart_fixture` | DNS cache, pools, suppression, mapping, probe target, and FSM are recreated; configured rows reload; an unsynchronized auto-added runtime-only green row does not. | -| Same-phase DNS retry follow-up | `dns_retry_same_post_per_pair` | — | Future resolver/unit coverage only: the unresolved pair retries while phase is unchanged; successful pairs are not redrained; the recovered pair is pinned and drained exactly once. Mutable DNS is outside simulator/TAP scope. | -| Partial pair progress follow-up | `one_pair_fails` | `multiple_reader_fixture` | Accepted follow-up only: successful pair state is retained worker-locally and only the failed pair retries. | +| Same-phase DNS retry follow-up | — | — | Implemented in source: the unresolved pair retries while the phase is unchanged, successful pairs are not redrained, and the recovered pair is pinned and drained once. Mutable DNS recovery is outside the existing simulator contract. | +| Partial pair progress follow-up | — | — | Implemented in source: successful pair state is retained worker-locally and only the unresolved pair retries. | `PROPOSED-POLICY`: The simulator cases previously proposed for durable effect ownership, compare-and-restore, retained `FAULTED` state, cleanup across stale @@ -1000,11 +998,11 @@ The source review remains open on implementation and verification: `healthy` field and does not add a second flag. `connection_unhealthy_unit-t` verifies that unhealthy connections remain terminal across reset and cannot enter either free pool. -3. Track the author-accepted same-phase DNS failure as required follow-up work - with focused resolver/unit coverage rather than simulator/TAP integration. - Until per-pair reconciliation exists, a transient first resolution failure - in POST_PROCESSING can leave traffic unpinned and old connections undrained. - Acceptance documents the risk; it does not make the failure safe. +3. **COMPLETED:** Retry green-IP resolution during eligible same-phase + observations and use worker-local per-pair completion state to prevent + repeated pin/drain work in POST_PROCESSING. The existing simulator cannot + verify mutable DNS failure and recovery; the author accepts this test + coverage limitation. 4. Add focused simulator and TAP coverage for the response commits and these follow-ups. Registration in `test/tap/groups/groups.json` is insufficient: an automatic PR check must build the BGD test flavor and execute the BGD @@ -1023,7 +1021,7 @@ proposed broad durable-ledger/controller PR is not part of this sequence. | PR2: BGD simulator foundation — COMPLETED | Add the TAP-controlled SQLite3-server simulator defined in [RDS_BGD_Simulator.md](RDS_BGD_Simulator.md): the `TEST_RDS_BGD` build mode, IP-keyed topology responses, common and BGD TAP helpers, a simulator group, and an end-to-end acceptance smoke test. | Completed after the isolated local Docker group passed. Provides the reusable harness required by PR6; GitHub workflow execution remains separate follow-up work under the review gate above. | | PR3: probe target and explicit TLS (**complete**) | Correct AWS-08 by selecting the exact supported explicit green writer row and its resolved `use_ssl`, including a row created or restored during discovery, while retaining the matched blue writer port and automatic-mode blue TLS fallback. | **Completed:** production behavior conforms to AWS-08. Existing-row and discovered-row simulator coverage remains part of PR6. | | PR4: terminal connection retirement (**complete**) | Preserve `healthy=false` across `MySQL_Connection::reset()` and destroy unhealthy connections in local and global pool-return paths. Do not introduce another flag or a new locking policy. | **Completed:** `connection_unhealthy_unit-t` proves a drained used connection cannot enter either free pool after reset or release. | -| PR5: same-phase per-pair reconciliation | Replace phase-equality no-op behavior with worker-local reconciliation for incomplete map/resolution/pin/drain work. Retry only incomplete pairs and never redrain a pair already completed in the current worker generation. | Depends on the accepted one-shot worker model; it must not introduce durable ownership or restart recovery. | +| PR5: same-phase per-pair reconciliation (**complete**) | Retry green-IP resolution on eligible equal-phase observations and never redrain a pair completed in the current worker generation. Configuration-derived pair mapping remains transition/generation driven. | **Completed:** production behavior conforms to AWS-12a. The existing simulator cannot verify mutable DNS recovery. | | PR6: simulator-driven BGD scenario suite | Use PR2's simulator to cover configuration and discovery order, automatic and explicit rows, worker replacement, normal lifecycle, late entry, cancellation and rollback, topology drain, direct probe destination/TLS for existing and discovered explicit green rows, offline exclusions, and terminal connection retirement where observable. | Depends on PR2 and should normally follow PR3-PR4 so the suite validates final behavior rather than encoding known failures. All payloads run in the BGD simulator GitHub workflow. | Any retained cleanup ledger, durable restart ownership, or alternative diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index f57803a9f0..1820faabc1 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -441,6 +441,7 @@ struct AWS_RDS_BlueGreenPair { int32_t green_use_ssl = -1; ///< Green server SSL; -1 means unset (use blue_use_ssl). std::string green_ip; ///< Green host IP resolved at SWITCHOVER_INITIATED and held warm. unsigned long long green_ip_ttl = 0; ///< Expiry for green_ip when resolved by the BGD thread; 0 means DNS_Cache-sourced. + bool green_ip_pinned = false; ///< True after green_ip has been pinned and blue_host connections drained/purged. bool is_writer = false; ///< True when this pair maps the blue writer. }; @@ -673,6 +674,12 @@ class MySQL_Monitor { */ void handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_Result& topology); /** + * @brief Pin green IPs and drain existing blue-host connections. + * + * @param st BGD switchover state. + */ + void aws_rds_bgd_pin_green_ips(AWS_RDS_BGD_State& st); + /** * @brief Run deferred switchover teardown or rollback cleanup. * * @details Restores post-switchover reader handling, unshuns readers, drops DNS pins, diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 589d05d1de..46dde3e4f0 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -7411,7 +7411,16 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo } if (topology_status == st.bgd_status) { - // no phase change + // Refresh or retry green IP resolution on every eligible same-phase observation. + if (topology_status >= AWS_RDS_BGD_Status::AVAILABLE + && topology_status <= AWS_RDS_BGD_Status::WRITER_SWITCHOVER_POST_PROCESSING) { + aws_rds_bgd_resolve_green_ips(st); + } + + // Retry pinning pairs whose green IP became available while remaining in POST_PROCESSING. + if (topology_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_POST_PROCESSING) { + aws_rds_bgd_pin_green_ips(st); + } return; } @@ -7455,6 +7464,12 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo aws_rds_bgd_add_green_writer_in_hg(st); aws_rds_bgd_set_bgd_in_progress(st); + // Repoint each mapped blue host onto its green IP and drain existing + // connections so new backend work resolves to green. + aws_rds_bgd_pin_green_ips(st); + + // Blue readers without a green counterpart must stop serving reads. + srv_addr_t writer; for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { if (p.is_writer) { @@ -7463,28 +7478,6 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo } } - // Repoint each mapped blue host onto its green IP and drain existing - // connections so new backend work resolves to green. - for (AWS_RDS_BlueGreenPair& p : st.bg_map) { - if (p.green_ip.empty()) { - proxy_warning( - "AWS RDS BGD [wHG=%u rHG=%u]: no green IP for blue '%s:%d'; cannot repoint\n", - st.writer_hg, st.reader_hg, p.blue_host.c_str(), p.port); - continue; - } - dns_cache->pin(p.blue_host, p.green_ip); - proxy_info( - "AWS RDS BGD [wHG=%u rHG=%u]: repointed blue '%s' to green IP %s\n", - st.writer_hg, st.reader_hg, p.blue_host.c_str(), p.green_ip.c_str()); - - MyHGM->wrlock(); - MyHGM->drain_server_connections(p.blue_host.c_str(), p.port); - MyHGM->wrunlock(); - My_Conn_Pool->purge_connections(p.blue_host.c_str(), p.port); - } - - // Blue readers without a green counterpart must stop serving reads. - std::vector blue_readers; MyHGM->wrlock(); MyHGC* rhgc = MyHGM->MyHGC_lookup(st.reader_hg); @@ -7552,6 +7545,38 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo } } +/** +* @brief Pin green IPs and drain existing blue-host connections. +* +* @param st BGD switchover state. +*/ +void MySQL_Monitor::aws_rds_bgd_pin_green_ips(AWS_RDS_BGD_State& st) { + for (AWS_RDS_BlueGreenPair& pair : st.bg_map) { + if (pair.green_ip_pinned) { + continue; + } + + if (pair.green_ip.empty()) { + proxy_debug(PROXY_DEBUG_MONITOR, 7, + "AWS RDS BGD [wHG=%u rHG=%u]: green host '%s' remains unresolved; " + "deferring pin/drain for blue '%s:%d'\n", + st.writer_hg, st.reader_hg, pair.green_host.c_str(), pair.blue_host.c_str(), pair.port); + continue; + } + + dns_cache->pin(pair.blue_host, pair.green_ip); + MyHGM->wrlock(); + MyHGM->drain_server_connections(pair.blue_host.c_str(), pair.port); + MyHGM->wrunlock(); + My_Conn_Pool->purge_connections(pair.blue_host.c_str(), pair.port); + pair.green_ip_pinned = true; + + proxy_info( + "AWS RDS BGD [wHG=%u rHG=%u]: repointed blue '%s' to green IP %s\n", + st.writer_hg, st.reader_hg, pair.blue_host.c_str(), pair.green_ip.c_str()); + } +} + /** * @brief Apply BGD hostgroup changes for the current switchover status. * From 251f820910cd67241e6bd5a8ad6d5e1ec3d5ec42 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Fri, 24 Jul 2026 07:03:48 +0000 Subject: [PATCH 57/81] ci: Add workflow for cluster simulation groups - Build ProxySQL runtime with all simulation flags in the Ubuntu 22 packaging image - Discover simulation groups and TAP binaries from groups.json - Cache exact-SHA runtime outputs for independent matrix jobs - Add a documented helper for local and workflow operations --- .github/workflows/CI-cluster-simulator.yml | 129 +++++++ .gitignore | 2 + test/infra/control/cluster-simulator-ci.bash | 339 +++++++++++++++++++ 3 files changed, 470 insertions(+) create mode 100644 .github/workflows/CI-cluster-simulator.yml create mode 100755 test/infra/control/cluster-simulator-ci.bash diff --git a/.github/workflows/CI-cluster-simulator.yml b/.github/workflows/CI-cluster-simulator.yml new file mode 100644 index 0000000000..deafe3a0ac --- /dev/null +++ b/.github/workflows/CI-cluster-simulator.yml @@ -0,0 +1,129 @@ +# Builds ProxySQL once with every cluster simulation flag enabled, then runs +# each registered cluster_sim_* TAP group as an independent matrix job. +# +# Maintenance notes: +# - Groups and TAP binaries are discovered from test/tap/groups/groups.json. +# - `testall` is intentional: every matrix job shares one ProxySQL binary built +# with all simulation flags. +# - Registering a new simulation group and TAP binary requires no YAML changes. +# - The exact-SHA cache contains only the runtime files used by matrix jobs. +# - Command details and local examples: test/infra/control/cluster-simulator-ci.bash help. + +name: CI-cluster-simulator +run-name: '${{ github.head_ref || github.ref_name }} ${{ github.workflow }} ${{ github.sha }}' + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }} + cancel-in-progress: true + +env: + BUILD_CACHE_KEY: cluster-simulator-v3-ubuntu22-${{ github.sha }} + RUNTIME_CACHE_DIR: .cluster-simulator-runtime + +jobs: + build: + name: Build ProxySQL with simulation support + runs-on: ubuntu-22.04 + outputs: + groups: ${{ steps.simulator-groups.outputs.groups }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Discover simulation groups + id: simulator-groups + run: test/infra/control/cluster-simulator-ci.bash discover + + - name: Restore simulation build + id: simulator-build + uses: actions/cache/restore@v4 + with: + key: ${{ env.BUILD_CACHE_KEY }} + path: ${{ env.RUNTIME_CACHE_DIR }} + + - name: Install cached simulation runtime + if: steps.simulator-build.outputs.cache-hit == 'true' + run: test/infra/control/cluster-simulator-ci.bash install + + - name: Build simulation test runtime + if: steps.simulator-build.outputs.cache-hit != 'true' + run: test/infra/control/cluster-simulator-ci.bash build + + - name: Verify simulation build + run: test/infra/control/cluster-simulator-ci.bash verify + + - name: Stage simulation runtime + if: steps.simulator-build.outputs.cache-hit != 'true' + run: test/infra/control/cluster-simulator-ci.bash stage + + - name: Save simulation build + if: steps.simulator-build.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + key: ${{ env.BUILD_CACHE_KEY }} + path: ${{ env.RUNTIME_CACHE_DIR }} + + test: + name: ${{ matrix.group }} + needs: build + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + group: ${{ fromJSON(needs.build.outputs.groups) }} + env: + INFRA_ID: ${{ matrix.group }}-${{ github.run_id }}-${{ github.run_attempt }} + TAP_GROUP: ${{ matrix.group }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Restore simulation build + uses: actions/cache/restore@v4 + with: + key: ${{ env.BUILD_CACHE_KEY }} + fail-on-cache-miss: true + path: ${{ env.RUNTIME_CACHE_DIR }} + + - name: Install simulation runtime + run: test/infra/control/cluster-simulator-ci.bash install + + - name: Verify simulation build + run: test/infra/control/cluster-simulator-ci.bash verify "${TAP_GROUP}" + + - name: Build CI base image + run: docker build --network host -t proxysql-ci-base:latest test/infra/docker-base + + - name: Start infrastructure + run: test/infra/control/ensure-infras.bash + + - name: Run simulation tests + run: test/infra/control/run-tests-isolated.bash + + - name: Cleanup + if: always() + run: | + test/infra/control/stop-proxysql-isolated.bash || true + test/infra/control/destroy-infras.bash || true + + - name: Archive failure logs + if: ${{ failure() && !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.group }}-${{ github.sha }}-logs-run${{ github.run_number }} + path: ci_infra_logs/ diff --git a/.gitignore b/.gitignore index 0177cb872b..e27f4e6798 100644 --- a/.gitignore +++ b/.gitignore @@ -198,6 +198,8 @@ pkgroot/ #files generated during CI run proxysql-save.cfg +.cluster-simulator-binaries +.cluster-simulator-runtime/ test/tap/tests/test_cluster_sync_config/cluster_sync_node_stderr.txt test/tap/tests/test_cluster_sync_config/proxysql*.pem test/tap/tests/test_cluster_sync_config/test_cluster_sync.cnf diff --git a/test/infra/control/cluster-simulator-ci.bash b/test/infra/control/cluster-simulator-ci.bash new file mode 100755 index 0000000000..a55f0d30e2 --- /dev/null +++ b/test/infra/control/cluster-simulator-ci.bash @@ -0,0 +1,339 @@ +#!/usr/bin/env bash +# +# Centralizes cluster simulation workflow operations so build and runtime +# packaging behavior is readable, reusable locally, and kept out of YAML. + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../../.." && pwd -P)" +GROUPS_FILE="${REPO_ROOT}/test/tap/groups/groups.json" +SIMULATOR_BINARIES_FILE="${REPO_ROOT}/.cluster-simulator-binaries" +RUNTIME_CACHE_DIR_VALUE="${RUNTIME_CACHE_DIR:-.cluster-simulator-runtime}" + +if [[ "${RUNTIME_CACHE_DIR_VALUE}" = /* ]]; then + RUNTIME_CACHE_PATH="$(realpath -m -- "${RUNTIME_CACHE_DIR_VALUE}")" +else + RUNTIME_CACHE_PATH="$(realpath -m -- "${REPO_ROOT}/${RUNTIME_CACHE_DIR_VALUE}")" +fi + +case "${RUNTIME_CACHE_PATH}" in + "${REPO_ROOT}/"*) ;; + *) + echo "ERROR: RUNTIME_CACHE_DIR must resolve inside ${REPO_ROOT}." >&2 + exit 1 + ;; +esac + +STAGE_TEMP_DIR="" + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +usage_error() { + echo "ERROR: $*" >&2 + echo "Run '$0 help' for usage." >&2 + exit 2 +} + +expect_no_arguments() { + local command="${1}" + local argument_count="${2}" + + [[ "${argument_count}" -eq 0 ]] || + usage_error "'${command}' does not accept arguments." +} + +require_command() { + command -v "${1}" >/dev/null 2>&1 || + die "Required command '${1}' was not found." +} + +require_executable() { + [[ -x "${1}" ]] || die "Required executable is missing: ${1}" +} + +require_directory() { + [[ -d "${1}" ]] || die "Required directory is missing: ${1}" +} + +discover_groups_json() { + jq -ce ' + [.[] | .[] | select(type == "string" and startswith("cluster_sim_"))] + | unique + | if length > 0 then . else error("no cluster simulation groups found") end + ' "${GROUPS_FILE}" +} + +refresh_binaries_manifest() { + local temporary_manifest + + temporary_manifest="$(mktemp "${SIMULATOR_BINARIES_FILE}.tmp.XXXXXX")" + if ! jq -er ' + [ + to_entries[] + | select(any(.value[]; type == "string" and startswith("cluster_sim_"))) + | .key + ] + | unique + | if length > 0 then .[] else error("no cluster simulation TAP binaries found") end + ' "${GROUPS_FILE}" > "${temporary_manifest}"; then + rm -f -- "${temporary_manifest}" + die "Failed to discover cluster simulation TAP binaries from ${GROUPS_FILE}." + fi + + mv -- "${temporary_manifest}" "${SIMULATOR_BINARIES_FILE}" +} + +load_manifest_binaries() { + [[ -s "${SIMULATOR_BINARIES_FILE}" ]] || + die "Simulation binary manifest is missing: ${SIMULATOR_BINARIES_FILE}" + mapfile -t SIMULATOR_BINARIES < "${SIMULATOR_BINARIES_FILE}" + [[ "${#SIMULATOR_BINARIES[@]}" -gt 0 ]] || + die "No TAP binaries were written to ${SIMULATOR_BINARIES_FILE}." +} + +load_group_binaries() { + local group="${1}" + local binaries_json + + [[ "${group}" == cluster_sim_* ]] || + die "'${group}' is not a cluster simulation group." + + binaries_json="$(jq -ce --arg group "${group}" ' + [ + to_entries[] + | select(.value | index($group)) + | .key + ] + | unique + | if length > 0 then . else error("group is not registered") end + ' "${GROUPS_FILE}")" || + die "Cluster simulation group '${group}' is not registered in ${GROUPS_FILE}." + + mapfile -t SIMULATOR_BINARIES < <(jq -r '.[]' <<< "${binaries_json}") +} + +verify_runtime_paths() { + local root="${1}" + shift + local binary + + require_executable "${root}/src/proxysql" + require_executable "${root}/test/deps/cluster_simulator/cluster_simulator" + require_directory "${root}/test/tap/tap" + + for binary in "$@"; do + require_executable "${root}/test/tap/tests/${binary}" + done +} + +cleanup_stage_temp() { + if [[ -n "${STAGE_TEMP_DIR}" && -d "${STAGE_TEMP_DIR}" ]]; then + rm -rf -- "${STAGE_TEMP_DIR}" + fi +} + +trap cleanup_stage_temp EXIT + +# discover +# Purpose: Generate the matrix group JSON and the TAP-binary build manifest. +# Local use: Run `cluster-simulator-ci.bash discover` to inspect registry output. +# GitHub use: Supplies the build job's matrix output before cache restoration. +handle_discover() { + expect_no_arguments "discover" "$#" + require_command jq + + local groups_json + local group_count + local binary_count + + groups_json="$(discover_groups_json)" || + die "Failed to discover cluster simulation groups from ${GROUPS_FILE}." + refresh_binaries_manifest + + group_count="$(jq 'length' <<< "${groups_json}")" + binary_count="$(wc -l < "${SIMULATOR_BINARIES_FILE}")" + + if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + printf 'groups=%s\n' "${groups_json}" >> "${GITHUB_OUTPUT}" + fi + + printf 'Discovered %s simulation groups and %s TAP binaries.\n' \ + "${group_count}" "${binary_count}" + jq -r '.[] | " group: \(.)"' <<< "${groups_json}" + sed 's/^/ binary: /' "${SIMULATOR_BINARIES_FILE}" +} + +# build +# Purpose: Build ProxySQL runtime with all simulation flags and every registered TAP binary. +# Local use: Run `cluster-simulator-ci.bash build` to reproduce the CI build. +# GitHub use: Invoked on an exact-SHA cache miss in the build job. +handle_build() { + expect_no_arguments "build" "$#" + require_command docker + require_command git + require_command jq + refresh_binaries_manifest + + local git_version + git_version="$(git -C "${REPO_ROOT}" describe --long --abbrev=7 2>/dev/null || + git -C "${REPO_ROOT}" describe --long --abbrev=7 --always)" || + die "Failed to derive the ProxySQL build version from Git." + + ( + cd "${REPO_ROOT}" + docker compose run --rm --no-deps \ + --env "GIT_VERSION_BASE=${git_version}" \ + --entrypoint /opt/proxysql/test/infra/control/cluster-simulator-ci.bash \ + --workdir /opt/proxysql \ + ubuntu22_build _build + ) +} + +# _build +# Purpose: Execute the compiler commands inside the Ubuntu 22 packaging image. +# Local use: Internal only; use the public `build` command from the host. +# GitHub use: Called by `build` as the packaging container entrypoint. +handle_internal_build() { + expect_no_arguments "_build" "$#" + load_manifest_binaries + [[ -n "${GIT_VERSION_BASE:-}" ]] || + die "GIT_VERSION_BASE was not provided by the host build command." + + cd "${REPO_ROOT}" + make -j"$(nproc)" GIT_VERSION_BASE="${GIT_VERSION_BASE}" testall + make -j"$(nproc)" GIT_VERSION_BASE="${GIT_VERSION_BASE}" build_cluster_simulator + make -C test/tap -j"$(nproc)" GIT_VERSION="${GIT_VERSION_BASE}" tap + make -C test/tap/tests -j"$(nproc)" \ + GIT_VERSION="${GIT_VERSION_BASE}" "${SIMULATOR_BINARIES[@]}" +} + +# verify +# Purpose: Check the complete runtime, or only the TAP binaries for one group. +# Local use: Run `verify` after a build, optionally with a cluster_sim_* group. +# GitHub use: Checks the build job runtime and each restored matrix-job runtime. +handle_verify() { + [[ "$#" -le 1 ]] || + usage_error "'verify' accepts at most one simulation group." + require_command jq + + local group="${1:-}" + + if [[ -n "${group}" ]]; then + load_group_binaries "${group}" + else + refresh_binaries_manifest + load_manifest_binaries + fi + + verify_runtime_paths "${REPO_ROOT}" "${SIMULATOR_BINARIES[@]}" + + if [[ -n "${group}" ]]; then + printf 'Verified simulation runtime for %s.\n' "${group}" + else + printf 'Verified simulation runtime for all registered groups.\n' + fi +} + +# stage +# Purpose: Assemble only the runtime files that matrix jobs need in the cache. +# Local use: Optional; run after `build` to inspect the cache payload locally. +# GitHub use: Creates the exact-SHA cache payload after a successful build. +handle_stage() { + expect_no_arguments "stage" "$#" + require_command jq + handle_verify + + local binary + local runtime_parent + + runtime_parent="$(dirname -- "${RUNTIME_CACHE_PATH}")" + mkdir -p -- "${runtime_parent}" + STAGE_TEMP_DIR="$(mktemp -d "${RUNTIME_CACHE_PATH}.tmp.XXXXXX")" + + install -D -m 0755 \ + "${REPO_ROOT}/src/proxysql" \ + "${STAGE_TEMP_DIR}/src/proxysql" + install -D -m 0755 \ + "${REPO_ROOT}/test/deps/cluster_simulator/cluster_simulator" \ + "${STAGE_TEMP_DIR}/test/deps/cluster_simulator/cluster_simulator" + install -d "${STAGE_TEMP_DIR}/test/tap" + cp -a "${REPO_ROOT}/test/tap/tap" "${STAGE_TEMP_DIR}/test/tap/" + + for binary in "${SIMULATOR_BINARIES[@]}"; do + install -D -m 0755 \ + "${REPO_ROOT}/test/tap/tests/${binary}" \ + "${STAGE_TEMP_DIR}/test/tap/tests/${binary}" + done + + if [[ -e "${RUNTIME_CACHE_PATH}" || -L "${RUNTIME_CACHE_PATH}" ]]; then + rm -rf -- "${RUNTIME_CACHE_PATH}" + fi + mv -- "${STAGE_TEMP_DIR}" "${RUNTIME_CACHE_PATH}" + STAGE_TEMP_DIR="" + + printf 'Staged simulation runtime in %s.\n' "${RUNTIME_CACHE_PATH}" +} + +# install +# Purpose: Restore a staged simulation runtime into the current checkout. +# Local use: Usually unnecessary; use it only to validate a staged cache payload. +# GitHub use: Installs files immediately after actions/cache restores the payload. +handle_install() { + expect_no_arguments "install" "$#" + require_command jq + require_directory "${RUNTIME_CACHE_PATH}" + refresh_binaries_manifest + load_manifest_binaries + verify_runtime_paths "${RUNTIME_CACHE_PATH}" "${SIMULATOR_BINARIES[@]}" + + cp -a "${RUNTIME_CACHE_PATH}/." "${REPO_ROOT}/" + printf 'Installed simulation runtime from %s.\n' "${RUNTIME_CACHE_PATH}" +} + +# help +# Purpose: Document the command interface, generated files, and common examples. +# Local use: Run `cluster-simulator-ci.bash help` when reproducing workflow steps. +# GitHub use: Not called by the workflow; it is maintainer-facing documentation. +handle_help() { + expect_no_arguments "help" "$#" + + cat < [arguments] + +Commands: + discover Print registered simulation groups and write: + ${SIMULATOR_BINARIES_FILE} + build Build ProxySQL with simulation support in ubuntu22_build. + verify [group] Verify all runtime files, or one matrix group. + stage Create the cache payload at: + ${RUNTIME_CACHE_PATH} + install Restore that cache payload into the checkout. + help Show this help. + +Examples: + $0 discover + $0 build + $0 verify + $0 verify cluster_sim_galera-g1 +EOF +} + +command_name="${1:-help}" +if [[ "$#" -gt 0 ]]; then + shift +fi + +case "${command_name}" in + discover) handle_discover "$@" ;; + build) handle_build "$@" ;; + _build) handle_internal_build "$@" ;; + verify) handle_verify "$@" ;; + stage) handle_stage "$@" ;; + install) handle_install "$@" ;; + help|-h|--help) handle_help "$@" ;; + *) usage_error "Unknown command '${command_name}'." ;; +esac From 1633b07e4029621e9ddf82a22fa609a7f61a64b6 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Fri, 24 Jul 2026 08:09:52 +0000 Subject: [PATCH 58/81] ci: Simplify cluster simulator job names --- .github/workflows/CI-cluster-simulator.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI-cluster-simulator.yml b/.github/workflows/CI-cluster-simulator.yml index deafe3a0ac..ff4235c510 100644 --- a/.github/workflows/CI-cluster-simulator.yml +++ b/.github/workflows/CI-cluster-simulator.yml @@ -29,7 +29,7 @@ env: jobs: build: - name: Build ProxySQL with simulation support + name: build runs-on: ubuntu-22.04 outputs: groups: ${{ steps.simulator-groups.outputs.groups }} @@ -75,7 +75,7 @@ jobs: path: ${{ env.RUNTIME_CACHE_DIR }} test: - name: ${{ matrix.group }} + name: test / ${{ matrix.group }} needs: build runs-on: ubuntu-22.04 strategy: From 1de71f95eafe55a0c31b63b35f43541f0f33ff4b Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Mon, 27 Jul 2026 08:12:55 +0000 Subject: [PATCH 59/81] fix: Isolate AWS RDS BGD workers by hostgroup - Compute BGD hosts list with per-cluster checksums. - Start, stop, or refresh only the worker affected by a configuration change while preserving other active workers. Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 8 +- include/MySQL_Monitor.hpp | 265 ++++++++++--- lib/MySQL_HostGroups_Manager.cpp | 143 +++---- lib/MySQL_Monitor.cpp | 584 +++++++++++++++++++---------- 4 files changed, 652 insertions(+), 348 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 1d33bae58c..2dce864181 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -1214,12 +1214,10 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { /** * @brief Rebuilds the AWS RDS BGD monitor's host resultset. * - * @details Rebuilds `GloMyMon->AWS_RDS_Blue_Hosts_resultset` and publishes a checksum combining - * the blue hosts with the green hosts. - * - * @param lock When true, the monitor's `aws_rds_bgd_mutex` is taken internally. + * @details Rebuilds `GloMyMon->AWS_RDS_BGD_Hosts_resultset` and publishes both the full BGD hosts + * checksum and one checksum per writer hostgroup. */ - void update_aws_rds_bgd_hosts_monitor_resultset(bool lock=false); + void update_aws_rds_bgd_hosts_monitor_resultset(); /** * @brief Auto-generate a runtime `mysql_aws_rds_bgd_hostgroups` entry for a server's writer hostgroup. * diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 1820faabc1..1c84e10bf2 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -402,6 +402,88 @@ struct srv_addr_t { int port = 0; }; +/** +* @brief State of the per-host RDS topology probe. +*/ +enum RDS_BGD_Topology_Monitor_State { + TOPOLOGY_TABLE_CHECK, ///< verify mysql.rds_topology exists + TOPOLOGY_METADATA_FETCH ///< table confirmed present; fetch and branch on its metadata +}; + +/** + * @brief Column positions in `AWS_RDS_BGD_Hosts_resultset`. + */ +enum AWS_RDS_BGD_Hosts_Column { + AWS_RDS_BGD_HOSTNAME = 0, + AWS_RDS_BGD_PORT, + AWS_RDS_BGD_USE_SSL, + AWS_RDS_BGD_WRITER_HOSTGROUP, + AWS_RDS_BGD_READER_HOSTGROUP, + AWS_RDS_BGD_GREEN_WRITER_HOSTGROUP, + AWS_RDS_BGD_GREEN_READER_HOSTGROUP, + AWS_RDS_BGD_CHECK_INTERVAL_MS, + AWS_RDS_BGD_CHECK_TIMEOUT_MS, + AWS_RDS_BGD_WRITER_IS_ALSO_READER, + AWS_RDS_BGD_SRV_TYPE, + AWS_RDS_BGD_IS_WRITER, + AWS_RDS_BGD_HOSTS_COLUMNS +}; + +/** + * @brief Switchover phase for an RDS blue/green deployment. + * + * @details AWS's mysql.rds_topology status only captures the writer switchover. As of 2026/07/03 + * the table exposes no read-replica switchover status; ProxySQL infers that the replicas have + * switched over from the table draining to empty (or disappearing) after it last reported + * SWITCHOVER_COMPLETED. + * + * Observed table lifecycle across one switchover: + * - Steady state: two rows (SOURCE = blue, TARGET = green), both AVAILABLE. + * - Switching: both rows step through SWITCHOVER_INITIATED -> _IN_PROGRESS -> _IN_POST_PROCESSING. + * - Writer done: the SOURCE row drops; a lone TARGET row reports SWITCHOVER_COMPLETED. + * - Replicas done: the table drains to empty (blue-reader DNS has propagated). + * + * The WRITER_SWITCHOVER_* values map 1:1 onto the mysql.rds_topology status strings. + * READER_SWITCHOVER_IN_PROGRESS is a ProxySQL inferred status entered after + * WRITER_SWITCHOVER_COMPLETED; it defers reader/DNS cleanup until the topology table drains + * to empty. SWITCHOVER_COMPLETED is a short-lived status used for final cleanup before + * returning to NONE. + */ +enum class AWS_RDS_BGD_Status { + NONE = 0, ///< no BGD topology / baseline + AVAILABLE = 1, ///< "AVAILABLE" + WRITER_SWITCHOVER_INITIATED = 2, ///< "SWITCHOVER_INITIATED" + WRITER_SWITCHOVER_IN_PROGRESS = 3, ///< "SWITCHOVER_IN_PROGRESS" + WRITER_SWITCHOVER_POST_PROCESSING = 4, ///< "SWITCHOVER_IN_POST_PROCESSING" + WRITER_SWITCHOVER_COMPLETED = 5, ///< "SWITCHOVER_COMPLETED" + READER_SWITCHOVER_IN_PROGRESS = 6, ///< ProxySQL inferred status; awaiting topology drain + deferred cleanup + SWITCHOVER_COMPLETED = 7, ///< short-lived status used for final cleanup before returning to NONE +}; + +enum class AWS_RDS_BGD_Server_Status { + NONE = 0, + IN_PROGRESS = 1 +}; + +// AWS RDS blue/green role and switchover-status column values (mysql.rds_topology). +inline const char* const BGD_ROLE_SOURCE = "BLUE_GREEN_DEPLOYMENT_SOURCE"; // blue +inline const char* const BGD_ROLE_TARGET = "BLUE_GREEN_DEPLOYMENT_TARGET"; // green +inline const char* const BGD_STATUS_AVAILABLE = "AVAILABLE"; +inline const char* const BGD_STATUS_INITIATED = "SWITCHOVER_INITIATED"; +inline const char* const BGD_STATUS_IN_PROGRESS = "SWITCHOVER_IN_PROGRESS"; +inline const char* const BGD_STATUS_POST_PROC = "SWITCHOVER_IN_POST_PROCESSING"; +inline const char* const BGD_STATUS_COMPLETED = "SWITCHOVER_COMPLETED"; + +/** +* @brief BGD Monitor state for one AWS RDS BGD worker. +*/ +struct AWS_RDS_BGD_Worker { + int writer_hg = 0; + pthread_t thread {}; + std::atomic_bool worker_stop {false}; + std::atomic current_checksum {0}; +}; + /** * @brief A single node (row) of a 'SELECT * FROM mysql.rds_topology' result. */ @@ -418,9 +500,24 @@ struct AWS_RDS_Topology_Node { * shared by the read_only monitor's discovery path and the AWS RDS BGD * monitor thread. */ -struct AWS_RDS_Topology_Result { +class AWS_RDS_Topology_Result { +public: bool blue_green = false; ///< 'role' and 'status' present AND non-NULL std::vector nodes; + + /** + * @brief Find the blue/green deployment TARGET node. + * + * @return The TARGET node, or nullptr when it is not present. + */ + AWS_RDS_Topology_Node* target() { + for (AWS_RDS_Topology_Node& node : nodes) { + if (strcasecmp(node.role.c_str(), BGD_ROLE_TARGET) == 0) { + return &node; + } + } + return nullptr; + } }; /** @@ -446,44 +543,14 @@ struct AWS_RDS_BlueGreenPair { }; /** - * @brief Switchover phase for an RDS blue/green deployment. - * - * @details AWS's mysql.rds_topology status only captures the writer switchover. As of 2026/07/03 - * the table exposes no read-replica switchover status; ProxySQL infers that the replicas have - * switched over from the table draining to empty (or disappearing) after it last reported - * SWITCHOVER_COMPLETED. - * - * Observed table lifecycle across one switchover: - * - Steady state: two rows (SOURCE = blue, TARGET = green), both AVAILABLE. - * - Switching: both rows step through SWITCHOVER_INITIATED -> _IN_PROGRESS -> _IN_POST_PROCESSING. - * - Writer done: the SOURCE row drops; a lone TARGET row reports SWITCHOVER_COMPLETED. - * - Replicas done: the table drains to empty (blue-reader DNS has propagated). - * - * The WRITER_SWITCHOVER_* values map 1:1 onto the mysql.rds_topology status strings. - * READER_SWITCHOVER_IN_PROGRESS is a ProxySQL inferred status entered after - * WRITER_SWITCHOVER_COMPLETED; it defers reader/DNS cleanup until the topology table drains - * to empty. SWITCHOVER_COMPLETED is a short-lived status used for final cleanup before - * returning to NONE. + * @brief Host used by a BGD worker to probe `mysql.rds_topology`. */ -enum class AWS_RDS_BGD_Status { - NONE = 0, ///< no BGD topology / baseline - AVAILABLE = 1, ///< "AVAILABLE" - WRITER_SWITCHOVER_INITIATED = 2, ///< "SWITCHOVER_INITIATED" - WRITER_SWITCHOVER_IN_PROGRESS = 3, ///< "SWITCHOVER_IN_PROGRESS" - WRITER_SWITCHOVER_POST_PROCESSING = 4, ///< "SWITCHOVER_IN_POST_PROCESSING" - WRITER_SWITCHOVER_COMPLETED = 5, ///< "SWITCHOVER_COMPLETED" - READER_SWITCHOVER_IN_PROGRESS = 6, ///< ProxySQL inferred status; awaiting topology drain + deferred cleanup - SWITCHOVER_COMPLETED = 7, ///< short-lived status used for final cleanup before returning to NONE -}; - -enum class AWS_RDS_BGD_Server_Status { - NONE = 0, - IN_PROGRESS = 1 +struct AWS_RDS_BGD_Probe_Host { + std::string hostname; + int port = 0; + int use_ssl = 0; }; -// Maps a switchover status enum to its stored/display string. -const char* aws_rds_bgd_status_str(AWS_RDS_BGD_Status s); - /** * @brief Switchover state carried by RDS BGD worker thread. * @@ -491,8 +558,8 @@ const char* aws_rds_bgd_status_str(AWS_RDS_BGD_Status s); * one blue/green deployment, so this struct lives on the worker's stack and is * single-owner (no locking on the struct itself). It is passed by reference to * handle_aws_rds_bgd, which runs the status-driven switchover FSM and mutates it - * across poll cycles. Config-derived fields are loaded once from the resultset; - * the rest carries topology, resolved IPs, and one-shot enforcement bookkeeping. + * across poll cycles. Config-derived fields can be refreshed in place; the rest + * carries resolved IPs and state for switchover actions and cleanup. */ struct AWS_RDS_BGD_State { unsigned int writer_hg = 0; ///< blue/current writer hostgroup @@ -500,13 +567,17 @@ struct AWS_RDS_BGD_State { int green_writer_hg = -1; ///< -1 when NULL (auto-discovery path) int green_reader_hg = -1; ///< -1 when NULL int writer_is_also_reader = 0; ///< drives post-switchover writer cleanup + unsigned int check_interval_ms = 0; ///< configured baseline check interval + unsigned int check_timeout_ms = 0; ///< configured topology-check timeout - std::string last_topology_status; ///< raw mysql.rds_topology TARGET status from the previous poll (verbatim) std::vector bg_map; ///< [writer] always; [readers] only when green_reader_hg is configured + std::vector probe_hosts; ///< hosts eligible for topology probes + std::vector shunned_readers; ///< readers we shunned AWS_RDS_BGD_Status bgd_status = AWS_RDS_BGD_Status::NONE; ///< drives the FSM and the deferred cleanup bool bgd_in_progress_set = false; ///< deployment's servers flagged in aws_rds_bgd_server_status + bool config_refresh_pending = false; ///< bg_map must be rebuilt from the next topology result unsigned int next_check_interval_ms = 0; ///< FSM-controlled interval; 0 => baseline std::string next_check_host; ///< FSM-pinned probe host; when set (the green IP), the worker @@ -514,22 +585,8 @@ struct AWS_RDS_BGD_State { unsigned int next_check_host_failures = 0; ///< consecutive failures polling next_check_host; clears it after 3 }; -/** -* @brief State of the per-host RDS topology probe. -*/ -enum RDS_BGD_Topology_Monitor_State { - TOPOLOGY_TABLE_CHECK, ///< verify mysql.rds_topology exists - TOPOLOGY_METADATA_FETCH ///< table confirmed present; fetch and branch on its metadata -}; - -// AWS RDS blue/green role and switchover-status column values (mysql.rds_topology). -inline const char* const BGD_ROLE_SOURCE = "BLUE_GREEN_DEPLOYMENT_SOURCE"; // blue -inline const char* const BGD_ROLE_TARGET = "BLUE_GREEN_DEPLOYMENT_TARGET"; // green -inline const char* const BGD_STATUS_AVAILABLE = "AVAILABLE"; -inline const char* const BGD_STATUS_INITIATED = "SWITCHOVER_INITIATED"; -inline const char* const BGD_STATUS_IN_PROGRESS = "SWITCHOVER_IN_PROGRESS"; -inline const char* const BGD_STATUS_POST_PROC = "SWITCHOVER_IN_POST_PROCESSING"; -inline const char* const BGD_STATUS_COMPLETED = "SWITCHOVER_COMPLETED"; +// Maps a switchover status enum to its stored/display string. +const char* aws_rds_bgd_status_str(AWS_RDS_BGD_Status s); // read_only monitor server-enumeration query. // Every server that belongs to a replication hostgroup and status NOT IN (OFFLINE_SOFT, OFFLINE_HARD) @@ -590,6 +647,7 @@ class MySQL_Monitor { pthread_mutex_t galera_mutex; // for simplicity, a mutex instead of a rwlock pthread_mutex_t aws_aurora_mutex; // for simplicity, a mutex instead of a rwlock pthread_mutex_t aws_rds_bgd_mutex; + pthread_mutex_t aws_rds_bgd_hosts_mutex; pthread_mutex_t mysql_servers_mutex; // for simplicity, a mutex instead of a rwlock pthread_mutex_t proxysql_servers_mutex; //std::map Group_Replication_Hosts_Map; @@ -601,8 +659,9 @@ class MySQL_Monitor { SQLite3_result *AWS_Aurora_Hosts_resultset; uint64_t AWS_Aurora_Hosts_resultset_checksum; std::unordered_map aws_rds_bgd_server_status; - SQLite3_result *AWS_RDS_Blue_Hosts_resultset; + std::shared_ptr AWS_RDS_BGD_Hosts_resultset; uint64_t AWS_RDS_BGD_Hosts_checksum; + std::unordered_map AWS_RDS_BGD_Cluster_checksum; unsigned int num_threads; unsigned int aux_threads; unsigned int started_threads; @@ -653,12 +712,40 @@ class MySQL_Monitor { /** * @brief AWS RDS BGD monitor thread entry point. * - * @details Spawns one worker (monitor_RDS_BGD_thread_HG) per writer hostgroup; each worker picks a pingable writer, - * probes 'mysql.rds_topology' and dispatches based on the detected topology shape. - * Workers are (re)spawned whenever the AWS_RDS_BGD_Hosts_checksum changes. + * @details Maintains one worker (monitor_RDS_BGD_thread_HG) per active writer hostgroup. The parent starts + * and stops workers and signals configuration changes. Each worker selects a pingable probe host, + * probes 'mysql.rds_topology', and runs the switchover state machine. */ void * monitor_aws_rds_bgd(); /** + * @brief Run an asynchronous query and store its result on a BGD monitor connection. + * + * @param mmsd Monitor state data holding the connection, timing, and result. + * @param query SQL text to execute. + * @param worker_stop Per-worker shutdown signal. + * + * @return 0 on success, 1 on timeout or query error, and 2 when shutdown is requested. + */ + int aws_rds_bgd_async_query( + MySQL_Monitor_State_Data* mmsd, const char* query, std::atomic_bool& worker_stop); + /** + * @brief Apply changed configuration to one running BGD worker. + * + * @details Before writer post-processing, applies the configuration and schedules mapping + * reconciliation after the next topology poll. At or after post-processing, rolls back the + * deployment and restarts its topology state machine. + * + * @param st Worker-owned BGD state. + * @param current_checksum Per-cluster checksum captured for this refresh. + * @param topology_state Current topology query state. + * @param next_loop_at Next scheduled worker iteration. + * + * @return true when the configuration was applied; false when it must be retried. + */ + bool aws_rds_bgd_refresh_worker_config( + AWS_RDS_BGD_State& st, uint64_t current_checksum, + RDS_BGD_Topology_Monitor_State& topology_state, unsigned long long& next_loop_at); + /** * @brief Run the status-driven blue/green switchover FSM for one deployment. * * @details Invoked each poll cycle by the BGD worker after it fetches the @@ -672,7 +759,7 @@ class MySQL_Monitor { * @param st BGD switchover state. * @param topology Parsed mysql.rds_topology result for this cycle. */ - void handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_Result& topology); + void handle_aws_rds_bgd(AWS_RDS_BGD_State& st, AWS_RDS_Topology_Result& topology); /** * @brief Pin green IPs and drain existing blue-host connections. * @@ -786,6 +873,64 @@ class MySQL_Monitor { void monitor_gr_async_actions_handler(const vector>& mmsds); private: + /** + * @brief Load one BGD worker's configuration from the published host rows. + * + * @details Copies the cluster rows, verifies their checksum, copies configuration fields from + * the first row, and builds the probe host list. + * + * @param writer_hg Writer hostgroup identifying the deployment. + * @param current_checksum Per-cluster checksum captured for this refresh. + * @param candidate State populated from the published rows. + * + * @return true when the checksum matches and the rows contain a probe host. + */ + bool aws_rds_bgd_load_worker_config(int writer_hg, uint64_t current_checksum, AWS_RDS_BGD_State& candidate); + /** + * @brief Replace the configuration-derived fields in a live BGD worker state. + * + * @param st Live worker state. + * @param candidate Parsed configuration to apply. + */ + void aws_rds_bgd_apply_cluster_config(AWS_RDS_BGD_State& st, AWS_RDS_BGD_State& candidate); + /** + * @brief Rebuild the mapping and reconcile writer state after a configuration refresh. + * + * @details Called only when config_refresh_pending is set. + * + * @param st Worker-owned BGD state. + * @param topology Fresh topology used to rebuild the mapping. + */ + void aws_rds_bgd_config_refresh_action(AWS_RDS_BGD_State& st, AWS_RDS_Topology_Result& topology); + /** + * @brief Build the blue-to-green host mapping for a BGD worker. + * + * @param st Worker-owned BGD state. + * @param topology Parsed topology used to identify the green target. + */ + void aws_rds_bgd_build_map(AWS_RDS_BGD_State& st, AWS_RDS_Topology_Result& topology); + /** + * @brief Resolve green host IPs and select the next topology probe host. + * + * @param st Worker-owned BGD state. + */ + void aws_rds_bgd_resolve_green_ips(AWS_RDS_BGD_State& st); + /** + * @brief Add the green writer to its configured hostgroup. + * + * @param st Worker-owned BGD state. + */ + void aws_rds_bgd_add_green_writer_in_hg(AWS_RDS_BGD_State& st); + /** + * @brief Find the writer pair in a blue-to-green host mapping. + * + * @param bg_map Host mapping to inspect. + * @param writer Writer address populated when a pair is found. + * + * @return true when the map contains a writer pair. + */ + bool aws_rds_bgd_find_writer(std::vector& bg_map, srv_addr_t& writer); + /** * @brief Handling of monitor tasks asyncronously * @details Basic workflow is same for all monitor_*_async methods: diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 682f02fad7..01d2d461e6 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1617,7 +1617,7 @@ bool MySQL_HostGroups_Manager::commit( // calls to 'generate_mysql_servers'. update_table_mysql_servers_for_monitor(false); // Refresh BGD monitoring after all runtime server changes are applied. - update_aws_rds_bgd_hosts_monitor_resultset(true); + update_aws_rds_bgd_hosts_monitor_resultset(); wrunlock(); unsigned long long curtime2=monotonic_time(); @@ -7139,105 +7139,78 @@ void MySQL_HostGroups_Manager::update_aws_aurora_hosts_monitor_resultset(bool lo } } -const char SELECT_AWS_RDS_BGD_BLUE_SERVERS_FOR_MONITOR[] { - "SELECT writer_hostgroup, reader_hostgroup, hostname, port, MAX(use_ssl) use_ssl, green_writer_hostgroup," - " green_reader_hostgroup, check_interval_ms, check_timeout_ms, writer_is_also_reader," - " MAX(hostgroup_id=writer_hostgroup) is_writer" - " FROM mysql_servers" - " JOIN mysql_aws_rds_bgd_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup" - " WHERE active=1 AND mysql_servers.status NOT IN (2,3)" - " GROUP BY writer_hostgroup, hostname, port" -}; - -const char SELECT_AWS_RDS_BGD_GREEN_SERVERS_FOR_MONITOR[] { - "SELECT bgd.writer_hostgroup, srv.hostgroup_id, srv.hostname, srv.port, srv.use_ssl FROM mysql_servers AS srv" - " JOIN mysql_aws_rds_bgd_hostgroups AS bgd ON srv.hostgroup_id=bgd.green_writer_hostgroup" - " OR srv.hostgroup_id=bgd.green_reader_hostgroup" - " WHERE bgd.active=1 AND srv.status NOT IN (2,3)" - " ORDER BY bgd.writer_hostgroup, srv.hostgroup_id, srv.hostname, srv.port" +const char SELECT_AWS_RDS_BGD_SERVERS_FOR_MONITOR[] { + "SELECT srv.hostname, srv.port, MAX(srv.use_ssl) AS use_ssl, " + "bgd.writer_hostgroup, bgd.reader_hostgroup, bgd.green_writer_hostgroup, bgd.green_reader_hostgroup, " + "bgd.check_interval_ms, bgd.check_timeout_ms, bgd.writer_is_also_reader, " + "'B' AS srv_type, MAX(srv.hostgroup_id=bgd.writer_hostgroup) AS is_writer " + "FROM mysql_servers AS srv " + "JOIN mysql_aws_rds_bgd_hostgroups AS bgd " + "ON srv.hostgroup_id=bgd.writer_hostgroup OR srv.hostgroup_id=bgd.reader_hostgroup " + "WHERE bgd.active=1 AND srv.status NOT IN (2,3) " + "GROUP BY bgd.writer_hostgroup, srv.hostname, srv.port " + "UNION ALL " + "SELECT srv.hostname, srv.port, srv.use_ssl, " + "bgd.writer_hostgroup, bgd.reader_hostgroup, bgd.green_writer_hostgroup, bgd.green_reader_hostgroup, " + "bgd.check_interval_ms, bgd.check_timeout_ms, bgd.writer_is_also_reader, " + "'G' AS srv_type, srv.hostgroup_id=bgd.green_writer_hostgroup AS is_writer " + "FROM mysql_servers AS srv " + "JOIN mysql_aws_rds_bgd_hostgroups AS bgd " + "ON srv.hostgroup_id=bgd.green_writer_hostgroup OR srv.hostgroup_id=bgd.green_reader_hostgroup " + "WHERE bgd.active=1 AND srv.status NOT IN (2,3) " + "ORDER BY writer_hostgroup, srv_type, is_writer DESC, hostname, port" }; /** * @brief Rebuilds the AWS RDS BGD monitor's host resultset. * - * @details Rebuilds `GloMyMon->AWS_RDS_Blue_Hosts_resultset` and publishes a checksum combining - * the blue hosts with the green hosts. - * - * @param lock When true, the monitor's `aws_rds_bgd_mutex` is taken internally. + * @details Rebuilds `GloMyMon->AWS_RDS_BGD_Hosts_resultset` and publishes both the full BGD hosts + * checksum and one checksum per writer hostgroup. The previous result remains active when the + * query fails. */ -void MySQL_HostGroups_Manager::update_aws_rds_bgd_hosts_monitor_resultset(bool lock) { +void MySQL_HostGroups_Manager::update_aws_rds_bgd_hosts_monitor_resultset() { if (!GloMyMon) { return; } - if (lock) { - pthread_mutex_lock(&GloMyMon->aws_rds_bgd_mutex); - } - - // Unlike other monitor resultset/checksum pairs, BGD intentionally tracks different data in each. - // - // AWS_RDS_Blue_Hosts_resultset contains only blue hosts. The BGD monitor dispatcher uses it to start - // workers, and each worker uses it to select its `mysql.rds_topology` probe candidates. - // - // AWS_RDS_BGD_Hosts_checksum combines the blue and green resultset checksums. Workers and the dispatcher - // use it as a generation signal: relevant changes in mysql_servers or mysql_aws_rds_bgd_hostgroups - // stop the old workers so replacements rebuild the blue/green map from the current runtime configuration. - - SQLite3_result* blue_resultset = nullptr; - SQLite3_result* green_resultset = nullptr; - char* blue_error = nullptr; - char* green_error = nullptr; - int blue_cols = 0; - int green_cols = 0; - int blue_affected_rows = 0; - int green_affected_rows = 0; - - mydb->execute_statement( - SELECT_AWS_RDS_BGD_BLUE_SERVERS_FOR_MONITOR, - &blue_error, &blue_cols, &blue_affected_rows, &blue_resultset); - mydb->execute_statement( - SELECT_AWS_RDS_BGD_GREEN_SERVERS_FOR_MONITOR, - &green_error, &green_cols, &green_affected_rows, &green_resultset); - - if (blue_error || green_error || !blue_resultset || !green_resultset) { - if (blue_error) { - proxy_error("Error refreshing AWS RDS BGD blue hosts: %s\n", blue_error); - } - if (green_error) { - proxy_error("Error refreshing AWS RDS BGD green hosts: %s\n", green_error); - } - free(blue_error); - free(green_error); - delete blue_resultset; - delete green_resultset; + SQLite3_result* resultset = nullptr; + char* error = nullptr; + int cols = 0; + int affected_rows = 0; + mydb->execute_statement(SELECT_AWS_RDS_BGD_SERVERS_FOR_MONITOR, &error, &cols, &affected_rows, &resultset); - if (lock) { - pthread_mutex_unlock(&GloMyMon->aws_rds_bgd_mutex); - } + if (error || !resultset) { + proxy_error("Error refreshing AWS RDS BGD hosts: %s\n", error ? error : "empty resultset"); + free(error); + delete resultset; return; } + free(error); - const uint64_t blue_checksum = blue_resultset->raw_checksum(); - const uint64_t green_checksum = green_resultset->raw_checksum(); - SpookyHash hash; - hash.Init(19, 3); - hash.Update(&blue_checksum, sizeof(blue_checksum)); - hash.Update(&green_checksum, sizeof(green_checksum)); - - uint64_t combined_checksum = 0; - uint64_t ignored = 0; - hash.Final(&combined_checksum, &ignored); - - if (GloMyMon->AWS_RDS_Blue_Hosts_resultset) { - delete GloMyMon->AWS_RDS_Blue_Hosts_resultset; + std::unordered_map cluster_checksums; + std::unordered_map cluster_resultsets; + for (SQLite3_row* row : resultset->rows) { + const int writer_hg = atoi(row->fields[AWS_RDS_BGD_WRITER_HOSTGROUP]); + auto cluster_it = cluster_resultsets.find(writer_hg); + if (cluster_it == cluster_resultsets.end()) { + cluster_it = cluster_resultsets.emplace( + writer_hg, new SQLite3_result(resultset->columns)).first; + } + cluster_it->second->add_row(row); } - GloMyMon->AWS_RDS_Blue_Hosts_resultset = blue_resultset; - GloMyMon->AWS_RDS_BGD_Hosts_checksum = combined_checksum; - delete green_resultset; - - if (lock) { - pthread_mutex_unlock(&GloMyMon->aws_rds_bgd_mutex); + for (const auto& [writer_hg, cluster_resultset] : cluster_resultsets) { + cluster_checksums[writer_hg] = cluster_resultset->raw_checksum(); + delete cluster_resultset; } + + const uint64_t hosts_checksum = resultset->raw_checksum(); + std::shared_ptr hosts_resultset { resultset }; + + pthread_mutex_lock(&GloMyMon->aws_rds_bgd_hosts_mutex); + GloMyMon->AWS_RDS_BGD_Hosts_resultset.swap(hosts_resultset); + GloMyMon->AWS_RDS_BGD_Hosts_checksum = hosts_checksum; + GloMyMon->AWS_RDS_BGD_Cluster_checksum.swap(cluster_checksums); + pthread_mutex_unlock(&GloMyMon->aws_rds_bgd_hosts_mutex); } /** @@ -7306,7 +7279,7 @@ bool MySQL_HostGroups_Manager::add_aws_rds_bgd_hostgroup_entry(const std::string if (added) { // publish the refreshed host list to the BGD monitor thread - update_aws_rds_bgd_hosts_monitor_resultset(true); + update_aws_rds_bgd_hosts_monitor_resultset(); } wrunlock(); diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 46dde3e4f0..8a8c756575 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -1141,12 +1141,14 @@ MySQL_Monitor::MySQL_Monitor() { pthread_mutex_init(&aws_aurora_mutex,NULL); pthread_mutex_init(&aws_rds_bgd_mutex,NULL); + pthread_mutex_init(&aws_rds_bgd_hosts_mutex,NULL); pthread_mutex_init(&mysql_servers_mutex,NULL); pthread_mutex_init(&proxysql_servers_mutex, NULL); AWS_Aurora_Hosts_resultset=NULL; AWS_Aurora_Hosts_resultset_checksum = 0; - AWS_RDS_Blue_Hosts_resultset=NULL; + AWS_RDS_BGD_Hosts_resultset.reset(); AWS_RDS_BGD_Hosts_checksum = 0; + AWS_RDS_BGD_Cluster_checksum.clear(); shutdown=false; monitor_enabled=true; // default // create new SQLite datatabase @@ -1247,10 +1249,9 @@ MySQL_Monitor::~MySQL_Monitor() { delete AWS_Aurora_Hosts_resultset; AWS_Aurora_Hosts_resultset=NULL; } - if (AWS_RDS_Blue_Hosts_resultset) { - delete AWS_RDS_Blue_Hosts_resultset; - AWS_RDS_Blue_Hosts_resultset=NULL; - } + AWS_RDS_BGD_Hosts_resultset.reset(); + AWS_RDS_BGD_Cluster_checksum.clear(); + pthread_mutex_destroy(&aws_rds_bgd_hosts_mutex); std::map::iterator it2; AWS_Aurora_monitor_node *node=NULL; for (it2 = AWS_Aurora_Hosts_Map.begin(); it2 != AWS_Aurora_Hosts_Map.end(); ++it2) { @@ -6622,12 +6623,13 @@ void * MySQL_Monitor::monitor_aws_aurora() { /** * @brief Runs an async query + store_result on the monitor connection. * -* @param mmsd Monitor state data holding the connection, timing, and result. -* @param query SQL text to execute. +* @param mmsd Monitor state data holding the connection, timing, and result. +* @param query SQL text to execute. +* @param worker_stop Per-worker shutdown signal. * -* @return 0 on success, 1 on timeout/query-error, 2 if shutdown was requested. +* @return 0 on success, 1 on timeout/query-error, 2 if global or worker shutdown was requested. */ -static int aws_rds_bgd_async_query(MySQL_Monitor_State_Data *mmsd, const char *query) { +int MySQL_Monitor::aws_rds_bgd_async_query(MySQL_Monitor_State_Data *mmsd, const char *query, std::atomic_bool& worker_stop) { mmsd->t1 = monotonic_time(); mmsd->interr = 0; mmsd->async_exit_status = mysql_query_start(&mmsd->interr, mmsd->mysql, query); @@ -6638,7 +6640,7 @@ static int aws_rds_bgd_async_query(MySQL_Monitor_State_Data *mmsd, const char *q mmsd->mysql_error_msg = strdup("timeout check"); return 1; } - if (GloMyMon->shutdown == true) { + if (shutdown == true || worker_stop.load()) { return 2; } if ((mmsd->async_exit_status & MYSQL_WAIT_TIMEOUT) == 0) { @@ -6653,7 +6655,7 @@ static int aws_rds_bgd_async_query(MySQL_Monitor_State_Data *mmsd, const char *q mmsd->mysql_error_msg = strdup("timeout check"); return 1; } - if (GloMyMon->shutdown == true) { + if (shutdown == true || worker_stop.load()) { return 2; } if ((mmsd->async_exit_status & MYSQL_WAIT_TIMEOUT) == 0) { @@ -6715,12 +6717,108 @@ static void aws_rds_bgd_set_status(AWS_RDS_BGD_State& st, AWS_RDS_BGD_Status sta } } -void * monitor_RDS_BGD_thread_HG(void *arg) { - unsigned int wHG = *(unsigned int *)arg; - unsigned int num_hosts = 0; +/** +* @brief Load one BGD worker's configuration from the published host rows. +* +* @details Copies the cluster rows, verifies their checksum, copies configuration fields from +* the first row, and builds the probe host list. FSM fields retain their defaults and must not +* replace the corresponding fields in the live state. +* +* @param writer_hg Writer hostgroup identifying the deployment. +* @param current_checksum Per-cluster checksum captured for this refresh. +* @param candidate State populated from the published rows. +* +* @return true when the checksum matches and the rows contain a blue writer; false otherwise. +*/ +bool MySQL_Monitor::aws_rds_bgd_load_worker_config(int writer_hg, uint64_t current_checksum, AWS_RDS_BGD_State& candidate) { + SQLite3_result result(AWS_RDS_BGD_HOSTS_COLUMNS); + std::shared_ptr hosts_resultset; + + pthread_mutex_lock(&aws_rds_bgd_hosts_mutex); + hosts_resultset = AWS_RDS_BGD_Hosts_resultset; + pthread_mutex_unlock(&aws_rds_bgd_hosts_mutex); + + if (hosts_resultset) { + for (SQLite3_row* row : hosts_resultset->rows) { + if (atoi(row->fields[AWS_RDS_BGD_WRITER_HOSTGROUP]) == writer_hg) { + result.add_row(row); + } + } + } + if (result.raw_checksum() != current_checksum) { + return false; + } + + candidate.writer_hg = writer_hg; + bool first_row = true; + + for (SQLite3_row* row : result.rows) { + unsigned int reader_hg = atoi(row->fields[AWS_RDS_BGD_READER_HOSTGROUP]); + int green_writer_hg = row->fields[AWS_RDS_BGD_GREEN_WRITER_HOSTGROUP] + && row->fields[AWS_RDS_BGD_GREEN_WRITER_HOSTGROUP][0] + ? atoi(row->fields[AWS_RDS_BGD_GREEN_WRITER_HOSTGROUP]) : -1; + int green_reader_hg = row->fields[AWS_RDS_BGD_GREEN_READER_HOSTGROUP] + && row->fields[AWS_RDS_BGD_GREEN_READER_HOSTGROUP][0] + ? atoi(row->fields[AWS_RDS_BGD_GREEN_READER_HOSTGROUP]) : -1; + unsigned int check_interval_ms = atoi(row->fields[AWS_RDS_BGD_CHECK_INTERVAL_MS]); + unsigned int check_timeout_ms = atoi(row->fields[AWS_RDS_BGD_CHECK_TIMEOUT_MS]); + int writer_is_also_reader = atoi(row->fields[AWS_RDS_BGD_WRITER_IS_ALSO_READER]); + + if (first_row) { + candidate.reader_hg = reader_hg; + candidate.green_writer_hg = green_writer_hg; + candidate.green_reader_hg = green_reader_hg; + candidate.check_interval_ms = check_interval_ms; + candidate.check_timeout_ms = check_timeout_ms; + candidate.writer_is_also_reader = writer_is_also_reader; + first_row = false; + } + + char* srv_type = row->fields[AWS_RDS_BGD_SRV_TYPE]; + if (srv_type[0] == 'B' && atoi(row->fields[AWS_RDS_BGD_IS_WRITER]) != 0) { + candidate.probe_hosts.push_back(AWS_RDS_BGD_Probe_Host { + row->fields[AWS_RDS_BGD_HOSTNAME], + atoi(row->fields[AWS_RDS_BGD_PORT]), + atoi(row->fields[AWS_RDS_BGD_USE_SSL]) + }); + } + } + + if (first_row || candidate.probe_hosts.empty()) { + proxy_error("AWS RDS BGD [wHG=%d]: no blue writer available for topology checks\n", writer_hg); + return false; + } + + return true; +} + +/** +* @brief Replace only configuration-derived fields in a live BGD worker state. +* +* @param st Live worker state. +* @param candidate Parsed configuration to apply. +*/ +void MySQL_Monitor::aws_rds_bgd_apply_cluster_config(AWS_RDS_BGD_State& st, AWS_RDS_BGD_State& candidate) { + st.reader_hg = candidate.reader_hg; + st.green_writer_hg = candidate.green_writer_hg; + st.green_reader_hg = candidate.green_reader_hg; + st.writer_is_also_reader = candidate.writer_is_also_reader; + st.check_interval_ms = candidate.check_interval_ms; + st.check_timeout_ms = candidate.check_timeout_ms; + st.probe_hosts = candidate.probe_hosts; +} + +/** +* @brief Run the monitor loop for one AWS RDS BGD writer hostgroup. +* +* @param arg Pointer to the worker state owned by the parent monitor thread. +* +* @return nullptr when the worker exits. +*/ +void* monitor_RDS_BGD_thread_HG(void* arg) { + AWS_RDS_BGD_Worker* worker = static_cast(arg); + unsigned int wHG = worker->writer_hg; unsigned int cur_host_idx = 0; - unsigned int check_interval_ms = 0; - unsigned int check_timeout_ms = 0; set_thread_name("MonitorRdsBgdHG", GloVars.set_thread_name); proxy_info("Started Monitor thread for AWS RDS writer HG %u\n", wHG); @@ -6739,70 +6837,19 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { MySQL_Monitor__thread_MySQL_Thread_Variables_version = GloMTH->get_global_version(); mysql_thr->refresh_variables(); - uint64_t initial_checksum = 0; - - // initial data load from the monitor resultset - // Columns: - // 0 writer_hostgroup, 1 reader_hostgroup, 2 hostname, 3 port, 4 use_ssl, - // 5 green_writer_hostgroup, 6 green_reader_hostgroup, 7 check_interval_ms, - // 8 check_timeout_ms, 9 writer_is_also_reader, 10 is_writer - pthread_mutex_lock(&GloMyMon->aws_rds_bgd_mutex); - initial_checksum = GloMyMon->AWS_RDS_BGD_Hosts_checksum; - for (SQLite3_row *r : GloMyMon->AWS_RDS_Blue_Hosts_resultset->rows) { - if (atoi(r->fields[0]) == (int)wHG) { - if (atoi(r->fields[10]) != 0) { - num_hosts++; - } - if (st.reader_hg == 0) { - st.reader_hg = atoi(r->fields[1]); - } - if (st.green_writer_hg < 0 && r->fields[5] && r->fields[5][0]) { - st.green_writer_hg = atoi(r->fields[5]); - } - if (st.green_reader_hg < 0 && r->fields[6] && r->fields[6][0]) { - st.green_reader_hg = atoi(r->fields[6]); - } - if (check_interval_ms == 0) { - check_interval_ms = atoi(r->fields[7]); - } - if (check_timeout_ms == 0) { - check_timeout_ms = atoi(r->fields[8]); - } - if (r->fields[9] && r->fields[9][0]) { - st.writer_is_also_reader = atoi(r->fields[9]); - } - } - } - - host_def_t *hpa = (host_def_t *)malloc(sizeof(host_def_t)*(num_hosts ? num_hosts : 1)); - for (SQLite3_row *r : GloMyMon->AWS_RDS_Blue_Hosts_resultset->rows) { - // r->writer_hostgroup == wHG && r->is_writer != 0 - if (atoi(r->fields[0]) == (int)wHG && atoi(r->fields[10]) != 0) { - hpa[cur_host_idx].host = strdup(r->fields[2]); - hpa[cur_host_idx].port = atoi(r->fields[3]); - hpa[cur_host_idx].use_ssl = atoi(r->fields[4]); - cur_host_idx++; - } - } - if (num_hosts && cur_host_idx >= num_hosts) { - cur_host_idx = num_hosts - 1; - } - pthread_mutex_unlock(&GloMyMon->aws_rds_bgd_mutex); - - bool exit_now = false; unsigned long long t1 = 0; unsigned long long next_loop_at = 0; bool crc = false; - uint64_t current_checksum = 0; + uint64_t last_checksum = 0; size_t rnd; bool found_pingable_host = false; - bool rc_ping = false; MySQL_Monitor_State_Data *mmsd = NULL; RDS_BGD_Topology_Monitor_State topology_state = TOPOLOGY_TABLE_CHECK; t1 = monotonic_time(); - while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true && exit_now==false) { + while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true + && worker->worker_stop.load()==false) { unsigned int glover; t1 = monotonic_time(); bool poll_success = false; @@ -6818,17 +6865,20 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { next_loop_at = 0; } - // if the host list/definition changed, terminate so the dispatcher respawns - pthread_mutex_lock(&GloMyMon->aws_rds_bgd_mutex); - current_checksum = GloMyMon->AWS_RDS_BGD_Hosts_checksum; - pthread_mutex_unlock(&GloMyMon->aws_rds_bgd_mutex); - if (current_checksum != initial_checksum) { - exit_now = true; - break; + uint64_t current_checksum = worker->current_checksum.load(); + if (current_checksum != last_checksum) { + if (!GloMyMon->aws_rds_bgd_refresh_worker_config(st, current_checksum, topology_state, next_loop_at)) { + usleep(50000); + continue; + } + last_checksum = current_checksum; + if (cur_host_idx >= st.probe_hosts.size()) { + cur_host_idx = 0; + } } - if (num_hosts == 0) { - next_loop_at = t1 + (check_interval_ms ? check_interval_ms : 1000) * 1000; + if (st.probe_hosts.empty()) { + next_loop_at = t1 + (st.check_interval_ms ? st.check_interval_ms : 1000) * 1000; usleep(50000); continue; } @@ -6844,7 +6894,7 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { // Determine the host to probe. If the FSM pinned a host (the green IP, during a // switchover), poll it directly and skip ping/random selection; otherwise pick a - // pingable host (random first, then shuffle and scan). + // pingable host, starting at a random position. const char* poll_host; int poll_port; bool poll_use_ssl; @@ -6870,43 +6920,35 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { if (st.next_check_host.empty()) { found_pingable_host = false; rnd = (size_t) rand(); - rnd %= num_hosts; - rc_ping = GloMyMon->server_responds_to_ping(hpa[rnd].host, hpa[rnd].port); - if (rc_ping) { - found_pingable_host = true; - cur_host_idx = rnd; - } else { - MyHGM->p_update_mysql_error_counter( - p_mysql_error_type::proxysql, wHG, hpa[rnd].host, hpa[rnd].port, ER_PROXYSQL_AWS_NO_PINGABLE_SRV - ); - shuffle_hosts(hpa, num_hosts); - for (unsigned int i=0; (found_pingable_host == false && iserver_responds_to_ping(hpa[i].host, hpa[i].port); - if (rc_ping) { - found_pingable_host = true; - cur_host_idx = i; - } else { - MyHGM->p_update_mysql_error_counter( - p_mysql_error_type::proxysql, wHG, hpa[i].host, hpa[i].port, ER_PROXYSQL_AWS_NO_PINGABLE_SRV - ); - } + rnd %= st.probe_hosts.size(); + for (size_t i = 0; found_pingable_host == false && i < st.probe_hosts.size(); i++) { + size_t host_idx = (rnd + i) % st.probe_hosts.size(); + AWS_RDS_BGD_Probe_Host& host = st.probe_hosts[host_idx]; + if (GloMyMon->server_responds_to_ping(host.hostname.data(), host.port)) { + found_pingable_host = true; + cur_host_idx = host_idx; + } else { + MyHGM->p_update_mysql_error_counter( + p_mysql_error_type::proxysql, wHG, host.hostname.data(), host.port, + ER_PROXYSQL_AWS_NO_PINGABLE_SRV + ); } } if (found_pingable_host == false) { proxy_error("No node is pingable for AWS RDS cluster with writer HG %u\n", wHG); - next_loop_at = t1 + check_interval_ms * 1000; + next_loop_at = t1 + st.check_interval_ms * 1000; continue; } - poll_host = hpa[cur_host_idx].host; - poll_port = hpa[cur_host_idx].port; - poll_use_ssl = hpa[cur_host_idx].use_ssl; + poll_host = st.probe_hosts[cur_host_idx].hostname.c_str(); + poll_port = st.probe_hosts[cur_host_idx].port; + poll_use_ssl = st.probe_hosts[cur_host_idx].use_ssl; } mmsd = new MySQL_Monitor_State_Data( MON_AWS_RDS_BGD, (char*)poll_host, poll_port, poll_use_ssl ); mmsd->writer_hostgroup = wHG; - mmsd->aws_aurora_check_timeout_ms = check_timeout_ms; + mmsd->aws_aurora_check_timeout_ms = st.check_timeout_ms; mmsd->mysql = GloMyMon->My_Conn_Pool->get_connection(mmsd->hostname, mmsd->port, mmsd); mmsd->t1 = t1; @@ -6932,7 +6974,7 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { // we advance to TOPOLOGY_METADATA_FETCH and skip this check on subsequent // iterations, until a fetch reports the table is gone. - int qrc = aws_rds_bgd_async_query(mmsd, QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK); + int qrc = GloMyMon->aws_rds_bgd_async_query(mmsd, QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK, worker->worker_stop); if (qrc == 2) { goto __exit_monitor_RDS_BGD_thread_HG_now; } @@ -6963,7 +7005,7 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { // differs by RDS type (the Multi-AZ Cluster topology table may not expose // 'role'/'status' at all), so dump all columns and detect what is present. - int qrc = aws_rds_bgd_async_query(mmsd, QUERY_AWS_RDS_TOPOLOGY_DISCOVERY); + int qrc = GloMyMon->aws_rds_bgd_async_query(mmsd, QUERY_AWS_RDS_TOPOLOGY_DISCOVERY, worker->worker_stop); if (qrc == 2) { goto __exit_monitor_RDS_BGD_thread_HG_now; } @@ -7032,7 +7074,7 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { mmsd->t2 = monotonic_time(); // the FSM tightens the interval to 100ms while a switchover is in flight // (st.next_check_interval_ms); otherwise fall back to the configured baseline. - unsigned int eff = st.next_check_interval_ms ? st.next_check_interval_ms : check_interval_ms; + unsigned int eff = st.next_check_interval_ms ? st.next_check_interval_ms : st.check_interval_ms; next_loop_at = t1 + (eff * 1000); if (mmsd->t2 > t1) { next_loop_at -= (mmsd->t2 - t1); @@ -7064,11 +7106,6 @@ void * monitor_RDS_BGD_thread_HG(void *arg) { mmsd = NULL; } - for (unsigned int i=0; igreen map once (refreshed only after a switchover completes and -// clears it, which also covers a worker that starts mid-switchover). The topology -// exposes only primaries, so the writer pair is always present; reader pairs exist -// only when green_reader_hostgroup is configured (the user populated it). -static void aws_rds_bgd_build_map(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_Result& topo) { +/** +* @brief Build the blue-to-green host mapping for a BGD worker. +* +* @details Builds the map only when it is empty. The topology exposes only primaries, so the +* writer pair is always present; reader pairs are added when green_reader_hostgroup is configured. +* +* @param st Worker-owned BGD state. +* @param topo Parsed topology used to identify the green target. +*/ +void MySQL_Monitor::aws_rds_bgd_build_map(AWS_RDS_BGD_State& st, AWS_RDS_Topology_Result& topo) { if (!st.bg_map.empty()) { return; } - std::string green_writer_host; - for (const AWS_RDS_Topology_Node& n : topo.nodes) { - if (strcasecmp(n.role.c_str(), BGD_ROLE_TARGET) == 0) { - green_writer_host = n.endpoint; - break; - } - } - if (green_writer_host.empty()) { + AWS_RDS_Topology_Node* target = topo.target(); + if (!target || target->endpoint.empty()) { return; } + std::string green_writer_host = target->endpoint; + MyHGM->wrlock(); // blue writer: the writer_hostgroup member whose name matches the green TARGET. @@ -7236,8 +7274,10 @@ static void aws_rds_bgd_build_map(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_ * green primary BY IP: green stays reachable through the entire cutover (blue has a connectivity * gap), and the green IP survives the post-COMPLETED name swap (it becomes the promoted primary), * whereas the green DNS name is retired. 'next_check_host' is cleared at COMPLETED. +* +* @param st Worker-owned BGD state. */ -static void aws_rds_bgd_resolve_green_ips(AWS_RDS_BGD_State& st) { +void MySQL_Monitor::aws_rds_bgd_resolve_green_ips(AWS_RDS_BGD_State& st) { int ai_family = mysql_resolution_family_to_ai_family(mysql_thread___resolution_family); for (auto &p : st.bg_map) { // Always check the cache first: a green host that is a monitored server may be there. @@ -7281,8 +7321,10 @@ static void aws_rds_bgd_resolve_green_ips(AWS_RDS_BGD_State& st) { /** * @brief Add the green writer to green_writer_hostgroup, when that hostgroup is configured. +* +* @param st Worker-owned BGD state. */ -static void aws_rds_bgd_add_green_writer_in_hg(AWS_RDS_BGD_State& st) { +void MySQL_Monitor::aws_rds_bgd_add_green_writer_in_hg(AWS_RDS_BGD_State& st) { if (st.green_writer_hg < 0) { return; } @@ -7307,6 +7349,126 @@ static void aws_rds_bgd_add_green_writer_in_hg(AWS_RDS_BGD_State& st) { } } +/** +* @brief Find the writer pair in a blue/green map. +* +* @param bg_map Blue/green host mapping. +* @param writer Writer address populated when a pair is found. +* +* @return true when the map contains a writer pair; false otherwise. +*/ +bool MySQL_Monitor::aws_rds_bgd_find_writer(std::vector& bg_map, srv_addr_t& writer) { + for (AWS_RDS_BlueGreenPair& p : bg_map) { + if (p.is_writer) { + writer = srv_addr_t { p.blue_host, p.port }; + return true; + } + } + return false; +} + +/** +* @brief Apply a changed configuration to one running BGD worker. +* +* @details Before writer post-processing, applies the configuration and schedules mapping +* reconciliation after the next topology poll. At or after post-processing, rolls back the +* deployment and restarts its topology FSM without replacing the worker thread. +* +* @param st Worker-owned BGD state. +* @param current_checksum Per-cluster checksum captured for this refresh. +* @param topology_state Current topology query state. +* @param next_loop_at Next scheduled worker iteration. +* +* @return true when the captured configuration was applied; false when it must be retried. +*/ +bool MySQL_Monitor::aws_rds_bgd_refresh_worker_config( + AWS_RDS_BGD_State& st, uint64_t current_checksum, + RDS_BGD_Topology_Monitor_State& topology_state, unsigned long long& next_loop_at +) { + AWS_RDS_BGD_State candidate; + if (!aws_rds_bgd_load_worker_config(st.writer_hg, current_checksum, candidate)) { + return false; + } + + // Changes at or after writer post-processing require a full rollback and FSM restart. + if (st.bgd_status >= AWS_RDS_BGD_Status::WRITER_SWITCHOVER_POST_PROCESSING) { + AWS_RDS_BGD_Status old_status = st.bgd_status; + handle_aws_rds_bgd_post_switchover(st, true); + aws_rds_bgd_apply_cluster_config(st, candidate); + topology_state = TOPOLOGY_TABLE_CHECK; + next_loop_at = 0; + proxy_info( + "AWS RDS BGD [wHG=%u rHG=%u]: applied checksum %llu with full rollback from %s\n", + st.writer_hg, st.reader_hg, (unsigned long long)current_checksum, + aws_rds_bgd_status_str(old_status)); + return true; + } + + AWS_RDS_BGD_Status status = st.bgd_status; + unsigned int old_reader_hg = st.reader_hg; + bool refresh_in_progress = st.bgd_in_progress_set; + bool hostgroups_changed = old_reader_hg != candidate.reader_hg; + + // Clear the in-progress marker from the old reader hostgroup before applying the new configuration. + if (refresh_in_progress && hostgroups_changed) { + aws_rds_bgd_clear_bgd_in_progress(st); + } + + // Apply the new configuration. + aws_rds_bgd_apply_cluster_config(st, candidate); + st.next_check_host.clear(); + st.next_check_host_failures = 0; + next_loop_at = 0; + // Rebuild bg_map from the next topology probe result. + st.config_refresh_pending = true; + + // Apply the in-progress marker to the new reader hostgroup after the refresh. + if (refresh_in_progress && hostgroups_changed) { + aws_rds_bgd_set_bgd_in_progress(st); + } + + proxy_info( + "AWS RDS BGD [wHG=%u rHG=%u]: applied checksum %llu with in-place refresh at %s\n", + st.writer_hg, st.reader_hg, (unsigned long long)current_checksum, + aws_rds_bgd_status_str(status)); + return true; +} + +/** +* @brief Rebuild the mapping and reconcile writer state after a configuration refresh. +* +* @details Called only when config_refresh_pending is set. +* +* @param st Worker-owned BGD state. +* @param topology Fresh topology used to rebuild the mapping. +*/ +void MySQL_Monitor::aws_rds_bgd_config_refresh_action(AWS_RDS_BGD_State& st, AWS_RDS_Topology_Result& topology) { + srv_addr_t old_writer; + bool had_old_writer = aws_rds_bgd_find_writer(st.bg_map, old_writer); + st.bg_map.clear(); + aws_rds_bgd_build_map(st, topology); + + srv_addr_t new_writer; + bool has_new_writer = aws_rds_bgd_find_writer(st.bg_map, new_writer); + aws_rds_bgd_add_green_writer_in_hg(st); + + // Transfer the writer demotion when the refreshed configuration maps a different writer. + if (st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_IN_PROGRESS + && (had_old_writer != has_new_writer + || (had_old_writer && (old_writer.host != new_writer.host || old_writer.port != new_writer.port)))) { + if (had_old_writer) { + MyHGM->read_only_action_v2(std::list { + read_only_server_t { old_writer.host, (port_t)old_writer.port, 0 } + }); + } + if (has_new_writer) { + MyHGM->read_only_action_v2(std::list { + read_only_server_t { new_writer.host, (port_t)new_writer.port, 1 } + }); + } + } +} + // Map a raw mysql.rds_topology TARGET status string onto BGD phase enum. static AWS_RDS_BGD_Status aws_rds_bgd_status_from_topology(const std::string& status) { if (strcasecmp(status.c_str(), BGD_STATUS_AVAILABLE) == 0) { @@ -7361,28 +7523,21 @@ const char* aws_rds_bgd_status_str(AWS_RDS_BGD_Status s) { * @param st BGD switchover state (worker-owned, mutated here). * @param topology Parsed mysql.rds_topology result for this cycle. */ -void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topology_Result& topology) { +void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, AWS_RDS_Topology_Result& topology) { if (!topology.blue_green) { st.next_check_interval_ms = 0; aws_rds_bgd_set_status(st, AWS_RDS_BGD_Status::NONE); return; } - std::string status; - for (const AWS_RDS_Topology_Node& n : topology.nodes) { - if (strcasecmp(n.role.c_str(), BGD_ROLE_TARGET) == 0) { - status = n.status; - break; - } - } - if (status.empty()) { + AWS_RDS_Topology_Node* target = topology.target(); + if (!target || target->status.empty()) { st.next_check_interval_ms = 0; aws_rds_bgd_set_status(st, AWS_RDS_BGD_Status::NONE); return; } - st.last_topology_status = status; - AWS_RDS_BGD_Status topology_status = aws_rds_bgd_status_from_topology(status); + AWS_RDS_BGD_Status topology_status = aws_rds_bgd_status_from_topology(target->status); // Once we advance to READER_SWITCHOVER_IN_PROGRESS phase, AWS keeps reporting // WRITER_SWITCHOVER_COMPLETED (a single green row) until mysql.rds_topology drains. Ignore @@ -7410,6 +7565,12 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, const AWS_RDS_Topo return; } + // Rebuild a refreshed worker's mapping only after receiving this current topology result. + if (st.config_refresh_pending) { + aws_rds_bgd_config_refresh_action(st, topology); + st.config_refresh_pending = false; + } + if (topology_status == st.bgd_status) { // Refresh or retry green IP resolution on every eligible same-phase observation. if (topology_status >= AWS_RDS_BGD_Status::AVAILABLE @@ -7706,7 +7867,7 @@ void MySQL_Monitor::handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st, bo // state cleanup st.bg_map.clear(); - st.last_topology_status.clear(); + st.config_refresh_pending = false; st.next_check_host.clear(); st.next_check_interval_ms = 0; aws_rds_bgd_set_status(st, AWS_RDS_BGD_Status::NONE); @@ -7851,9 +8012,9 @@ void MySQL_Monitor::set_aws_rds_bgd_server_in_progress(unsigned int writer_hg, u /** * @brief AWS RDS BGD monitor thread entry point. * -* @details Spawns one worker (monitor_RDS_BGD_thread_HG) per writer hostgroup; each worker picks a pingable -* writer, probes 'mysql.rds_topology' and dispatches based on the detected topology shape. -* Workers are (re)spawned whenever the AWS_RDS_BGD_Hosts_checksum changes. +* @details Maintains one worker (monitor_RDS_BGD_thread_HG) per active writer hostgroup. The parent starts +* and stops workers and signals configuration changes. Each worker selects a pingable probe host, +* probes 'mysql.rds_topology', and runs the switchover state machine. */ void * MySQL_Monitor::monitor_aws_rds_bgd() { // Wait for GloMTH to be initialized @@ -7867,14 +8028,12 @@ void * MySQL_Monitor::monitor_aws_rds_bgd() { mysql_thr->refresh_variables(); uint64_t last_checksum = 0; - unsigned int *hgs_array = NULL; - pthread_t *pthreads_array = NULL; - unsigned int hgs_num = 0; + std::unordered_map> workers; while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true) { unsigned int glover; if (!GloMTH) - return NULL; + break; glover = GloMTH->get_global_version(); if (MySQL_Monitor__thread_MySQL_Thread_Variables_version < glover) { @@ -7882,70 +8041,99 @@ void * MySQL_Monitor::monitor_aws_rds_bgd() { mysql_thr->refresh_variables(); } - // respawn the per-writer-HG workers when the host list/definition changes - pthread_mutex_lock(&aws_rds_bgd_mutex); - uint64_t new_checksum = AWS_RDS_BGD_Hosts_checksum; - pthread_mutex_unlock(&aws_rds_bgd_mutex); + uint64_t new_checksum = 0; + std::shared_ptr hosts_resultset; + std::unordered_map cluster_checksums; + + pthread_mutex_lock(&aws_rds_bgd_hosts_mutex); + new_checksum = AWS_RDS_BGD_Hosts_checksum; + if (new_checksum != last_checksum && AWS_RDS_BGD_Hosts_resultset) { + hosts_resultset = AWS_RDS_BGD_Hosts_resultset; + cluster_checksums = AWS_RDS_BGD_Cluster_checksum; + } + pthread_mutex_unlock(&aws_rds_bgd_hosts_mutex); + + std::unordered_map active_cluster_checksums; + if (hosts_resultset) { + for (SQLite3_row* row : hosts_resultset->rows) { + char* srv_type = row->fields[AWS_RDS_BGD_SRV_TYPE]; + if (srv_type && srv_type[0] == 'B' + && atoi(row->fields[AWS_RDS_BGD_IS_WRITER]) != 0) { + int writer_hg = atoi(row->fields[AWS_RDS_BGD_WRITER_HOSTGROUP]); + auto checksum_it = cluster_checksums.find(writer_hg); + if (checksum_it != cluster_checksums.end()) { + active_cluster_checksums[writer_hg] = checksum_it->second; + } + } + } + } + if (new_checksum != last_checksum) { - proxy_info("Detected new/changed definition for AWS RDS monitoring\n"); + proxy_info("Detected changed definition for AWS RDS Blue Green monitoring\n"); last_checksum = new_checksum; - if (pthreads_array) { - for (unsigned int i=0; i < hgs_num; i++) { - pthread_join(pthreads_array[i], NULL); - proxy_info("Stopped Monitor thread for AWS RDS writer HG %u\n", hgs_array[i]); + std::vector stopped_workers; + + for (auto& [writer_hg, worker] : workers) { + auto cluster_it = active_cluster_checksums.find(writer_hg); + if (cluster_it == active_cluster_checksums.end()) { + worker->worker_stop.store(true); + stopped_workers.push_back(writer_hg); + proxy_info( + "AWS RDS BGD [wHG=%d]: stopping worker; deployment is inactive, removed, or has no blue writer\n", + writer_hg); + continue; + } + + uint64_t old_cluster_checksum = worker->current_checksum.load(); + if (old_cluster_checksum != cluster_it->second) { + worker->current_checksum.store(cluster_it->second); + proxy_info( + "AWS RDS BGD [wHG=%d]: signaling config refresh, checksum %llu -> %llu\n", + writer_hg, (unsigned long long)old_cluster_checksum, + (unsigned long long)cluster_it->second); } - free(pthreads_array); - free(hgs_array); - pthreads_array = NULL; - hgs_array = NULL; } - hgs_num = 0; - pthread_mutex_lock(&aws_rds_bgd_mutex); - unsigned int num_rows = AWS_RDS_Blue_Hosts_resultset->rows_count; - if (num_rows) { - unsigned int *tmp_hgs_array = (unsigned int *)malloc(sizeof(unsigned int)*num_rows); - for (SQLite3_row *r : AWS_RDS_Blue_Hosts_resultset->rows) { - int wHG = atoi(r->fields[0]); - bool found = false; - for (unsigned int i=0; i < hgs_num; i++) { - if (tmp_hgs_array[i] == (unsigned int)wHG) { - found = true; - } - } - if (found == false) { - tmp_hgs_array[hgs_num] = wHG; - hgs_num++; - } + for (auto& [writer_hg, checksum] : active_cluster_checksums) { + if (workers.find(writer_hg) != workers.end()) { + continue; } - proxy_info("Activating Monitoring of %u AWS RDS clusters\n", hgs_num); - hgs_array = (unsigned int *)malloc(sizeof(unsigned int)*hgs_num); - pthreads_array = (pthread_t *)malloc(sizeof(pthread_t)*hgs_num); - for (unsigned int i=0; i < hgs_num; i++) { - hgs_array[i] = tmp_hgs_array[i]; - proxy_info("Starting Monitor thread for AWS RDS writer HG %u\n", hgs_array[i]); - if (pthread_create(&pthreads_array[i], NULL, monitor_RDS_BGD_thread_HG, &hgs_array[i]) != 0) { - // LCOV_EXCL_START - proxy_error("Thread creation\n"); - assert(0); - // LCOV_EXCL_STOP - } + + std::unique_ptr worker(new AWS_RDS_BGD_Worker); + worker->writer_hg = writer_hg; + worker->current_checksum.store(checksum); + AWS_RDS_BGD_Worker* worker_arg = worker.get(); + workers.emplace(writer_hg, std::move(worker)); + proxy_info("Starting Monitor thread for AWS RDS writer HG %d\n", writer_hg); + if (pthread_create(&worker_arg->thread, NULL, monitor_RDS_BGD_thread_HG, worker_arg) != 0) { + // LCOV_EXCL_START + proxy_error("Thread creation\n"); + assert(0); + // LCOV_EXCL_STOP } - free(tmp_hgs_array); } - pthread_mutex_unlock(&aws_rds_bgd_mutex); + + for (int writer_hg : stopped_workers) { + auto worker_it = workers.find(writer_hg); + if (worker_it == workers.end()) { + continue; + } + pthread_join(worker_it->second->thread, NULL); + proxy_info("Stopped Monitor thread for AWS RDS writer HG %d\n", writer_hg); + workers.erase(worker_it); + } } usleep(10000); } - // on shutdown, join any running per-HG workers - if (pthreads_array) { - for (unsigned int i=0; i < hgs_num; i++) { - pthread_join(pthreads_array[i], NULL); - } - free(pthreads_array); - free(hgs_array); + for (auto& [writer_hg, worker] : workers) { + worker->worker_stop.store(true); + } + for (auto& [writer_hg, worker] : workers) { + pthread_join(worker->thread, NULL); + proxy_info("Stopped Monitor thread for AWS RDS writer HG %d\n", writer_hg); } + workers.clear(); if (mysql_thr) { delete mysql_thr; mysql_thr = NULL; From 75ee87227ad6c82634431fb8367c6b6124d4aaa1 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 28 Jul 2026 07:05:53 +0000 Subject: [PATCH 60/81] fix: preserve RDS BGD runtime status on reload - Preserve existing runtime BGD status while updating configured fields. - Delete removed rows and replace rows whose reader hostgroup changed. Signed-off-by: Wazir Ahmed --- lib/MySQL_HostGroups_Manager.cpp | 124 +++++++++++++++++++++++++------ 1 file changed, 103 insertions(+), 21 deletions(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 01d2d461e6..a021d4f0f3 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1551,7 +1551,6 @@ bool MySQL_HostGroups_Manager::commit( // AWS RDS if (incoming_aws_rds_bgd_hostgroups) { proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_aws_rds_bgd_hostgroups\n"); - mydb->execute("DELETE FROM mysql_aws_rds_bgd_hostgroups"); generate_mysql_aws_rds_bgd_hostgroups_table(); } @@ -6500,24 +6499,88 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { * @details The incoming resultset comes from the admin config table (11 columns, no `auto_generated`); config-loaded * entries are user-defined, so `auto_generated` is stored as 0. `green_writer_hostgroup` and * `green_reader_hostgroup` are optional and bound as SQL NULL when absent. + * + * @note Existing deployments preserve their runtime `status` while configured fields are reloaded. */ void MySQL_HostGroups_Manager::generate_mysql_aws_rds_bgd_hostgroups_table() { if (incoming_aws_rds_bgd_hostgroups==NULL) { return; } + struct RuntimeRow { + int reader_hostgroup; + int status; + }; + + std::map runtime_rows; + std::map incoming_reader_hostgroups; + + for (SQLite3_row* row : incoming_aws_rds_bgd_hostgroups->rows) { + incoming_reader_hostgroups.emplace(atoi(row->fields[0]), atoi(row->fields[1])); + } + + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + const char* select_query = "SELECT writer_hostgroup, reader_hostgroup, status FROM mysql_aws_rds_bgd_hostgroups"; + mydb->execute_statement(select_query, &error, &cols, &affected_rows, &resultset); + if (error) { + proxy_error("Error on %s : %s\n", select_query, error); + free(error); + error = NULL; + assert(0); + } + if (resultset) { + for (SQLite3_row* row : resultset->rows) { + runtime_rows.emplace(atoi(row->fields[0]), RuntimeRow {atoi(row->fields[1]), atoi(row->fields[2])}); + } + delete resultset; + resultset = NULL; + } + int rc; - char *query=(char *)"INSERT INTO mysql_aws_rds_bgd_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," - "writer_is_also_reader,check_interval_ms,check_timeout_ms,comment,auto_generated) VALUES " - "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)"; + const char* delete_query = "DELETE FROM mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=?1"; + auto [delete_rc, delete_statement_unique] = mydb->prepare_v2(delete_query); + ASSERT_SQLITE_OK(delete_rc, mydb); + sqlite3_stmt* delete_statement = delete_statement_unique.get(); + + // Remove missing deployments and release changed reader hostgroups before inserting their replacements. + for (const auto& [writer_hostgroup, runtime_row] : runtime_rows) { + auto incoming_it = incoming_reader_hostgroups.find(writer_hostgroup); + bool removed = incoming_it == incoming_reader_hostgroups.end(); + bool reader_changed = !removed && incoming_it->second != runtime_row.reader_hostgroup; + if (!removed && !reader_changed) { + continue; + } + + rc=(*proxy_sqlite3_bind_int64)(delete_statement, 1, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + SAFE_SQLITE3_STEP2(delete_statement); + rc=(*proxy_sqlite3_clear_bindings)(delete_statement); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(delete_statement); ASSERT_SQLITE_OK(rc, mydb); + } + + const char* update_query = + "UPDATE mysql_aws_rds_bgd_hostgroups SET " + "reader_hostgroup=?1, green_writer_hostgroup=?2, green_reader_hostgroup=?3, active=?4, " + "writer_is_also_reader=?5, check_interval_ms=?6, check_timeout_ms=?7, comment=?8, auto_generated=?9 " + "WHERE writer_hostgroup=?10"; + auto [update_rc, update_statement_unique] = mydb->prepare_v2(update_query); + ASSERT_SQLITE_OK(update_rc, mydb); + sqlite3_stmt* update_statement = update_statement_unique.get(); + + const char* insert_query = + "INSERT INTO mysql_aws_rds_bgd_hostgroups(" + "writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active," + "writer_is_also_reader, check_interval_ms, check_timeout_ms, comment, auto_generated, status" + ") VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"; + auto [insert_rc, insert_statement_unique] = mydb->prepare_v2(insert_query); + ASSERT_SQLITE_OK(insert_rc, mydb); + sqlite3_stmt* insert_statement = insert_statement_unique.get(); - auto [rc1, statement_unique] = mydb->prepare_v2(query); - ASSERT_SQLITE_OK(rc1, mydb); - sqlite3_stmt *statement = statement_unique.get(); proxy_info("New mysql_aws_rds_bgd_hostgroups table\n"); - for (std::vector::iterator it = incoming_aws_rds_bgd_hostgroups->rows.begin() ; it != incoming_aws_rds_bgd_hostgroups->rows.end(); ++it) { - SQLite3_row *r=*it; + for (SQLite3_row* r : incoming_aws_rds_bgd_hostgroups->rows) { int writer_hostgroup=atoi(r->fields[0]); int reader_hostgroup=atoi(r->fields[1]); const char *gw_str = r->fields[2]; @@ -6533,26 +6596,45 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_rds_bgd_hostgroups_table() { proxy_info("Loading AWS RDS info for (%d,%d,%d,%d,%s,%d,%d,%d,%d,\"%s\")\n", writer_hostgroup,reader_hostgroup, green_writer_hostgroup,green_reader_hostgroup,(active ? "on" : "off"),writer_is_also_reader, check_interval_ms,check_timeout_ms,auto_generated,r->fields[8]); - rc=(*proxy_sqlite3_bind_int64)(statement, 1, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 2, reader_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + + auto runtime_it = runtime_rows.find(writer_hostgroup); + bool update_existing = + runtime_it != runtime_rows.end() && + runtime_it->second.reader_hostgroup == reader_hostgroup; + sqlite3_stmt* statement = update_existing ? update_statement : insert_statement; + int field_offset = update_existing ? 0 : 1; + + if (!update_existing) { + rc=(*proxy_sqlite3_bind_int64)(statement, 1, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + } + rc=(*proxy_sqlite3_bind_int64)(statement, 1 + field_offset, reader_hostgroup); ASSERT_SQLITE_OK(rc, mydb); if (green_writer_hostgroup >= 0) { - rc=(*proxy_sqlite3_bind_int64)(statement, 3, green_writer_hostgroup); + rc=(*proxy_sqlite3_bind_int64)(statement, 2 + field_offset, green_writer_hostgroup); } else { - rc=(*proxy_sqlite3_bind_null)(statement, 3); + rc=(*proxy_sqlite3_bind_null)(statement, 2 + field_offset); } ASSERT_SQLITE_OK(rc, mydb); if (green_reader_hostgroup >= 0) { - rc=(*proxy_sqlite3_bind_int64)(statement, 4, green_reader_hostgroup); + rc=(*proxy_sqlite3_bind_int64)(statement, 3 + field_offset, green_reader_hostgroup); } else { - rc=(*proxy_sqlite3_bind_null)(statement, 4); + rc=(*proxy_sqlite3_bind_null)(statement, 3 + field_offset); } ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 5, active); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 6, writer_is_also_reader); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 7, check_interval_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 8, check_timeout_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_text)(statement, 9, r->fields[8], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 10, auto_generated); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 4 + field_offset, active); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 5 + field_offset, writer_is_also_reader); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 6 + field_offset, check_interval_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 7 + field_offset, check_timeout_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 8 + field_offset, r->fields[8], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 9 + field_offset, auto_generated); ASSERT_SQLITE_OK(rc, mydb); + + if (update_existing) { + rc=(*proxy_sqlite3_bind_int64)(statement, 10, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + } else { + int status = runtime_it == runtime_rows.end() + ? static_cast(AWS_RDS_BGD_Status::NONE) + : runtime_it->second.status; + rc=(*proxy_sqlite3_bind_int64)(statement, 11, status); ASSERT_SQLITE_OK(rc, mydb); + } SAFE_SQLITE3_STEP2(statement); rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, mydb); From 61790f9063bf3cd9d99440d64e82977688add312 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 28 Jul 2026 07:06:24 +0000 Subject: [PATCH 61/81] fix: stabilize RDS BGD monitor transitions - Release an active monitor connection before deleting worker state on exit. - Reapply writer demotion after configuration refresh and reconcile completed placement. - Purge green endpoint connections after post-switchover processing. Signed-off-by: Wazir Ahmed --- lib/MySQL_Monitor.cpp | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 8a8c756575..a019afefb8 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -7102,6 +7102,9 @@ void* monitor_RDS_BGD_thread_HG(void* arg) { } if (mmsd) { + if (mmsd->mysql) { + GloMyMon->My_Conn_Pool->destroy_mysql_connection(mmsd); + } delete mmsd; mmsd = NULL; } @@ -7452,11 +7455,12 @@ void MySQL_Monitor::aws_rds_bgd_config_refresh_action(AWS_RDS_BGD_State& st, AWS bool has_new_writer = aws_rds_bgd_find_writer(st.bg_map, new_writer); aws_rds_bgd_add_green_writer_in_hg(st); - // Transfer the writer demotion when the refreshed configuration maps a different writer. - if (st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_IN_PROGRESS - && (had_old_writer != has_new_writer - || (had_old_writer && (old_writer.host != new_writer.host || old_writer.port != new_writer.port)))) { - if (had_old_writer) { + // Reapply the in-progress demotion after the configuration reload restores configured placement. + if (st.bgd_status == AWS_RDS_BGD_Status::WRITER_SWITCHOVER_IN_PROGRESS) { + bool writer_changed = + had_old_writer != has_new_writer || + (had_old_writer && (old_writer.host != new_writer.host || old_writer.port != new_writer.port)); + if (writer_changed && had_old_writer) { MyHGM->read_only_action_v2(std::list { read_only_server_t { old_writer.host, (port_t)old_writer.port, 0 } }); @@ -7766,9 +7770,7 @@ void MySQL_Monitor::aws_rds_bgd_hostgroup_action( changed |= MyHGM->aws_rds_bgd_configure_writer(writer.host.c_str(), writer.port, writer_is_also_reader); shun_readers = true; } else if (bgd_status == AWS_RDS_BGD_Status::SWITCHOVER_COMPLETED) { - if (!writer_is_also_reader) { - changed |= (MyHGM->remove_server_in_hg(reader_hg, writer.host, writer.port) == 0); - } + changed |= MyHGM->aws_rds_bgd_configure_writer(writer.host.c_str(), writer.port, writer_is_also_reader); } else { MyHGM->wrunlock(); return; @@ -7859,6 +7861,9 @@ void MySQL_Monitor::handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st, bo for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { dns_cache->remove(p.blue_host); My_Conn_Pool->purge_connections(p.blue_host.c_str(), p.port); + if (!p.green_ip.empty()) { + My_Conn_Pool->purge_connections(p.green_ip.c_str(), p.port); + } } if (!rollback) { From a646af9f58d4176c8f6c7877f22000f7ce0d2561 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 28 Jul 2026 07:06:32 +0000 Subject: [PATCH 62/81] fix: preserve TLS-specific monitor connections - Skip pooled connections whose TLS mode does not match the monitor task. - Return skipped connections to the pool after searching for a compatible connection. Signed-off-by: Wazir Ahmed --- lib/MySQL_Monitor.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index a019afefb8..75a6a4dd6d 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -413,6 +413,7 @@ MYSQL * MySQL_Monitor_Connection_Pool::get_connection(char *hostname, int port, } } #endif // DEBUG + std::vector skipped_conn; while (srv->conns->len) { unsigned int idx = rand() % srv->conns->len; MYSQL* mysql = (MYSQL*)srv->conns->remove_index_fast(idx); @@ -428,9 +429,23 @@ MYSQL * MySQL_Monitor_Connection_Pool::get_connection(char *hostname, int port, continue; } + // The pool is grouped by hostname and port, but the same server can + // be monitored over plaintext and TLS. Keep connections that may + // match another monitor task and continue searching for this one. + bool connection_uses_ssl = mysql->options.use_ssl != 0; + if (mmsd && connection_uses_ssl != mmsd->use_ssl) { + skipped_conn.push_back(mysql); + continue; + } + my = mysql; break; } + + // Return skipped connections to the pool + for (MYSQL* mysql : skipped_conn) { + srv->conns->add(mysql); + } #ifdef DEBUG // 'my' can be NULL due to connection cleanup, and can cause crash if (my) { From c163855c183f852dabba39084af6b19b4259a735 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 28 Jul 2026 07:06:42 +0000 Subject: [PATCH 63/81] fix: track offline RDS BGD green writers - Track whether the configured green writer is OFFLINE_SOFT or OFFLINE_HARD. - Skip DNS resolution and probe pinning only for an explicitly offline green writer. Signed-off-by: Wazir Ahmed --- include/MySQL_Monitor.hpp | 1 + lib/MySQL_Monitor.cpp | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 1c84e10bf2..e2a5f5ed0d 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -536,6 +536,7 @@ struct AWS_RDS_BlueGreenPair { int64_t blue_max_conns = 1000; ///< Blue server max_connections mirrored onto the green server when it is added. int32_t blue_use_ssl = 0; ///< Blue server SSL setting mirrored onto the green server when it is added. int32_t green_use_ssl = -1; ///< Green server SSL; -1 means unset (use blue_use_ssl). + bool green_offline = false; ///< True when the configured green writer is OFFLINE_SOFT/OFFLINE_HARD. std::string green_ip; ///< Green host IP resolved at SWITCHOVER_INITIATED and held warm. unsigned long long green_ip_ttl = 0; ///< Expiry for green_ip when resolved by the BGD thread; 0 means DNS_Cache-sourced. bool green_ip_pinned = false; ///< True after green_ip has been pinned and blue_host connections drained/purged. diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 75a6a4dd6d..ab871f46ed 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -7216,14 +7216,17 @@ void MySQL_Monitor::aws_rds_bgd_build_map(AWS_RDS_BGD_State& st, AWS_RDS_Topolog if (gwhgc && gwhgc->mysrvs) { for (unsigned int k = 0; k < gwhgc->mysrvs->cnt(); k++) { MySrvC* gs = gwhgc->mysrvs->idx(k); - if (gs->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD - || gs->get_status() == MYSQL_SERVER_STATUS_OFFLINE_SOFT) { + if (strcasecmp(gs->address, green_writer_host.c_str()) != 0 || gs->port != p.port) { continue; } - if (strcasecmp(gs->address, green_writer_host.c_str()) == 0 && gs->port == p.port) { + + p.green_offline = + gs->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD + || gs->get_status() == MYSQL_SERVER_STATUS_OFFLINE_SOFT; + if (!p.green_offline) { p.green_use_ssl = gs->use_ssl; - break; } + break; } } } @@ -7298,6 +7301,10 @@ void MySQL_Monitor::aws_rds_bgd_build_map(AWS_RDS_BGD_State& st, AWS_RDS_Topolog void MySQL_Monitor::aws_rds_bgd_resolve_green_ips(AWS_RDS_BGD_State& st) { int ai_family = mysql_resolution_family_to_ai_family(mysql_thread___resolution_family); for (auto &p : st.bg_map) { + if (p.green_offline) { + continue; + } + // Always check the cache first: a green host that is a monitored server may be there. size_t n = 0; std::string ip = MySQL_Monitor::dns_lookup(p.green_host, false, &n); @@ -7326,7 +7333,7 @@ void MySQL_Monitor::aws_rds_bgd_resolve_green_ips(AWS_RDS_BGD_State& st) { // Pin the worker's next probe to the green writer's IP (observe the switchover from green). for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { - if (p.is_writer && !p.green_ip.empty()) { + if (p.is_writer && !p.green_offline && !p.green_ip.empty()) { if (st.next_check_host != p.green_ip) { st.next_check_host = p.green_ip; proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: pinning rds_topology probe to green IP %s\n", @@ -7359,6 +7366,7 @@ void MySQL_Monitor::aws_rds_bgd_add_green_writer_in_hg(AWS_RDS_BGD_State& st) { MySrvC* s = MyHGM->find_server_in_hg((unsigned int)st.green_writer_hg, p.green_host, p.port); if (s) { p.green_use_ssl = s->use_ssl; + p.green_offline = false; } MyHGM->publish_mysql_servers_to_runtime(); } From 8886603af3a74546ed692c670bb8890e6648e0c5 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 28 Jul 2026 07:07:06 +0000 Subject: [PATCH 64/81] test: wait for ProxySQL simulator ports - Add a common readiness check for ProxySQL ports used by simulator groups. - Run the readiness check after simulator infrastructure and ProxySQL start. Signed-off-by: Wazir Ahmed --- test/infra/control/ensure-infras.bash | 10 ++++- test/infra/control/readiness.bash | 42 +++++++++++++++++++ test/tap/groups/cluster_sim_aurora/env.sh | 1 + test/tap/groups/cluster_sim_galera/env.sh | 1 + test/tap/groups/cluster_sim_group_repl/env.sh | 1 + test/tap/groups/cluster_sim_rds_bgd/env.sh | 1 + test/tap/groups/cluster_sim_read_only/env.sh | 1 + test/tap/groups/cluster_sim_repl_lag/env.sh | 1 + 8 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 test/infra/control/readiness.bash diff --git a/test/infra/control/ensure-infras.bash b/test/infra/control/ensure-infras.bash index b1baa5e8f9..3a7d865380 100755 --- a/test/infra/control/ensure-infras.bash +++ b/test/infra/control/ensure-infras.bash @@ -6,6 +6,7 @@ set -o pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" export WORKSPACE="${REPO_ROOT}" +source "${SCRIPT_DIR}/readiness.bash" # Default INFRA_ID if not provided export INFRA_ID="${INFRA_ID:-dev-$USER}" @@ -101,6 +102,13 @@ for EXT in bash sql; do fi done +PROXYSQL_READY_PORTS=(6032 6033 6132 6133) +if [ -n "${PROXYSQL_READY_PORTS_EXTRA:-}" ]; then + read -r -a EXTRA_READY_PORTS <<< "${PROXYSQL_READY_PORTS_EXTRA}" + PROXYSQL_READY_PORTS+=("${EXTRA_READY_PORTS[@]}") +fi +wait_for_proxysql_ports "${PROXY_CONTAINER}" 30 "${PROXYSQL_READY_PORTS[@]}" + # 4. Ensure Docker Compose helper is available COMPOSE_CMD="docker compose" if ! $COMPOSE_CMD version &>/dev/null; then COMPOSE_CMD="docker-compose"; fi @@ -159,4 +167,4 @@ if [ -f "${SETUP_HOOK}" ]; then "${SETUP_HOOK}" fi -# ensure-infras.bash completed successfully \ No newline at end of file +# ensure-infras.bash completed successfully diff --git a/test/infra/control/readiness.bash b/test/infra/control/readiness.bash new file mode 100644 index 0000000000..5813346814 --- /dev/null +++ b/test/infra/control/readiness.bash @@ -0,0 +1,42 @@ +#!/bin/bash + +wait_for_proxysql_ports() { + local container="$1" + local timeout_seconds="$2" + shift 2 + + local port + local attempt + + if [[ ! "${timeout_seconds}" =~ ^[1-9][0-9]*$ ]]; then + echo "ERROR: Invalid ProxySQL readiness timeout: ${timeout_seconds}" >&2 + return 1 + fi + + echo ">>> Running readiness checks for ProxySQL ports: $*" + + for port in "$@"; do + if [[ ! "${port}" =~ ^[0-9]+$ ]]; then + echo "ERROR: Invalid ProxySQL readiness port: ${port}" >&2 + return 1 + fi + + echo -n ">>> Waiting for ${container}:${port} " + for ((attempt = 0; attempt < timeout_seconds; attempt++)); do + if docker exec "${container}" \ + bash -c "exec 3<>/dev/tcp/127.0.0.1/${port}" \ + >/dev/null 2>&1; then + echo "Ready." + break + fi + echo -n "." + sleep 1 + done + + if [ "${attempt}" -ge "${timeout_seconds}" ]; then + echo " TIMEOUT" + docker logs --tail=60 "${container}" >&2 || true + return 1 + fi + done +} diff --git a/test/tap/groups/cluster_sim_aurora/env.sh b/test/tap/groups/cluster_sim_aurora/env.sh index 45220ddd6e..17560db941 100644 --- a/test/tap/groups/cluster_sim_aurora/env.sh +++ b/test/tap/groups/cluster_sim_aurora/env.sh @@ -9,6 +9,7 @@ export CLUSTER_SIM_HOST_FILE="${WORKSPACE}/test/tap/groups/cluster_sim_aurora/ad # username/password match what enable_aurora_testing() inserts. export AURORA_HOSTNAME=proxysql export AURORA_PORT=3306 +export PROXYSQL_READY_PORTS_EXTRA="3306" # Skip the background cluster nodes: they are built without TEST_AURORA and # their empty mysql_users sync back to the primary, wiping aurora1/2/3. diff --git a/test/tap/groups/cluster_sim_galera/env.sh b/test/tap/groups/cluster_sim_galera/env.sh index e7b9ea3fff..2a37bfd682 100644 --- a/test/tap/groups/cluster_sim_galera/env.sh +++ b/test/tap/groups/cluster_sim_galera/env.sh @@ -8,6 +8,7 @@ export CLUSTER_SIM_TESTS_ROOT="${WORKSPACE}/test/deps/cluster_simulator/tests" # username/password match what enable_galera_testing() inserts (galera1/pass1). export GALERA_HOSTNAME=proxysql export GALERA_PORT=3306 +export PROXYSQL_READY_PORTS_EXTRA="3306" # Skip the background cluster nodes so their empty mysql_users do not sync # back to the primary and wipe galera1/2/galera. diff --git a/test/tap/groups/cluster_sim_group_repl/env.sh b/test/tap/groups/cluster_sim_group_repl/env.sh index 5a82453574..6aae7caa17 100644 --- a/test/tap/groups/cluster_sim_group_repl/env.sh +++ b/test/tap/groups/cluster_sim_group_repl/env.sh @@ -8,6 +8,7 @@ export CLUSTER_SIM_TESTS_ROOT="${WORKSPACE}/test/deps/cluster_simulator/tests" # for username/password (grouprep1/pass1) match what enable_grouprep_testing() inserts. export GROUPREP_HOSTNAME=proxysql export GROUPREP_PORT=3306 +export PROXYSQL_READY_PORTS_EXTRA="3306" # Skip the background cluster nodes so their empty mysql_users do not sync # back to the primary and wipe grouprep1. diff --git a/test/tap/groups/cluster_sim_rds_bgd/env.sh b/test/tap/groups/cluster_sim_rds_bgd/env.sh index 5071a61929..898b34322e 100644 --- a/test/tap/groups/cluster_sim_rds_bgd/env.sh +++ b/test/tap/groups/cluster_sim_rds_bgd/env.sh @@ -3,6 +3,7 @@ # Inject AWS-style endpoint aliases into the ProxySQL container. export CLUSTER_SIM_HOST_FILE="${WORKSPACE}/test/tap/groups/cluster_sim_rds_bgd/add-hosts" +export PROXYSQL_READY_PORTS_EXTRA="3306" # Skip background cluster nodes: the TAP test drives the primary ProxySQL's # built-in SQLite3-server simulator directly. diff --git a/test/tap/groups/cluster_sim_read_only/env.sh b/test/tap/groups/cluster_sim_read_only/env.sh index 39d25fd56b..a5fee6bb39 100644 --- a/test/tap/groups/cluster_sim_read_only/env.sh +++ b/test/tap/groups/cluster_sim_read_only/env.sh @@ -8,6 +8,7 @@ export CLUSTER_SIM_TESTS_ROOT="${WORKSPACE}/test/deps/cluster_simulator/tests" # defaults for username/password are 'root/root' (provisioned by pre-proxysql.sql). export READONLY_HOSTNAME=proxysql export READONLY_PORT=3306 +export PROXYSQL_READY_PORTS_EXTRA="3306" # Skip the background cluster nodes so their empty mysql_users do not sync # back to the primary and wipe the root user we inject. diff --git a/test/tap/groups/cluster_sim_repl_lag/env.sh b/test/tap/groups/cluster_sim_repl_lag/env.sh index 584475fba8..4ef92d774f 100644 --- a/test/tap/groups/cluster_sim_repl_lag/env.sh +++ b/test/tap/groups/cluster_sim_repl_lag/env.sh @@ -8,6 +8,7 @@ export CLUSTER_SIM_TESTS_ROOT="${WORKSPACE}/test/deps/cluster_simulator/tests" # defaults for username/password are 'root/root' (provisioned by pre-proxysql.sql). export REPL_LAG_HOSTNAME=proxysql export REPL_LAG_PORT=3306 +export PROXYSQL_READY_PORTS_EXTRA="3306" # Skip the background cluster nodes so their empty mysql_users do not sync # back to the primary and wipe the root user we inject. From d24d64ff295594ac4987f8c46f7fc699edb5a217 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 28 Jul 2026 07:07:16 +0000 Subject: [PATCH 65/81] test: add shared RDS BGD simulator support - Add shared endpoint, topology, probe, wait, and cleanup helpers for BGD tests. - Document the approved BGD TAP structure, naming, comments, and cleanup rules. Signed-off-by: Wazir Ahmed --- bgd-test-style.md | 157 ++++++++++ test/tap/tap/rds_bgd_simulator.cpp | 29 +- test/tap/tap/rds_bgd_simulator.h | 31 +- test/tap/tap/rds_bgd_tap.h | 473 ++++++++++++++++++++++++++++- 4 files changed, 683 insertions(+), 7 deletions(-) create mode 100644 bgd-test-style.md diff --git a/bgd-test-style.md b/bgd-test-style.md new file mode 100644 index 0000000000..3c34b2e211 --- /dev/null +++ b/bgd-test-style.md @@ -0,0 +1,157 @@ +# AWS RDS BGD TAP Style Rules + +These rules apply to every new or refactored AWS RDS BGD TAP file. + +## Test Scope And Documentation + +- Use one TAP executable per independently reportable behavior. +- Keep each TAP executable focused on the behavior named by the file. Do not + add connection-pool, TLS, server-status, or other property mutations unless + that behavior requires them. +- Move independently useful coverage into a dedicated TAP executable instead + of carrying it through an unrelated scenario. +- File and test headers must name the exact BGD configuration change, topology + state, and expected ProxySQL result. +- Use names from the BGD implementation and Admin tables. Avoid generic wording + such as "surface", "departure", or "input replacement". +- Document multi-step tests with setup, mutation, and verification bullets. +- Test public configuration and observable routing, runtime, connection-pool, + and probe behavior. Do not test or document non-public internal server states. + +## Local Test Harness + +- Every BGD TAP file defines local `setup()` and `cleanup()` functions. +- `setup()` initializes only the test harness: load the TAP environment, + connect to ProxySQL Admin, and connect to the SQLite simulator. +- `setup()` must not configure BGD topology, Admin rows, simulated writer + state, or other scenario prerequisites. +- `setup()` must not contain TAP assertions. +- `setup()` releases any partially created connection before returning a + failure. `main()` then returns `exit_status()` without calling `cleanup()`. +- `cleanup()` clears the ProxySQL Admin/runtime and simulator state created by + the test, then closes test connections. +- `cleanup()` is called only after `setup()` succeeds and may assume its + connections are valid. +- `cleanup()` attempts every cleanup operation, closes test connections, and + returns `EXIT_FAILURE` if any Admin or simulator cleanup operation fails. +- A cleanup failure must fail the TAP executable so the developer and CI can + see that the test did not leave a clean state. +- Store the Admin and simulator cleanup results separately. Log each failure + with an `Error:` diagnostic, continue with the remaining cleanup operations, + and return one combined result. +- Close the Admin connection immediately after its cleanup operation. Do not + skip connection closure because cleanup failed. +- Do not wait for probe quiescence during cleanup. +- After harness setup succeeds, failures must flow to a single + `exit_cleanup:` label in `main()`. That label always calls the local + `cleanup()` function. +- Declare the TAP plan before calling `setup()`. After `exit_cleanup:`, return + `EXIT_FAILURE` when cleanup fails; otherwise return `exit_status()`. A setup + failure is then reported as missing planned assertions instead of bypassing + TAP result handling. +- At `exit_cleanup:`, check `cleanup()` directly without a temporary result. + Return `EXIT_FAILURE` immediately when cleanup fails. +- Do not use `BAIL_OUT()` after test resources have been created or test state + has been changed. Log the failure, set the process result, and continue to + `exit_cleanup:`. +- Do not register global pointers or `atexit` handlers for test cleanup. +- A test assumes clean ProxySQL and simulator state at entry. Do not clear + state at the start of `setup()` or the test function. +- Scenario topology, ProxySQL configuration, mutations, waits, and TAP + assertions belong in the named test function. +- Keep `main()` limited to `plan()`, `setup()`, the named test, the + `exit_cleanup:` path, and the TAP exit status. + +## C++ Layout + +- Do not wrap TAP test files in an anonymous namespace. Each TAP file builds as + its own executable, so file-local namespace isolation is unnecessary. +- When phases share local state, name the struct `TestState`. Do not include + the test or file name in the local state type. +- Call the named test phases directly from `main()`. Do not add a wrapper test + function whose only job is to call the phases. +- The comment before each phase call in `main()` must make the phase + understandable without opening the function. State the concrete simulator + or ProxySQL configuration, the BGD status or server placement being + produced, and the observable result being verified. +- Do not write call-site comments that merely restate the function name or use + vague verbs such as "establish", "prepare", "handle", or "process". +- Keep call-site phase comments concise, but use exact BGD statuses, + hostgroups, tables, or server roles where they matter. +- Name simulator state helpers after the exact variable and value being set, + such as `set_writer_read_only_0()`. Avoid interpreted names such as + `set_writers_writable()`. +- Organize phase comments by the system being acted on, using labels such as + `Simulator:`, `ProxySQL:`, `Client:`, and `Verify:`. Include only the + applicable systems, and state the concrete action or expected result for + each one. +- Always use braces for `if` statements, including one-line bodies: + + ```cpp + if (condition) { + action(); + } + ``` + +- Keep complete function declarations, definitions, and calls on one line + whenever they remain readable. Use 120 characters as a guideline, not a + hard limit; prefer a small overrun to splitting a simple call across lines. + Wrap only when the complete statement is materially too long. +- Do not add `const` qualifiers to function parameters in these TAP files. + Prefer shorter, simpler signatures over strict const-correctness in + test-local helpers. +- Build SQL strings, expected-result text, and list arguments in named local + variables before calling a helper. Do not mix string concatenation, + temporary lists, and the function call in one statement. +- Format query helpers in four visible blocks separated by blank lines: + construct the query, construct related arguments, call the helper, and + return the stored result. +- When a helper call must wrap, group related arguments across as few lines as + possible and place the closing `);` on its own line. +- Keep return statements simple. Store a function result in a local variable + and return that variable instead of returning a large function call or a + heavily combined expression. +- Do not create duplicate query helpers that differ only by one expected + value. Pass that expected value as an argument to one clearly named helper. +- Do not place multiple function calls inside one `ok()` condition. Evaluate + each call into a clearly named local variable first, then combine those + boolean variables in the assertion. +- When a helper performs multiple function calls, execute them one at a time. + Return `EXIT_FAILURE` immediately after the specific call that fails, then + return `EXIT_SUCCESS` after all calls succeed. Do not combine calls with + `&&` and a ternary return. +- Apply the same sequential pattern inside test phases. Separate each + call-and-failure-check block with a blank line; do not conditionally invoke + later operations through ternary expressions. +- Before returning `EXIT_FAILURE` from setup, helper, phase, or `main()`, emit + a diagnostic beginning with `Error:` that names the failed operation. + Cleanup may log its individual failures before returning one combined + result. +- Shared condition and probe wait helpers must only return their result. They + must not dump timeout diagnostics; the calling phase logs the relevant + `Error:` message. +- Use `bgd_expect_no_table_check()` and `bgd_expect_no_metadata_probe()` for + negative probe checks instead of duplicating timeout logic in TAP files. + Pass the bounded timeout explicitly; these helpers return success only when + the unwanted probe wait returns `ETIMEDOUT`. +- Pass wait helpers only the values required to perform the wait. Do not pass + scenario names, phase names, expected text, probe sequences, or hostgroup + lists solely for generic diagnostics. +- Do not add `ok(false, ...)` to a failure-return path. Emit the `Error:` + diagnostic and return `EXIT_FAILURE`; the incomplete TAP plan will fail the + executable. +- Assertion messages and phase names must identify concrete BGD statuses, + hostgroups, tables, or server movements. Avoid undefined relational wording + such as "original definition", "current state", or "changed setup". +- In assertion messages, describe a runtime BGD status as + `BGD status for wHG ` instead of the longer + `runtime BGD row for writer hostgroup `. +- Separate environment loading, Admin connection, and simulator connection + into distinct blocks with blank lines. +- Separate setup, test execution, and cleanup calls in `main()` with blank + lines. +- Use `RDS_BGD_Cluster::get_endpoints()`, `get_blue_endpoints()`, and + `get_green_endpoints()` instead of rebuilding endpoint lists with reader + loops in individual tests. +- Keep positive condition and probe waits at three seconds or less. Use bounded + waits instead of fixed sleeps. diff --git a/test/tap/tap/rds_bgd_simulator.cpp b/test/tap/tap/rds_bgd_simulator.cpp index be18689b6b..5b53f0aad4 100644 --- a/test/tap/tap/rds_bgd_simulator.cpp +++ b/test/tap/tap/rds_bgd_simulator.cpp @@ -35,8 +35,23 @@ vector RDS_BGD_Cluster::get_writers() { return { blue_writer.endpoint(), green_writer.endpoint() }; } -vector RDS_BGD_Cluster::get_writer_hosts() { - return { blue_writer.host_endpoint(), green_writer.host_endpoint() }; +vector RDS_BGD_Cluster::get_blue_endpoints() { + vector endpoints { blue_writer.endpoint() }; + for (RDS_BGD_Host& host : blue_readers) endpoints.push_back(host.endpoint()); + return endpoints; +} + +vector RDS_BGD_Cluster::get_green_endpoints() { + vector endpoints { green_writer.endpoint() }; + for (RDS_BGD_Host& host : green_readers) endpoints.push_back(host.endpoint()); + return endpoints; +} + +vector RDS_BGD_Cluster::get_endpoints() { + vector endpoints = get_blue_endpoints(); + vector green_endpoints = get_green_endpoints(); + endpoints.insert(endpoints.end(), green_endpoints.begin(), green_endpoints.end()); + return endpoints; } vector RDS_BGD_Cluster::get_topology(string status) { @@ -120,6 +135,16 @@ int RDS_BGD_Simulator::topology_error(vector backends, int error_code, return execute_transaction(statements); } +int RDS_BGD_Simulator::cleanup() { + vector statements { + "DELETE FROM READONLY_STATUS", + "DELETE FROM RDS_BGD_TOPOLOGY", + "DELETE FROM RDS_BGD_CONTROL", + "DELETE FROM RDS_BGD_PROBE_LOG", + }; + return execute_transaction(statements); +} + rc_t RDS_BGD_Simulator::probe_log_last_sequence() { if (connection() == nullptr) { return { EXIT_FAILURE, 0 }; diff --git a/test/tap/tap/rds_bgd_simulator.h b/test/tap/tap/rds_bgd_simulator.h index cd3233df5c..2827d39435 100644 --- a/test/tap/tap/rds_bgd_simulator.h +++ b/test/tap/tap/rds_bgd_simulator.h @@ -66,11 +66,29 @@ class RDS_BGD_Cluster { vector get_writers(); /** - * @brief Returns both writer hostname/port endpoints for read-only simulation. + * @brief Returns the blue writer and configured blue readers. * - * @return Blue and green writer endpoints keyed by RDS hostname. + * @return Blue deployment endpoints keyed by simulator IP address. */ - vector get_writer_hosts(); + vector get_blue_endpoints(); + + /** + * @brief Returns the green writer and configured green readers. + * + * @return Green deployment endpoints keyed by simulator IP address. + */ + vector get_green_endpoints(); + + /** + * @brief Returns every simulator IP/port endpoint in this cluster. + * + * @details Includes both writers and all configured blue and green readers. + * Tests use this list when resetting or publishing topology for a complete + * simulated deployment. + * + * @return Writer and reader endpoints keyed by simulator IP address. + */ + vector get_endpoints(); /** * @brief Builds the topology rows published by the simulated writers. @@ -162,6 +180,13 @@ class RDS_BGD_Simulator : public Cluster_Simulator { */ int topology_error(vector backends, int error_code, string error_msg); + /** + * @brief Removes all read-only, topology-control, topology-row, and probe state. + * + * @return EXIT_SUCCESS when the simulator state is empty; EXIT_FAILURE otherwise. + */ + int cleanup(); + /** * @brief Reads the latest sequence from the RDS BGD probe log. * diff --git a/test/tap/tap/rds_bgd_tap.h b/test/tap/tap/rds_bgd_tap.h index abfe2118eb..8dd80847bf 100644 --- a/test/tap/tap/rds_bgd_tap.h +++ b/test/tap/tap/rds_bgd_tap.h @@ -2,6 +2,8 @@ #define TAP_TESTS_RDS_BGD_TAP_H #include +#include +#include #include #include @@ -10,8 +12,10 @@ using namespace std; +inline int execute_all(MYSQL* admin, vector queries); + inline RDS_BGD_Cluster bgd_cluster_init() { - return { + RDS_BGD_Cluster cluster { { "db-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.11", 3306 }, { "db-1-green-iqu47r.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.14", 3306 }, { @@ -23,12 +27,477 @@ inline RDS_BGD_Cluster bgd_cluster_init() { { "db-1-reader-2-green-3fpjuu.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.16", 3306 }, }, }; + return cluster; +} + +inline RDS_BGD_Cluster bgd_cluster_1_deployment_b_init() { + RDS_BGD_Cluster cluster { + { "db-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.11", 3306 }, + { "db-1-green-s7m2kx.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.17", 3306 }, + { + { "db-1-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.12", 3306 }, + { "db-1-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.13", 3306 }, + }, + { + { "db-1-reader-1-green-v4n8qp.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.18", 3306 }, + { "db-1-reader-2-green-w6h3rz.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.19", 3306 }, + }, + }; + return cluster; +} + +inline RDS_BGD_Cluster bgd_cluster_2_init() { + RDS_BGD_Cluster cluster { + { "db-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.20", 3306 }, + { "db-2-green-iqu47r.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.23", 3306 }, + { + { "db-2-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.21", 3306 }, + { "db-2-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.22", 3306 }, + }, + { + { "db-2-reader-1-green-dlzky7.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.24", 3306 }, + { "db-2-reader-2-green-3fpjuu.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.25", 3306 }, + }, + }; + return cluster; +} + +inline RDS_BGD_Cluster bgd_cluster_3_init() { + RDS_BGD_Cluster cluster { + { "db-3.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.26", 3306 }, + { "db-3-green-iqu47r.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.29", 3306 }, + { + { "db-3-reader-1.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.27", 3306 }, + { "db-3-reader-2.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.28", 3306 }, + }, + { + { "db-3-reader-1-green-dlzky7.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.30", 3306 }, + { "db-3-reader-2-green-3fpjuu.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com", "127.10.0.31", 3306 }, + }, + }; + return cluster; +} + +enum class BGD_Admin_Mode { + automatic, + explicit_configuration, +}; + +struct BGD_Hostgroups { + int blue_writer; + int blue_reader; + int green_writer; + int green_reader; +}; + +inline vector bgd_topology_with_readers(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + for (RDS_BGD_Host& host : cluster.blue_readers) { + rows.push_back({ host.hostname, host.hostname, host.port, "BLUE_GREEN_DEPLOYMENT_SOURCE", status }); + } + for (RDS_BGD_Host& host : cluster.green_readers) { + rows.push_back({ host.hostname, host.hostname, host.port, "BLUE_GREEN_DEPLOYMENT_TARGET", status }); + } + return rows; +} + +inline int bgd_set_writer_read_only_0(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (sim.read_only_update(cluster.blue_writer.host_endpoint(), false) != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated blue writer"); + return EXIT_FAILURE; + } + + if (sim.read_only_update(cluster.green_writer.host_endpoint(), false) != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated green writer"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +inline int bgd_set_host_read_only_0(RDS_BGD_Simulator& sim, RDS_BGD_Host& host) { + int rc = sim.read_only_update(host.host_endpoint(), false); + return rc; +} + +inline int bgd_set_host_read_only_1(RDS_BGD_Simulator& sim, RDS_BGD_Host& host) { + int rc = sim.read_only_update(host.host_endpoint(), true); + return rc; +} + +inline string bgd_sql_quote(string value) { + string quoted { "'" }; + for (char c : value) { + quoted += c; + if (c == '\'') { + quoted += '\''; + } + } + quoted += '\''; + return quoted; +} + +inline int bgd_admin_cleanup(MYSQL* admin) { + vector config_queries { + "SET mysql-aws_blue_green_deployment_auto_discovery='false'", + "LOAD MYSQL VARIABLES TO RUNTIME", + "DELETE FROM mysql_aws_rds_bgd_hostgroups", + "LOAD MYSQL SERVERS TO RUNTIME", + }; + int config_rc = execute_all(admin, config_queries); + + vector state_queries { + "DELETE FROM mysql_servers", + "DELETE FROM mysql_replication_hostgroups", + "UPDATE mysql_users SET default_hostgroup=0 WHERE username='testuser'", + "LOAD MYSQL SERVERS TO RUNTIME", + "LOAD MYSQL USERS TO RUNTIME", + }; + int state_rc = execute_all(admin, state_queries); + + if (config_rc != EXIT_SUCCESS || state_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +inline int bgd_admin_add_servers( + MYSQL* admin, RDS_BGD_Cluster cluster, BGD_Hostgroups hostgroups, + vector hosts, bool green, int use_ssl) +{ + vector queries {}; + for (RDS_BGD_Host& host : hosts) { + int hostgroup = hostgroups.blue_reader; + if (!green && host.hostname == cluster.blue_writer.hostname) { + hostgroup = hostgroups.blue_writer; + } else if (green && host.hostname == cluster.green_writer.hostname) { + hostgroup = hostgroups.green_writer; + } else if (green) { + hostgroup = hostgroups.green_reader; + } + + string color = green ? "green " : "blue "; + string comment = bgd_sql_quote("BGD TAP " + color + host.ip); + string query = + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + + to_string(hostgroup) + "," + bgd_sql_quote(host.hostname) + "," + + to_string(host.port) + ",'ONLINE'," + to_string(use_ssl) + "," + + comment + ")"; + queries.push_back(query); + } + + int rc = execute_all(admin, queries); + return rc; +} + +inline int bgd_admin_setup( + MYSQL* admin, RDS_BGD_Cluster cluster, BGD_Hostgroups hostgroups, + BGD_Admin_Mode mode, vector blue_hosts, + vector green_hosts = {}, int blue_use_ssl = 0, int green_use_ssl = 0) +{ + string auto_discovery = mode == BGD_Admin_Mode::automatic ? "true" : "false"; + vector queries { + "INSERT INTO mysql_replication_hostgroups(writer_hostgroup,reader_hostgroup) VALUES (" + + to_string(hostgroups.blue_writer) + "," + to_string(hostgroups.blue_reader) + ")", + "SET mysql-monitor_username='testuser'", + "SET mysql-monitor_password='testuser'", + "SET mysql-monitor_enabled='true'", + "SET mysql-monitor_read_only_interval=100", + "SET mysql-monitor_aws_rds_topology_discovery_interval=1", + "SET mysql-aws_blue_green_deployment_auto_discovery='" + auto_discovery + "'", + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroups.blue_writer) + + " WHERE username='testuser'", + }; + + if (mode == BGD_Admin_Mode::explicit_configuration) { + string bgd_query = + "INSERT INTO mysql_aws_rds_bgd_hostgroups(" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment) VALUES (" + + to_string(hostgroups.blue_writer) + "," + to_string(hostgroups.blue_reader) + "," + + to_string(hostgroups.green_writer) + "," + to_string(hostgroups.green_reader) + + ",1,0,100,800,'BGD TAP explicit configuration')"; + queries.push_back(bgd_query); + } + + int config_rc = execute_all(admin, queries); + if (config_rc != EXIT_SUCCESS) { + diag("Error: failed to configure ProxySQL BGD variables and hostgroups"); + return EXIT_FAILURE; + } + + int blue_rc = bgd_admin_add_servers(admin, cluster, hostgroups, blue_hosts, false, blue_use_ssl); + if (blue_rc != EXIT_SUCCESS) { + diag("Error: failed to configure blue servers"); + return EXIT_FAILURE; + } + + int green_rc = bgd_admin_add_servers(admin, cluster, hostgroups, green_hosts, true, green_use_ssl); + if (green_rc != EXIT_SUCCESS) { + diag("Error: failed to configure green servers"); + return EXIT_FAILURE; + } + + vector load_queries { + "LOAD MYSQL VARIABLES TO RUNTIME", + "LOAD MYSQL USERS TO RUNTIME", + "LOAD MYSQL SERVERS TO RUNTIME", + }; + int load_rc = execute_all(admin, load_queries); + if (load_rc != EXIT_SUCCESS) { + diag("Error: failed to load ProxySQL BGD configuration to runtime"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +inline rc_t> bgd_runtime_rows(MYSQL* admin, int writer_hostgroup) { + string query = + "SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "auto_generated,status FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hostgroup); + + rc_t> result = mysql_query_ext_rows(admin, query); + return result; +} + +inline rc_t> bgd_runtime_servers(MYSQL* admin, vector hostgroups) { + string predicate {}; + for (size_t i = 0; i < hostgroups.size(); ++i) { + if (i != 0) { + predicate += ","; + } + predicate += to_string(hostgroups[i]); + } + + string query = + "SELECT hostgroup_id,hostname,port,status,use_ssl FROM runtime_mysql_servers WHERE hostgroup_id IN (" + + predicate + ") ORDER BY hostgroup_id,hostname,port"; + + rc_t> result = mysql_query_ext_rows(admin, query); + return result; +} + +inline rc_t bgd_connection_pool_count(MYSQL* admin, int hostgroup, string hostname = "") { + string query = + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0) FROM stats_mysql_connection_pool WHERE hostgroup=" + + to_string(hostgroup); + if (!hostname.empty()) { + query += " AND srv_host=" + bgd_sql_quote(hostname); + } + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + rc_t result { EXIT_FAILURE, 0 }; + return result; + } + + int64_t count = strtoll(rows[0][0].c_str(), nullptr, 10); + rc_t result { EXIT_SUCCESS, count }; + return result; +} + +inline rc_t bgd_backend_ip_echo(MYSQL* proxy) { + string query = "SELECT @@version_comment LIMIT 1"; + + auto [rc, rows] = mysql_query_ext_rows(proxy, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + rc_t result { EXIT_FAILURE, {} }; + return result; + } + + rc_t result { EXIT_SUCCESS, rows[0][0] }; + return result; +} + +inline rc_t bgd_probe_count_since( + RDS_BGD_Simulator& sim, uint64_t sequence, Endpoint backend, RDS_BGD_Probe_Kind kind) +{ + auto [rc, logs] = sim.probe_log_since(sequence); + if (rc != EXIT_SUCCESS) { + rc_t result { EXIT_FAILURE, 0 }; + return result; + } + + uint64_t count = 0; + for (const RDS_BGD_Probe_Log& log : logs) { + bool backend_matches = + log.backend.host == backend.host && + log.backend.port == backend.port; + bool kind_matches = log.probe_kind == kind; + if (backend_matches && kind_matches) { + ++count; + } + } + + rc_t result { EXIT_SUCCESS, count }; + return result; +} + +inline int bgd_wait_for_condition(MYSQL* admin, string query, uint32_t timeout_seconds) { + int rc = wait_for_cond(admin, query, timeout_seconds); + return rc; +} + +inline int bgd_wait_for_status(MYSQL* admin, BGD_Hostgroups& hostgroups, string status, uint32_t timeout_seconds) { + string query = + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(hostgroups.blue_writer) + " AND status=" + bgd_sql_quote(status); + + int rc = bgd_wait_for_condition(admin, query, timeout_seconds); + return rc; +} + +inline int bgd_wait_for_server_placement( + MYSQL* admin, int writer_hostgroup, int reader_hostgroup, RDS_BGD_Host& host, + bool in_reader_hostgroup, uint32_t timeout_seconds) +{ + string writer_count = in_reader_hostgroup ? "0" : "1"; + string reader_count = in_reader_hostgroup ? "1" : "0"; + + string query = "SELECT " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(writer_hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + ")=" + + writer_count + " AND " + + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(reader_hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + ")=" + + reader_count; + + int rc = bgd_wait_for_condition(admin, query, timeout_seconds); + return rc; +} + +inline rc_t bgd_wait_for_probe_from_backends( + RDS_BGD_Simulator& sim, uint64_t sequence, vector backends, + RDS_BGD_Probe_Kind kind, uint32_t timeout_ms, int encrypted = -1) +{ + uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; + do { + auto [rc, logs] = sim.probe_log_since(sequence); + if (rc != EXIT_SUCCESS) { + rc_t result { EXIT_FAILURE, {} }; + return result; + } + + for (const RDS_BGD_Probe_Log& log : logs) { + for (const Endpoint& backend : backends) { + bool backend_matches = + log.backend.host == backend.host && + log.backend.port == backend.port; + bool kind_matches = log.probe_kind == kind; + bool encryption_matches = + encrypted < 0 || + log.encrypted == (encrypted != 0); + if (backend_matches && kind_matches && encryption_matches) { + rc_t result { EXIT_SUCCESS, log }; + return result; + } + } + } + + usleep(50000); + } while (monotonic_time() < deadline); + + rc_t result { ETIMEDOUT, {} }; + return result; +} + +/** + * Verify that a configuration change does not restart BGD discovery. + * + * The expected result is ETIMEDOUT because no table-check probe should appear + * after the given sequence. + */ +inline int bgd_expect_no_table_check( + RDS_BGD_Simulator& sim, uint64_t sequence, vector backends, uint32_t timeout_ms) +{ + auto [probe_rc, probe] = bgd_wait_for_probe_from_backends( + sim, sequence, backends, RDS_BGD_Probe_Kind::table_check, timeout_ms + ); + + if (probe_rc == ETIMEDOUT) { + return EXIT_SUCCESS; + } + return EXIT_FAILURE; +} + +/** + * Verify that one endpoint does not receive metadata probes. + * + * The expected result is ETIMEDOUT because no metadata probe should reach the + * endpoint after the given sequence. + */ +inline int bgd_expect_no_metadata_probe( + RDS_BGD_Simulator& sim, uint64_t sequence, Endpoint backend, uint32_t timeout_ms) +{ + vector backends { backend }; + + auto [probe_rc, probe] = bgd_wait_for_probe_from_backends( + sim, sequence, backends, RDS_BGD_Probe_Kind::metadata, timeout_ms + ); + + if (probe_rc == ETIMEDOUT) { + return EXIT_SUCCESS; + } + return EXIT_FAILURE; +} + +/** + * Verify that none of the supplied endpoints receives a metadata probe. + * + * The expected result is ETIMEDOUT because no metadata probe should reach any + * endpoint after the given sequence. + */ +inline int bgd_expect_no_metadata_probe_from_backends( + RDS_BGD_Simulator& sim, uint64_t sequence, vector backends, uint32_t timeout_ms) +{ + auto [probe_rc, probe] = bgd_wait_for_probe_from_backends( + sim, sequence, backends, RDS_BGD_Probe_Kind::metadata, timeout_ms + ); + + if (probe_rc == ETIMEDOUT) { + return EXIT_SUCCESS; + } + return EXIT_FAILURE; +} + +/** + * Verify that read_only monitoring remains suppressed for the full observation window. + * + * The helper fails immediately if a new read_only log row appears after the + * supplied baseline. + */ +inline int bgd_expect_no_read_only_log(MYSQL* admin, RDS_BGD_Host& host, int64_t baseline, uint32_t timeout_ms) { + if (baseline < 0) { + return EXIT_FAILURE; + } + + uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; + do { + string query = + "SELECT COUNT(*) FROM mysql_server_read_only_log WHERE hostname=" + + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + + " AND time_start_us>" + to_string(baseline); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return EXIT_FAILURE; + } + + if (rows[0][0] != "0") { + return EXIT_FAILURE; + } + + usleep(50000); + } while (monotonic_time() < deadline); + + return EXIT_SUCCESS; } inline int execute_all(MYSQL* admin, vector queries) { for (string& query : queries) { if (mysql_query(admin, query.c_str()) != 0) { - diag("Admin query failed (%u): %s; query: %s", mysql_errno(admin), mysql_error(admin), query.c_str()); + diag("Error: Admin query failed (%u): %s; query: %s", + mysql_errno(admin), mysql_error(admin), query.c_str()); return EXIT_FAILURE; } } From 84e416c7a72fba7e546efd060dbe056260c8033e Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 28 Jul 2026 07:07:23 +0000 Subject: [PATCH 66/81] test: modernize RDS BGD smoke coverage - Use the approved local setup, cleanup, phase, and failure-handling structure. - Retain the existing smoke coverage with bounded waits and readable assertions. Signed-off-by: Wazir Ahmed --- test/tap/tests/test_rds_bgd_smoke-t.cpp | 248 +++++++++++++++++------- 1 file changed, 179 insertions(+), 69 deletions(-) diff --git a/test/tap/tests/test_rds_bgd_smoke-t.cpp b/test/tap/tests/test_rds_bgd_smoke-t.cpp index eb7ce5f971..8b72573e6b 100644 --- a/test/tap/tests/test_rds_bgd_smoke-t.cpp +++ b/test/tap/tests/test_rds_bgd_smoke-t.cpp @@ -1,108 +1,218 @@ /** * @file test_rds_bgd_smoke-t.cpp - * @brief Smoke test for TAP-controlled AWS RDS BGD simulation. + * @brief Explicitly configured BGD worker reaching AVAILABLE and probing the green writer. * - * Test steps: - * 1. Connect to ProxySQL Admin and the SQLite3-server simulator. - * 2. Configure both simulated writers as writable. - * 3. Publish an AVAILABLE topology on the blue and green writer IPs. - * 4. Configure ProxySQL with the blue writer and BGD hostgroups. - * 5. Verify that ProxySQL reaches AVAILABLE and probes the green writer IP. + * Steps: + * + * 1. Set read_only=0 for the blue and green writers and publish AVAILABLE topology. + * 2. Configure BGD hostgroups 10-40 with the blue writer in hostgroup 10. + * 3. Verify BGD status AVAILABLE and a plaintext metadata probe to the green writer. */ +#include #include #include #include -#include "rds_bgd_tap.h" #include "command_line.h" +#include "rds_bgd_tap.h" #include "utils.h" -int configure_proxysql_for_bgd(MYSQL* admin, RDS_BGD_Cluster& cluster) { - RDS_BGD_Host& writer = cluster.blue_writer; - return execute_all(admin, { - "DELETE FROM mysql_servers", - "DELETE FROM mysql_replication_hostgroups", - "DELETE FROM mysql_aws_rds_bgd_hostgroups", - "INSERT INTO mysql_replication_hostgroups(writer_hostgroup,reader_hostgroup) " - "VALUES (10,20)", +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; + +struct TestState { + RDS_BGD_Cluster cluster { bgd_cluster_init() }; + BGD_Hostgroups hostgroups { 10, 20, 30, 40 }; + vector topology_endpoints { cluster.get_writers() }; + uint64_t probe_sequence { 0 }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int configure_explicit_bgd(MYSQL* admin, TestState& state) { + RDS_BGD_Host& writer = state.cluster.blue_writer; + BGD_Hostgroups& hg = state.hostgroups; + + string add_replication_hostgroups = + "INSERT INTO mysql_replication_hostgroups(writer_hostgroup,reader_hostgroup) VALUES (" + + to_string(hg.blue_writer) + "," + to_string(hg.blue_reader) + ")"; + string add_bgd_hostgroups = "INSERT INTO mysql_aws_rds_bgd_hostgroups(" - "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," - "active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment) " - "VALUES (10,20,30,40,1,0,100,800,'BGD simulator smoke test')", - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,use_ssl,comment) VALUES (10,'" + - writer.hostname + "'," + std::to_string(writer.port) + ",0,'blue writer')", + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment) VALUES (" + + to_string(hg.blue_writer) + "," + to_string(hg.blue_reader) + "," + + to_string(hg.green_writer) + "," + to_string(hg.green_reader) + + ",1,0,100,800,'BGD simulator smoke test')"; + string add_blue_writer = + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,use_ssl,comment) VALUES (" + + to_string(hg.blue_writer) + "," + bgd_sql_quote(writer.hostname) + "," + + to_string(writer.port) + ",0,'blue writer')"; + vector queries { + add_replication_hostgroups, + add_bgd_hostgroups, + add_blue_writer, "SET mysql-monitor_username='testuser'", "SET mysql-monitor_password='testuser'", "SET mysql-monitor_enabled='true'", + "SET mysql-aws_blue_green_deployment_auto_discovery='false'", "LOAD MYSQL VARIABLES TO RUNTIME", "LOAD MYSQL SERVERS TO RUNTIME", - }); + }; + + int rc = execute_all(admin, queries); + return rc; } -int main() { - plan(3); +/** + * Publish AVAILABLE topology for writable blue and green writers. + * + * - Set read_only=0 for both simulated writers. + * - Record the probe sequence before publishing topology. + * - Publish AVAILABLE topology to the blue and green writer endpoints. + */ +int publish_available_topology(RDS_BGD_Simulator& sim, TestState& state) { + int writer_rc = bgd_set_writer_read_only_0(sim, state.cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated writers"); + return EXIT_FAILURE; + } - CommandLine cl {}; - if (cl.getEnv()) { - BAIL_OUT("failed to load TAP environment"); + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before publishing AVAILABLE topology"); + return EXIT_FAILURE; } + state.probe_sequence = seq; - MYSQL* admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); - if (admin == nullptr) { - BAIL_OUT("failed to connect to ProxySQL Admin"); + vector topology = state.cluster.get_topology("AVAILABLE"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology"); + return EXIT_FAILURE; } - RDS_BGD_Simulator sim {}; - if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { - mysql_close(admin); - BAIL_OUT("failed to connect to the SQLite3-server simulator"); + ok(true, "simulator publishes AVAILABLE topology to the blue and green writers"); + return EXIT_SUCCESS; +} + +/** + * Configure an explicit BGD worker for writer hostgroup 10. + * + * - Insert mysql_replication_hostgroups and mysql_aws_rds_bgd_hostgroups rows. + * - Insert the blue writer in mysql_servers hostgroup 10. + * - Verify that the runtime BGD status reaches AVAILABLE. + */ +int configure_bgd_available(MYSQL* admin, TestState& state) { + int config_rc = configure_explicit_bgd(admin, state); + if (config_rc != EXIT_SUCCESS) { + diag("Error: failed to configure mysql_servers and mysql_aws_rds_bgd_hostgroups"); + return EXIT_FAILURE; } - // Initialize the test cluster and make both simulated writers writable. - RDS_BGD_Cluster cluster = bgd_cluster_init(); - for (Endpoint& writer : cluster.get_writer_hosts()) { - if (sim.read_only_update(writer, false) != EXIT_SUCCESS) { - mysql_close(admin); - BAIL_OUT("failed to configure writer read_only state"); - } + int status_rc = bgd_wait_for_status(admin, state.hostgroups, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 10 did not reach AVAILABLE"); + return EXIT_FAILURE; } - // Record the last probe sequence before enabling BGD monitoring. - auto [rc, last_seq] = sim.probe_log_last_sequence(); - if (rc != EXIT_SUCCESS) { - mysql_close(admin); - BAIL_OUT("failed to read the last BGD probe-log sequence"); + ok(true, "BGD status for wHG 10 reports AVAILABLE"); + return EXIT_SUCCESS; +} + +/** + * Verify the AVAILABLE worker probes the green writer. + * + * - Wait for a metadata probe after the topology publication sequence. + * - Require the probe on the green writer IP without TLS. + */ +int test_plaintext_green_writer_probe(RDS_BGD_Simulator& sim, TestState& state) { + auto [probe_rc, probe] = sim.wait_for_probe_log( + state.probe_sequence, state.cluster.green_writer.endpoint(), + RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + ); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: green writer did not receive a plaintext metadata probe"); + return EXIT_FAILURE; } - // Publish the AVAILABLE topology on both simulated writer IPs. - rc = sim.topology_update(cluster.get_writers(), cluster.get_topology("AVAILABLE")); - ok(rc == EXIT_SUCCESS, "publish AVAILABLE topology to both writer IPs"); - if (rc != EXIT_SUCCESS) { - mysql_close(admin); - BAIL_OUT("failed to publish BGD topology"); + ok(true, "BGD worker probes the green writer IP over plaintext"); + return EXIT_SUCCESS; +} + +int main() { + plan(3); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); } - // Configure ProxySQL with the blue writer and BGD hostgroups. - if (configure_proxysql_for_bgd(admin, cluster) != EXIT_SUCCESS) { - mysql_close(admin); - BAIL_OUT("failed to configure ProxySQL for BGD monitoring"); + TestState state {}; + + // Simulator: set blue/green writer read_only=0 and publish AVAILABLE topology. + // Verify: topology publication succeeds for both writer endpoints. + if (publish_available_topology(sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; } - // Wait for topology discovery to place the BGD hostgroups in AVAILABLE. - rc = wait_for_cond( - admin, - "SELECT COUNT(*)=1 FROM runtime_mysql_aws_rds_bgd_hostgroups " - "WHERE writer_hostgroup=10 AND status='AVAILABLE'", - 3); - ok(rc == EXIT_SUCCESS, "ProxySQL enters the AVAILABLE BGD state"); + // ProxySQL: configure mysql_servers and mysql_aws_rds_bgd_hostgroups for wHG 10. + // Verify: BGD status for wHG 10 reports AVAILABLE. + if (configure_bgd_available(admin, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } - // Verify that ProxySQL probes metadata directly on the green writer IP. - auto [probe_rc, green_probe] = sim.wait_for_probe_log( - last_seq, cluster.green_writer.endpoint(), - RDS_BGD_Probe_Kind::metadata, 3000, 0); - ok(probe_rc == EXIT_SUCCESS, "ProxySQL probes topology directly on the green writer IP over plaintext"); + // ProxySQL: run the explicitly configured BGD worker without TLS. + // Verify: the green writer IP receives a plaintext metadata probe. + if (test_plaintext_green_writer_probe(sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } - mysql_close(admin); +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } return exit_status(); } From 0471cd6f23937f24354d3b99388a174dc59eaa4d Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 28 Jul 2026 07:08:30 +0000 Subject: [PATCH 67/81] test: cover RDS BGD discovery and configuration - Cover automatic discovery, explicit startup, and persistent configuration behavior. - Verify green membership ordering and the TLS mode used by BGD metadata probes. - Register and lint the focused discovery and configuration tests. Signed-off-by: Wazir Ahmed --- test/tap/groups/groups.json | 5 + .../test_rds_bgd_automatic_discovery-t.cpp | 326 +++++++++ ...st_rds_bgd_configuration_persistence-t.cpp | 650 ++++++++++++++++++ .../tests/test_rds_bgd_explicit_startup-t.cpp | 359 ++++++++++ ...st_rds_bgd_green_membership_ordering-t.cpp | 421 ++++++++++++ test/tap/tests/test_rds_bgd_probe_tls-t.cpp | 459 +++++++++++++ 6 files changed, 2220 insertions(+) create mode 100644 test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp create mode 100644 test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp create mode 100644 test/tap/tests/test_rds_bgd_explicit_startup-t.cpp create mode 100644 test/tap/tests/test_rds_bgd_green_membership_ordering-t.cpp create mode 100644 test/tap/tests/test_rds_bgd_probe_tls-t.cpp diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index e0a29a010f..bbabf99245 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -430,6 +430,11 @@ "test_query_rules_fast_routing_algorithm-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g4","mysql90-g4","mysql95-g4" ], "test_query_rules_routing-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g4","mysql90-g4","mysql95-g4" ], "test_query_timeout-t" : [ "legacy-g9","mariadb10-galera-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql84-gr-g9","mysql90-g4","mysql95-g4" ], + "test_rds_bgd_automatic_discovery-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_configuration_persistence-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_explicit_startup-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_green_membership_ordering-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_probe_tls-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_smoke-t" : [ "cluster_sim_rds_bgd-g1" ], "test_read_only_actions_offline_hard_servers-t" : [ "legacy-g5","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g5","mysql84-g9","mysql90-g4","mysql90-g5","mysql95-g4","mysql95-g5" ], "test_rw_binary_data-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql90-g4","mysql95-g4" ], diff --git a/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp b/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp new file mode 100644 index 0000000000..30c30c3be2 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp @@ -0,0 +1,326 @@ +/** + * @file test_rds_bgd_automatic_discovery-t.cpp + * @brief Automatic BGD row creation from AVAILABLE topology. + * + * Steps: + * + * 1. Publish AVAILABLE topology before loading blue hostgroups 810 and 811. + * 2. Verify one runtime-only BGD row with derived blue hostgroups and NULL + * green hostgroups. + * 3. Load blue hostgroups 820 and 821 while topology is absent. + * 4. Verify no BGD row exists until AVAILABLE topology is published. + * 5. Verify repeated discovery keeps one runtime-only BGD row. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; + +struct TestState { + RDS_BGD_Cluster topology_first { bgd_cluster_init() }; + RDS_BGD_Cluster absent_first { bgd_cluster_2_init() }; + BGD_Hostgroups topology_first_hg { 810, 811, 812, 813 }; + BGD_Hostgroups absent_first_hg { 820, 821, 822, 823 }; + vector topology_first_endpoints { topology_first.get_endpoints() }; + vector absent_first_endpoints { absent_first.get_endpoints() }; + uint64_t absent_available_sequence { 0 }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +bool runtime_auto_row_matches(MYSQL* admin, BGD_Hostgroups& hg) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(hg.blue_writer) + " AND reader_hostgroup=" + to_string(hg.blue_reader) + + " AND green_writer_hostgroup IS NULL AND green_reader_hostgroup IS NULL AND auto_generated=1"; + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool matches = rows[0][0] == "1"; + return matches; +} + +bool runtime_bgd_row_absent(MYSQL* admin, int writer_hostgroup) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hostgroup); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool absent = rows[0][0] == "0"; + return absent; +} + +bool persistent_bgd_row_absent(MYSQL* admin, int writer_hostgroup) { + string query = + "SELECT COUNT(*) FROM mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hostgroup); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool absent = rows[0][0] == "0"; + return absent; +} + +bool runtime_bgd_row_count_matches(MYSQL* admin, int writer_hostgroup, int expected_count) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hostgroup); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool matches = rows[0][0] == to_string(expected_count); + return matches; +} + +/** + * Discover a deployment whose AVAILABLE topology exists before its blue writer. + * + * - Set read_only=0 for both simulated writers. + * - Publish AVAILABLE topology before loading blue hostgroups 810 and 811. + * - Verify one auto-generated runtime row with NULL green hostgroups. + * - Verify automatic discovery does not create a persistent BGD row. + */ +int test_topology_before_writer(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.topology_first; + BGD_Hostgroups& hg = state.topology_first_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure topology-first simulated writers"); + return EXIT_FAILURE; + } + + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.topology_first_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish topology-first AVAILABLE topology"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer }; + vector green_servers {}; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::automatic, blue_servers, green_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure automatic discovery for blue hostgroups 810 and 811"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 810 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + bool row_matches = runtime_auto_row_matches(admin, hg); + ok(row_matches, "automatic discovery derives hostgroups 810 and 811 with NULL green hostgroups"); + + bool persistent_absent = persistent_bgd_row_absent(admin, hg.blue_writer); + ok(persistent_absent, "automatic discovery keeps wHG 810 out of mysql_aws_rds_bgd_hostgroups"); + return EXIT_SUCCESS; +} + +/** + * Start automatic discovery while topology is absent. + * + * - Set read_only=0 for both simulated writers. + * - Remove topology for blue hostgroups 820 and 821. + * - Load the blue writer and wait for the absent-table metadata probe. + * - Verify no runtime or persistent BGD row is created. + * - Publish AVAILABLE topology and verify automatic row creation. + */ +int test_topology_absent_then_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.absent_first; + BGD_Hostgroups& hg = state.absent_first_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure topology-absent simulated writers"); + return EXIT_FAILURE; + } + + int drop_rc = sim.topology_drop(state.absent_first_endpoints); + if (drop_rc != EXIT_SUCCESS) { + diag("Error: failed to publish absent topology for blue hostgroups 820 and 821"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before topology-absent discovery"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers {}; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::automatic, blue_servers, green_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure automatic discovery for blue hostgroups 820 and 821"); + return EXIT_FAILURE; + } + + auto [absent_probe_rc, absent_probe] = + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (absent_probe_rc != EXIT_SUCCESS) { + diag("Error: automatic discovery did not issue the absent-table metadata probe"); + return EXIT_FAILURE; + } + + bool runtime_absent = runtime_bgd_row_absent(admin, hg.blue_writer); + bool persistent_absent = persistent_bgd_row_absent(admin, hg.blue_writer); + ok(runtime_absent && persistent_absent, "absent topology creates no runtime or persistent BGD row for wHG 820"); + + auto [available_seq_rc, available_seq] = sim.probe_log_last_sequence(); + if (available_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before AVAILABLE topology"); + return EXIT_FAILURE; + } + + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.absent_first_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 820"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 820 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + auto [green_probe_rc, green_probe] = + sim.wait_for_probe_log(available_seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (green_probe_rc != EXIT_SUCCESS) { + diag("Error: wHG 820 did not probe the AVAILABLE green writer"); + return EXIT_FAILURE; + } + + state.absent_available_sequence = green_probe.sequence_id; + bool row_matches = runtime_auto_row_matches(admin, hg); + ok(row_matches, "AVAILABLE topology creates the derived automatic BGD row for wHG 820"); + return EXIT_SUCCESS; +} + +/** + * Observe steady metadata polling after automatic discovery reaches AVAILABLE. + * + * - Wait for another green-writer metadata probe. + * - Verify runtime contains one auto-generated BGD row. + * - Verify the automatic row remains absent from persistent configuration. + */ +int test_repeated_discovery(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.absent_first; + BGD_Hostgroups& hg = state.absent_first_hg; + + auto [probe_rc, probe] = sim.wait_for_probe_log( + state.absent_available_sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: automatic wHG 820 did not continue green-writer metadata polling"); + return EXIT_FAILURE; + } + + bool one_runtime_row = runtime_bgd_row_count_matches(admin, hg.blue_writer, 1); + bool persistent_absent = persistent_bgd_row_absent(admin, hg.blue_writer); + ok(one_runtime_row && persistent_absent, "steady metadata polling keeps one runtime-only BGD row for wHG 820"); + return EXIT_SUCCESS; +} + +int main() { + plan(5); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish AVAILABLE topology before configuring blue hostgroups 810 and 811. + // ProxySQL: enable automatic discovery and load only the blue writer. + // Verify: one runtime-only auto-generated row uses derived blue hostgroups and NULL green hostgroups. + if (test_topology_before_writer(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish absent topology, then AVAILABLE topology for the second deployment. + // ProxySQL: load blue hostgroups 820 and 821 while topology is absent. + // Verify: no row exists while absent; AVAILABLE creates one auto-generated runtime row. + if (test_topology_absent_then_available(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: allow another table-check for the AVAILABLE deployment. + // Verify: repeated discovery keeps one runtime-only BGD row for wHG 820. + if (test_repeated_discovery(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp b/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp new file mode 100644 index 0000000000..7638e03c0a --- /dev/null +++ b/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp @@ -0,0 +1,650 @@ +/** + * @file test_rds_bgd_configuration_persistence-t.cpp + * @brief BGD runtime and persistent configuration ownership. + * + * Steps: + * + * 1. Convert an auto-generated row for wHG 890 into explicit configuration. + * 2. Verify persistent BGD rows reject NULL green hostgroups and accept a + * complete row. + * 3. SAVE runtime BGD state and verify only the explicit row is persisted. + * 4. Run automatic discovery beside administrator-owned configuration and + * verify its BGD row and green-server status remain unchanged. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; + +struct TestState { + RDS_BGD_Cluster conversion { bgd_cluster_2_init() }; + RDS_BGD_Cluster explicit_save { bgd_cluster_3_init() }; + RDS_BGD_Cluster automatic_save { bgd_cluster_1_deployment_b_init() }; + RDS_BGD_Cluster admin_owned { bgd_cluster_init() }; + BGD_Hostgroups conversion_hg { 890, 891, 892, 893 }; + BGD_Hostgroups valid_hg { 910, 911, 912, 913 }; + BGD_Hostgroups explicit_save_hg { 920, 921, 922, 923 }; + BGD_Hostgroups automatic_save_hg { 930, 931, 932, 933 }; + BGD_Hostgroups admin_owned_hg { 1310, 1311, 1312, 1313 }; + vector conversion_endpoints { conversion.get_endpoints() }; + vector explicit_save_endpoints { explicit_save.get_endpoints() }; + vector automatic_save_endpoints { automatic_save.get_endpoints() }; + vector admin_owned_endpoints { admin_owned.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int configure_monitor(MYSQL* admin, BGD_Hostgroups& hg, bool automatic) { + string automatic_value = automatic ? "true" : "false"; + vector queries { + "INSERT INTO mysql_replication_hostgroups(writer_hostgroup,reader_hostgroup) VALUES (" + + to_string(hg.blue_writer) + "," + to_string(hg.blue_reader) + ")", + "SET mysql-monitor_username='testuser'", + "SET mysql-monitor_password='testuser'", + "SET mysql-monitor_enabled='true'", + "SET mysql-monitor_read_only_interval=100", + "SET mysql-monitor_aws_rds_topology_discovery_interval=1", + "SET mysql-aws_blue_green_deployment_auto_discovery='" + automatic_value + "'", + "LOAD MYSQL VARIABLES TO RUNTIME", + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +int insert_explicit_bgd_row(MYSQL* admin, BGD_Hostgroups& hg, string comment, int active = 1) { + string query = + "INSERT INTO mysql_aws_rds_bgd_hostgroups(" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment) VALUES (" + + to_string(hg.blue_writer) + "," + to_string(hg.blue_reader) + "," + + to_string(hg.green_writer) + "," + to_string(hg.green_reader) + "," + + to_string(active) + ",0,100,800," + bgd_sql_quote(comment) + ")"; + + int rc = mysql_query(admin, query.c_str()); + if (rc != 0) { + diag("Error: failed to insert mysql_aws_rds_bgd_hostgroups row for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int add_all_servers(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + int blue_rc = bgd_admin_add_servers(admin, cluster, hg, blue_servers, false, 0); + if (blue_rc != EXIT_SUCCESS) { + diag("Error: failed to add blue servers for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + vector green_servers { cluster.green_writer, cluster.green_readers[0], cluster.green_readers[1] }; + int green_rc = bgd_admin_add_servers(admin, cluster, hg, green_servers, true, 0); + if (green_rc != EXIT_SUCCESS) { + diag("Error: failed to add green servers for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + vector load_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_rc = execute_all(admin, load_queries); + if (load_rc != EXIT_SUCCESS) { + diag("Error: failed to load servers for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +bool persistent_bgd_row_matches(MYSQL* admin, BGD_Hostgroups& hg) { + string query = + "SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup " + "FROM mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + to_string(hg.blue_writer); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 4) { + return false; + } + + bool matches = + rows[0][0] == to_string(hg.blue_writer) && + rows[0][1] == to_string(hg.blue_reader) && + rows[0][2] == to_string(hg.green_writer) && + rows[0][3] == to_string(hg.green_reader); + return matches; +} + +bool persistent_bgd_row_absent(MYSQL* admin, int writer_hostgroup) { + string query = + "SELECT COUNT(*) FROM mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hostgroup); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool absent = rows[0][0] == "0"; + return absent; +} + +bool runtime_explicit_bgd_row_matches(MYSQL* admin, BGD_Hostgroups& hg) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(hg.blue_writer) + " AND reader_hostgroup=" + to_string(hg.blue_reader) + + " AND green_writer_hostgroup=" + to_string(hg.green_writer) + + " AND green_reader_hostgroup=" + to_string(hg.green_reader) + " AND auto_generated=0"; + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool matches = rows[0][0] == "1"; + return matches; +} + +rc_t> bgd_admin_snapshot(MYSQL* admin, int writer_hostgroup) { + string query = + "SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment " + "FROM mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + to_string(writer_hostgroup); + + rc_t> result = mysql_query_ext_rows(admin, query); + return result; +} + +rc_t> runtime_bgd_ownership_snapshot(MYSQL* admin, int writer_hostgroup) { + string query = + "SELECT green_writer_hostgroup,green_reader_hostgroup,active,auto_generated " + "FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + to_string(writer_hostgroup); + + rc_t> result = mysql_query_ext_rows(admin, query); + return result; +} + +rc_t> green_server_snapshot(MYSQL* admin, string table, BGD_Hostgroups& hg) { + string query = + "SELECT hostgroup_id,hostname,port,status,use_ssl,weight,max_connections FROM " + table + + " WHERE hostgroup_id IN (" + to_string(hg.green_writer) + "," + to_string(hg.green_reader) + + ") ORDER BY hostgroup_id,hostname,port"; + + rc_t> result = mysql_query_ext_rows(admin, query); + return result; +} + +/** + * Convert the automatic runtime row for wHG 890 to explicit configuration. + * + * - Enable automatic discovery with only the blue writer configured. + * - Verify the runtime row has NULL green hostgroups and auto_generated=1. + * - Disable automatic discovery and load explicit hostgroups 890-893. + * - Verify explicit values replace the automatic row and persist. + */ +int test_automatic_to_explicit(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.conversion; + BGD_Hostgroups& hg = state.conversion_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure automatic-conversion simulated writers"); + return EXIT_FAILURE; + } + + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.conversion_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 890"); + return EXIT_FAILURE; + } + + int monitor_rc = configure_monitor(admin, hg, true); + if (monitor_rc != EXIT_SUCCESS) { + diag("Error: failed to enable automatic discovery for wHG 890"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer }; + int server_rc = bgd_admin_add_servers(admin, cluster, hg, blue_servers, false, 0); + if (server_rc != EXIT_SUCCESS) { + diag("Error: failed to add the blue writer for wHG 890"); + return EXIT_FAILURE; + } + + vector load_server_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_server_rc = execute_all(admin, load_server_queries); + if (load_server_rc != EXIT_SUCCESS) { + diag("Error: failed to load the blue writer for wHG 890"); + return EXIT_FAILURE; + } + + string automatic_query = + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=890 " + "AND auto_generated=1 AND green_writer_hostgroup IS NULL AND green_reader_hostgroup IS NULL"; + int automatic_rc = bgd_wait_for_condition(admin, automatic_query, kTimeoutSeconds); + if (automatic_rc != EXIT_SUCCESS) { + diag("Error: automatic discovery did not create the nullable runtime row for wHG 890"); + return EXIT_FAILURE; + } + + ok(true, "automatic discovery records NULL green hostgroups for wHG 890"); + + vector disable_queries { + "SET mysql-aws_blue_green_deployment_auto_discovery='false'", + "LOAD MYSQL VARIABLES TO RUNTIME", + }; + int disable_rc = execute_all(admin, disable_queries); + if (disable_rc != EXIT_SUCCESS) { + diag("Error: failed to disable automatic discovery before converting wHG 890"); + return EXIT_FAILURE; + } + + int row_rc = insert_explicit_bgd_row(admin, hg, "converted automatic BGD row"); + if (row_rc != EXIT_SUCCESS) { + diag("Error: failed to insert explicit configuration for wHG 890"); + return EXIT_FAILURE; + } + + vector load_row_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_row_rc = execute_all(admin, load_row_queries); + if (load_row_rc != EXIT_SUCCESS) { + diag("Error: failed to load explicit configuration for wHG 890"); + return EXIT_FAILURE; + } + + string explicit_query = + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=890 " + "AND auto_generated=0 AND green_writer_hostgroup=892 AND green_reader_hostgroup=893"; + int explicit_rc = bgd_wait_for_condition(admin, explicit_query, kTimeoutSeconds); + if (explicit_rc != EXIT_SUCCESS) { + diag("Error: explicit configuration did not replace the automatic row for wHG 890"); + return EXIT_FAILURE; + } + + bool runtime_matches = runtime_explicit_bgd_row_matches(admin, hg); + bool persistent_matches = persistent_bgd_row_matches(admin, hg); + ok(runtime_matches && persistent_matches, "explicit hostgroups 890-893 replace and persist the automatic row"); + return EXIT_SUCCESS; +} + +/** + * Validate persistent green-hostgroup requirements. + * + * - Attempt persistent rows with a NULL green writer or reader hostgroup. + * - Verify both invalid rows are rejected. + * - Load a complete row for hostgroups 910-913. + * - Verify it exists in persistent and runtime configuration. + */ +int test_persistent_row_validation(MYSQL* admin, TestState& state) { + string null_writer_query = + "INSERT INTO mysql_aws_rds_bgd_hostgroups(" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup) " + "VALUES (900,901,NULL,903)"; + int null_writer_rc = mysql_query(admin, null_writer_query.c_str()); + bool null_writer_absent = persistent_bgd_row_absent(admin, 900); + ok(null_writer_rc != 0 && null_writer_absent, "persistent BGD configuration rejects a NULL green writer hostgroup"); + + string null_reader_query = + "INSERT INTO mysql_aws_rds_bgd_hostgroups(" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup) " + "VALUES (904,905,906,NULL)"; + int null_reader_rc = mysql_query(admin, null_reader_query.c_str()); + bool null_reader_absent = persistent_bgd_row_absent(admin, 904); + ok(null_reader_rc != 0 && null_reader_absent, "persistent BGD configuration rejects a NULL green reader hostgroup"); + + BGD_Hostgroups& hg = state.valid_hg; + int monitor_rc = configure_monitor(admin, hg, false); + if (monitor_rc != EXIT_SUCCESS) { + diag("Error: failed to configure monitoring for hostgroups 910-913"); + return EXIT_FAILURE; + } + + int row_rc = insert_explicit_bgd_row(admin, hg, "valid persistent BGD row"); + if (row_rc != EXIT_SUCCESS) { + diag("Error: failed to insert valid persistent BGD row for hostgroups 910-913"); + return EXIT_FAILURE; + } + + vector load_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_rc = execute_all(admin, load_queries); + if (load_rc != EXIT_SUCCESS) { + diag("Error: failed to load valid BGD row for hostgroups 910-913"); + return EXIT_FAILURE; + } + + bool persistent_matches = persistent_bgd_row_matches(admin, hg); + bool runtime_matches = runtime_explicit_bgd_row_matches(admin, hg); + ok(persistent_matches && runtime_matches, "complete persistent BGD configuration loads hostgroups 910-913"); + return EXIT_SUCCESS; +} + +/** + * Save explicit and automatic runtime rows back to persistent configuration. + * + * - Run explicit wHG 920 and automatic wHG 930 together. + * - Remove the persistent explicit row. + * - Execute SAVE MYSQL SERVERS FROM RUNTIME. + * - Verify SAVE restores wHG 920 and skips auto-generated wHG 930. + */ +int test_save_from_runtime(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& explicit_cluster = state.explicit_save; + RDS_BGD_Cluster& automatic_cluster = state.automatic_save; + BGD_Hostgroups& explicit_hg = state.explicit_save_hg; + BGD_Hostgroups& automatic_hg = state.automatic_save_hg; + + int explicit_writer_rc = bgd_set_writer_read_only_0(sim, explicit_cluster); + if (explicit_writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure explicit SAVE simulated writers"); + return EXIT_FAILURE; + } + + int automatic_writer_rc = bgd_set_writer_read_only_0(sim, automatic_cluster); + if (automatic_writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure automatic SAVE simulated writers"); + return EXIT_FAILURE; + } + + vector explicit_topology = bgd_topology_with_readers(explicit_cluster, "AVAILABLE"); + int explicit_topology_rc = sim.topology_update(state.explicit_save_endpoints, explicit_topology); + if (explicit_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 920"); + return EXIT_FAILURE; + } + + vector automatic_topology = bgd_topology_with_readers(automatic_cluster, "AVAILABLE"); + int automatic_topology_rc = sim.topology_update(state.automatic_save_endpoints, automatic_topology); + if (automatic_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 930"); + return EXIT_FAILURE; + } + + int monitor_rc = configure_monitor(admin, explicit_hg, true); + if (monitor_rc != EXIT_SUCCESS) { + diag("Error: failed to configure SAVE monitoring for wHG 920"); + return EXIT_FAILURE; + } + + int explicit_row_rc = insert_explicit_bgd_row(admin, explicit_hg, "explicit SAVE row"); + if (explicit_row_rc != EXIT_SUCCESS) { + diag("Error: failed to insert explicit SAVE row for wHG 920"); + return EXIT_FAILURE; + } + + int explicit_servers_rc = add_all_servers(admin, explicit_cluster, explicit_hg); + if (explicit_servers_rc != EXIT_SUCCESS) { + diag("Error: failed to load servers for wHG 920"); + return EXIT_FAILURE; + } + + string replication_query = + "INSERT INTO mysql_replication_hostgroups(writer_hostgroup,reader_hostgroup) VALUES (" + + to_string(automatic_hg.blue_writer) + "," + to_string(automatic_hg.blue_reader) + ")"; + vector automatic_config_queries { replication_query }; + int automatic_config_rc = execute_all(admin, automatic_config_queries); + if (automatic_config_rc != EXIT_SUCCESS) { + diag("Error: failed to configure replication hostgroups 930 and 931"); + return EXIT_FAILURE; + } + + vector automatic_blue_servers { automatic_cluster.blue_writer }; + int automatic_server_rc = bgd_admin_add_servers(admin, automatic_cluster, automatic_hg, automatic_blue_servers, false, 0); + if (automatic_server_rc != EXIT_SUCCESS) { + diag("Error: failed to add the automatic blue writer for wHG 930"); + return EXIT_FAILURE; + } + + vector load_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_rc = execute_all(admin, load_queries); + if (load_rc != EXIT_SUCCESS) { + diag("Error: failed to load explicit and automatic SAVE scenarios"); + return EXIT_FAILURE; + } + + string explicit_runtime_query = + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_rds_bgd_hostgroups " + "WHERE writer_hostgroup=920 AND auto_generated=0"; + int explicit_runtime_rc = bgd_wait_for_condition(admin, explicit_runtime_query, kTimeoutSeconds); + if (explicit_runtime_rc != EXIT_SUCCESS) { + diag("Error: explicit wHG 920 did not reach runtime before SAVE"); + return EXIT_FAILURE; + } + + string automatic_runtime_query = + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_rds_bgd_hostgroups " + "WHERE writer_hostgroup=930 AND auto_generated=1"; + int automatic_runtime_rc = bgd_wait_for_condition(admin, automatic_runtime_query, kTimeoutSeconds); + if (automatic_runtime_rc != EXIT_SUCCESS) { + diag("Error: automatic wHG 930 did not reach runtime before SAVE"); + return EXIT_FAILURE; + } + + string delete_query = + "DELETE FROM mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(explicit_hg.blue_writer); + vector save_queries { + delete_query, + "SAVE MYSQL SERVERS FROM RUNTIME", + }; + int save_rc = execute_all(admin, save_queries); + if (save_rc != EXIT_SUCCESS) { + diag("Error: failed to save runtime BGD rows to persistent configuration"); + return EXIT_FAILURE; + } + + bool explicit_persisted = persistent_bgd_row_matches(admin, explicit_hg); + bool automatic_absent = persistent_bgd_row_absent(admin, automatic_hg.blue_writer); + ok(explicit_persisted && automatic_absent, "SAVE restores explicit wHG 920 and skips auto-generated wHG 930"); + return EXIT_SUCCESS; +} + +/** + * Run automatic discovery beside administrator-owned BGD and server rows. + * + * - Configure inactive explicit hostgroups 1310-1313. + * - Set the configured green writer to SHUNNED. + * - Enable automatic discovery and publish AVAILABLE topology. + * - Verify the BGD row and green-server status remain unchanged. + */ +int test_admin_server_status_preserved(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.admin_owned; + BGD_Hostgroups& hg = state.admin_owned_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure administrator-owned simulated writers"); + return EXIT_FAILURE; + } + + int monitor_rc = configure_monitor(admin, hg, false); + if (monitor_rc != EXIT_SUCCESS) { + diag("Error: failed to configure monitoring for administrator-owned wHG 1310"); + return EXIT_FAILURE; + } + + int row_rc = insert_explicit_bgd_row(admin, hg, "administrator-owned inactive BGD row", 0); + if (row_rc != EXIT_SUCCESS) { + diag("Error: failed to insert administrator-owned wHG 1310"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0] }; + int blue_rc = bgd_admin_add_servers(admin, cluster, hg, blue_servers, false, 0); + if (blue_rc != EXIT_SUCCESS) { + diag("Error: failed to load administrator-owned blue servers"); + return EXIT_FAILURE; + } + + vector green_servers { cluster.green_writer }; + int green_rc = bgd_admin_add_servers(admin, cluster, hg, green_servers, true, 0); + if (green_rc != EXIT_SUCCESS) { + diag("Error: failed to load administrator-owned green writer"); + return EXIT_FAILURE; + } + + string shun_query = + "UPDATE mysql_servers SET status='SHUNNED' WHERE hostgroup_id=" + + to_string(hg.green_writer) + " AND hostname=" + bgd_sql_quote(cluster.green_writer.hostname) + + " AND port=3306"; + vector ownership_queries { + shun_query, + "LOAD MYSQL SERVERS TO RUNTIME", + "SET mysql-aws_blue_green_deployment_auto_discovery='true'", + "LOAD MYSQL VARIABLES TO RUNTIME", + }; + int ownership_rc = execute_all(admin, ownership_queries); + if (ownership_rc != EXIT_SUCCESS) { + diag("Error: failed to enable automatic discovery beside administrator-owned wHG 1310"); + return EXIT_FAILURE; + } + + auto [bgd_before_rc, bgd_before] = bgd_admin_snapshot(admin, hg.blue_writer); + auto [runtime_bgd_before_rc, runtime_bgd_before] = runtime_bgd_ownership_snapshot(admin, hg.blue_writer); + auto [admin_before_rc, admin_before] = green_server_snapshot(admin, "mysql_servers", hg); + auto [runtime_before_rc, runtime_before] = green_server_snapshot(admin, "runtime_mysql_servers", hg); + if (bgd_before_rc != EXIT_SUCCESS || runtime_bgd_before_rc != EXIT_SUCCESS || + admin_before_rc != EXIT_SUCCESS || runtime_before_rc != EXIT_SUCCESS) { + diag("Error: failed to snapshot administrator-owned BGD and green rows"); + return EXIT_FAILURE; + } + + bool explicit_runtime_bgd = + runtime_bgd_before.size() == 1 && + runtime_bgd_before[0].size() == 4 && + runtime_bgd_before[0][0] == "1312" && + runtime_bgd_before[0][1] == "1313" && + runtime_bgd_before[0][2] == "0" && + runtime_bgd_before[0][3] == "0"; + if (!explicit_runtime_bgd) { + diag("Error: runtime BGD row does not contain the administrator-owned hostgroups and flags"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before administrator-owned discovery"); + return EXIT_FAILURE; + } + + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.admin_owned_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology beside administrator-owned wHG 1310"); + return EXIT_FAILURE; + } + + auto [probe_rc, probe] = + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: automatic discovery did not probe beside administrator-owned wHG 1310"); + return EXIT_FAILURE; + } + + auto [bgd_after_rc, bgd_after] = bgd_admin_snapshot(admin, hg.blue_writer); + auto [runtime_bgd_after_rc, runtime_bgd_after] = runtime_bgd_ownership_snapshot(admin, hg.blue_writer); + auto [admin_after_rc, admin_after] = green_server_snapshot(admin, "mysql_servers", hg); + auto [runtime_after_rc, runtime_after] = green_server_snapshot(admin, "runtime_mysql_servers", hg); + if (bgd_after_rc != EXIT_SUCCESS || runtime_bgd_after_rc != EXIT_SUCCESS || + admin_after_rc != EXIT_SUCCESS || runtime_after_rc != EXIT_SUCCESS) { + diag("Error: failed to read administrator-owned rows after discovery"); + return EXIT_FAILURE; + } + + bool bgd_unchanged = bgd_before == bgd_after; + bool runtime_bgd_unchanged = runtime_bgd_before == runtime_bgd_after; + bool admin_servers_unchanged = admin_before == admin_after; + bool runtime_servers_unchanged = runtime_before == runtime_after; + ok(bgd_unchanged && runtime_bgd_unchanged && admin_servers_unchanged && runtime_servers_unchanged, + "automatic discovery preserves administrator-owned wHG 1310 and its SHUNNED green writer"); + return EXIT_SUCCESS; +} + +int main() { + plan(7); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish AVAILABLE topology for the blue writer in wHG 890. + // ProxySQL: create an automatic row, disable discovery, and load explicit hostgroups 890-893. + // Verify: explicit green hostgroups replace the nullable automatic row and persist. + if (test_automatic_to_explicit(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: insert two BGD rows with one NULL green hostgroup, then one complete row. + // Verify: invalid rows are rejected and complete hostgroups 910-913 load as explicit configuration. + if (test_persistent_row_validation(admin, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: run explicit wHG 920 and auto-generated wHG 930, then SAVE runtime state. + // Verify: SAVE persists only the explicit BGD row. + if (test_save_from_runtime(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: configure inactive administrator-owned wHG 1310 with a SHUNNED green writer. + // Simulator: publish AVAILABLE while automatic discovery is enabled. + // Verify: the BGD row and green-server status remain unchanged. + if (test_admin_server_status_preserved(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_explicit_startup-t.cpp b/test/tap/tests/test_rds_bgd_explicit_startup-t.cpp new file mode 100644 index 0000000000..2ddc6ee15f --- /dev/null +++ b/test/tap/tests/test_rds_bgd_explicit_startup-t.cpp @@ -0,0 +1,359 @@ +/** + * @file test_rds_bgd_explicit_startup-t.cpp + * @brief Starting an explicit BGD worker after both required inputs exist. + * + * Steps: + * + * 1. Load the BGD row for hostgroups 840-843 before loading its servers. + * 2. Verify no table-check occurs until an eligible blue server is loaded. + * 3. Load servers for hostgroups 850-853 before loading their BGD row. + * 4. Verify no table-check occurs until the explicit BGD row is loaded. + */ + +#include +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; +const uint32_t kNegativeProbeTimeoutMs = 800; + +struct TestState { + RDS_BGD_Cluster row_first { bgd_cluster_init() }; + RDS_BGD_Cluster servers_first { bgd_cluster_2_init() }; + BGD_Hostgroups row_first_hg { 840, 841, 842, 843 }; + BGD_Hostgroups servers_first_hg { 850, 851, 852, 853 }; + vector row_first_endpoints { row_first.get_endpoints() }; + vector servers_first_endpoints { servers_first.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int configure_monitor(MYSQL* admin, BGD_Hostgroups& hg) { + vector queries { + "INSERT INTO mysql_replication_hostgroups(writer_hostgroup,reader_hostgroup) VALUES (" + + to_string(hg.blue_writer) + "," + to_string(hg.blue_reader) + ")", + "SET mysql-monitor_username='testuser'", + "SET mysql-monitor_password='testuser'", + "SET mysql-monitor_enabled='true'", + "SET mysql-monitor_read_only_interval=100", + "SET mysql-monitor_aws_rds_topology_discovery_interval=1", + "SET mysql-aws_blue_green_deployment_auto_discovery='false'", + "LOAD MYSQL VARIABLES TO RUNTIME", + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +int insert_explicit_bgd_row(MYSQL* admin, BGD_Hostgroups& hg, string comment) { + string query = + "INSERT INTO mysql_aws_rds_bgd_hostgroups(" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment) VALUES (" + + to_string(hg.blue_writer) + "," + to_string(hg.blue_reader) + "," + + to_string(hg.green_writer) + "," + to_string(hg.green_reader) + + ",1,0,100,800," + bgd_sql_quote(comment) + ")"; + + int rc = mysql_query(admin, query.c_str()); + if (rc != 0) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int add_cluster_servers(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + int blue_rc = bgd_admin_add_servers(admin, cluster, hg, blue_servers, false, 0); + if (blue_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector green_servers { cluster.green_writer, cluster.green_readers[0], cluster.green_readers[1] }; + int green_rc = bgd_admin_add_servers(admin, cluster, hg, green_servers, true, 0); + if (green_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_rc = execute_all(admin, queries); + if (load_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +bool runtime_membership_matches(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { + vector hostgroups { hg.blue_writer, hg.blue_reader, hg.green_writer, hg.green_reader }; + auto [rc, rows] = bgd_runtime_servers(admin, hostgroups); + if (rc != EXIT_SUCCESS || rows.size() != 6) { + return false; + } + + vector> expected { + { hg.blue_writer, cluster.blue_writer.hostname }, + { hg.blue_reader, cluster.blue_readers[0].hostname }, + { hg.blue_reader, cluster.blue_readers[1].hostname }, + { hg.green_writer, cluster.green_writer.hostname }, + { hg.green_reader, cluster.green_readers[0].hostname }, + { hg.green_reader, cluster.green_readers[1].hostname }, + }; + + for (pair& server : expected) { + bool found = false; + for (mysql_res_row& row : rows) { + if (row.size() == 5 && row[0] == to_string(server.first) && row[1] == server.second) { + found = true; + break; + } + } + if (!found) { + return false; + } + } + return true; +} + +/** + * Load the explicit BGD row before any eligible blue server. + * + * - Publish AVAILABLE topology for hostgroups 840-843. + * - Load the explicit BGD row without mysql_servers membership. + * - Verify no table-check probe starts. + * - Load all servers and verify AVAILABLE with explicit runtime membership. + */ +int test_bgd_row_before_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.row_first; + BGD_Hostgroups& hg = state.row_first_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure row-first simulated writers"); + return EXIT_FAILURE; + } + + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.row_first_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for hostgroups 840-843"); + return EXIT_FAILURE; + } + + int monitor_rc = configure_monitor(admin, hg); + if (monitor_rc != EXIT_SUCCESS) { + diag("Error: failed to configure monitoring for hostgroups 840-843"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before loading wHG 840"); + return EXIT_FAILURE; + } + + int row_rc = insert_explicit_bgd_row(admin, hg, "BGD row before servers"); + if (row_rc != EXIT_SUCCESS) { + diag("Error: failed to insert the explicit BGD row for wHG 840"); + return EXIT_FAILURE; + } + + vector load_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_row_rc = execute_all(admin, load_queries); + if (load_row_rc != EXIT_SUCCESS) { + diag("Error: failed to load wHG 840 before its servers"); + return EXIT_FAILURE; + } + + int no_probe_rc = bgd_expect_no_table_check(sim, seq, state.row_first_endpoints, kNegativeProbeTimeoutMs); + if (no_probe_rc != EXIT_SUCCESS) { + diag("Error: wHG 840 started before an eligible blue server existed"); + return EXIT_FAILURE; + } + + ok(true, "wHG 840 does not start before an eligible blue server exists"); + + int servers_rc = add_cluster_servers(admin, cluster, hg); + if (servers_rc != EXIT_SUCCESS) { + diag("Error: failed to load servers for hostgroups 840-843"); + return EXIT_FAILURE; + } + + auto [probe_rc, probe] = + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: loading the blue writer did not start the wHG 840 table check"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 840 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + bool membership_matches = runtime_membership_matches(admin, cluster, hg); + ok(membership_matches, "loading servers starts wHG 840 with explicit runtime membership"); + return EXIT_SUCCESS; +} + +/** + * Load all servers before their explicit BGD row. + * + * - Publish AVAILABLE topology for hostgroups 850-853. + * - Load mysql_servers membership without a BGD row. + * - Verify no table-check probe starts. + * - Load the explicit row and verify AVAILABLE with explicit membership. + */ +int test_servers_before_bgd_row(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.servers_first; + BGD_Hostgroups& hg = state.servers_first_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure servers-first simulated writers"); + return EXIT_FAILURE; + } + + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.servers_first_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for hostgroups 850-853"); + return EXIT_FAILURE; + } + + int monitor_rc = configure_monitor(admin, hg); + if (monitor_rc != EXIT_SUCCESS) { + diag("Error: failed to configure monitoring for hostgroups 850-853"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before loading hostgroups 850-853"); + return EXIT_FAILURE; + } + + int servers_rc = add_cluster_servers(admin, cluster, hg); + if (servers_rc != EXIT_SUCCESS) { + diag("Error: failed to load servers before wHG 850"); + return EXIT_FAILURE; + } + + int no_probe_rc = bgd_expect_no_table_check(sim, seq, state.servers_first_endpoints, kNegativeProbeTimeoutMs); + if (no_probe_rc != EXIT_SUCCESS) { + diag("Error: servers in hostgroups 850-853 started without an explicit BGD row"); + return EXIT_FAILURE; + } + + ok(true, "servers in hostgroups 850-853 do not start without an explicit BGD row"); + + int row_rc = insert_explicit_bgd_row(admin, hg, "servers before BGD row"); + if (row_rc != EXIT_SUCCESS) { + diag("Error: failed to insert the explicit BGD row for wHG 850"); + return EXIT_FAILURE; + } + + vector load_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_row_rc = execute_all(admin, load_queries); + if (load_row_rc != EXIT_SUCCESS) { + diag("Error: failed to load the explicit BGD row for wHG 850"); + return EXIT_FAILURE; + } + + auto [probe_rc, probe] = + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: loading wHG 850 did not start the blue table check"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 850 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + bool membership_matches = runtime_membership_matches(admin, cluster, hg); + ok(membership_matches, "loading wHG 850 starts the worker with explicit runtime membership"); + return EXIT_SUCCESS; +} + +int main() { + plan(4); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish AVAILABLE topology for hostgroups 840-843. + // ProxySQL: load the explicit BGD row before loading mysql_servers. + // Verify: no table-check starts until eligible blue membership exists. + if (test_bgd_row_before_servers(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish AVAILABLE topology for hostgroups 850-853. + // ProxySQL: load mysql_servers before loading the explicit BGD row. + // Verify: no table-check starts until wHG 850 is loaded. + if (test_servers_before_bgd_row(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_green_membership_ordering-t.cpp b/test/tap/tests/test_rds_bgd_green_membership_ordering-t.cpp new file mode 100644 index 0000000000..08f1e2bfc4 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_green_membership_ordering-t.cpp @@ -0,0 +1,421 @@ +/** + * @file test_rds_bgd_green_membership_ordering-t.cpp + * @brief Loading configured green membership at three supported times. + * + * Steps: + * + * 1. Load green membership for hostgroups 862 and 863 before AVAILABLE. + * 2. Load green membership for hostgroups 872 and 873 after discovery. + * 3. Start wHG 880 against absent topology, then load green membership for + * hostgroups 882 and 883 before publishing AVAILABLE. + * 4. Verify all three orders produce complete runtime green membership. + */ + +#include +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; + +struct TestState { + RDS_BGD_Cluster before_available { bgd_cluster_3_init() }; + RDS_BGD_Cluster after_discovery { bgd_cluster_1_deployment_b_init() }; + RDS_BGD_Cluster after_worker_start { bgd_cluster_init() }; + BGD_Hostgroups before_available_hg { 860, 861, 862, 863 }; + BGD_Hostgroups after_discovery_hg { 870, 871, 872, 873 }; + BGD_Hostgroups after_worker_start_hg { 880, 881, 882, 883 }; + vector before_available_endpoints { before_available.get_endpoints() }; + vector after_discovery_endpoints { after_discovery.get_endpoints() }; + vector after_worker_start_endpoints { after_worker_start.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int configure_monitor(MYSQL* admin, BGD_Hostgroups& hg) { + vector queries { + "INSERT INTO mysql_replication_hostgroups(writer_hostgroup,reader_hostgroup) VALUES (" + + to_string(hg.blue_writer) + "," + to_string(hg.blue_reader) + ")", + "SET mysql-monitor_username='testuser'", + "SET mysql-monitor_password='testuser'", + "SET mysql-monitor_enabled='true'", + "SET mysql-monitor_read_only_interval=100", + "SET mysql-monitor_aws_rds_topology_discovery_interval=1", + "SET mysql-aws_blue_green_deployment_auto_discovery='false'", + "LOAD MYSQL VARIABLES TO RUNTIME", + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +int insert_explicit_bgd_row(MYSQL* admin, BGD_Hostgroups& hg, string comment) { + string query = + "INSERT INTO mysql_aws_rds_bgd_hostgroups(" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment) VALUES (" + + to_string(hg.blue_writer) + "," + to_string(hg.blue_reader) + "," + + to_string(hg.green_writer) + "," + to_string(hg.green_reader) + + ",1,0,100,800," + bgd_sql_quote(comment) + ")"; + + int rc = mysql_query(admin, query.c_str()); + if (rc != 0) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int add_blue_servers(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + int add_rc = bgd_admin_add_servers(admin, cluster, hg, blue_servers, false, 0); + if (add_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_rc = execute_all(admin, queries); + if (load_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int add_green_servers(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { + vector green_servers { cluster.green_writer, cluster.green_readers[0], cluster.green_readers[1] }; + int add_rc = bgd_admin_add_servers(admin, cluster, hg, green_servers, true, 0); + if (add_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_rc = execute_all(admin, queries); + if (load_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +bool runtime_green_membership_matches(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { + vector hostgroups { hg.green_writer, hg.green_reader }; + auto [rc, rows] = bgd_runtime_servers(admin, hostgroups); + if (rc != EXIT_SUCCESS || rows.size() != 3) { + return false; + } + + vector> expected { + { hg.green_writer, cluster.green_writer.hostname }, + { hg.green_reader, cluster.green_readers[0].hostname }, + { hg.green_reader, cluster.green_readers[1].hostname }, + }; + + for (pair& server : expected) { + bool found = false; + for (mysql_res_row& row : rows) { + if (row.size() == 5 && row[0] == to_string(server.first) && row[1] == server.second) { + found = true; + break; + } + } + if (!found) { + return false; + } + } + return true; +} + +/** + * Load complete green membership before the first AVAILABLE observation. + * + * - Configure wHG 860 and all blue/green mysql_servers rows. + * - Publish AVAILABLE topology after all membership exists. + * - Verify runtime hostgroups 862 and 863 contain the configured green set. + */ +int test_green_before_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.before_available; + BGD_Hostgroups& hg = state.before_available_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure green-before-AVAILABLE simulated writers"); + return EXIT_FAILURE; + } + + int monitor_rc = configure_monitor(admin, hg); + if (monitor_rc != EXIT_SUCCESS) { + diag("Error: failed to configure monitoring for wHG 860"); + return EXIT_FAILURE; + } + + int row_rc = insert_explicit_bgd_row(admin, hg, "green membership before AVAILABLE"); + if (row_rc != EXIT_SUCCESS) { + diag("Error: failed to insert the explicit BGD row for wHG 860"); + return EXIT_FAILURE; + } + + int blue_rc = add_blue_servers(admin, cluster, hg); + if (blue_rc != EXIT_SUCCESS) { + diag("Error: failed to load blue membership for wHG 860"); + return EXIT_FAILURE; + } + + int green_rc = add_green_servers(admin, cluster, hg); + if (green_rc != EXIT_SUCCESS) { + diag("Error: failed to load green membership for hostgroups 862 and 863"); + return EXIT_FAILURE; + } + + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.before_available_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 860"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 860 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + bool membership_matches = runtime_green_membership_matches(admin, cluster, hg); + ok(membership_matches, "green membership loaded before AVAILABLE appears in hostgroups 862 and 863"); + return EXIT_SUCCESS; +} + +/** + * Load green membership after the explicit worker discovers AVAILABLE. + * + * - Publish AVAILABLE and start wHG 870 with blue membership only. + * - Load the configured green writer/readers into hostgroups 872 and 873. + * - Verify runtime_mysql_servers contains the complete green membership after + * the worker observes the configuration change. + */ +int test_green_after_discovery(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.after_discovery; + BGD_Hostgroups& hg = state.after_discovery_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure green-after-discovery simulated writers"); + return EXIT_FAILURE; + } + + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.after_discovery_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 870"); + return EXIT_FAILURE; + } + + int monitor_rc = configure_monitor(admin, hg); + if (monitor_rc != EXIT_SUCCESS) { + diag("Error: failed to configure monitoring for wHG 870"); + return EXIT_FAILURE; + } + + int row_rc = insert_explicit_bgd_row(admin, hg, "green membership after discovery"); + if (row_rc != EXIT_SUCCESS) { + diag("Error: failed to insert the explicit BGD row for wHG 870"); + return EXIT_FAILURE; + } + + int blue_rc = add_blue_servers(admin, cluster, hg); + if (blue_rc != EXIT_SUCCESS) { + diag("Error: failed to load blue membership for wHG 870"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 870 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before loading hostgroups 872 and 873"); + return EXIT_FAILURE; + } + + int green_rc = add_green_servers(admin, cluster, hg); + if (green_rc != EXIT_SUCCESS) { + diag("Error: failed to load green membership for hostgroups 872 and 873"); + return EXIT_FAILURE; + } + + auto [probe_rc, probe] = + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: wHG 870 did not probe the green writer after membership load"); + return EXIT_FAILURE; + } + + bool membership_matches = runtime_green_membership_matches(admin, cluster, hg); + ok(membership_matches, "green membership loaded after discovery appears in hostgroups 872 and 873"); + return EXIT_SUCCESS; +} + +/** + * Start an explicit worker before topology and green membership exist. + * + * - Start wHG 880 with blue membership against absent topology. + * - Load green membership into hostgroups 882 and 883. + * - Publish AVAILABLE and verify complete runtime green membership. + */ +int test_green_after_worker_start(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.after_worker_start; + BGD_Hostgroups& hg = state.after_worker_start_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure green-after-worker-start simulated writers"); + return EXIT_FAILURE; + } + + int drop_rc = sim.topology_drop(state.after_worker_start_endpoints); + if (drop_rc != EXIT_SUCCESS) { + diag("Error: failed to publish absent topology for wHG 880"); + return EXIT_FAILURE; + } + + int monitor_rc = configure_monitor(admin, hg); + if (monitor_rc != EXIT_SUCCESS) { + diag("Error: failed to configure monitoring for wHG 880"); + return EXIT_FAILURE; + } + + int row_rc = insert_explicit_bgd_row(admin, hg, "green membership after worker start"); + if (row_rc != EXIT_SUCCESS) { + diag("Error: failed to insert the explicit BGD row for wHG 880"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before starting wHG 880"); + return EXIT_FAILURE; + } + + int blue_rc = add_blue_servers(admin, cluster, hg); + if (blue_rc != EXIT_SUCCESS) { + diag("Error: failed to load blue membership for wHG 880"); + return EXIT_FAILURE; + } + + auto [start_rc, start_probe] = + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + if (start_rc != EXIT_SUCCESS) { + diag("Error: wHG 880 did not start the blue table-check probe"); + return EXIT_FAILURE; + } + + int green_rc = add_green_servers(admin, cluster, hg); + if (green_rc != EXIT_SUCCESS) { + diag("Error: failed to load green membership for hostgroups 882 and 883"); + return EXIT_FAILURE; + } + + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.after_worker_start_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 880"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 880 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + bool membership_matches = runtime_green_membership_matches(admin, cluster, hg); + ok(membership_matches, "green membership loaded after worker start appears in hostgroups 882 and 883"); + return EXIT_SUCCESS; +} + +int main() { + plan(3); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // ProxySQL: load wHG 860 and complete blue/green membership before AVAILABLE. + // Simulator: publish AVAILABLE topology after membership exists. + // Verify: runtime_mysql_servers contains the configured green writer/readers in hostgroups 862 and 863. + if (test_green_before_available(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish AVAILABLE topology and start wHG 870 with blue membership. + // ProxySQL: load green writer/readers into hostgroups 872 and 873 after discovery. + // Verify: runtime_mysql_servers converges on complete green membership. + if (test_green_after_discovery(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: start wHG 880 against absent topology, then load green hostgroups 882 and 883. + // Simulator: publish AVAILABLE after the worker and green membership exist. + // Verify: runtime_mysql_servers converges on complete green membership. + if (test_green_after_worker_start(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_probe_tls-t.cpp b/test/tap/tests/test_rds_bgd_probe_tls-t.cpp new file mode 100644 index 0000000000..4751272c62 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_probe_tls-t.cpp @@ -0,0 +1,459 @@ +/** + * @file test_rds_bgd_probe_tls-t.cpp + * @brief Selecting BGD metadata targets and their TLS values. + * + * Steps: + * + * 1. Configure a plaintext blue reader before a TLS blue writer and verify + * automatic discovery uses the matched writer TLS. + * 2. Configure an exact TLS TARGET beside a plaintext distractor and verify + * the recorded TARGET is selected. + * 3. Leave green writer hostgroup 962 empty, set use_ssl=1 defaults, and + * verify the created TARGET row and metadata probe use TLS. + * 4. Verify table-check, blue metadata, and green metadata probe order. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; +const uint32_t kNegativeProbeTimeoutMs = 1200; + +struct TestState { + RDS_BGD_Cluster automatic { bgd_cluster_init() }; + RDS_BGD_Cluster explicit_target { bgd_cluster_2_init() }; + RDS_BGD_Cluster distractor { bgd_cluster_1_deployment_b_init() }; + RDS_BGD_Cluster created_target { bgd_cluster_3_init() }; + BGD_Hostgroups automatic_hg { 940, 941, 942, 943 }; + BGD_Hostgroups explicit_target_hg { 950, 951, 952, 953 }; + BGD_Hostgroups created_target_hg { 960, 961, 962, 963 }; + vector automatic_endpoints { automatic.get_endpoints() }; + vector explicit_target_endpoints { explicit_target.get_endpoints() }; + vector created_target_endpoints { created_target.get_endpoints() }; +}; + +struct ProbeChain { + RDS_BGD_Probe_Log table; + RDS_BGD_Probe_Log blue; + RDS_BGD_Probe_Log green; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + vector attribute_queries { + "DELETE FROM mysql_hostgroup_attributes", + "LOAD MYSQL SERVERS TO RUNTIME", + }; + int attribute_rc = execute_all(admin, attribute_queries); + if (attribute_rc != EXIT_SUCCESS) { + diag("Error: failed to clean BGD TLS hostgroup attributes"); + } + + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (attribute_rc != EXIT_SUCCESS || admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int wait_for_probe_chain(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster, + int blue_use_ssl, int green_use_ssl, ProbeChain& chain) +{ + auto [table_rc, table] = + sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, blue_use_ssl); + if (table_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + auto [blue_rc, blue] = + sim.wait_for_probe_log(table.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, blue_use_ssl); + if (blue_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + auto [green_rc, green] = + sim.wait_for_probe_log(blue.sequence_id, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, green_use_ssl); + if (green_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + chain = { table, blue, green }; + return EXIT_SUCCESS; +} + +bool probe_chain_ordered(ProbeChain& chain) { + bool ordered = + chain.table.sequence_id < chain.blue.sequence_id && + chain.blue.sequence_id < chain.green.sequence_id && + chain.table.probe_kind == RDS_BGD_Probe_Kind::table_check && + chain.blue.probe_kind == RDS_BGD_Probe_Kind::metadata && + chain.green.probe_kind == RDS_BGD_Probe_Kind::metadata; + return ordered; +} + +bool runtime_server_tls_matches(MYSQL* admin, int hostgroup, RDS_BGD_Host& host, int use_ssl) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + + " AND use_ssl=" + to_string(use_ssl); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool matches = rows[0][0] == "1"; + return matches; +} + +/** + * Select the automatic blue writer when a reader appears first. + * + * - Load a blue reader with use_ssl=0 before the blue writer with use_ssl=1. + * - Publish AVAILABLE topology. + * - Verify table-check and blue metadata use the writer with TLS. + * - Verify the mapped green writer metadata probe also uses TLS. + */ +int test_automatic_writer_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.automatic; + BGD_Hostgroups& hg = state.automatic_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure automatic TLS simulated writers"); + return EXIT_FAILURE; + } + + vector no_servers {}; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::automatic, no_servers, no_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure automatic TLS discovery for hostgroups 940-943"); + return EXIT_FAILURE; + } + + vector reader { cluster.blue_readers[0] }; + int reader_rc = bgd_admin_add_servers(admin, cluster, hg, reader, false, 0); + if (reader_rc != EXIT_SUCCESS) { + diag("Error: failed to load the plaintext blue reader in hostgroup 941"); + return EXIT_FAILURE; + } + + vector writer { cluster.blue_writer }; + int server_writer_rc = bgd_admin_add_servers(admin, cluster, hg, writer, false, 1); + if (server_writer_rc != EXIT_SUCCESS) { + diag("Error: failed to load the TLS blue writer in hostgroup 940"); + return EXIT_FAILURE; + } + + vector load_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_rc = execute_all(admin, load_queries); + if (load_rc != EXIT_SUCCESS) { + diag("Error: failed to load automatic TLS server rows"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the automatic TLS probe sequence"); + return EXIT_FAILURE; + } + + vector topology = cluster.get_topology("AVAILABLE"); + int topology_rc = sim.topology_update(state.automatic_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish automatic AVAILABLE topology"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 940 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + ProbeChain chain {}; + int chain_rc = wait_for_probe_chain(sim, seq, cluster, 1, 1, chain); + if (chain_rc != EXIT_SUCCESS) { + diag("Error: automatic TLS probe chain did not complete"); + return EXIT_FAILURE; + } + + bool writer_tls = runtime_server_tls_matches(admin, hg.blue_writer, cluster.blue_writer, 1); + bool selected_writer = chain.table.backend.host == cluster.blue_writer.ip && chain.blue.backend.host == cluster.blue_writer.ip; + ok(writer_tls && selected_writer && chain.table.encrypted && chain.blue.encrypted, + "automatic discovery selects the TLS writer in hostgroup 940 instead of the plaintext reader"); + + bool green_target = chain.green.backend.host == cluster.green_writer.ip && chain.green.encrypted; + bool ordered = probe_chain_ordered(chain); + ok(green_target && ordered, "automatic probes run table-check, blue metadata, then TLS green metadata in order"); + return EXIT_SUCCESS; +} + +/** + * Select the exact explicit TARGET beside a valid-looking distractor. + * + * - Configure the exact green writer with use_ssl=1. + * - Configure a different green writer hostname with use_ssl=0 in hostgroup 952. + * - Publish AVAILABLE topology for the exact deployment. + * - Verify the exact TARGET and TLS value are used in probe order. + */ +int test_explicit_target_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.explicit_target; + RDS_BGD_Cluster& distractor = state.distractor; + BGD_Hostgroups& hg = state.explicit_target_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure explicit TLS simulated writers"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer }; + vector no_green_servers {}; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, + blue_servers, no_green_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure explicit TLS hostgroups 950-953"); + return EXIT_FAILURE; + } + + string distractor_query = + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + + to_string(hg.green_writer) + "," + bgd_sql_quote(distractor.green_writer.hostname) + + ",3306,'ONLINE',0,'BGD TAP TLS distractor')"; + string target_query = + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + + to_string(hg.green_writer) + "," + bgd_sql_quote(cluster.green_writer.hostname) + + ",3306,'ONLINE',1,'BGD TAP exact TARGET')"; + vector target_queries { + distractor_query, + target_query, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + int target_rc = execute_all(admin, target_queries); + if (target_rc != EXIT_SUCCESS) { + diag("Error: failed to load exact and distractor TARGET rows in hostgroup 952"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the explicit TLS probe sequence"); + return EXIT_FAILURE; + } + + vector topology_endpoints = state.explicit_target_endpoints; + vector distractor_endpoints = distractor.get_endpoints(); + topology_endpoints.insert(topology_endpoints.end(), distractor_endpoints.begin(), distractor_endpoints.end()); + vector topology = cluster.get_topology("AVAILABLE"); + int topology_rc = sim.topology_update(topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish explicit AVAILABLE topology"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 950 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + ProbeChain chain {}; + int chain_rc = wait_for_probe_chain(sim, seq, cluster, 0, 1, chain); + if (chain_rc != EXIT_SUCCESS) { + diag("Error: explicit TLS probe chain did not complete"); + return EXIT_FAILURE; + } + + bool exact_target = + chain.green.backend.host == cluster.green_writer.ip && + chain.green.backend.host != distractor.green_writer.ip && + chain.green.encrypted; + int no_distractor_rc = bgd_expect_no_metadata_probe(sim, seq, distractor.green_writer.endpoint(), kNegativeProbeTimeoutMs); + if (no_distractor_rc != EXIT_SUCCESS) { + diag("Error: explicit discovery probed the plaintext green-writer distractor"); + return EXIT_FAILURE; + } + + ok(exact_target, "explicit discovery selects only the exact TLS TARGET and rejects the plaintext distractor"); + + bool ordered = probe_chain_ordered(chain); + bool blue_plaintext = !chain.table.encrypted && !chain.blue.encrypted; + ok(ordered && blue_plaintext, "explicit probes run plaintext table-check, blue metadata, then TLS TARGET metadata"); + return EXIT_SUCCESS; +} + +/** + * Apply green hostgroup TLS defaults when ProxySQL creates the TARGET row. + * + * - Leave green writer hostgroup 962 empty. + * - Configure servers_defaults use_ssl=1. + * - Publish AVAILABLE topology. + * - Verify the created TARGET runtime row and metadata probe use TLS. + */ +int test_created_target_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.created_target; + BGD_Hostgroups& hg = state.created_target_hg; + + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure created-TARGET simulated writers"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer }; + vector no_green_servers {}; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, + blue_servers, no_green_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure created-TARGET hostgroups 960-963"); + return EXIT_FAILURE; + } + + string defaults_query = + "INSERT INTO mysql_hostgroup_attributes(hostgroup_id,servers_defaults) VALUES (" + + to_string(hg.green_writer) + ",' {\"use_ssl\":1 }')"; + vector defaults_queries { + defaults_query, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + int defaults_rc = execute_all(admin, defaults_queries); + if (defaults_rc != EXIT_SUCCESS) { + diag("Error: failed to configure TLS defaults for green writer hostgroup 962"); + return EXIT_FAILURE; + } + + string empty_query = + "SELECT COUNT(*)=0 FROM mysql_servers WHERE hostgroup_id=" + + to_string(hg.green_writer); + int empty_rc = bgd_wait_for_condition(admin, empty_query, kTimeoutSeconds); + if (empty_rc != EXIT_SUCCESS) { + diag("Error: green writer hostgroup 962 was not empty before discovery"); + return EXIT_FAILURE; + } + + ok(true, "green writer hostgroup 962 starts empty with use_ssl=1 defaults"); + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the created-TARGET probe sequence"); + return EXIT_FAILURE; + } + + vector topology = cluster.get_topology("AVAILABLE"); + int topology_rc = sim.topology_update(state.created_target_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish created-TARGET AVAILABLE topology"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 960 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + string created_query = + "SELECT COUNT(*)=1 FROM runtime_mysql_servers WHERE hostgroup_id=962 AND hostname=" + + bgd_sql_quote(cluster.green_writer.hostname) + " AND port=3306 AND use_ssl=1"; + int created_rc = bgd_wait_for_condition(admin, created_query, kTimeoutSeconds); + if (created_rc != EXIT_SUCCESS) { + diag("Error: discovery did not create the TLS TARGET row in hostgroup 962"); + return EXIT_FAILURE; + } + + bool runtime_tls = runtime_server_tls_matches(admin, hg.green_writer, cluster.green_writer, 1); + ok(runtime_tls, "AVAILABLE creates the TARGET runtime row with hostgroup 962 TLS defaults"); + + ProbeChain chain {}; + int chain_rc = wait_for_probe_chain(sim, seq, cluster, 0, 1, chain); + if (chain_rc != EXIT_SUCCESS) { + diag("Error: created-TARGET TLS probe chain did not complete"); + return EXIT_FAILURE; + } + + bool green_tls = chain.green.backend.host == cluster.green_writer.ip && chain.green.encrypted; + bool ordered = probe_chain_ordered(chain); + ok(green_tls && ordered, "created TARGET probes run table-check, blue metadata, then TLS green metadata"); + return EXIT_SUCCESS; +} + +int main() { + plan(7); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // ProxySQL: load a plaintext reader before the TLS writer in automatic hostgroups 940 and 941. + // Simulator: publish AVAILABLE topology. + // Verify: table-check and metadata probes select the writer and use TLS. + if (test_automatic_writer_tls(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: load an exact TLS TARGET and plaintext distractor into green writer hostgroup 952. + // Simulator: publish AVAILABLE for the exact TARGET deployment. + // Verify: metadata probing selects the exact TARGET and preserves probe order. + if (test_explicit_target_tls(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: leave hostgroup 962 empty and configure use_ssl=1 server defaults. + // Simulator: publish AVAILABLE topology. + // Verify: ProxySQL creates and probes the TARGET row with TLS. + if (test_created_target_tls(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} From 6843d78fe8598c7238acbb4ba5ac038121a2d937 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 28 Jul 2026 07:08:56 +0000 Subject: [PATCH 68/81] test: cover RDS BGD lifecycle and rollback - Cover forward writer switchover and cancellation rollback. - Verify writer placement after disabling or removing BGD during switchover. - Register and lint the focused lifecycle tests. Signed-off-by: Wazir Ahmed --- test/tap/groups/groups.json | 4 + ...st_rds_bgd_disable_during_switchover-t.cpp | 228 +++++++ ...est_rds_bgd_remove_during_switchover-t.cpp | 251 +++++++ test/tap/tests/test_rds_bgd_rollback-t.cpp | 631 ++++++++++++++++++ .../test_rds_bgd_writer_switchover-t.cpp | 558 ++++++++++++++++ 5 files changed, 1672 insertions(+) create mode 100644 test/tap/tests/test_rds_bgd_disable_during_switchover-t.cpp create mode 100644 test/tap/tests/test_rds_bgd_remove_during_switchover-t.cpp create mode 100644 test/tap/tests/test_rds_bgd_rollback-t.cpp create mode 100644 test/tap/tests/test_rds_bgd_writer_switchover-t.cpp diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index bbabf99245..f3ed4d88ca 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -432,10 +432,14 @@ "test_query_timeout-t" : [ "legacy-g9","mariadb10-galera-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql84-gr-g9","mysql90-g4","mysql95-g4" ], "test_rds_bgd_automatic_discovery-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_configuration_persistence-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_disable_during_switchover-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_explicit_startup-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_green_membership_ordering-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_probe_tls-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_remove_during_switchover-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_rollback-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_smoke-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_writer_switchover-t" : [ "cluster_sim_rds_bgd-g1" ], "test_read_only_actions_offline_hard_servers-t" : [ "legacy-g5","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g5","mysql84-g9","mysql90-g4","mysql90-g5","mysql95-g4","mysql95-g5" ], "test_rw_binary_data-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql90-g4","mysql95-g4" ], "test_server_sess_status-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql90-g4","mysql95-g4" ], diff --git a/test/tap/tests/test_rds_bgd_disable_during_switchover-t.cpp b/test/tap/tests/test_rds_bgd_disable_during_switchover-t.cpp new file mode 100644 index 0000000000..07c7327bf9 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_disable_during_switchover-t.cpp @@ -0,0 +1,228 @@ +/** + * @file test_rds_bgd_disable_during_switchover-t.cpp + * @brief Disabling BGD during writer switchover. + * + * Steps: + * + * 1. Configure BGD hostgroups 1340-1343 and reach `AVAILABLE`. + * 2. Publish `SWITCHOVER_IN_PROGRESS` and verify that the blue writer moves + * from hostgroup 1340 to hostgroup 1341. + * 3. Set `active=0` without changing the configured hostgroups. + * 4. Verify that the blue writer returns from hostgroup 1341 to hostgroup 1340. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; + +struct TestState { + RDS_BGD_Cluster cluster { bgd_cluster_init() }; + BGD_Hostgroups hostgroups { 1340, 1341, 1342, 1343 }; + vector topology_endpoints { cluster.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +/** + * Configure BGD hostgroups 1340-1343. + * + * - Set `read_only=0` for the simulated blue and green writers. + * - Publish `AVAILABLE` topology. + * - Configure `mysql_servers` and `mysql_aws_rds_bgd_hostgroups`. + * - Verify that the runtime BGD row reaches `AVAILABLE`. + */ +int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Set read_only=0 for the simulated blue and green writers. + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated writer read_only values"); + return EXIT_FAILURE; + } + + // Publish AVAILABLE topology. + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology"); + return EXIT_FAILURE; + } + + // Configure mysql_servers and mysql_aws_rds_bgd_hostgroups. + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0], cluster.green_readers[1] }; + + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure mysql_servers and mysql_aws_rds_bgd_hostgroups"); + return EXIT_FAILURE; + } + + // Wait for the runtime BGD row to report AVAILABLE. + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: runtime BGD status did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + ok(true, "BGD status for wHG 1340 reports AVAILABLE"); + return EXIT_SUCCESS; +} + +/** + * Move the BGD row for writer hostgroup 1340 into writer switchover. + * + * - Publish `SWITCHOVER_IN_PROGRESS`. + * - Verify `WRITER_SWITCHOVER_IN_PROGRESS`. + * - Verify that the blue writer moves to the blue reader hostgroup. + */ +int test_writer_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Publish SWITCHOVER_IN_PROGRESS and wait for the runtime BGD status. + vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: runtime BGD status did not reach WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + ok(true, "BGD status for wHG 1340 reports WRITER_SWITCHOVER_IN_PROGRESS"); + + // Verify the blue writer was moved from the writer to the reader hostgroup. + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, true, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer did not move to the blue reader hostgroup"); + return EXIT_FAILURE; + } + + ok(true, "SWITCHOVER_IN_PROGRESS moves the blue writer from hostgroup 1340 to 1341"); + return EXIT_SUCCESS; +} + +/** + * Disable BGD during writer switchover. + * + * - Set `active=0` in `mysql_aws_rds_bgd_hostgroups`. + * - Load the configuration to runtime without changing any hostgroups. + * - Verify that the blue writer returns from hostgroup 1341 to hostgroup 1340. + */ +int test_disable_during_switchover(MYSQL* admin, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Disable BGD without changing mysql_servers or the configured hostgroups. + string update_bgd = "UPDATE mysql_aws_rds_bgd_hostgroups SET active=0 WHERE writer_hostgroup=" + to_string(hg.blue_writer); + vector queries { update_bgd, "LOAD MYSQL SERVERS TO RUNTIME" }; + + int rc = execute_all(admin, queries); + if (rc != EXIT_SUCCESS) { + diag("Error: failed to set active=0 and load the BGD configuration to runtime"); + return EXIT_FAILURE; + } + + // Wait until disabling restores the blue writer to its writer hostgroup. + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer did not return to the blue writer hostgroup"); + return EXIT_FAILURE; + } + + ok(true, "setting active=0 restores the blue writer from hostgroup 1341 to 1340"); + return EXIT_SUCCESS; +} + +int main() { + plan(4); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: set the blue/green writers to read_only=0 and publish AVAILABLE topology. + // ProxySQL: update mysql_servers and mysql_aws_rds_bgd_hostgroups with BGD configuration. + // Verify: runtime_mysql_aws_rds_bgd_hostgroups status reports AVAILABLE. + if (test_bgd_status_available(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish SWITCHOVER_IN_PROGRESS topology. + // Verify: runtime_mysql_aws_rds_bgd_hostgroups status reports WRITER_SWITCHOVER_IN_PROGRESS. + // Verify: runtime_mysql_servers moves the blue writer from writer hostgroup to reader hostgroup. + if (test_writer_switchover_in_progress(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: set active=0 without changing mysql_servers or the configured BGD hostgroups. + // Verify: runtime_mysql_servers returns the blue writer from reader hostgroup to writer hostgroup. + if (test_disable_during_switchover(admin, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_remove_during_switchover-t.cpp b/test/tap/tests/test_rds_bgd_remove_during_switchover-t.cpp new file mode 100644 index 0000000000..dbfa042c38 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_remove_during_switchover-t.cpp @@ -0,0 +1,251 @@ +/** + * @file test_rds_bgd_remove_during_switchover-t.cpp + * @brief Removing BGD configuration during writer switchover. + * + * Steps: + * + * 1. Configure BGD hostgroups 1350-1353 and reach `AVAILABLE`. + * 2. Publish `SWITCHOVER_IN_PROGRESS` and verify that the blue writer moves + * from hostgroup 1350 to hostgroup 1351. + * 3. Delete writer hostgroup 1350 from `mysql_aws_rds_bgd_hostgroups`. + * 4. Verify that the blue writer returns to hostgroup 1350 and the runtime BGD + * row is removed. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; + +struct TestState { + RDS_BGD_Cluster cluster { bgd_cluster_init() }; + BGD_Hostgroups hostgroups { 1350, 1351, 1352, 1353 }; + vector topology_endpoints { cluster.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int wait_for_bgd_row_absent(MYSQL* admin, BGD_Hostgroups& hg) { + string query = + "SELECT COUNT(*)=0 FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(hg.blue_writer); + + int rc = bgd_wait_for_condition(admin, query, kTimeoutSeconds); + return rc; +} + +/** + * Configure BGD hostgroups 1350-1353. + * + * - Set `read_only=0` for the simulated blue and green writers. + * - Publish `AVAILABLE` topology. + * - Configure `mysql_servers` and `mysql_aws_rds_bgd_hostgroups`. + * - Verify that the runtime BGD row reaches `AVAILABLE`. + */ +int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Set read_only=0 for the simulated blue and green writers. + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated writer read_only values"); + return EXIT_FAILURE; + } + + // Publish AVAILABLE topology. + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology"); + return EXIT_FAILURE; + } + + // Configure mysql_servers and mysql_aws_rds_bgd_hostgroups. + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0], cluster.green_readers[1] }; + + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure mysql_servers and mysql_aws_rds_bgd_hostgroups"); + return EXIT_FAILURE; + } + + // Wait for the runtime BGD row to report AVAILABLE. + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: runtime BGD status did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + ok(true, "BGD status for wHG 1350 reports AVAILABLE"); + return EXIT_SUCCESS; +} + +/** + * Move the BGD row for writer hostgroup 1350 into writer switchover. + * + * - Publish `SWITCHOVER_IN_PROGRESS`. + * - Verify `WRITER_SWITCHOVER_IN_PROGRESS`. + * - Verify that the blue writer moves to the blue reader hostgroup. + */ +int test_writer_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Publish SWITCHOVER_IN_PROGRESS and wait for the runtime BGD status. + vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: runtime BGD status did not reach WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + ok(true, "BGD status for wHG 1350 reports WRITER_SWITCHOVER_IN_PROGRESS"); + + // Verify the blue writer was moved from the writer to the reader hostgroup. + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, true, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer did not move to the blue reader hostgroup"); + return EXIT_FAILURE; + } + + ok(true, "SWITCHOVER_IN_PROGRESS moves the blue writer from hostgroup 1350 to 1351"); + return EXIT_SUCCESS; +} + +/** + * Remove BGD configuration during writer switchover. + * + * - Delete writer hostgroup 1350 from `mysql_aws_rds_bgd_hostgroups`. + * - Load the configuration to runtime without changing `mysql_servers`. + * - Verify that the blue writer returns to hostgroup 1350. + * - Verify that the runtime BGD row for writer hostgroup 1350 is removed. + */ +int test_remove_during_switchover(MYSQL* admin, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Delete the BGD row without changing mysql_servers or the configured hostgroups. + string delete_bgd = "DELETE FROM mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(hg.blue_writer); + vector queries { + delete_bgd, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + if (rc != EXIT_SUCCESS) { + diag("Error: failed to delete and load the BGD configuration"); + return EXIT_FAILURE; + } + + // Wait until deleting the row restores the blue writer to its writer hostgroup. + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer did not return to the blue writer hostgroup"); + return EXIT_FAILURE; + } + + // Wait until the deleted BGD row is absent from runtime. + int row_rc = wait_for_bgd_row_absent(admin, hg); + if (row_rc != EXIT_SUCCESS) { + diag("Error: deleted BGD configuration remains in the runtime table"); + return EXIT_FAILURE; + } + + ok(true, "deleting BGD configuration restores the blue writer from hostgroup 1351 to 1350"); + ok(true, "deleting wHG 1350 removes it from runtime_mysql_aws_rds_bgd_hostgroups"); + return EXIT_SUCCESS; +} + +int main() { + plan(5); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: set the blue/green writers to read_only=0 and publish AVAILABLE topology. + // ProxySQL: update mysql_servers and mysql_aws_rds_bgd_hostgroups with BGD configuration. + // Verify: runtime_mysql_aws_rds_bgd_hostgroups status reports AVAILABLE. + if (test_bgd_status_available(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish SWITCHOVER_IN_PROGRESS topology. + // Verify: runtime_mysql_aws_rds_bgd_hostgroups status reports WRITER_SWITCHOVER_IN_PROGRESS. + // Verify: runtime_mysql_servers moves the blue writer from writer hostgroup to reader hostgroup. + if (test_writer_switchover_in_progress(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: delete wHG 1350 from mysql_aws_rds_bgd_hostgroups without changing mysql_servers. + // Verify: the blue writer returns to hostgroup 1350 and the runtime BGD row is absent. + if (test_remove_during_switchover(admin, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_rollback-t.cpp b/test/tap/tests/test_rds_bgd_rollback-t.cpp new file mode 100644 index 0000000000..1e1bbe270f --- /dev/null +++ b/test/tap/tests/test_rds_bgd_rollback-t.cpp @@ -0,0 +1,631 @@ +/** + * @file test_rds_bgd_rollback-t.cpp + * @brief Returning from writer switchover to AVAILABLE. + * + * Steps: + * + * 1. Enter SWITCHOVER_INITIATED with a monitor-created green writer. + * 2. Return to AVAILABLE and verify blue-writer placement, read_only + * processing, and the monitor-created green writer. + * 3. Enter SWITCHOVER_IN_PROGRESS with explicit green servers and pools. + * 4. Return to AVAILABLE and verify blue routing without removing explicit + * green servers or draining their pools. + * 5. Repeat AVAILABLE and verify that rollback remains stable. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; + +struct GreenRows { + vector admin_writer {}; + vector runtime_writer {}; + vector admin_reader {}; + vector runtime_reader {}; +}; + +struct TestState { + RDS_BGD_Cluster initiated_cluster { bgd_cluster_init() }; + BGD_Hostgroups initiated_hg { 980, 981, 982, 983 }; + vector initiated_endpoints { initiated_cluster.get_endpoints() }; + + RDS_BGD_Cluster progress_cluster { bgd_cluster_2_init() }; + BGD_Hostgroups progress_hg { 990, 991, 992, 993 }; + vector progress_endpoints { progress_cluster.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + rows.push_back({ + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + rows.push_back({ + cluster.green_readers[0].hostname, + cluster.green_readers[0].hostname, + cluster.green_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + return rows; +} + +int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_0(sim, cluster.green_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[1]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int publish_topology(RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { + vector topology = topology_with_reader_pair(cluster, status); + + int rc = sim.topology_update(endpoints, topology); + return rc; +} + +int wait_for_green_writer(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { + auto [probe_rc, probe] = + sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + return probe_rc; +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + vector queries { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +rc_t connect_and_echo(CommandLine& cl) { + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + rc_t result { EXIT_FAILURE, {} }; + return result; + } + + rc_t result = bgd_backend_ip_echo(client); + mysql_close(client); + return result; +} + +int64_t last_read_only_log_time(MYSQL* admin, RDS_BGD_Host& host) { + string query = + "SELECT COALESCE(MAX(time_start_us),0) FROM mysql_server_read_only_log WHERE hostname=" + + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return -1; + } + + int64_t time = strtoll(rows[0][0].c_str(), nullptr, 10); + return time; +} + +int wait_for_read_only_log(MYSQL* admin, RDS_BGD_Host& host, int64_t baseline) { + string query = + "SELECT COUNT(*)>0 FROM mysql_server_read_only_log WHERE hostname=" + + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + + " AND time_start_us>" + to_string(baseline); + + int rc = bgd_wait_for_condition(admin, query, kTimeoutSeconds); + return rc; +} + +rc_t> server_row_snapshot(MYSQL* admin, string table, int hostgroup, RDS_BGD_Host& host) { + string query = + "SELECT hostgroup_id,hostname,port,status,use_ssl,weight,max_connections FROM " + table + + " WHERE hostgroup_id=" + to_string(hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port); + + rc_t> result = mysql_query_ext_rows(admin, query); + return result; +} + +bool server_row_matches(MYSQL* admin, string table, int hostgroup, RDS_BGD_Host& host, vector expected) { + auto [rc, rows] = server_row_snapshot(admin, table, hostgroup, host); + if (rc != EXIT_SUCCESS) { + return false; + } + + bool matches = rows == expected; + return matches; +} + +rc_t green_rows_snapshot(MYSQL* admin, BGD_Hostgroups& hg, RDS_BGD_Cluster& cluster, bool include_reader) { + GreenRows rows {}; + + auto [admin_writer_rc, admin_writer] = server_row_snapshot(admin, "mysql_servers", hg.green_writer, cluster.green_writer); + if (admin_writer_rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + rows.admin_writer = admin_writer; + + auto [runtime_writer_rc, runtime_writer] = + server_row_snapshot(admin, "runtime_mysql_servers", hg.green_writer, cluster.green_writer); + if (runtime_writer_rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + rows.runtime_writer = runtime_writer; + + if (include_reader) { + auto [admin_reader_rc, admin_reader] = + server_row_snapshot(admin, "mysql_servers", hg.green_reader, cluster.green_readers[0]); + if (admin_reader_rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + rows.admin_reader = admin_reader; + + auto [runtime_reader_rc, runtime_reader] = + server_row_snapshot(admin, "runtime_mysql_servers", hg.green_reader, cluster.green_readers[0]); + if (runtime_reader_rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + rows.runtime_reader = runtime_reader; + } + + return { EXIT_SUCCESS, rows }; +} + +bool green_rows_match(MYSQL* admin, BGD_Hostgroups& hg, RDS_BGD_Cluster& cluster, GreenRows& expected, bool include_reader) { + bool admin_writer = server_row_matches(admin, "mysql_servers", hg.green_writer, cluster.green_writer, expected.admin_writer); + bool runtime_writer = + server_row_matches(admin, "runtime_mysql_servers", hg.green_writer, cluster.green_writer, expected.runtime_writer); + + bool admin_reader = true; + bool runtime_reader = true; + if (include_reader) { + admin_reader = + server_row_matches(admin, "mysql_servers", hg.green_reader, cluster.green_readers[0], expected.admin_reader); + runtime_reader = server_row_matches( + admin, "runtime_mysql_servers", hg.green_reader, cluster.green_readers[0], expected.runtime_reader + ); + } + + bool matches = admin_writer && runtime_writer && admin_reader && runtime_reader; + return matches; +} + +/** + * Return from SWITCHOVER_INITIATED to AVAILABLE. + * + * - Configure BGD without a green mysql_servers row and let the worker create + * the green writer in runtime. + * - Publish SWITCHOVER_INITIATED, then return to AVAILABLE. + * - Verify blue-writer placement and normal read_only processing are restored. + * - Repeat AVAILABLE and verify the monitor-created green writer remains. + */ +int test_initiated_rollback(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.initiated_cluster; + BGD_Hostgroups& hg = state.initiated_hg; + + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG 980"); + return EXIT_FAILURE; + } + + int available_topology_rc = publish_topology(sim, state.initiated_endpoints, cluster, "AVAILABLE"); + if (available_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 980"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 980-983"); + return EXIT_FAILURE; + } + + int available_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (available_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 980 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + auto [created_rc, created_rows] = green_rows_snapshot(admin, hg, cluster, false); + if (created_rc != EXIT_SUCCESS || !created_rows.admin_writer.empty() || created_rows.runtime_writer.size() != 1) { + diag("Error: the green writer was not created only in runtime hostgroup 982"); + return EXIT_FAILURE; + } + + int initiated_topology_rc = publish_topology(sim, state.initiated_endpoints, cluster, "SWITCHOVER_INITIATED"); + if (initiated_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_INITIATED topology for wHG 980"); + return EXIT_FAILURE; + } + + int initiated_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_INITIATED", kTimeoutSeconds); + if (initiated_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 980 did not reach WRITER_SWITCHOVER_INITIATED"); + return EXIT_FAILURE; + } + + int64_t read_only_baseline = last_read_only_log_time(admin, cluster.blue_readers[0]); + if (read_only_baseline < 0) { + diag("Error: failed to read the blue-reader read_only log baseline"); + return EXIT_FAILURE; + } + + auto [return_seq_rc, return_seq] = sim.probe_log_last_sequence(); + if (return_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the AVAILABLE rollback probe sequence"); + return EXIT_FAILURE; + } + + int return_topology_rc = publish_topology(sim, state.initiated_endpoints, cluster, "AVAILABLE"); + if (return_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to return wHG 980 topology to AVAILABLE"); + return EXIT_FAILURE; + } + + int returned_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (returned_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 980 did not return to AVAILABLE"); + return EXIT_FAILURE; + } + + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: initiated rollback did not restore the blue writer to hostgroup 980"); + return EXIT_FAILURE; + } + + int probe_rc = wait_for_green_writer(sim, return_seq, cluster); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: initiated rollback did not resume green-writer probing"); + return EXIT_FAILURE; + } + + ok(true, "returning from SWITCHOVER_INITIATED restores the blue writer to hostgroup 980"); + + int reader_update_rc = bgd_set_host_read_only_0(sim, cluster.blue_readers[0]); + if (reader_update_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated blue reader"); + return EXIT_FAILURE; + } + + int reader_log_rc = wait_for_read_only_log(admin, cluster.blue_readers[0], read_only_baseline); + if (reader_log_rc != EXIT_SUCCESS) { + diag("Error: read_only monitoring remained suppressed after initiated rollback"); + return EXIT_FAILURE; + } + + ok(true, "returning to AVAILABLE restores normal read_only monitoring"); + + auto [repeat_seq_rc, repeat_seq] = sim.probe_log_last_sequence(); + if (repeat_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the repeated AVAILABLE probe sequence"); + return EXIT_FAILURE; + } + + int repeat_topology_rc = publish_topology(sim, state.initiated_endpoints, cluster, "AVAILABLE"); + if (repeat_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to repeat AVAILABLE topology for wHG 980"); + return EXIT_FAILURE; + } + + int repeat_status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (repeat_status_rc != EXIT_SUCCESS) { + diag("Error: repeated AVAILABLE did not keep BGD status for wHG 980"); + return EXIT_FAILURE; + } + + int repeat_placement_rc = + bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (repeat_placement_rc != EXIT_SUCCESS) { + diag("Error: repeated AVAILABLE changed blue-writer placement for wHG 980"); + return EXIT_FAILURE; + } + + int repeat_probe_rc = wait_for_green_writer(sim, repeat_seq, cluster); + if (repeat_probe_rc != EXIT_SUCCESS) { + diag("Error: repeated AVAILABLE did not probe the green writer"); + return EXIT_FAILURE; + } + + bool created_rows_match = green_rows_match(admin, hg, cluster, created_rows, false); + ok(created_rows_match, "repeated AVAILABLE keeps the monitor-created green writer in runtime hostgroup 982"); + return EXIT_SUCCESS; +} + +/** + * Return from SWITCHOVER_IN_PROGRESS to AVAILABLE. + * + * - Configure explicit green writer/reader rows and establish their pools. + * - Enter SWITCHOVER_IN_PROGRESS and require blue-writer demotion. + * - Return to AVAILABLE and verify blue routing is restored. + * - Verify explicit green rows and pools remain unchanged. + * - Repeat AVAILABLE and verify rollback remains stable. + */ +int test_in_progress_rollback(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.progress_cluster; + BGD_Hostgroups& hg = state.progress_hg; + + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG 990"); + return EXIT_FAILURE; + } + + int available_topology_rc = publish_topology(sim, state.progress_endpoints, cluster, "AVAILABLE"); + if (available_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 990"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 990-993"); + return EXIT_FAILURE; + } + + int available_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (available_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 990 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + auto [green_rows_rc, green_rows] = green_rows_snapshot(admin, hg, cluster, true); + if (green_rows_rc != EXIT_SUCCESS || green_rows.admin_writer.size() != 1 || + green_rows.runtime_writer.size() != 1 || green_rows.admin_reader.size() != 1 || + green_rows.runtime_reader.size() != 1) { + diag("Error: failed to snapshot explicit green servers for wHG 990"); + return EXIT_FAILURE; + } + + int writer_hg_rc = set_default_hostgroup(admin, hg.green_writer); + if (writer_hg_rc != EXIT_SUCCESS) { + diag("Error: failed to route the test user through green writer hostgroup 992"); + return EXIT_FAILURE; + } + + int writer_echo_rc = connect_and_echo(cl).first; + if (writer_echo_rc != EXIT_SUCCESS) { + diag("Error: failed to establish a green-writer connection pool"); + return EXIT_FAILURE; + } + + int reader_hg_rc = set_default_hostgroup(admin, hg.green_reader); + if (reader_hg_rc != EXIT_SUCCESS) { + diag("Error: failed to route the test user through green reader hostgroup 993"); + return EXIT_FAILURE; + } + + int reader_echo_rc = connect_and_echo(cl).first; + if (reader_echo_rc != EXIT_SUCCESS) { + diag("Error: failed to establish a green-reader connection pool"); + return EXIT_FAILURE; + } + + int restore_hg_rc = set_default_hostgroup(admin, hg.blue_writer); + if (restore_hg_rc != EXIT_SUCCESS) { + diag("Error: failed to restore the test user to blue writer hostgroup 990"); + return EXIT_FAILURE; + } + + auto [writer_pool_rc, writer_pool] = bgd_connection_pool_count(admin, hg.green_writer); + auto [reader_pool_rc, reader_pool] = bgd_connection_pool_count(admin, hg.green_reader); + if (writer_pool_rc != EXIT_SUCCESS || writer_pool < 1 || reader_pool_rc != EXIT_SUCCESS || reader_pool < 1) { + diag("Error: failed to establish green pools before in-progress rollback"); + return EXIT_FAILURE; + } + + int initiated_topology_rc = publish_topology(sim, state.progress_endpoints, cluster, "SWITCHOVER_INITIATED"); + if (initiated_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_INITIATED topology for wHG 990"); + return EXIT_FAILURE; + } + + int initiated_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_INITIATED", kTimeoutSeconds); + if (initiated_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 990 did not reach WRITER_SWITCHOVER_INITIATED"); + return EXIT_FAILURE; + } + + int progress_topology_rc = publish_topology(sim, state.progress_endpoints, cluster, "SWITCHOVER_IN_PROGRESS"); + if (progress_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for wHG 990"); + return EXIT_FAILURE; + } + + int progress_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (progress_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 990 did not reach WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + int demotion_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, true, kTimeoutSeconds); + if (demotion_rc != EXIT_SUCCESS) { + diag("Error: the blue writer did not move to reader hostgroup 991"); + return EXIT_FAILURE; + } + + auto [return_seq_rc, return_seq] = sim.probe_log_last_sequence(); + if (return_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the in-progress rollback probe sequence"); + return EXIT_FAILURE; + } + + int return_topology_rc = publish_topology(sim, state.progress_endpoints, cluster, "AVAILABLE"); + if (return_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to return wHG 990 topology to AVAILABLE"); + return EXIT_FAILURE; + } + + int returned_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (returned_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 990 did not return to AVAILABLE"); + return EXIT_FAILURE; + } + + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: in-progress rollback did not restore the blue writer to hostgroup 990"); + return EXIT_FAILURE; + } + + int probe_rc = wait_for_green_writer(sim, return_seq, cluster); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: in-progress rollback did not resume green-writer probing"); + return EXIT_FAILURE; + } + + auto [blue_echo_rc, blue_echo] = connect_and_echo(cl); + bool blue_routing = blue_echo_rc == EXIT_SUCCESS && blue_echo.find(cluster.blue_writer.ip) != string::npos; + ok(blue_routing, "returning from SWITCHOVER_IN_PROGRESS restores routing through blue writer hostgroup 990"); + + bool rows_match = green_rows_match(admin, hg, cluster, green_rows, true); + ok(rows_match, "in-progress rollback keeps explicit green servers in Admin and runtime hostgroups 992-993"); + + auto [post_writer_pool_rc, post_writer_pool] = bgd_connection_pool_count(admin, hg.green_writer); + auto [post_reader_pool_rc, post_reader_pool] = bgd_connection_pool_count(admin, hg.green_reader); + bool pools_match = post_writer_pool_rc == EXIT_SUCCESS && post_writer_pool >= writer_pool && + post_reader_pool_rc == EXIT_SUCCESS && post_reader_pool >= reader_pool; + ok(pools_match, "in-progress rollback does not drain green writer and reader pools"); + + auto [repeat_seq_rc, repeat_seq] = sim.probe_log_last_sequence(); + if (repeat_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the repeated AVAILABLE probe sequence for wHG 990"); + return EXIT_FAILURE; + } + + int repeat_topology_rc = publish_topology(sim, state.progress_endpoints, cluster, "AVAILABLE"); + if (repeat_topology_rc != EXIT_SUCCESS) { + diag("Error: failed to repeat AVAILABLE topology for wHG 990"); + return EXIT_FAILURE; + } + + int repeat_status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (repeat_status_rc != EXIT_SUCCESS) { + diag("Error: repeated AVAILABLE did not keep BGD status for wHG 990"); + return EXIT_FAILURE; + } + + int repeat_placement_rc = + bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (repeat_placement_rc != EXIT_SUCCESS) { + diag("Error: repeated AVAILABLE changed blue-writer placement for wHG 990"); + return EXIT_FAILURE; + } + + int repeat_probe_rc = wait_for_green_writer(sim, repeat_seq, cluster); + if (repeat_probe_rc != EXIT_SUCCESS) { + diag("Error: repeated AVAILABLE did not probe the green writer for wHG 990"); + return EXIT_FAILURE; + } + + auto [repeat_writer_pool_rc, repeat_writer_pool] = bgd_connection_pool_count(admin, hg.green_writer); + auto [repeat_reader_pool_rc, repeat_reader_pool] = bgd_connection_pool_count(admin, hg.green_reader); + bool repeat_rows = green_rows_match(admin, hg, cluster, green_rows, true); + bool repeat_pools = repeat_writer_pool_rc == EXIT_SUCCESS && repeat_writer_pool >= writer_pool && + repeat_reader_pool_rc == EXIT_SUCCESS && repeat_reader_pool >= reader_pool; + ok(repeat_rows && repeat_pools, "repeated AVAILABLE keeps blue placement, explicit green servers, and green pools"); + return EXIT_SUCCESS; +} + +int main() { + plan(7); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish AVAILABLE, SWITCHOVER_INITIATED, then AVAILABLE for a monitor-created green writer. + // Verify: blue writer returns to hostgroup 980 and normal read_only monitoring resumes. + // Verify: repeated AVAILABLE keeps the monitor-created green writer in runtime hostgroup 982. + if (test_initiated_rollback(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish AVAILABLE, SWITCHOVER_INITIATED, SWITCHOVER_IN_PROGRESS, then AVAILABLE. + // ProxySQL: configure explicit green servers and establish green writer/reader pools. + // Verify: blue routing returns without removing green servers or draining their pools. + if (test_in_progress_rollback(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_writer_switchover-t.cpp b/test/tap/tests/test_rds_bgd_writer_switchover-t.cpp new file mode 100644 index 0000000000..9bf8b8ee94 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_writer_switchover-t.cpp @@ -0,0 +1,558 @@ +/** + * @file test_rds_bgd_writer_switchover-t.cpp + * @brief BGD writer switchover from AVAILABLE through POST_PROCESSING. + * + * Steps: + * + * 1. Configure BGD hostgroups 970-973 and reach AVAILABLE. + * 2. Publish SWITCHOVER_INITIATED and verify read-only placement suppression. + * 3. Publish SWITCHOVER_IN_PROGRESS and verify blue-writer demotion. + * 4. Create a blue-writer pool through normal routing hostgroup 974. + * 5. Publish SWITCHOVER_IN_POST_PROCESSING and verify writer restoration, + * blue-pool drain, and green backend routing. + * 6. Create a post-cutover pool and verify repeated POST_PROCESSING does not + * drain it again. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; +const uint32_t kReadOnlyObservationMs = 500; + +struct TestState { + RDS_BGD_Cluster cluster { bgd_cluster_init() }; + BGD_Hostgroups hostgroups { 970, 971, 972, 973 }; + int pool_hostgroup { 974 }; + vector topology_endpoints { cluster.get_endpoints() }; + int64_t reader_log_baseline { -1 }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + rows.push_back({ + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + rows.push_back({ + cluster.green_readers[0].hostname, + cluster.green_readers[0].hostname, + cluster.green_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + return rows; +} + +int wait_for_green_observation(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { + auto [probe_rc, probe] = + sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + return probe_rc; +} + +int64_t last_read_only_log_time(MYSQL* admin, RDS_BGD_Host& host) { + string query = + "SELECT COALESCE(MAX(time_start_us),0) FROM mysql_server_read_only_log WHERE hostname=" + + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return -1; + } + + int64_t time = strtoll(rows[0][0].c_str(), nullptr, 10); + return time; +} + +bool server_match_count(MYSQL* admin, int hostgroup, RDS_BGD_Host& host, int expected_count) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool matches = rows[0][0] == to_string(expected_count); + return matches; +} + +int wait_for_blue_writer_pool_drain(MYSQL* admin, RDS_BGD_Cluster& cluster) { + string query = + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0)=0 FROM stats_mysql_connection_pool WHERE srv_host=" + + bgd_sql_quote(cluster.blue_writer.hostname); + + int rc = bgd_wait_for_condition(admin, query, kTimeoutSeconds); + return rc; +} + +rc_t connect_and_echo(CommandLine& cl) { + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + rc_t result { EXIT_FAILURE, {} }; + return result; + } + + rc_t result = bgd_backend_ip_echo(client); + mysql_close(client); + return result; +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + vector queries { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +int create_blue_writer_pool(CommandLine& cl, MYSQL* admin, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + + string add_server = + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(state.pool_hostgroup) + "," + bgd_sql_quote(cluster.blue_writer.hostname) + + "," + to_string(cluster.blue_writer.port) + ",'ONLINE','BGD TAP blue pool router')"; + vector queries { + add_server, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int server_rc = execute_all(admin, queries); + if (server_rc != EXIT_SUCCESS) { + diag("Error: failed to configure blue-pool routing hostgroup 974"); + return EXIT_FAILURE; + } + + int user_rc = set_default_hostgroup(admin, state.pool_hostgroup); + if (user_rc != EXIT_SUCCESS) { + diag("Error: failed to route testuser through blue-pool hostgroup 974"); + return EXIT_FAILURE; + } + + auto [echo_rc, echo] = connect_and_echo(cl); + if (echo_rc != EXIT_SUCCESS || echo.find(cluster.blue_writer.ip) == string::npos) { + diag("Error: failed to create a blue-writer connection through hostgroup 974"); + return EXIT_FAILURE; + } + + int restore_rc = set_default_hostgroup(admin, state.hostgroups.blue_writer); + if (restore_rc != EXIT_SUCCESS) { + diag("Error: failed to restore testuser to writer hostgroup 970"); + return EXIT_FAILURE; + } + + string query = + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0)>=1 FROM stats_mysql_connection_pool WHERE hostgroup=" + + to_string(state.pool_hostgroup) + " AND srv_host=" + bgd_sql_quote(cluster.blue_writer.hostname); + + int pool_rc = bgd_wait_for_condition(admin, query, kTimeoutSeconds); + return pool_rc; +} + +/** + * Configure wHG 970 and reach AVAILABLE. + * + * - Set writer read_only=0 and reader read_only=1 values. + * - Publish AVAILABLE topology with one reader pair. + * - Configure mysql_servers and mysql_aws_rds_bgd_hostgroups. + * - Verify BGD status AVAILABLE. + */ +int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + int blue_writer_rc = bgd_set_host_read_only_0(sim, cluster.blue_writer); + if (blue_writer_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated blue writer"); + return EXIT_FAILURE; + } + + int green_writer_rc = bgd_set_host_read_only_0(sim, cluster.green_writer); + if (green_writer_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated green writer"); + return EXIT_FAILURE; + } + + int blue_reader_0_rc = bgd_set_host_read_only_1(sim, cluster.blue_readers[0]); + if (blue_reader_0_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=1 for the first simulated blue reader"); + return EXIT_FAILURE; + } + + int blue_reader_1_rc = bgd_set_host_read_only_1(sim, cluster.blue_readers[1]); + if (blue_reader_1_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=1 for the second simulated blue reader"); + return EXIT_FAILURE; + } + + vector topology = topology_with_reader_pair(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 970"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 970-973"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 970 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + ok(true, "BGD status for wHG 970 reports AVAILABLE"); + return EXIT_SUCCESS; +} + +/** + * Enter writer switchover initiated. + * + * - Publish SWITCHOVER_INITIATED. + * - Verify WRITER_SWITCHOVER_INITIATED. + * - Change simulated blue writer/reader read_only values. + * - Verify BGD suppresses their normal placement changes. + */ +int test_switchover_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the SWITCHOVER_INITIATED probe sequence"); + return EXIT_FAILURE; + } + + vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_INITIATED topology"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_INITIATED", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 970 did not reach WRITER_SWITCHOVER_INITIATED"); + return EXIT_FAILURE; + } + + ok(true, "BGD status for wHG 970 reports WRITER_SWITCHOVER_INITIATED"); + + int64_t writer_baseline = last_read_only_log_time(admin, cluster.blue_writer); + state.reader_log_baseline = last_read_only_log_time(admin, cluster.blue_readers[0]); + + int writer_ro_rc = bgd_set_host_read_only_1(sim, cluster.blue_writer); + if (writer_ro_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=1 for the simulated blue writer"); + return EXIT_FAILURE; + } + + int reader_ro_rc = bgd_set_host_read_only_0(sim, cluster.blue_readers[0]); + if (reader_ro_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated blue reader"); + return EXIT_FAILURE; + } + + auto [suppression_seq_rc, suppression_seq] = sim.probe_log_last_sequence(); + if (suppression_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the initiated suppression probe sequence"); + return EXIT_FAILURE; + } + + vector repeat_topology = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); + int repeat_rc = sim.topology_update(state.topology_endpoints, repeat_topology); + if (repeat_rc != EXIT_SUCCESS) { + diag("Error: failed to repeat SWITCHOVER_INITIATED topology"); + return EXIT_FAILURE; + } + + int observation_rc = wait_for_green_observation(sim, suppression_seq, cluster); + if (observation_rc != EXIT_SUCCESS) { + diag("Error: BGD did not observe repeated SWITCHOVER_INITIATED topology"); + return EXIT_FAILURE; + } + + int writer_suppression_rc = + bgd_expect_no_read_only_log(admin, cluster.blue_writer, writer_baseline, kReadOnlyObservationMs); + if (writer_suppression_rc != EXIT_SUCCESS) { + diag("Error: blue-writer read_only monitoring was not suppressed during SWITCHOVER_INITIATED"); + return EXIT_FAILURE; + } + + int reader_suppression_rc = + bgd_expect_no_read_only_log(admin, cluster.blue_readers[0], state.reader_log_baseline, kReadOnlyObservationMs); + if (reader_suppression_rc != EXIT_SUCCESS) { + diag("Error: blue-reader read_only monitoring was not suppressed during SWITCHOVER_INITIATED"); + return EXIT_FAILURE; + } + + bool writer_in_writer_hg = server_match_count(admin, hg.blue_writer, cluster.blue_writer, 1); + bool writer_absent_reader_hg = server_match_count(admin, hg.blue_reader, cluster.blue_writer, 0); + bool reader_in_reader_hg = server_match_count(admin, hg.blue_reader, cluster.blue_readers[0], 1); + bool reader_absent_writer_hg = server_match_count(admin, hg.blue_writer, cluster.blue_readers[0], 0); + ok(writer_in_writer_hg && writer_absent_reader_hg && reader_in_reader_hg && reader_absent_writer_hg, + "SWITCHOVER_INITIATED suppresses blue writer and reader placement changes"); + return EXIT_SUCCESS; +} + +/** + * Enter writer switchover in progress. + * + * - Publish SWITCHOVER_IN_PROGRESS. + * - Verify WRITER_SWITCHOVER_IN_PROGRESS. + * - Verify the blue writer moves from hostgroup 970 to 971. + * - Verify the mapped blue reader remains suppressed in hostgroup 971. + */ +int test_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 970 did not reach WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + ok(true, "BGD status for wHG 970 reports WRITER_SWITCHOVER_IN_PROGRESS"); + + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, true, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer did not move from hostgroup 970 to 971"); + return EXIT_FAILURE; + } + + ok(true, "SWITCHOVER_IN_PROGRESS moves the blue writer from hostgroup 970 to 971"); + + int reader_suppression_rc = + bgd_expect_no_read_only_log(admin, cluster.blue_readers[0], state.reader_log_baseline, kReadOnlyObservationMs); + if (reader_suppression_rc != EXIT_SUCCESS) { + diag("Error: blue-reader read_only monitoring was not suppressed during SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + bool reader_in_reader_hg = server_match_count(admin, hg.blue_reader, cluster.blue_readers[0], 1); + bool reader_absent_writer_hg = server_match_count(admin, hg.blue_writer, cluster.blue_readers[0], 0); + ok(reader_in_reader_hg && reader_absent_writer_hg, + "SWITCHOVER_IN_PROGRESS keeps the mapped blue reader suppressed in hostgroup 971"); + return EXIT_SUCCESS; +} + +/** + * Enter writer switchover post-processing. + * + * - Create a blue-writer pool through normal routing hostgroup 974. + * - Publish SWITCHOVER_IN_POST_PROCESSING. + * - Verify WRITER_SWITCHOVER_POST_PROCESSING. + * - Verify writer restoration, blue-pool drain, and green backend routing. + * - Repeat POST_PROCESSING and verify the post-cutover pool is not drained. + */ +int test_switchover_post_processing(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + int blue_pool_rc = create_blue_writer_pool(cl, admin, state); + if (blue_pool_rc != EXIT_SUCCESS) { + diag("Error: failed to establish the blue-writer pool before SWITCHOVER_IN_POST_PROCESSING"); + return EXIT_FAILURE; + } + + vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING topology"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_POST_PROCESSING", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 970 did not reach WRITER_SWITCHOVER_POST_PROCESSING"); + return EXIT_FAILURE; + } + + ok(true, "BGD status for wHG 970 reports WRITER_SWITCHOVER_POST_PROCESSING"); + + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer did not return from hostgroup 971 to 970"); + return EXIT_FAILURE; + } + + int reader_suppression_rc = + bgd_expect_no_read_only_log(admin, cluster.blue_readers[0], state.reader_log_baseline, kReadOnlyObservationMs); + if (reader_suppression_rc != EXIT_SUCCESS) { + diag("Error: blue-reader read_only monitoring was not suppressed during SWITCHOVER_IN_POST_PROCESSING"); + return EXIT_FAILURE; + } + + bool reader_in_reader_hg = server_match_count(admin, hg.blue_reader, cluster.blue_readers[0], 1); + bool reader_absent_writer_hg = server_match_count(admin, hg.blue_writer, cluster.blue_readers[0], 0); + ok(reader_in_reader_hg && reader_absent_writer_hg, + "POST_PROCESSING restores the blue writer to hostgroup 970 and keeps the reader in 971"); + + int pool_drain_rc = wait_for_blue_writer_pool_drain(admin, cluster); + if (pool_drain_rc != EXIT_SUCCESS) { + diag("Error: POST_PROCESSING did not drain the old blue-writer pool"); + return EXIT_FAILURE; + } + + ok(true, "POST_PROCESSING drains the old blue-writer connection pool"); + + auto [echo_rc, echo] = connect_and_echo(cl); + if (echo_rc != EXIT_SUCCESS) { + diag("Error: failed to connect through wHG 970 after POST_PROCESSING"); + return EXIT_FAILURE; + } + + bool green_routing = echo.find(cluster.green_writer.ip) != string::npos; + ok(green_routing, "POST_PROCESSING routes the blue writer hostname to the green backend IP"); + + auto [pool_before_rc, pool_before] = bgd_connection_pool_count(admin, hg.blue_writer, cluster.blue_writer.hostname); + if (pool_before_rc != EXIT_SUCCESS || pool_before < 1) { + diag("Error: failed to establish the post-cutover pool before repeated POST_PROCESSING"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the repeated POST_PROCESSING probe sequence"); + return EXIT_FAILURE; + } + + vector repeat_topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); + int repeat_rc = sim.topology_update(state.topology_endpoints, repeat_topology); + if (repeat_rc != EXIT_SUCCESS) { + diag("Error: failed to repeat SWITCHOVER_IN_POST_PROCESSING topology"); + return EXIT_FAILURE; + } + + int observation_rc = wait_for_green_observation(sim, seq, cluster); + if (observation_rc != EXIT_SUCCESS) { + diag("Error: BGD did not observe repeated POST_PROCESSING topology"); + return EXIT_FAILURE; + } + + auto [pool_after_rc, pool_after] = bgd_connection_pool_count(admin, hg.blue_writer, cluster.blue_writer.hostname); + if (pool_after_rc != EXIT_SUCCESS) { + diag("Error: failed to read the pool after repeated POST_PROCESSING"); + return EXIT_FAILURE; + } + + ok(pool_after >= pool_before, "repeated POST_PROCESSING does not drain the post-cutover connection pool"); + return EXIT_SUCCESS; +} + +int main() { + plan(11); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: set writer/reader read_only values and publish AVAILABLE topology. + // ProxySQL: configure BGD hostgroups 970-973. + // Verify: BGD status for wHG 970 reports AVAILABLE. + if (test_bgd_status_available(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish SWITCHOVER_INITIATED and reverse one blue writer/reader read_only pair. + // Verify: BGD status is WRITER_SWITCHOVER_INITIATED and placement changes remain suppressed. + if (test_switchover_initiated(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish SWITCHOVER_IN_PROGRESS topology. + // Verify: BGD status for wHG 970 reports WRITER_SWITCHOVER_IN_PROGRESS. + // Verify: the blue writer moves from hostgroup 970 to 971. + // Verify: the mapped blue reader remains suppressed in hostgroup 971. + if (test_switchover_in_progress(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: create a blue-writer pool through normal routing hostgroup 974. + // Simulator: publish SWITCHOVER_IN_POST_PROCESSING twice. + // Verify: writer placement is restored, the old pool drains, and routing reaches the green IP. + // Verify: repeated POST_PROCESSING preserves a connection created after cutover. + if (test_switchover_post_processing(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} From 7d2e462fc907ab5b879be49d32bd33a0e71ce007 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 28 Jul 2026 07:09:18 +0000 Subject: [PATCH 69/81] test: cover RDS BGD late entry and topology failures - Cover worker startup during writer phases and after deployment completion. - Verify empty, absent, malformed, and failed topology responses. - Register and lint the focused late-entry and topology tests. Signed-off-by: Wazir Ahmed --- test/tap/groups/groups.json | 4 + .../test_rds_bgd_late_entry_completed-t.cpp | 349 ++++++++++ ...est_rds_bgd_late_entry_writer_phases-t.cpp | 624 ++++++++++++++++++ .../test_rds_bgd_topology_empty_absent-t.cpp | 560 ++++++++++++++++ .../tests/test_rds_bgd_topology_errors-t.cpp | 497 ++++++++++++++ 5 files changed, 2034 insertions(+) create mode 100644 test/tap/tests/test_rds_bgd_late_entry_completed-t.cpp create mode 100644 test/tap/tests/test_rds_bgd_late_entry_writer_phases-t.cpp create mode 100644 test/tap/tests/test_rds_bgd_topology_empty_absent-t.cpp create mode 100644 test/tap/tests/test_rds_bgd_topology_errors-t.cpp diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index f3ed4d88ca..1694fb1e1d 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -435,10 +435,14 @@ "test_rds_bgd_disable_during_switchover-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_explicit_startup-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_green_membership_ordering-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_late_entry_completed-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_late_entry_writer_phases-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_probe_tls-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_remove_during_switchover-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_rollback-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_smoke-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_topology_empty_absent-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_topology_errors-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_writer_switchover-t" : [ "cluster_sim_rds_bgd-g1" ], "test_read_only_actions_offline_hard_servers-t" : [ "legacy-g5","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g5","mysql84-g9","mysql90-g4","mysql90-g5","mysql95-g4","mysql95-g5" ], "test_rw_binary_data-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql90-g4","mysql95-g4" ], diff --git a/test/tap/tests/test_rds_bgd_late_entry_completed-t.cpp b/test/tap/tests/test_rds_bgd_late_entry_completed-t.cpp new file mode 100644 index 0000000000..6e212f11b2 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_late_entry_completed-t.cpp @@ -0,0 +1,349 @@ +/** + * @file test_rds_bgd_late_entry_completed-t.cpp + * @brief Starting a BGD worker from target-only SWITCHOVER_COMPLETED. + * + * Steps: + * + * 1. Publish target-only SWITCHOVER_COMPLETED before configuring wHG 1200. + * 2. Verify the first observation enters READER_SWITCHOVER_IN_PROGRESS + * without rebuilding writer-switchover routing or placement. + * 3. Establish green writer/reader pools. + * 4. Publish empty topology and verify status NONE and green-pool cleanup. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; +const uint32_t kNoProbeTimeoutMs = 1200; + +struct TestState { + RDS_BGD_Cluster cluster { bgd_cluster_1_deployment_b_init() }; + BGD_Hostgroups hostgroups { 1200, 1201, 1202, 1203 }; + vector topology_endpoints { cluster.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ + cluster.green_writer.hostname, + cluster.green_writer.hostname, + cluster.green_writer.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + "SWITCHOVER_COMPLETED", + }}; + return rows; +} + +int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_0(sim, cluster.green_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[1]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.green_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int wait_for_blue_writer(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { + auto [probe_rc, probe] = + sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + return probe_rc; +} + +bool runtime_server_match(MYSQL* admin, int hostgroup, RDS_BGD_Host& host, string status) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + + " AND status=" + bgd_sql_quote(status); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool matches = rows[0][0] == "1"; + return matches; +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + vector queries { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +rc_t connect_and_echo(CommandLine& cl) { + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + rc_t result { EXIT_FAILURE, {} }; + return result; + } + + rc_t result = bgd_backend_ip_echo(client); + mysql_close(client); + return result; +} + +int wait_for_green_pool_drain(MYSQL* admin, BGD_Hostgroups& hg) { + string query = + "SELECT " + "(SELECT COALESCE(SUM(ConnUsed+ConnFree),0) FROM stats_mysql_connection_pool WHERE hostgroup=" + + to_string(hg.green_writer) + ")=0 AND " + "(SELECT COALESCE(SUM(ConnUsed+ConnFree),0) FROM stats_mysql_connection_pool WHERE hostgroup=" + + to_string(hg.green_reader) + ")=0"; + + int rc = bgd_wait_for_condition(admin, query, kTimeoutSeconds); + return rc; +} + +/** + * Start wHG 1200 from target-only SWITCHOVER_COMPLETED. + * + * - Publish SWITCHOVER_COMPLETED before configuring the BGD row. + * - Verify READER_SWITCHOVER_IN_PROGRESS and a blue metadata probe. + * - Verify no direct green metadata probe, blue routing remains active, and + * the blue writer is not demoted. + * - Establish green writer/reader pools for terminal cleanup. + */ +int test_first_completed(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG 1200"); + return EXIT_FAILURE; + } + + auto [publish_seq_rc, publish_seq] = sim.probe_log_last_sequence(); + if (publish_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the first COMPLETED probe sequence"); + return EXIT_FAILURE; + } + + vector topology = target_only_completed(cluster); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 1200-1203"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "READER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1200 did not reach READER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + int blue_probe_rc = wait_for_blue_writer(sim, publish_seq, cluster); + if (blue_probe_rc != EXIT_SUCCESS) { + diag("Error: first COMPLETED observation did not probe the blue writer"); + return EXIT_FAILURE; + } + + ok(true, "first SWITCHOVER_COMPLETED observation reports READER_SWITCHOVER_IN_PROGRESS for wHG 1200"); + + int no_green_rc = bgd_expect_no_metadata_probe(sim, publish_seq, cluster.green_writer.endpoint(), kNoProbeTimeoutMs); + if (no_green_rc != EXIT_SUCCESS) { + diag("Error: first COMPLETED observation rebuilt a direct green metadata probe"); + return EXIT_FAILURE; + } + + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: first COMPLETED observation changed blue-writer placement"); + return EXIT_FAILURE; + } + + auto [blue_echo_rc, blue_echo] = connect_and_echo(cl); + bool blue_routing = blue_echo_rc == EXIT_SUCCESS && blue_echo.find(cluster.blue_writer.ip) != string::npos; + bool reader_online = runtime_server_match(admin, hg.blue_reader, cluster.blue_readers[1], "ONLINE"); + ok(blue_routing && reader_online, + "first SWITCHOVER_COMPLETED observation keeps blue routing without writer-phase pins or demotion"); + + int writer_hg_rc = set_default_hostgroup(admin, hg.green_writer); + if (writer_hg_rc != EXIT_SUCCESS) { + diag("Error: failed to route the test user through green writer hostgroup 1202"); + return EXIT_FAILURE; + } + + int writer_echo_rc = connect_and_echo(cl).first; + if (writer_echo_rc != EXIT_SUCCESS) { + diag("Error: failed to establish a green-writer pool before terminal cleanup"); + return EXIT_FAILURE; + } + + int reader_hg_rc = set_default_hostgroup(admin, hg.green_reader); + if (reader_hg_rc != EXIT_SUCCESS) { + diag("Error: failed to route the test user through green reader hostgroup 1203"); + return EXIT_FAILURE; + } + + int reader_echo_rc = connect_and_echo(cl).first; + if (reader_echo_rc != EXIT_SUCCESS) { + diag("Error: failed to establish a green-reader pool before terminal cleanup"); + return EXIT_FAILURE; + } + + int restore_hg_rc = set_default_hostgroup(admin, hg.blue_writer); + if (restore_hg_rc != EXIT_SUCCESS) { + diag("Error: failed to restore the test user to blue writer hostgroup 1200"); + return EXIT_FAILURE; + } + + auto [writer_pool_rc, writer_pool] = bgd_connection_pool_count(admin, hg.green_writer); + auto [reader_pool_rc, reader_pool] = bgd_connection_pool_count(admin, hg.green_reader); + if (writer_pool_rc != EXIT_SUCCESS || writer_pool < 1 || reader_pool_rc != EXIT_SUCCESS || reader_pool < 1) { + diag("Error: green pools are empty before terminal completed cleanup"); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +/** + * Publish empty topology during direct-entry reader switchover. + * + * - Remove the simulated topology after first-observation COMPLETED. + * - Verify BGD status NONE. + * - Verify eligible green writer/reader pools are drained. + * - Verify blue-writer and blue-reader placement remains available. + */ +int test_completed_empty_topology(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + int topology_rc = sim.topology_delete(state.topology_endpoints); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish empty topology for wHG 1200"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "NONE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1200 did not reach NONE after empty topology"); + return EXIT_FAILURE; + } + + int drain_rc = wait_for_green_pool_drain(admin, hg); + if (drain_rc != EXIT_SUCCESS) { + diag("Error: empty topology did not drain green pools for hostgroups 1202-1203"); + return EXIT_FAILURE; + } + + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: empty topology changed blue-writer placement for wHG 1200"); + return EXIT_FAILURE; + } + + bool reader_online = runtime_server_match(admin, hg.blue_reader, cluster.blue_readers[1], "ONLINE"); + ok(reader_online, "empty topology reaches NONE, drains green pools, and keeps blue hostgroups 1200-1201 available"); + return EXIT_SUCCESS; +} + +int main() { + plan(3); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish target-only SWITCHOVER_COMPLETED before wHG 1200 is configured. + // Verify: first observation enters READER_SWITCHOVER_IN_PROGRESS without green pins or writer demotion. + // Client: establish green writer/reader pools for terminal cleanup. + if (test_first_completed(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish empty topology during READER_SWITCHOVER_IN_PROGRESS. + // Verify: BGD status reaches NONE and green writer/reader pools are drained. + // Verify: blue writer and reader hostgroups remain available. + if (test_completed_empty_topology(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_late_entry_writer_phases-t.cpp b/test/tap/tests/test_rds_bgd_late_entry_writer_phases-t.cpp new file mode 100644 index 0000000000..970378605e --- /dev/null +++ b/test/tap/tests/test_rds_bgd_late_entry_writer_phases-t.cpp @@ -0,0 +1,624 @@ +/** + * @file test_rds_bgd_late_entry_writer_phases-t.cpp + * @brief Starting a BGD worker from each writer switchover phase. + * + * Steps: + * + * 1. Start wHG 1170 when topology already reports SWITCHOVER_INITIATED and + * verify read_only suppression without blue-writer demotion. + * 2. Start wHG 1180 when topology already reports SWITCHOVER_IN_PROGRESS and + * verify prerequisite construction before blue-writer demotion. + * 3. Create a blue-writer pool, then start wHG 1190 when topology already + * reports SWITCHOVER_IN_POST_PROCESSING. + * 4. Verify green routing, blue-pool drain, writer placement, and reader + * read_only suppression from the first POST_PROCESSING observation. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; +const uint32_t kReadOnlyObservationMs = 500; + +struct TestState { + RDS_BGD_Cluster initiated_cluster { bgd_cluster_init() }; + BGD_Hostgroups initiated_hg { 1170, 1171, 1172, 1173 }; + vector initiated_endpoints { initiated_cluster.get_endpoints() }; + + RDS_BGD_Cluster progress_cluster { bgd_cluster_2_init() }; + BGD_Hostgroups progress_hg { 1180, 1181, 1182, 1183 }; + vector progress_endpoints { progress_cluster.get_endpoints() }; + + RDS_BGD_Cluster post_cluster { bgd_cluster_3_init() }; + BGD_Hostgroups post_hg { 1190, 1191, 1192, 1193 }; + vector post_endpoints { post_cluster.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + rows.push_back({ + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + rows.push_back({ + cluster.green_readers[0].hostname, + cluster.green_readers[0].hostname, + cluster.green_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + return rows; +} + +int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_0(sim, cluster.green_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[1]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.green_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int publish_topology(RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { + vector topology = topology_with_reader_pair(cluster, status); + + int rc = sim.topology_update(endpoints, topology); + return rc; +} + +int wait_for_green_writer(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { + auto [probe_rc, probe] = + sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + return probe_rc; +} + +bool runtime_server_match(MYSQL* admin, int hostgroup, RDS_BGD_Host& host, string status) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + + " AND status=" + bgd_sql_quote(status); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool matches = rows[0][0] == "1"; + return matches; +} + +int64_t last_read_only_log_time(MYSQL* admin, RDS_BGD_Host& host) { + string query = + "SELECT COALESCE(MAX(time_start_us),0) FROM mysql_server_read_only_log WHERE hostname=" + + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return -1; + } + + int64_t time = strtoll(rows[0][0].c_str(), nullptr, 10); + return time; +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + vector queries { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +rc_t connect_and_echo(CommandLine& cl) { + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + rc_t result { EXIT_FAILURE, {} }; + return result; + } + + rc_t result = bgd_backend_ip_echo(client); + mysql_close(client); + return result; +} + +int wait_for_blue_writer_pool_drain(MYSQL* admin, RDS_BGD_Cluster& cluster) { + string query = + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0)=0 FROM stats_mysql_connection_pool WHERE srv_host=" + + bgd_sql_quote(cluster.blue_writer.hostname); + + int rc = bgd_wait_for_condition(admin, query, kTimeoutSeconds); + return rc; +} + +int configure_servers_without_worker(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { + vector config_queries { + "INSERT INTO mysql_replication_hostgroups(writer_hostgroup,reader_hostgroup) VALUES (" + + to_string(hg.blue_writer) + "," + to_string(hg.blue_reader) + ")", + "SET mysql-monitor_username='testuser'", + "SET mysql-monitor_password='testuser'", + "SET mysql-monitor_enabled='true'", + "SET mysql-monitor_read_only_interval=100", + "SET mysql-monitor_aws_rds_topology_discovery_interval=1", + "SET mysql-aws_blue_green_deployment_auto_discovery='false'", + "UPDATE mysql_users SET default_hostgroup=" + to_string(hg.blue_writer) + " WHERE username='testuser'", + }; + + int config_rc = execute_all(admin, config_queries); + if (config_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + int blue_rc = bgd_admin_add_servers(admin, cluster, hg, blue_servers, false, 0); + if (blue_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int green_rc = bgd_admin_add_servers(admin, cluster, hg, green_servers, true, 0); + if (green_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector load_queries { + "LOAD MYSQL VARIABLES TO RUNTIME", + "LOAD MYSQL USERS TO RUNTIME", + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int load_rc = execute_all(admin, load_queries); + if (load_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int enable_bgd_worker(MYSQL* admin, BGD_Hostgroups& hg) { + string insert_bgd = + "INSERT INTO mysql_aws_rds_bgd_hostgroups(" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,writer_is_also_reader,check_interval_ms,check_timeout_ms,comment) VALUES (" + + to_string(hg.blue_writer) + "," + to_string(hg.blue_reader) + "," + + to_string(hg.green_writer) + "," + to_string(hg.green_reader) + + ",1,0,100,800,'BGD TAP late writer phase')"; + vector queries { + insert_bgd, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +/** + * Start wHG 1170 from SWITCHOVER_INITIATED. + * + * - Publish SWITCHOVER_INITIATED before configuring the BGD row. + * - Verify WRITER_SWITCHOVER_INITIATED without blue-writer demotion. + * - Change simulated writer/reader read_only values. + * - Verify read_only monitoring is suppressed for deployment members. + */ +int test_first_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.initiated_cluster; + BGD_Hostgroups& hg = state.initiated_hg; + + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG 1170"); + return EXIT_FAILURE; + } + + auto [publish_seq_rc, publish_seq] = sim.probe_log_last_sequence(); + if (publish_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the first INITIATED probe sequence"); + return EXIT_FAILURE; + } + + int topology_rc = publish_topology(sim, state.initiated_endpoints, cluster, "SWITCHOVER_INITIATED"); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_INITIATED topology for wHG 1170"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 1170-1173"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_INITIATED", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1170 did not reach WRITER_SWITCHOVER_INITIATED"); + return EXIT_FAILURE; + } + + int probe_rc = wait_for_green_writer(sim, publish_seq, cluster); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: first INITIATED observation did not probe the green writer"); + return EXIT_FAILURE; + } + + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: first INITIATED observation changed blue-writer placement"); + return EXIT_FAILURE; + } + + ok(true, "first SWITCHOVER_INITIATED observation keeps the blue writer in hostgroup 1170"); + + int64_t writer_log = last_read_only_log_time(admin, cluster.blue_writer); + int64_t reader_log = last_read_only_log_time(admin, cluster.blue_readers[0]); + if (writer_log < 0 || reader_log < 0) { + diag("Error: failed to read INITIATED read_only log baselines"); + return EXIT_FAILURE; + } + + auto [suppression_seq_rc, suppression_seq] = sim.probe_log_last_sequence(); + if (suppression_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the INITIATED suppression probe sequence"); + return EXIT_FAILURE; + } + + int writer_update_rc = bgd_set_host_read_only_1(sim, cluster.blue_writer); + if (writer_update_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=1 for the simulated blue writer"); + return EXIT_FAILURE; + } + + int reader_update_rc = bgd_set_host_read_only_0(sim, cluster.blue_readers[0]); + if (reader_update_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated blue reader"); + return EXIT_FAILURE; + } + + int suppression_probe_rc = wait_for_green_writer(sim, suppression_seq, cluster); + if (suppression_probe_rc != EXIT_SUCCESS) { + diag("Error: INITIATED suppression check did not observe the green writer"); + return EXIT_FAILURE; + } + + int writer_suppression_rc = + bgd_expect_no_read_only_log(admin, cluster.blue_writer, writer_log, kReadOnlyObservationMs); + if (writer_suppression_rc != EXIT_SUCCESS) { + diag("Error: blue-writer read_only monitoring was not suppressed on first SWITCHOVER_INITIATED observation"); + return EXIT_FAILURE; + } + + int reader_suppression_rc = + bgd_expect_no_read_only_log(admin, cluster.blue_readers[0], reader_log, kReadOnlyObservationMs); + if (reader_suppression_rc != EXIT_SUCCESS) { + diag("Error: blue-reader read_only monitoring was not suppressed on first SWITCHOVER_INITIATED observation"); + return EXIT_FAILURE; + } + + bool reader_online = runtime_server_match(admin, hg.blue_reader, cluster.blue_readers[0], "ONLINE"); + ok(reader_online, "first SWITCHOVER_INITIATED observation suppresses writer and reader read_only placement changes"); + return EXIT_SUCCESS; +} + +/** + * Start wHG 1180 from SWITCHOVER_IN_PROGRESS. + * + * - Publish SWITCHOVER_IN_PROGRESS before configuring the BGD row. + * - Verify WRITER_SWITCHOVER_IN_PROGRESS and blue-writer demotion. + * - Change the simulated blue-reader read_only value. + * - Verify read_only monitoring remains suppressed after demotion. + */ +int test_first_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.progress_cluster; + BGD_Hostgroups& hg = state.progress_hg; + + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG 1180"); + return EXIT_FAILURE; + } + + auto [publish_seq_rc, publish_seq] = sim.probe_log_last_sequence(); + if (publish_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the first IN_PROGRESS probe sequence"); + return EXIT_FAILURE; + } + + int topology_rc = publish_topology(sim, state.progress_endpoints, cluster, "SWITCHOVER_IN_PROGRESS"); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for wHG 1180"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 1180-1183"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1180 did not reach WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + int probe_rc = wait_for_green_writer(sim, publish_seq, cluster); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: first IN_PROGRESS observation did not probe the green writer"); + return EXIT_FAILURE; + } + + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, true, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: first IN_PROGRESS observation did not demote the blue writer"); + return EXIT_FAILURE; + } + + ok(true, "first SWITCHOVER_IN_PROGRESS observation moves the blue writer from hostgroup 1180 to 1181"); + + int64_t reader_log = last_read_only_log_time(admin, cluster.blue_readers[0]); + if (reader_log < 0) { + diag("Error: failed to read the IN_PROGRESS blue-reader log baseline"); + return EXIT_FAILURE; + } + + auto [suppression_seq_rc, suppression_seq] = sim.probe_log_last_sequence(); + if (suppression_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the IN_PROGRESS suppression probe sequence"); + return EXIT_FAILURE; + } + + int reader_update_rc = bgd_set_host_read_only_0(sim, cluster.blue_readers[0]); + if (reader_update_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated blue reader"); + return EXIT_FAILURE; + } + + int suppression_probe_rc = wait_for_green_writer(sim, suppression_seq, cluster); + if (suppression_probe_rc != EXIT_SUCCESS) { + diag("Error: IN_PROGRESS suppression check did not observe the green writer"); + return EXIT_FAILURE; + } + + int reader_suppression_rc = + bgd_expect_no_read_only_log(admin, cluster.blue_readers[0], reader_log, kReadOnlyObservationMs); + if (reader_suppression_rc != EXIT_SUCCESS) { + diag("Error: blue-reader read_only monitoring was not suppressed on first SWITCHOVER_IN_PROGRESS observation"); + return EXIT_FAILURE; + } + + bool reader_online = runtime_server_match(admin, hg.blue_reader, cluster.blue_readers[0], "ONLINE"); + ok(reader_online, "first SWITCHOVER_IN_PROGRESS observation suppresses blue-reader read_only placement changes"); + return EXIT_SUCCESS; +} + +/** + * Start wHG 1190 from SWITCHOVER_IN_POST_PROCESSING. + * + * - Publish POST_PROCESSING and create a blue-writer pool before enabling BGD. + * - Verify WRITER_SWITCHOVER_POST_PROCESSING and restored blue-writer placement. + * - Verify the old blue pool drains and new connections route to green. + * - Verify mapped blue readers remain under read_only suppression. + */ +int test_first_post_processing(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.post_cluster; + BGD_Hostgroups& hg = state.post_hg; + + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG 1190"); + return EXIT_FAILURE; + } + + auto [publish_seq_rc, publish_seq] = sim.probe_log_last_sequence(); + if (publish_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the first POST_PROCESSING probe sequence"); + return EXIT_FAILURE; + } + + int topology_rc = publish_topology(sim, state.post_endpoints, cluster, "SWITCHOVER_IN_POST_PROCESSING"); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING topology for wHG 1190"); + return EXIT_FAILURE; + } + + int servers_rc = configure_servers_without_worker(admin, cluster, hg); + if (servers_rc != EXIT_SUCCESS) { + diag("Error: failed to configure hostgroups 1190-1193 without a BGD worker"); + return EXIT_FAILURE; + } + + auto [blue_echo_rc, blue_echo] = connect_and_echo(cl); + if (blue_echo_rc != EXIT_SUCCESS || blue_echo.find(cluster.blue_writer.ip) == string::npos) { + diag("Error: failed to establish the pre-worker blue-writer pool"); + return EXIT_FAILURE; + } + + auto [pool_before_rc, pool_before] = bgd_connection_pool_count(admin, hg.blue_writer, cluster.blue_writer.hostname); + if (pool_before_rc != EXIT_SUCCESS || pool_before < 1) { + diag("Error: blue-writer pool is empty before enabling wHG 1190"); + return EXIT_FAILURE; + } + + int worker_rc = enable_bgd_worker(admin, hg); + if (worker_rc != EXIT_SUCCESS) { + diag("Error: failed to enable BGD worker for wHG 1190"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_POST_PROCESSING", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1190 did not reach WRITER_SWITCHOVER_POST_PROCESSING"); + return EXIT_FAILURE; + } + + int probe_rc = wait_for_green_writer(sim, publish_seq, cluster); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: first POST_PROCESSING observation did not probe the green writer"); + return EXIT_FAILURE; + } + + ok(true, "first POST_PROCESSING observation reports WRITER_SWITCHOVER_POST_PROCESSING for wHG 1190"); + + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: first POST_PROCESSING observation did not keep the blue writer in hostgroup 1190"); + return EXIT_FAILURE; + } + + int drain_rc = wait_for_blue_writer_pool_drain(admin, cluster); + if (drain_rc != EXIT_SUCCESS) { + diag("Error: first POST_PROCESSING observation did not drain the old blue-writer pool"); + return EXIT_FAILURE; + } + + auto [green_echo_rc, green_echo] = connect_and_echo(cl); + bool green_routing = green_echo_rc == EXIT_SUCCESS && green_echo.find(cluster.green_writer.ip) != string::npos; + ok(green_routing, + "first POST_PROCESSING observation restores hostgroup 1190, drains its blue pool, and routes to green"); + + int64_t reader_log = last_read_only_log_time(admin, cluster.blue_readers[0]); + if (reader_log < 0) { + diag("Error: failed to read the POST_PROCESSING blue-reader log baseline"); + return EXIT_FAILURE; + } + + auto [suppression_seq_rc, suppression_seq] = sim.probe_log_last_sequence(); + if (suppression_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the POST_PROCESSING suppression probe sequence"); + return EXIT_FAILURE; + } + + int reader_update_rc = bgd_set_host_read_only_0(sim, cluster.blue_readers[0]); + if (reader_update_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated blue reader"); + return EXIT_FAILURE; + } + + int suppression_probe_rc = wait_for_green_writer(sim, suppression_seq, cluster); + if (suppression_probe_rc != EXIT_SUCCESS) { + diag("Error: POST_PROCESSING suppression check did not observe the green writer"); + return EXIT_FAILURE; + } + + int reader_suppression_rc = + bgd_expect_no_read_only_log(admin, cluster.blue_readers[0], reader_log, kReadOnlyObservationMs); + if (reader_suppression_rc != EXIT_SUCCESS) { + diag("Error: blue-reader read_only monitoring was not suppressed on first POST_PROCESSING observation"); + return EXIT_FAILURE; + } + + bool reader_online = runtime_server_match(admin, hg.blue_reader, cluster.blue_readers[0], "ONLINE"); + ok(reader_online, "first POST_PROCESSING observation keeps the mapped blue reader ONLINE in hostgroup 1191"); + return EXIT_SUCCESS; +} + +int main() { + plan(7); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish SWITCHOVER_INITIATED before wHG 1170 is configured. + // Verify: first observation reports WRITER_SWITCHOVER_INITIATED without writer demotion. + // Verify: writer and reader read_only placement changes are suppressed. + if (test_first_initiated(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish SWITCHOVER_IN_PROGRESS before wHG 1180 is configured. + // Verify: first observation reports WRITER_SWITCHOVER_IN_PROGRESS and moves the writer to hostgroup 1181. + // Verify: blue-reader read_only placement changes are suppressed. + if (test_first_in_progress(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish SWITCHOVER_IN_POST_PROCESSING before wHG 1190 is configured. + // Client: create a blue-writer pool before enabling the BGD worker. + // Verify: first observation drains the pool, routes to green, restores writer placement, and retains reader placement. + if (test_first_post_processing(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_topology_empty_absent-t.cpp b/test/tap/tests/test_rds_bgd_topology_empty_absent-t.cpp new file mode 100644 index 0000000000..1c938b306f --- /dev/null +++ b/test/tap/tests/test_rds_bgd_topology_empty_absent-t.cpp @@ -0,0 +1,560 @@ +/** + * @file test_rds_bgd_topology_empty_absent-t.cpp + * @brief Empty and absent BGD topology before and after writer completion. + * + * Steps: + * + * 1. Delete topology rows during WRITER_SWITCHOVER_IN_PROGRESS and verify + * rollback through a successful metadata probe. + * 2. Drop the topology table during WRITER_SWITCHOVER_IN_PROGRESS and verify + * rollback followed by a blue-writer table check. + * 3. Delete topology rows during READER_SWITCHOVER_IN_PROGRESS and verify + * reader cleanup through a successful metadata probe. + * 4. Drop the topology table during READER_SWITCHOVER_IN_PROGRESS and verify + * reader cleanup followed by a blue-writer table check. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; + +struct TestState { + RDS_BGD_Cluster empty_before { bgd_cluster_init() }; + BGD_Hostgroups empty_before_hg { 1100, 1101, 1102, 1103 }; + vector empty_before_endpoints { empty_before.get_endpoints() }; + + RDS_BGD_Cluster absent_before { bgd_cluster_2_init() }; + BGD_Hostgroups absent_before_hg { 1110, 1111, 1112, 1113 }; + vector absent_before_endpoints { absent_before.get_endpoints() }; + + RDS_BGD_Cluster empty_reader { bgd_cluster_3_init() }; + BGD_Hostgroups empty_reader_hg { 1120, 1121, 1122, 1123 }; + vector empty_reader_endpoints { empty_reader.get_endpoints() }; + + RDS_BGD_Cluster absent_reader { bgd_cluster_1_deployment_b_init() }; + BGD_Hostgroups absent_reader_hg { 1130, 1131, 1132, 1133 }; + vector absent_reader_endpoints { absent_reader.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + rows.push_back({ + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + rows.push_back({ + cluster.green_readers[0].hostname, + cluster.green_readers[0].hostname, + cluster.green_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + return rows; +} + +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ + cluster.green_writer.hostname, + cluster.green_writer.hostname, + cluster.green_writer.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + "SWITCHOVER_COMPLETED", + }}; + return rows; +} + +int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_0(sim, cluster.green_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[1]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.green_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +bool runtime_server_online(MYSQL* admin, int hostgroup, RDS_BGD_Host& host) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + + " AND status='ONLINE'"; + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool online = rows[0][0] == "1"; + return online; +} + +int configure_bgd(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int admin_rc = bgd_admin_setup( + admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0 + ); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int publish_topology(RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { + vector topology = topology_with_reader_pair(cluster, status); + + int rc = sim.topology_update(endpoints, topology); + return rc; +} + +int enter_writer_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, + BGD_Hostgroups& hg, vector endpoints) +{ + int config_rc = configure_bgd(admin, sim, cluster, hg); + if (config_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + int available_rc = publish_topology(sim, endpoints, cluster, "AVAILABLE"); + if (available_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + int available_status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (available_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG %d did not reach AVAILABLE", hg.blue_writer); + return EXIT_FAILURE; + } + + int progress_rc = publish_topology(sim, endpoints, cluster, "SWITCHOVER_IN_PROGRESS"); + if (progress_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + int progress_status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (progress_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG %d did not reach WRITER_SWITCHOVER_IN_PROGRESS", hg.blue_writer); + return EXIT_FAILURE; + } + + int placement_rc = + bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, true, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer for wHG %d did not move to its reader hostgroup", hg.blue_writer); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int enter_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, + BGD_Hostgroups& hg, vector endpoints) +{ + int progress_rc = enter_writer_switchover(admin, sim, cluster, hg, endpoints); + if (progress_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + int post_rc = publish_topology(sim, endpoints, cluster, "SWITCHOVER_IN_POST_PROCESSING"); + if (post_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING topology for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + int post_status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_POST_PROCESSING", kTimeoutSeconds); + if (post_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG %d did not reach WRITER_SWITCHOVER_POST_PROCESSING", hg.blue_writer); + return EXIT_FAILURE; + } + + int placement_rc = + bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer for wHG %d did not return to its writer hostgroup", hg.blue_writer); + return EXIT_FAILURE; + } + + vector completed = target_only_completed(cluster); + int completed_rc = sim.topology_update(endpoints, completed); + if (completed_rc != EXIT_SUCCESS) { + diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + int reader_status_rc = bgd_wait_for_status(admin, hg, "READER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (reader_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG %d did not reach READER_SWITCHOVER_IN_PROGRESS", hg.blue_writer); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int disable_bgd(MYSQL* admin, BGD_Hostgroups& hg) { + string query = + "UPDATE mysql_aws_rds_bgd_hostgroups SET active=0 WHERE writer_hostgroup=" + + to_string(hg.blue_writer); + vector queries { + query, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +/** + * Delete topology rows during writer switchover. + * + * - Reach WRITER_SWITCHOVER_IN_PROGRESS for wHG 1100. + * - Delete every topology row while the topology table remains present. + * - Verify BGD status NONE, restored blue-writer placement, and metadata + * telemetry from the pinned green writer. + */ +int test_empty_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.empty_before; + BGD_Hostgroups& hg = state.empty_before_hg; + + int progress_rc = enter_writer_switchover(admin, sim, cluster, hg, state.empty_before_endpoints); + if (progress_rc != EXIT_SUCCESS) { + diag("Error: failed to reach writer switchover for wHG 1100"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before empty topology for wHG 1100"); + return EXIT_FAILURE; + } + + int empty_rc = sim.topology_delete(state.empty_before_endpoints); + if (empty_rc != EXIT_SUCCESS) { + diag("Error: failed to delete topology rows for wHG 1100"); + return EXIT_FAILURE; + } + + int none_rc = bgd_wait_for_status(admin, hg, "NONE", kTimeoutSeconds); + if (none_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1100 did not reach NONE after empty topology"); + return EXIT_FAILURE; + } + + int placement_rc = + bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: empty topology did not restore the blue writer for wHG 1100"); + return EXIT_FAILURE; + } + + ok(true, "empty topology restores the blue writer and sets BGD status for wHG 1100 to NONE"); + + auto [probe_rc, probe] = + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: empty topology for wHG 1100 was not observed through green-writer metadata"); + return EXIT_FAILURE; + } + + ok(true, "empty topology for wHG 1100 is observed through a successful green-writer metadata probe"); + + int disable_rc = disable_bgd(admin, hg); + if (disable_rc != EXIT_SUCCESS) { + diag("Error: failed to stop wHG 1100 before the next topology scenario"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +/** + * Drop the topology table during writer switchover. + * + * - Reach WRITER_SWITCHOVER_IN_PROGRESS for wHG 1110. + * - Drop the topology table on the simulated blue and green endpoints. + * - Verify BGD status NONE, restored blue-writer placement, and a new + * blue-writer table-check probe. + */ +int test_absent_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.absent_before; + BGD_Hostgroups& hg = state.absent_before_hg; + + int progress_rc = enter_writer_switchover(admin, sim, cluster, hg, state.absent_before_endpoints); + if (progress_rc != EXIT_SUCCESS) { + diag("Error: failed to reach writer switchover for wHG 1110"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before absent topology for wHG 1110"); + return EXIT_FAILURE; + } + + int absent_rc = sim.topology_drop(state.absent_before_endpoints); + if (absent_rc != EXIT_SUCCESS) { + diag("Error: failed to drop the topology table for wHG 1110"); + return EXIT_FAILURE; + } + + int none_rc = bgd_wait_for_status(admin, hg, "NONE", kTimeoutSeconds); + if (none_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1110 did not reach NONE after absent topology"); + return EXIT_FAILURE; + } + + int placement_rc = + bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: absent topology did not restore the blue writer for wHG 1110"); + return EXIT_FAILURE; + } + + ok(true, "absent topology restores the blue writer and sets BGD status for wHG 1110 to NONE"); + + auto [probe_rc, probe] = + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: absent topology for wHG 1110 did not return to blue-writer table checks"); + return EXIT_FAILURE; + } + + ok(true, "absent topology for wHG 1110 returns probing to the blue-writer table check"); + + int disable_rc = disable_bgd(admin, hg); + if (disable_rc != EXIT_SUCCESS) { + diag("Error: failed to stop wHG 1110 before the next topology scenario"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +/** + * Delete topology rows during reader switchover. + * + * - Reach READER_SWITCHOVER_IN_PROGRESS for wHG 1120. + * - Delete every topology row while the topology table remains present. + * - Verify BGD status NONE, restored blue-reader routing, retained green + * rows, and metadata telemetry from the pinned green writer. + */ +int test_empty_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.empty_reader; + BGD_Hostgroups& hg = state.empty_reader_hg; + + int reader_rc = enter_reader_switchover(admin, sim, cluster, hg, state.empty_reader_endpoints); + if (reader_rc != EXIT_SUCCESS) { + diag("Error: failed to reach reader switchover for wHG 1120"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before empty topology for wHG 1120"); + return EXIT_FAILURE; + } + + int empty_rc = sim.topology_delete(state.empty_reader_endpoints); + if (empty_rc != EXIT_SUCCESS) { + diag("Error: failed to delete topology rows for wHG 1120"); + return EXIT_FAILURE; + } + + int none_rc = bgd_wait_for_status(admin, hg, "NONE", kTimeoutSeconds); + if (none_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1120 did not reach NONE after empty topology"); + return EXIT_FAILURE; + } + + bool blue_reader_online = runtime_server_online(admin, hg.blue_reader, cluster.blue_readers[1]); + bool green_writer_online = runtime_server_online(admin, hg.green_writer, cluster.green_writer); + bool green_reader_online = runtime_server_online(admin, hg.green_reader, cluster.green_readers[0]); + ok(blue_reader_online && green_writer_online && green_reader_online, + "empty topology completes reader cleanup for wHG 1120 and retains configured green rows"); + + auto [probe_rc, probe] = + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: empty topology for wHG 1120 was not observed through green-writer metadata"); + return EXIT_FAILURE; + } + + ok(true, "reader cleanup for wHG 1120 starts from a successful green-writer metadata probe"); + + int disable_rc = disable_bgd(admin, hg); + if (disable_rc != EXIT_SUCCESS) { + diag("Error: failed to stop wHG 1120 before the next topology scenario"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +/** + * Drop the topology table during reader switchover. + * + * - Reach READER_SWITCHOVER_IN_PROGRESS for wHG 1130. + * - Drop the topology table on the simulated blue and green endpoints. + * - Verify BGD status NONE, restored blue-reader routing, retained green + * rows, and a new blue-writer table-check probe. + */ +int test_absent_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.absent_reader; + BGD_Hostgroups& hg = state.absent_reader_hg; + + int reader_rc = enter_reader_switchover(admin, sim, cluster, hg, state.absent_reader_endpoints); + if (reader_rc != EXIT_SUCCESS) { + diag("Error: failed to reach reader switchover for wHG 1130"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before absent topology for wHG 1130"); + return EXIT_FAILURE; + } + + int absent_rc = sim.topology_drop(state.absent_reader_endpoints); + if (absent_rc != EXIT_SUCCESS) { + diag("Error: failed to drop the topology table for wHG 1130"); + return EXIT_FAILURE; + } + + int none_rc = bgd_wait_for_status(admin, hg, "NONE", kTimeoutSeconds); + if (none_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1130 did not reach NONE after absent topology"); + return EXIT_FAILURE; + } + + bool blue_reader_online = runtime_server_online(admin, hg.blue_reader, cluster.blue_readers[1]); + bool green_writer_online = runtime_server_online(admin, hg.green_writer, cluster.green_writer); + bool green_reader_online = runtime_server_online(admin, hg.green_reader, cluster.green_readers[0]); + ok(blue_reader_online && green_writer_online && green_reader_online, + "absent topology completes reader cleanup for wHG 1130 and retains configured green rows"); + + auto [probe_rc, probe] = + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: absent topology for wHG 1130 did not return to blue-writer table checks"); + return EXIT_FAILURE; + } + + ok(true, "reader cleanup for wHG 1130 returns probing to the blue-writer table check"); + return EXIT_SUCCESS; +} + +int main() { + plan(8); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish SWITCHOVER_IN_PROGRESS, then delete all topology rows. + // Verify: wHG 1100 reaches NONE, restores its blue writer, and records green-writer metadata. + if (test_empty_before_completion(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish SWITCHOVER_IN_PROGRESS, then drop the topology table. + // Verify: wHG 1110 reaches NONE, restores its blue writer, and returns to blue-writer table checks. + if (test_absent_before_completion(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish target-only SWITCHOVER_COMPLETED, then delete all topology rows. + // Verify: wHG 1120 completes reader cleanup through successful green-writer metadata. + if (test_empty_during_reader_switchover(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish target-only SWITCHOVER_COMPLETED, then drop the topology table. + // Verify: wHG 1130 completes reader cleanup and returns to blue-writer table checks. + if (test_absent_during_reader_switchover(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_topology_errors-t.cpp b/test/tap/tests/test_rds_bgd_topology_errors-t.cpp new file mode 100644 index 0000000000..06bc414ffe --- /dev/null +++ b/test/tap/tests/test_rds_bgd_topology_errors-t.cpp @@ -0,0 +1,497 @@ +/** + * @file test_rds_bgd_topology_errors-t.cpp + * @brief BGD metadata error 1146 and generic metadata-error handling. + * + * Steps: + * + * 1. Return metadata error 1146 during WRITER_SWITCHOVER_IN_PROGRESS and + * verify rollback followed by blue-writer table checks. + * 2. Return metadata error 1146 during READER_SWITCHOVER_IN_PROGRESS and + * verify reader cleanup followed by blue-writer table checks. + * 3. Return a generic metadata error during WRITER_SWITCHOVER_IN_PROGRESS and + * verify that the active status and blue-writer demotion remain unchanged. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; +const uint32_t kNegativeProbeTimeoutMs = 800; + +struct TestState { + RDS_BGD_Cluster before_completion { bgd_cluster_init() }; + BGD_Hostgroups before_completion_hg { 1140, 1141, 1142, 1143 }; + vector before_completion_endpoints { before_completion.get_endpoints() }; + + RDS_BGD_Cluster reader_switchover { bgd_cluster_2_init() }; + BGD_Hostgroups reader_switchover_hg { 1150, 1151, 1152, 1153 }; + vector reader_switchover_endpoints { reader_switchover.get_endpoints() }; + + RDS_BGD_Cluster generic_error { bgd_cluster_3_init() }; + BGD_Hostgroups generic_error_hg { 1160, 1161, 1162, 1163 }; + vector generic_error_endpoints { generic_error.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + rows.push_back({ + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + rows.push_back({ + cluster.green_readers[0].hostname, + cluster.green_readers[0].hostname, + cluster.green_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + return rows; +} + +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ + cluster.green_writer.hostname, + cluster.green_writer.hostname, + cluster.green_writer.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + "SWITCHOVER_COMPLETED", + }}; + return rows; +} + +int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_0(sim, cluster.green_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[1]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.green_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +bool runtime_server_online(MYSQL* admin, int hostgroup, RDS_BGD_Host& host) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + + " AND status='ONLINE'"; + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool online = rows[0][0] == "1"; + return online; +} + +int configure_bgd(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int admin_rc = bgd_admin_setup( + admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0 + ); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int publish_topology(RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { + vector topology = topology_with_reader_pair(cluster, status); + + int rc = sim.topology_update(endpoints, topology); + return rc; +} + +int enter_writer_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, + BGD_Hostgroups& hg, vector endpoints) +{ + int config_rc = configure_bgd(admin, sim, cluster, hg); + if (config_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + int available_rc = publish_topology(sim, endpoints, cluster, "AVAILABLE"); + if (available_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + int available_status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (available_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG %d did not reach AVAILABLE", hg.blue_writer); + return EXIT_FAILURE; + } + + int progress_rc = publish_topology(sim, endpoints, cluster, "SWITCHOVER_IN_PROGRESS"); + if (progress_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + int progress_status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (progress_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG %d did not reach WRITER_SWITCHOVER_IN_PROGRESS", hg.blue_writer); + return EXIT_FAILURE; + } + + int placement_rc = + bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, true, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer for wHG %d did not move to its reader hostgroup", hg.blue_writer); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int enter_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, + BGD_Hostgroups& hg, vector endpoints) +{ + int progress_rc = enter_writer_switchover(admin, sim, cluster, hg, endpoints); + if (progress_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + int post_rc = publish_topology(sim, endpoints, cluster, "SWITCHOVER_IN_POST_PROCESSING"); + if (post_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING topology for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + int post_status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_POST_PROCESSING", kTimeoutSeconds); + if (post_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG %d did not reach WRITER_SWITCHOVER_POST_PROCESSING", hg.blue_writer); + return EXIT_FAILURE; + } + + int placement_rc = + bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer for wHG %d did not return to its writer hostgroup", hg.blue_writer); + return EXIT_FAILURE; + } + + vector completed = target_only_completed(cluster); + int completed_rc = sim.topology_update(endpoints, completed); + if (completed_rc != EXIT_SUCCESS) { + diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + int reader_status_rc = bgd_wait_for_status(admin, hg, "READER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (reader_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG %d did not reach READER_SWITCHOVER_IN_PROGRESS", hg.blue_writer); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int wait_for_metadata_error(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster, + int error_number, string error_message, RDS_BGD_Probe_Log& probe) +{ + vector green_endpoint { cluster.green_writer.endpoint() }; + int error_rc = sim.topology_error(green_endpoint, error_number, error_message); + if (error_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + auto [probe_rc, metadata_probe] = + sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + probe = metadata_probe; + return EXIT_SUCCESS; +} + +int wait_for_blue_table_check(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { + vector blue_endpoint { cluster.blue_writer.endpoint() }; + int drop_rc = sim.topology_drop(blue_endpoint); + if (drop_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + auto [probe_rc, probe] = + sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + return probe_rc; +} + +/** + * Return metadata error 1146 before writer completion. + * + * - Reach WRITER_SWITCHOVER_IN_PROGRESS for wHG 1140. + * - Return error 1146 from the pinned green-writer metadata probe. + * - Verify BGD status NONE, restored blue-writer placement, and a subsequent + * blue-writer table check. + */ +int test_error_1146_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.before_completion; + BGD_Hostgroups& hg = state.before_completion_hg; + + int progress_rc = enter_writer_switchover(admin, sim, cluster, hg, state.before_completion_endpoints); + if (progress_rc != EXIT_SUCCESS) { + diag("Error: failed to reach writer switchover for wHG 1140"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before metadata error 1146 for wHG 1140"); + return EXIT_FAILURE; + } + + RDS_BGD_Probe_Log metadata {}; + int metadata_rc = + wait_for_metadata_error(sim, seq, cluster, 1146, "Table 'mysql.rds_topology' doesn't exist", metadata); + if (metadata_rc != EXIT_SUCCESS) { + diag("Error: wHG 1140 did not observe metadata error 1146 on the green writer"); + return EXIT_FAILURE; + } + + int none_rc = bgd_wait_for_status(admin, hg, "NONE", kTimeoutSeconds); + if (none_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1140 did not reach NONE after metadata error 1146"); + return EXIT_FAILURE; + } + + int placement_rc = + bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, false, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: metadata error 1146 did not restore the blue writer for wHG 1140"); + return EXIT_FAILURE; + } + + ok(true, "metadata error 1146 restores the blue writer and sets BGD status for wHG 1140 to NONE"); + + int table_rc = wait_for_blue_table_check(sim, metadata.sequence_id, cluster); + if (table_rc != EXIT_SUCCESS) { + diag("Error: wHG 1140 did not return to blue-writer table checks after metadata error 1146"); + return EXIT_FAILURE; + } + + ok(true, "metadata error 1146 returns wHG 1140 from green metadata to blue-writer table checks"); + return EXIT_SUCCESS; +} + +/** + * Return metadata error 1146 during reader switchover. + * + * - Reach READER_SWITCHOVER_IN_PROGRESS for wHG 1150. + * - Return error 1146 from the pinned green-writer metadata probe. + * - Verify BGD status NONE, restored blue-reader routing, retained green rows, + * and a subsequent blue-writer table check. + */ +int test_error_1146_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.reader_switchover; + BGD_Hostgroups& hg = state.reader_switchover_hg; + + int reader_rc = enter_reader_switchover(admin, sim, cluster, hg, state.reader_switchover_endpoints); + if (reader_rc != EXIT_SUCCESS) { + diag("Error: failed to reach reader switchover for wHG 1150"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before metadata error 1146 for wHG 1150"); + return EXIT_FAILURE; + } + + RDS_BGD_Probe_Log metadata {}; + int metadata_rc = + wait_for_metadata_error(sim, seq, cluster, 1146, "Table 'mysql.rds_topology' doesn't exist", metadata); + if (metadata_rc != EXIT_SUCCESS) { + diag("Error: wHG 1150 did not observe metadata error 1146 on the green writer"); + return EXIT_FAILURE; + } + + int none_rc = bgd_wait_for_status(admin, hg, "NONE", kTimeoutSeconds); + if (none_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1150 did not reach NONE after metadata error 1146"); + return EXIT_FAILURE; + } + + bool blue_reader_online = runtime_server_online(admin, hg.blue_reader, cluster.blue_readers[1]); + bool green_writer_online = runtime_server_online(admin, hg.green_writer, cluster.green_writer); + bool green_reader_online = runtime_server_online(admin, hg.green_reader, cluster.green_readers[0]); + ok(blue_reader_online && green_writer_online && green_reader_online, + "metadata error 1146 completes reader cleanup for wHG 1150 and retains configured green rows"); + + int table_rc = wait_for_blue_table_check(sim, metadata.sequence_id, cluster); + if (table_rc != EXIT_SUCCESS) { + diag("Error: wHG 1150 did not return to blue-writer table checks after metadata error 1146"); + return EXIT_FAILURE; + } + + ok(true, "metadata error 1146 returns wHG 1150 from green metadata to blue-writer table checks"); + return EXIT_SUCCESS; +} + +/** + * Return a generic metadata error before writer completion. + * + * - Reach WRITER_SWITCHOVER_IN_PROGRESS for wHG 1160. + * - Return error 1105 from the pinned green-writer metadata probe. + * - Verify that table checking does not restart and that the in-progress + * status and blue-writer demotion remain unchanged. + */ +int test_generic_metadata_error(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.generic_error; + BGD_Hostgroups& hg = state.generic_error_hg; + + int progress_rc = enter_writer_switchover(admin, sim, cluster, hg, state.generic_error_endpoints); + if (progress_rc != EXIT_SUCCESS) { + diag("Error: failed to reach writer switchover for wHG 1160"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before generic metadata error for wHG 1160"); + return EXIT_FAILURE; + } + + RDS_BGD_Probe_Log metadata {}; + int metadata_rc = wait_for_metadata_error(sim, seq, cluster, 1105, "simulated generic metadata failure", metadata); + if (metadata_rc != EXIT_SUCCESS) { + diag("Error: wHG 1160 did not observe the generic metadata error on the green writer"); + return EXIT_FAILURE; + } + + int no_table_rc = + bgd_expect_no_table_check(sim, metadata.sequence_id, state.generic_error_endpoints, kNegativeProbeTimeoutMs); + if (no_table_rc != EXIT_SUCCESS) { + diag("Error: generic metadata error restarted table checking for wHG 1160"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: generic metadata error changed BGD status for wHG 1160"); + return EXIT_FAILURE; + } + + int placement_rc = + bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, true, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: generic metadata error changed blue-writer placement for wHG 1160"); + return EXIT_FAILURE; + } + + ok(true, "generic metadata error keeps wHG 1160 in progress with the blue writer in hostgroup 1161"); + return EXIT_SUCCESS; +} + +int main() { + plan(5); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: return metadata error 1146 during SWITCHOVER_IN_PROGRESS. + // Verify: wHG 1140 reaches NONE, restores its blue writer, and returns to blue-writer table checks. + if (test_error_1146_before_completion(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: return metadata error 1146 during READER_SWITCHOVER_IN_PROGRESS. + // Verify: wHG 1150 completes reader cleanup and returns to blue-writer table checks. + if (test_error_1146_during_reader_switchover(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: return generic metadata error 1105 during SWITCHOVER_IN_PROGRESS. + // Verify: wHG 1160 remains in progress with its blue writer in reader hostgroup 1161. + if (test_generic_metadata_error(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} From b61de4c383d5d855bee611fb60cacd599f6d95ce Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 28 Jul 2026 07:09:43 +0000 Subject: [PATCH 70/81] test: cover RDS BGD server and connection policies - Verify reader placement and reader cleanup across BGD phases. - Verify green connection cleanup excludes explicitly offline servers. - Register and lint the focused server-policy tests. Signed-off-by: Wazir Ahmed --- doc/AWS_Blue_Green/RDS_BGD_Monitor.md | 2 +- test/tap/groups/groups.json | 3 + .../test_rds_bgd_green_pool_cleanup-t.cpp | 552 ++++++++++++++++++ .../tests/test_rds_bgd_reader_policy-t.cpp | 431 ++++++++++++++ ...st_rds_bgd_reader_switchover_cleanup-t.cpp | 417 +++++++++++++ 5 files changed, 1404 insertions(+), 1 deletion(-) create mode 100644 test/tap/tests/test_rds_bgd_green_pool_cleanup-t.cpp create mode 100644 test/tap/tests/test_rds_bgd_reader_policy-t.cpp create mode 100644 test/tap/tests/test_rds_bgd_reader_switchover_cleanup-t.cpp diff --git a/doc/AWS_Blue_Green/RDS_BGD_Monitor.md b/doc/AWS_Blue_Green/RDS_BGD_Monitor.md index eb12f2e651..edfadd25c5 100644 --- a/doc/AWS_Blue_Green/RDS_BGD_Monitor.md +++ b/doc/AWS_Blue_Green/RDS_BGD_Monitor.md @@ -964,7 +964,7 @@ durable-ledger design. | Eligible green generation checksum | `green_checksum_matrix` | `admin_green_add_remove_ssl_status` | Add/remove, `use_ssl`, and transitions into or out of `OFFLINE_SOFT`/`OFFLINE_HARD` change the checksum and replace workers; irrelevant changes do not. | | Admin commit during active phase | `config_change_exits_worker` | `load_mysql_servers_mid_switchover` | The old worker runs one-shot rollback, the dispatcher joins it, and the replacement builds a new map from the committed runtime rows. | | Green membership persistence | `green_row_persists_cancel_and_success` | `green_row_lifecycle` | Auto-added and user rows remain after rollback and success; no existing status is changed. | -| Green drain policy | `green_drain_status_matrix` | `green_hg_cleanup` | Rollback drains no green connections. Success drains `ONLINE`, `SHUNNED`, and `SHUNNED_AWS_BGD` green servers while leaving `OFFLINE_SOFT` and `OFFLINE_HARD` untouched. Rows remain present. | +| Green drain policy | `green_drain_status_matrix` | `green_hg_cleanup` | Rollback drains no green connections. Success drains eligible non-offline green servers while leaving `OFFLINE_SOFT` and `OFFLINE_HARD` untouched. Rows remain present. | | Offline status exclusions | `offline_servers_not_acted_on` | `offline_soft_hard_servers` | Blue servers in either offline status do not participate in mapping or unmatched-reader shunning; green servers in either status are not drained. | | Terminal connection retirement | `unhealthy_survives_reset` | `drained_used_connection_not_repooled` | After a drain marks a used connection unhealthy, reset does not revive it and neither local nor global pool return can place it in a free cache. | | Persistent/user green hostgroups | `user_green_hostgroups_not_null` | `user_configuration_requires_both_green_hgs` | Persistent user inserts with either green hostgroup `NULL` fail; a row with both values loads with `auto_generated=0`. | diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index 1694fb1e1d..a4da267a3c 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -435,9 +435,12 @@ "test_rds_bgd_disable_during_switchover-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_explicit_startup-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_green_membership_ordering-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_green_pool_cleanup-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_late_entry_completed-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_late_entry_writer_phases-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_probe_tls-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_reader_policy-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_reader_switchover_cleanup-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_remove_during_switchover-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_rollback-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_smoke-t" : [ "cluster_sim_rds_bgd-g1" ], diff --git a/test/tap/tests/test_rds_bgd_green_pool_cleanup-t.cpp b/test/tap/tests/test_rds_bgd_green_pool_cleanup-t.cpp new file mode 100644 index 0000000000..181915b473 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_green_pool_cleanup-t.cpp @@ -0,0 +1,552 @@ +/** + * @file test_rds_bgd_green_pool_cleanup-t.cpp + * @brief BGD rollback and successful cleanup for public green-server statuses. + * + * Steps: + * + * 1. Configure ONLINE, SHUNNED, OFFLINE_SOFT, and OFFLINE_HARD green rows and + * establish one causal connection pool for each hostname. + * 2. Roll back SWITCHOVER_IN_PROGRESS to AVAILABLE and verify that every + * green pool and configured status is preserved. + * 3. Complete writer and reader switchover, then publish empty topology. + * 4. Verify that cleanup drains ONLINE and SHUNNED pools, preserves + * OFFLINE_SOFT and OFFLINE_HARD pools, and retains all configured rows. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const int kRouterHostgroup = 1350; + +struct GreenServer { + int hostgroup; + RDS_BGD_Host host; + string status; +}; + +struct TestState { + RDS_BGD_Cluster cluster { bgd_cluster_3_init() }; + RDS_BGD_Cluster extra { bgd_cluster_1_deployment_b_init() }; + BGD_Hostgroups hostgroups { 1300, 1301, 1302, 1303 }; + vector topology_endpoints { cluster.get_endpoints() }; + vector servers { + { hostgroups.green_writer, cluster.green_writer, "ONLINE" }, + { hostgroups.green_reader, cluster.green_readers[0], "SHUNNED" }, + { hostgroups.green_reader, cluster.green_readers[1], "OFFLINE_SOFT" }, + { hostgroups.green_reader, extra.green_readers[0], "OFFLINE_HARD" }, + }; + vector pool_before {}; + vector pool_after {}; + vector admin_snapshot {}; + vector runtime_snapshot {}; + + TestState() { + topology_endpoints.push_back(extra.green_readers[0].endpoint()); + } +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_readers(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + for (RDS_BGD_Host& host : cluster.blue_readers) { + rows.push_back({ + host.hostname, + host.hostname, + host.port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + } + for (RDS_BGD_Host& host : cluster.green_readers) { + rows.push_back({ + host.hostname, + host.hostname, + host.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + } + return rows; +} + +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ + cluster.green_writer.hostname, + cluster.green_writer.hostname, + cluster.green_writer.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + "SWITCHOVER_COMPLETED", + }}; + return rows; +} + +int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_0(sim, cluster.green_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[1]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int add_server(MYSQL* admin, int hostgroup, RDS_BGD_Host& host, string status) { + string query = + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + + to_string(hostgroup) + "," + bgd_sql_quote(host.hostname) + "," + to_string(host.port) + + "," + bgd_sql_quote(status) + ",0," + bgd_sql_quote("BGD TAP pool " + host.ip) + ")"; + + int rc = mysql_query(admin, query.c_str()); + if (rc != 0) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int set_server_status(MYSQL* admin, GreenServer& server) { + string query = + "UPDATE mysql_servers SET status=" + bgd_sql_quote(server.status) + + " WHERE hostgroup_id=" + to_string(server.hostgroup) + + " AND hostname=" + bgd_sql_quote(server.host.hostname) + + " AND port=" + to_string(server.host.port); + + int rc = mysql_query(admin, query.c_str()); + if (rc != 0) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + vector queries { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +int create_pool(CommandLine& cl, MYSQL* admin, int hostgroup) { + int user_rc = set_default_hostgroup(admin, hostgroup); + if (user_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + return EXIT_FAILURE; + } + + auto [echo_rc, echo] = bgd_backend_ip_echo(client); + mysql_close(client); + return echo_rc; +} + +rc_t pool_for_hostname(MYSQL* admin, string hostname) { + string query = + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0) FROM stats_mysql_connection_pool WHERE srv_host=" + + bgd_sql_quote(hostname); + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + rc_t result { EXIT_FAILURE, 0 }; + return result; + } + + int64_t count = strtoll(rows[0][0].c_str(), nullptr, 10); + rc_t result { EXIT_SUCCESS, count }; + return result; +} + +rc_t> green_snapshot(MYSQL* admin, string table, BGD_Hostgroups& hg) { + string query = + "SELECT hostgroup_id,hostname,port,status,use_ssl,weight,max_connections FROM " + table + + " WHERE hostgroup_id IN (" + to_string(hg.green_writer) + "," + to_string(hg.green_reader) + + ") ORDER BY hostgroup_id,hostname,port"; + + rc_t> result = mysql_query_ext_rows(admin, query); + return result; +} + +int read_pools(MYSQL* admin, vector& servers, vector& pools) { + pools.clear(); + for (GreenServer& server : servers) { + auto [pool_rc, pool] = pool_for_hostname(admin, server.host.hostname); + if (pool_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + pools.push_back(pool); + } + return EXIT_SUCCESS; +} + +bool all_pools_nonzero(vector& pools) { + if (pools.size() != 4) { + return false; + } + + for (int64_t pool : pools) { + if (pool < 1) { + return false; + } + } + return true; +} + +int configure_status_matrix(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG 1300"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0], cluster.green_readers[1] }; + int admin_rc = bgd_admin_setup( + admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0 + ); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 1300-1303"); + return EXIT_FAILURE; + } + + int extra_rc = add_server(admin, hg.green_reader, state.extra.green_readers[0], "ONLINE"); + if (extra_rc != EXIT_SUCCESS) { + diag("Error: failed to add the OFFLINE_HARD green reader to hostgroup 1303"); + return EXIT_FAILURE; + } + + for (size_t i = 0; i < state.servers.size(); ++i) { + int router_rc = add_server(admin, kRouterHostgroup + static_cast(i), state.servers[i].host, "ONLINE"); + if (router_rc != EXIT_SUCCESS) { + diag("Error: failed to add pool-router row for green status index %zu", i); + return EXIT_FAILURE; + } + } + + for (GreenServer& server : state.servers) { + int status_rc = set_server_status(admin, server); + if (status_rc != EXIT_SUCCESS) { + diag("Error: failed to set %s for green server %s", server.status.c_str(), server.host.hostname.c_str()); + return EXIT_FAILURE; + } + } + + vector load_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int load_rc = execute_all(admin, load_queries); + if (load_rc != EXIT_SUCCESS) { + diag("Error: failed to load the green status matrix to runtime"); + return EXIT_FAILURE; + } + + auto [admin_snapshot_rc, admin_snapshot] = green_snapshot(admin, "mysql_servers", hg); + if (admin_snapshot_rc != EXIT_SUCCESS || admin_snapshot.size() != 4) { + diag("Error: failed to snapshot four persistent green status rows"); + return EXIT_FAILURE; + } + state.admin_snapshot = admin_snapshot; + + auto [runtime_snapshot_rc, runtime_snapshot] = green_snapshot(admin, "runtime_mysql_servers", hg); + if (runtime_snapshot_rc != EXIT_SUCCESS || runtime_snapshot.size() != 3) { + diag("Error: failed to snapshot ONLINE, SHUNNED, and OFFLINE_SOFT runtime rows"); + return EXIT_FAILURE; + } + state.runtime_snapshot = runtime_snapshot; + + for (size_t i = 0; i < state.servers.size(); ++i) { + int pool_rc = create_pool(cl, admin, kRouterHostgroup + static_cast(i)); + if (pool_rc != EXIT_SUCCESS) { + diag("Error: failed to create causal pool for green status index %zu", i); + return EXIT_FAILURE; + } + } + + int user_rc = set_default_hostgroup(admin, hg.blue_writer); + if (user_rc != EXIT_SUCCESS) { + diag("Error: failed to restore testuser to writer hostgroup 1300"); + return EXIT_FAILURE; + } + + int pools_rc = read_pools(admin, state.servers, state.pool_before); + if (pools_rc != EXIT_SUCCESS || !all_pools_nonzero(state.pool_before)) { + diag("Error: every green status must have a nonzero pool before lifecycle changes"); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int publish_topology(RDS_BGD_Simulator& sim, TestState& state, string status) { + vector topology = topology_with_readers(state.cluster, status); + + int rc = sim.topology_update(state.topology_endpoints, topology); + return rc; +} + +bool snapshots_match(MYSQL* admin, TestState& state) { + auto [admin_rc, admin_rows] = green_snapshot(admin, "mysql_servers", state.hostgroups); + auto [runtime_rc, runtime_rows] = green_snapshot(admin, "runtime_mysql_servers", state.hostgroups); + if (admin_rc != EXIT_SUCCESS || runtime_rc != EXIT_SUCCESS) { + return false; + } + + bool matches = admin_rows == state.admin_snapshot && runtime_rows == state.runtime_snapshot; + return matches; +} + +/** + * Roll back writer switchover with four green status pools. + * + * - Configure ONLINE, SHUNNED, OFFLINE_SOFT, and OFFLINE_HARD green rows. + * - Establish a nonzero causal pool for every green hostname. + * - Publish AVAILABLE, SWITCHOVER_IN_PROGRESS, then AVAILABLE. + * - Verify rollback preserves every green pool and exact configured row. + */ +int test_rollback_preserves_green_pools(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + int config_rc = configure_status_matrix(cl, admin, sim, state); + if (config_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + int available_rc = publish_topology(sim, state, "AVAILABLE"); + if (available_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for wHG 1300"); + return EXIT_FAILURE; + } + + int available_status_rc = bgd_wait_for_status(admin, state.hostgroups, "AVAILABLE", kTimeoutSeconds); + if (available_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1300 did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + int progress_rc = publish_topology(sim, state, "SWITCHOVER_IN_PROGRESS"); + if (progress_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for wHG 1300"); + return EXIT_FAILURE; + } + + int progress_status_rc = + bgd_wait_for_status(admin, state.hostgroups, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (progress_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1300 did not reach WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + int rollback_rc = publish_topology(sim, state, "AVAILABLE"); + if (rollback_rc != EXIT_SUCCESS) { + diag("Error: failed to publish rollback AVAILABLE topology for wHG 1300"); + return EXIT_FAILURE; + } + + int rollback_status_rc = bgd_wait_for_status(admin, state.hostgroups, "AVAILABLE", kTimeoutSeconds); + if (rollback_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1300 did not return to AVAILABLE"); + return EXIT_FAILURE; + } + + vector pools_after_rollback {}; + int pools_rc = read_pools(admin, state.servers, pools_after_rollback); + if (pools_rc != EXIT_SUCCESS || pools_after_rollback.size() != state.pool_before.size()) { + diag("Error: failed to read green pools after rollback"); + return EXIT_FAILURE; + } + + bool pools_preserved = true; + for (size_t i = 0; i < state.pool_before.size(); ++i) { + if (pools_after_rollback[i] < state.pool_before[i]) { + pools_preserved = false; + } + } + ok(pools_preserved, "AVAILABLE rollback preserves all four green status pools for wHG 1300"); + + bool rows_preserved = snapshots_match(admin, state); + ok(rows_preserved, "AVAILABLE rollback preserves the configured green rows and public statuses for wHG 1300"); + return EXIT_SUCCESS; +} + +/** + * Complete reader cleanup and drain eligible green pools. + * + * - Publish POST_PROCESSING and target-only SWITCHOVER_COMPLETED. + * - Require every green pool to remain nonzero immediately before cleanup. + * - Delete topology rows and verify that ONLINE and SHUNNED pools drain. + */ +int test_successful_cleanup_drains_non_offline(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + int post_rc = publish_topology(sim, state, "SWITCHOVER_IN_POST_PROCESSING"); + if (post_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING topology for wHG 1300"); + return EXIT_FAILURE; + } + + int post_status_rc = + bgd_wait_for_status(admin, state.hostgroups, "WRITER_SWITCHOVER_POST_PROCESSING", kTimeoutSeconds); + if (post_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1300 did not reach WRITER_SWITCHOVER_POST_PROCESSING"); + return EXIT_FAILURE; + } + + vector completed = target_only_completed(state.cluster); + int completed_rc = sim.topology_update(state.topology_endpoints, completed); + if (completed_rc != EXIT_SUCCESS) { + diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology for wHG 1300"); + return EXIT_FAILURE; + } + + int reader_status_rc = + bgd_wait_for_status(admin, state.hostgroups, "READER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (reader_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1300 did not reach READER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + int baseline_rc = read_pools(admin, state.servers, state.pool_before); + if (baseline_rc != EXIT_SUCCESS || !all_pools_nonzero(state.pool_before)) { + diag("Error: every green status must have a nonzero pool immediately before reader cleanup"); + return EXIT_FAILURE; + } + + int empty_rc = sim.topology_delete(state.topology_endpoints); + if (empty_rc != EXIT_SUCCESS) { + diag("Error: failed to publish empty topology for wHG 1300"); + return EXIT_FAILURE; + } + + int none_rc = bgd_wait_for_status(admin, state.hostgroups, "NONE", kTimeoutSeconds); + if (none_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1300 did not reach NONE during reader cleanup"); + return EXIT_FAILURE; + } + + int pools_rc = read_pools(admin, state.servers, state.pool_after); + if (pools_rc != EXIT_SUCCESS || state.pool_after.size() != 4) { + diag("Error: failed to read green pools after reader cleanup"); + return EXIT_FAILURE; + } + + bool non_offline_drained = state.pool_after[0] == 0 && state.pool_after[1] == 0; + ok(non_offline_drained, "reader cleanup drains ONLINE and SHUNNED green pools for wHG 1300"); + return EXIT_SUCCESS; +} + +/** + * Preserve offline pools and configured green rows during successful cleanup. + * + * - Compare OFFLINE_SOFT and OFFLINE_HARD pools with their causal baselines. + * - Verify persistent and runtime green rows still match their pre-lifecycle + * snapshots. + */ +int test_cleanup_preserves_offline_pools(MYSQL* admin, TestState& state) { + if (state.pool_before.size() != 4 || state.pool_after.size() != 4) { + diag("Error: green pool baselines are incomplete after reader cleanup"); + return EXIT_FAILURE; + } + + bool offline_pools_preserved = + state.pool_after[2] == state.pool_before[2] && + state.pool_after[3] == state.pool_before[3]; + ok(offline_pools_preserved, "reader cleanup preserves OFFLINE_SOFT and OFFLINE_HARD green pools for wHG 1300"); + + bool rows_preserved = snapshots_match(admin, state); + ok(rows_preserved, "reader cleanup retains all configured green rows and public statuses for wHG 1300"); + return EXIT_SUCCESS; +} + +int main() { + plan(5); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // ProxySQL: configure four public green statuses and establish one causal pool for each hostname. + // Simulator: publish AVAILABLE, SWITCHOVER_IN_PROGRESS, then AVAILABLE. + // Verify: rollback preserves all four pools and exact configured green rows. + if (test_rollback_preserves_green_pools(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish POST_PROCESSING, target-only SWITCHOVER_COMPLETED, then empty topology. + // Verify: reader cleanup drains ONLINE and SHUNNED green pools. + if (test_successful_cleanup_drains_non_offline(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Verify: reader cleanup preserves OFFLINE_SOFT/OFFLINE_HARD pools and configured green rows. + if (test_cleanup_preserves_offline_pools(admin, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_reader_policy-t.cpp b/test/tap/tests/test_rds_bgd_reader_policy-t.cpp new file mode 100644 index 0000000000..b274b4e9c5 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_reader_policy-t.cpp @@ -0,0 +1,431 @@ +/** + * @file test_rds_bgd_reader_policy-t.cpp + * @brief BGD matched-reader, offline-reader, and writer-fallback routing. + * + * Steps: + * + * 1. Publish SWITCHOVER_IN_POST_PROCESSING with one mapped and one unmapped + * blue reader. + * 2. Verify that the mapped blue reader remains ONLINE and reader traffic + * reaches its green target instead of the unmapped blue reader. + * 3. Publish SWITCHOVER_IN_POST_PROCESSING without reader pairs while both + * blue readers are OFFLINE_SOFT or OFFLINE_HARD, then verify that they do + * not trigger writer fallback. + * 4. Make one blue reader ONLINE but omit it from topology and verify that + * writer fallback keeps the reader hostgroup routable. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; + +struct TestState { + RDS_BGD_Cluster matched { bgd_cluster_init() }; + BGD_Hostgroups matched_hg { 1280, 1281, 1282, 1283 }; + vector matched_endpoints { matched.get_endpoints() }; + + RDS_BGD_Cluster fallback { bgd_cluster_2_init() }; + BGD_Hostgroups fallback_hg { 1290, 1291, 1292, 1293 }; + vector fallback_endpoints { fallback.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_reader_pairs(RDS_BGD_Cluster& cluster, string status, size_t pairs) { + vector rows = cluster.get_topology(status); + for (size_t i = 0; i < pairs; ++i) { + rows.push_back({ + cluster.blue_readers[i].hostname, + cluster.blue_readers[i].hostname, + cluster.blue_readers[i].port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + rows.push_back({ + cluster.green_readers[i].hostname, + cluster.green_readers[i].hostname, + cluster.green_readers[i].port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + } + return rows; +} + +int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_0(sim, cluster.green_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[1]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int configure_bgd( + MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, size_t green_reader_count) +{ + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer }; + for (size_t i = 0; i < green_reader_count; ++i) { + green_servers.push_back(cluster.green_readers[i]); + } + + int admin_rc = bgd_admin_setup( + admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0 + ); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups for wHG %d", hg.blue_writer); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int publish_post_processing( + RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, size_t pairs) +{ + vector topology = + topology_with_reader_pairs(cluster, "SWITCHOVER_IN_POST_PROCESSING", pairs); + + int rc = sim.topology_update(endpoints, topology); + return rc; +} + +int wait_for_reader_online(MYSQL* admin, BGD_Hostgroups& hg, RDS_BGD_Host& host) { + string query = + "SELECT COUNT(*)=1 FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.blue_reader) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + + " AND status='ONLINE'"; + + int rc = bgd_wait_for_condition(admin, query, kTimeoutSeconds); + return rc; +} + +int wait_for_writer_reader_membership(MYSQL* admin, BGD_Hostgroups& hg, RDS_BGD_Cluster& cluster, int expected) { + string query = + "SELECT COUNT(*)=" + to_string(expected) + + " FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.blue_reader) + + " AND hostname=" + bgd_sql_quote(cluster.blue_writer.hostname) + + " AND port=" + to_string(cluster.blue_writer.port); + + int rc = bgd_wait_for_condition(admin, query, kTimeoutSeconds); + return rc; +} + +int set_blue_reader_statuses(MYSQL* admin, BGD_Hostgroups& hg, RDS_BGD_Cluster& cluster, + string first_status, string second_status) +{ + string first_query = + "UPDATE mysql_servers SET status=" + bgd_sql_quote(first_status) + + " WHERE hostgroup_id=" + to_string(hg.blue_reader) + + " AND hostname=" + bgd_sql_quote(cluster.blue_readers[0].hostname) + + " AND port=" + to_string(cluster.blue_readers[0].port); + string second_query = + "UPDATE mysql_servers SET status=" + bgd_sql_quote(second_status) + + " WHERE hostgroup_id=" + to_string(hg.blue_reader) + + " AND hostname=" + bgd_sql_quote(cluster.blue_readers[1].hostname) + + " AND port=" + to_string(cluster.blue_readers[1].port); + vector queries { + first_query, + second_query, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + vector queries { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +rc_t connect_and_echo(CommandLine& cl) { + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + rc_t result { EXIT_FAILURE, {} }; + return result; + } + + rc_t result = bgd_backend_ip_echo(client); + mysql_close(client); + return result; +} + +/** + * Apply reader matching during SWITCHOVER_IN_POST_PROCESSING. + * + * - Configure blue readers 0 and 1 in hostgroup 1281. + * - Publish SWITCHOVER_IN_POST_PROCESSING topology with a pair only for blue + * reader 0. + * - Verify that the mapped reader remains ONLINE. + * - Route a client through hostgroup 1281 and verify that it reaches the + * mapped green reader instead of the unmapped blue reader. + */ +int test_matched_unmatched_readers(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.matched; + BGD_Hostgroups& hg = state.matched_hg; + + int config_rc = configure_bgd(admin, sim, cluster, hg, 1); + if (config_rc != EXIT_SUCCESS) { + diag("Error: failed to configure matched-reader scenario for wHG 1280"); + return EXIT_FAILURE; + } + + // Prefer the unmapped reader heavily so a routing check fails if BGD leaves it eligible. + string mapped_weight = + "UPDATE mysql_servers SET weight=1 WHERE hostgroup_id=" + to_string(hg.blue_reader) + + " AND hostname=" + bgd_sql_quote(cluster.blue_readers[0].hostname); + string unmapped_weight = + "UPDATE mysql_servers SET weight=1000000 WHERE hostgroup_id=" + to_string(hg.blue_reader) + + " AND hostname=" + bgd_sql_quote(cluster.blue_readers[1].hostname); + vector weight_queries { + mapped_weight, + unmapped_weight, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int weight_rc = execute_all(admin, weight_queries); + if (weight_rc != EXIT_SUCCESS) { + diag("Error: failed to configure deterministic reader weights in hostgroup 1281"); + return EXIT_FAILURE; + } + + int topology_rc = publish_post_processing(sim, state.matched_endpoints, cluster, 1); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish one-pair SWITCHOVER_IN_POST_PROCESSING topology for wHG 1280"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_POST_PROCESSING", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1280 did not reach WRITER_SWITCHOVER_POST_PROCESSING"); + return EXIT_FAILURE; + } + + int matched_rc = wait_for_reader_online(admin, hg, cluster.blue_readers[0]); + if (matched_rc != EXIT_SUCCESS) { + diag("Error: mapped blue reader did not remain ONLINE in hostgroup 1281"); + return EXIT_FAILURE; + } + + ok(true, "SWITCHOVER_IN_POST_PROCESSING keeps the mapped blue reader ONLINE in hostgroup 1281"); + + int user_rc = set_default_hostgroup(admin, hg.blue_reader); + if (user_rc != EXIT_SUCCESS) { + diag("Error: failed to route testuser through reader hostgroup 1281"); + return EXIT_FAILURE; + } + + auto [echo_rc, echo] = connect_and_echo(cl); + if (echo_rc != EXIT_SUCCESS) { + diag("Error: failed to connect through reader hostgroup 1281"); + return EXIT_FAILURE; + } + + bool mapped_reader_routing = echo.find(cluster.green_readers[0].ip) != string::npos; + ok(mapped_reader_routing, "SWITCHOVER_IN_POST_PROCESSING routes hostgroup 1281 through the mapped green reader"); + return EXIT_SUCCESS; +} + +/** + * Exclude offline blue readers from writer-fallback calculation. + * + * - Configure both blue readers in hostgroup 1291 as OFFLINE_SOFT and + * OFFLINE_HARD. + * - Publish SWITCHOVER_IN_POST_PROCESSING topology without reader pairs. + * - Verify that the blue writer is not added to reader hostgroup 1291. + */ +int test_offline_blue_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.fallback; + BGD_Hostgroups& hg = state.fallback_hg; + + int config_rc = configure_bgd(admin, sim, cluster, hg, 0); + if (config_rc != EXIT_SUCCESS) { + diag("Error: failed to configure offline-reader scenario for wHG 1290"); + return EXIT_FAILURE; + } + + int offline_rc = set_blue_reader_statuses(admin, hg, cluster, "OFFLINE_SOFT", "OFFLINE_HARD"); + if (offline_rc != EXIT_SUCCESS) { + diag("Error: failed to configure OFFLINE_SOFT and OFFLINE_HARD blue readers in hostgroup 1291"); + return EXIT_FAILURE; + } + + int topology_rc = publish_post_processing(sim, state.fallback_endpoints, cluster, 0); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish writer-only SWITCHOVER_IN_POST_PROCESSING topology for wHG 1290"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_POST_PROCESSING", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1290 did not reach WRITER_SWITCHOVER_POST_PROCESSING"); + return EXIT_FAILURE; + } + + int writer_absent_rc = wait_for_writer_reader_membership(admin, hg, cluster, 0); + if (writer_absent_rc != EXIT_SUCCESS) { + diag("Error: offline blue readers incorrectly triggered writer fallback in hostgroup 1291"); + return EXIT_FAILURE; + } + + ok(true, "OFFLINE_SOFT and OFFLINE_HARD blue readers do not trigger writer fallback in hostgroup 1291"); + return EXIT_SUCCESS; +} + +/** + * Route the reader hostgroup through writer fallback. + * + * - Make blue reader 0 ONLINE and publish SWITCHOVER_IN_POST_PROCESSING + * without reader pairs. + * - Verify that the blue writer is added to reader hostgroup 1291. + * - Connect through hostgroup 1291 and verify routing reaches the green writer + * IP pinned for the blue writer hostname. + */ +int test_writer_reader_fallback(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.fallback; + BGD_Hostgroups& hg = state.fallback_hg; + + int online_rc = set_blue_reader_statuses(admin, hg, cluster, "ONLINE", "OFFLINE_HARD"); + if (online_rc != EXIT_SUCCESS) { + diag("Error: failed to make one blue reader eligible for writer fallback"); + return EXIT_FAILURE; + } + + int topology_rc = publish_post_processing(sim, state.fallback_endpoints, cluster, 0); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish writer-only SWITCHOVER_IN_POST_PROCESSING topology for wHG 1290"); + return EXIT_FAILURE; + } + + int writer_present_rc = wait_for_writer_reader_membership(admin, hg, cluster, 1); + if (writer_present_rc != EXIT_SUCCESS) { + diag("Error: writer fallback did not add the blue writer to reader hostgroup 1291"); + return EXIT_FAILURE; + } + + int user_rc = set_default_hostgroup(admin, hg.blue_reader); + if (user_rc != EXIT_SUCCESS) { + diag("Error: failed to route testuser through reader hostgroup 1291"); + return EXIT_FAILURE; + } + + auto [echo_rc, echo] = connect_and_echo(cl); + if (echo_rc != EXIT_SUCCESS) { + diag("Error: failed to connect through writer fallback in reader hostgroup 1291"); + return EXIT_FAILURE; + } + + bool green_writer_routing = echo.find(cluster.green_writer.ip) != string::npos; + ok(green_writer_routing, "writer fallback keeps hostgroup 1291 routable through the green writer IP"); + return EXIT_SUCCESS; +} + +int main() { + plan(4); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish SWITCHOVER_IN_POST_PROCESSING with one reader pair for wHG 1280. + // Verify: only the mapped blue reader remains ONLINE in reader hostgroup 1281. + if (test_matched_unmatched_readers(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: configure both blue readers in hostgroup 1291 as OFFLINE_SOFT and OFFLINE_HARD. + // Simulator: publish SWITCHOVER_IN_POST_PROCESSING without reader pairs. + // Verify: offline blue readers do not add the blue writer to reader hostgroup 1291. + if (test_offline_blue_servers(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: make one blue reader ONLINE. + // Simulator: publish SWITCHOVER_IN_POST_PROCESSING without reader pairs. + // Verify: writer fallback keeps reader hostgroup 1291 routable through the green writer IP. + if (test_writer_reader_fallback(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_reader_switchover_cleanup-t.cpp b/test/tap/tests/test_rds_bgd_reader_switchover_cleanup-t.cpp new file mode 100644 index 0000000000..3d10a06c50 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_reader_switchover_cleanup-t.cpp @@ -0,0 +1,417 @@ +/** + * @file test_rds_bgd_reader_switchover_cleanup-t.cpp + * @brief BGD reader switchover and terminal empty-topology cleanup. + * + * Steps: + * + * 1. Configure hostgroups 980-983 and advance through writer post-processing. + * 2. Publish target-only SWITCHOVER_COMPLETED and verify + * READER_SWITCHOVER_IN_PROGRESS with green rows retained. + * 3. Repeat the completed observation and verify the reader phase is stable. + * 4. Publish empty topology and verify NONE, restored blue-reader routing, + * blue-IP probing, green-pool drain, and retained green rows. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; + +struct TestState { + RDS_BGD_Cluster cluster { bgd_cluster_init() }; + BGD_Hostgroups hostgroups { 980, 981, 982, 983 }; + vector topology_endpoints { cluster.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + rows.push_back({ + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + rows.push_back({ + cluster.green_readers[0].hostname, + cluster.green_readers[0].hostname, + cluster.green_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + return rows; +} + +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows { + { + cluster.green_writer.hostname, + cluster.green_writer.hostname, + cluster.green_writer.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + "SWITCHOVER_COMPLETED", + }, + }; + return rows; +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + vector queries { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +int create_pool(CommandLine& cl) { + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + return EXIT_FAILURE; + } + + rc_t echo = bgd_backend_ip_echo(client); + mysql_close(client); + return echo.first; +} + +bool runtime_server_online(MYSQL* admin, int hostgroup, RDS_BGD_Host& host) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + + " AND status='ONLINE'"; + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool online = rows[0][0] == "1"; + return online; +} + +bool green_rows_online(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { + bool writer_online = runtime_server_online(admin, hg.green_writer, cluster.green_writer); + bool reader_online = runtime_server_online(admin, hg.green_reader, cluster.green_readers[0]); + bool rows_online = writer_online && reader_online; + return rows_online; +} + +int advance_to_post_processing(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + vector available = topology_with_reader_pair(cluster, "AVAILABLE"); + int available_rc = sim.topology_update(state.topology_endpoints, available); + if (available_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + int available_status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (available_status_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector in_progress = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); + int progress_rc = sim.topology_update(state.topology_endpoints, in_progress); + if (progress_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + int progress_status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (progress_status_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector post_processing = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); + int post_rc = sim.topology_update(state.topology_endpoints, post_processing); + if (post_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + int post_status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_POST_PROCESSING", kTimeoutSeconds); + if (post_status_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +/** + * Enter reader switchover after writer post-processing. + * + * - Configure hostgroups 980-983 with one mapped reader pair. + * - Create pools in green writer and reader hostgroups. + * - Advance through writer post-processing. + * - Publish target-only SWITCHOVER_COMPLETED twice. + * - Verify READER_SWITCHOVER_IN_PROGRESS and retained green rows. + */ +int test_reader_switchover_in_progress(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + int blue_writer_rc = bgd_set_host_read_only_0(sim, cluster.blue_writer); + if (blue_writer_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated blue writer"); + return EXIT_FAILURE; + } + + int green_writer_rc = bgd_set_host_read_only_0(sim, cluster.green_writer); + if (green_writer_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=0 for the simulated green writer"); + return EXIT_FAILURE; + } + + int blue_reader_rc = bgd_set_host_read_only_1(sim, cluster.blue_readers[0]); + if (blue_reader_rc != EXIT_SUCCESS) { + diag("Error: failed to set read_only=1 for the simulated blue reader"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 980-983"); + return EXIT_FAILURE; + } + + int green_writer_hg_rc = set_default_hostgroup(admin, hg.green_writer); + if (green_writer_hg_rc != EXIT_SUCCESS) { + diag("Error: failed to route the test user through green writer hostgroup 982"); + return EXIT_FAILURE; + } + + int green_writer_pool_rc = create_pool(cl); + if (green_writer_pool_rc != EXIT_SUCCESS) { + diag("Error: failed to create a green-writer pool before reader cleanup"); + return EXIT_FAILURE; + } + + int green_reader_hg_rc = set_default_hostgroup(admin, hg.green_reader); + if (green_reader_hg_rc != EXIT_SUCCESS) { + diag("Error: failed to route the test user through green reader hostgroup 983"); + return EXIT_FAILURE; + } + + int green_reader_pool_rc = create_pool(cl); + if (green_reader_pool_rc != EXIT_SUCCESS) { + diag("Error: failed to create a green-reader pool before reader cleanup"); + return EXIT_FAILURE; + } + + int restore_hg_rc = set_default_hostgroup(admin, hg.blue_writer); + if (restore_hg_rc != EXIT_SUCCESS) { + diag("Error: failed to restore testuser to blue writer hostgroup 980"); + return EXIT_FAILURE; + } + + int post_rc = advance_to_post_processing(admin, sim, state); + if (post_rc != EXIT_SUCCESS) { + diag("Error: failed to reach WRITER_SWITCHOVER_POST_PROCESSING before reader switchover"); + return EXIT_FAILURE; + } + + vector completed = target_only_completed(cluster); + int completed_rc = sim.topology_update(state.topology_endpoints, completed); + if (completed_rc != EXIT_SUCCESS) { + diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology"); + return EXIT_FAILURE; + } + + int reader_status_rc = bgd_wait_for_status(admin, hg, "READER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (reader_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 980 did not reach READER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + ok(true, "target-only SWITCHOVER_COMPLETED sets BGD status for wHG 980 to READER_SWITCHOVER_IN_PROGRESS"); + + bool rows_online = green_rows_online(admin, cluster, hg); + ok(rows_online, "READER_SWITCHOVER_IN_PROGRESS retains configured green writer and reader rows"); + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the repeated reader-switchover probe sequence"); + return EXIT_FAILURE; + } + + vector repeated_completed = target_only_completed(cluster); + int repeat_rc = sim.topology_update(state.topology_endpoints, repeated_completed); + if (repeat_rc != EXIT_SUCCESS) { + diag("Error: failed to repeat target-only SWITCHOVER_COMPLETED topology"); + return EXIT_FAILURE; + } + + auto [probe_rc, probe] = + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: BGD did not observe repeated target-only SWITCHOVER_COMPLETED topology"); + return EXIT_FAILURE; + } + + int repeat_status_rc = bgd_wait_for_status(admin, hg, "READER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (repeat_status_rc != EXIT_SUCCESS) { + diag("Error: repeated completion changed BGD status for wHG 980"); + return EXIT_FAILURE; + } + + bool repeated_rows_online = green_rows_online(admin, cluster, hg); + ok(repeated_rows_online, "repeated SWITCHOVER_COMPLETED preserves reader switchover and green rows"); + return EXIT_SUCCESS; +} + +/** + * Complete reader cleanup with present-but-empty topology. + * + * - Delete every topology row while the topology table remains present. + * - Verify BGD status NONE and restored blue-reader routing. + * - Verify metadata probing returns from the green pin to the blue writer. + * - Verify green pools drain while configured green rows remain ONLINE. + */ +int test_reader_switchover_cleanup(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + auto [green_writer_pool_before_rc, green_writer_pool_before] = bgd_connection_pool_count(admin, hg.green_writer); + auto [green_reader_pool_before_rc, green_reader_pool_before] = bgd_connection_pool_count(admin, hg.green_reader); + if (green_writer_pool_before_rc != EXIT_SUCCESS || green_reader_pool_before_rc != EXIT_SUCCESS || + green_writer_pool_before < 1 || green_reader_pool_before < 1) { + diag("Error: green writer or reader pool is empty before reader cleanup"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before empty topology"); + return EXIT_FAILURE; + } + + int empty_rc = sim.topology_delete(state.topology_endpoints); + if (empty_rc != EXIT_SUCCESS) { + diag("Error: failed to publish present-but-empty topology"); + return EXIT_FAILURE; + } + + int none_rc = bgd_wait_for_status(admin, hg, "NONE", kTimeoutSeconds); + if (none_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 980 did not reach NONE"); + return EXIT_FAILURE; + } + + ok(true, "present-but-empty topology sets BGD status for wHG 980 to NONE"); + + bool unmatched_reader_online = runtime_server_online(admin, hg.blue_reader, cluster.blue_readers[1]); + ok(unmatched_reader_online, "reader cleanup restores the unmatched blue reader in hostgroup 981"); + + auto [green_probe_rc, green_probe] = + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (green_probe_rc != EXIT_SUCCESS) { + diag("Error: reader cleanup did not observe empty topology through the green pin"); + return EXIT_FAILURE; + } + + auto [blue_probe_rc, blue_probe] = + sim.wait_for_probe_log(green_probe.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + if (blue_probe_rc != EXIT_SUCCESS) { + diag("Error: metadata probing did not return to the blue writer after reader cleanup"); + return EXIT_FAILURE; + } + + bool probe_order = green_probe.sequence_id < blue_probe.sequence_id; + ok(probe_order, "reader cleanup removes the green pin and resumes blue-writer metadata probing"); + + auto [green_writer_pool_rc, green_writer_pool] = bgd_connection_pool_count(admin, hg.green_writer); + auto [green_reader_pool_rc, green_reader_pool] = bgd_connection_pool_count(admin, hg.green_reader); + if (green_writer_pool_rc != EXIT_SUCCESS || green_reader_pool_rc != EXIT_SUCCESS) { + diag("Error: failed to read green pools after reader cleanup"); + return EXIT_FAILURE; + } + + ok(green_writer_pool == 0 && green_reader_pool == 0, "reader cleanup drains eligible green writer and reader pools"); + + bool rows_online = green_rows_online(admin, cluster, hg); + ok(rows_online, "reader cleanup retains configured green writer and reader rows as ONLINE"); + return EXIT_SUCCESS; +} + +int main() { + plan(8); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // ProxySQL: configure hostgroups 980-983 and establish green writer/reader pools. + // Simulator: advance through writer post-processing, then publish target-only SWITCHOVER_COMPLETED twice. + // Verify: BGD remains in READER_SWITCHOVER_IN_PROGRESS and green rows remain ONLINE. + if (test_reader_switchover_in_progress(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: delete all rows while keeping the topology table present. + // Verify: BGD reaches NONE, blue-reader routing and blue probing resume, green pools drain, and rows remain. + if (test_reader_switchover_cleanup(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} From 630de5f4e242ba082ef3465438a95f9abb5e6948 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 28 Jul 2026 07:10:11 +0000 Subject: [PATCH 71/81] test: cover RDS BGD worker configuration refresh - Verify active workers consume scalar, membership, and hostgroup changes. - Verify refreshed configuration is applied after writer completion. - Register and lint the focused worker-refresh tests. Signed-off-by: Wazir Ahmed --- test/tap/groups/groups.json | 3 + ...nfig_refresh_after_writer_completion-t.cpp | 281 +++++++++ .../test_rds_bgd_worker_config_refresh-t.cpp | 589 ++++++++++++++++++ ...est_rds_bgd_worker_hostgroup_refresh-t.cpp | 359 +++++++++++ 4 files changed, 1232 insertions(+) create mode 100644 test/tap/tests/test_rds_bgd_config_refresh_after_writer_completion-t.cpp create mode 100644 test/tap/tests/test_rds_bgd_worker_config_refresh-t.cpp create mode 100644 test/tap/tests/test_rds_bgd_worker_hostgroup_refresh-t.cpp diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index a4da267a3c..24b5f0af6b 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -431,6 +431,7 @@ "test_query_rules_routing-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g4","mysql90-g4","mysql95-g4" ], "test_query_timeout-t" : [ "legacy-g9","mariadb10-galera-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql84-gr-g9","mysql90-g4","mysql95-g4" ], "test_rds_bgd_automatic_discovery-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_config_refresh_after_writer_completion-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_configuration_persistence-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_disable_during_switchover-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_explicit_startup-t" : [ "cluster_sim_rds_bgd-g1" ], @@ -446,6 +447,8 @@ "test_rds_bgd_smoke-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_topology_empty_absent-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_topology_errors-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_worker_config_refresh-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_worker_hostgroup_refresh-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_writer_switchover-t" : [ "cluster_sim_rds_bgd-g1" ], "test_read_only_actions_offline_hard_servers-t" : [ "legacy-g5","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g5","mysql84-g9","mysql90-g4","mysql90-g5","mysql95-g4","mysql95-g5" ], "test_rw_binary_data-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql90-g4","mysql95-g4" ], diff --git a/test/tap/tests/test_rds_bgd_config_refresh_after_writer_completion-t.cpp b/test/tap/tests/test_rds_bgd_config_refresh_after_writer_completion-t.cpp new file mode 100644 index 0000000000..0d375e13a1 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_config_refresh_after_writer_completion-t.cpp @@ -0,0 +1,281 @@ +/** + * @file test_rds_bgd_config_refresh_after_writer_completion-t.cpp + * @brief BGD configuration refresh during reader switchover. + * + * Steps: + * + * 1. Configure BGD hostgroups 1360-1363 and reach + * WRITER_SWITCHOVER_IN_PROGRESS. + * 2. Publish target-only SWITCHOVER_COMPLETED and verify + * READER_SWITCHOVER_IN_PROGRESS. + * 3. Change check_timeout_ms in mysql_aws_rds_bgd_hostgroups and load the + * configuration to runtime. + * 4. Verify that the refresh performs a blue table check before blue metadata + * and republishes READER_SWITCHOVER_IN_PROGRESS. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; + +struct TestState { + RDS_BGD_Cluster cluster { bgd_cluster_3_init() }; + BGD_Hostgroups hostgroups { 1360, 1361, 1362, 1363 }; + vector topology_endpoints { cluster.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_readers(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + for (RDS_BGD_Host& host : cluster.blue_readers) { + rows.push_back({ + host.hostname, + host.hostname, + host.port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + } + for (RDS_BGD_Host& host : cluster.green_readers) { + rows.push_back({ + host.hostname, + host.hostname, + host.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + } + return rows; +} + +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ + cluster.green_writer.hostname, + cluster.green_writer.hostname, + cluster.green_writer.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + "SWITCHOVER_COMPLETED", + }}; + return rows; +} + +int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_0(sim, cluster.green_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[1]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +/** + * Reach reader switchover before changing the BGD configuration. + * + * - Publish SWITCHOVER_IN_PROGRESS before loading BGD hostgroups 1360-1363. + * - Require WRITER_SWITCHOVER_IN_PROGRESS. + * - Publish target-only SWITCHOVER_COMPLETED. + * - Verify BGD status READER_SWITCHOVER_IN_PROGRESS. + */ +int test_reader_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + int read_only_rc = configure_read_only_values(sim, cluster); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for wHG 1360"); + return EXIT_FAILURE; + } + + vector progress = topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); + int topology_rc = sim.topology_update(state.topology_endpoints, progress); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for wHG 1360"); + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int admin_rc = bgd_admin_setup( + admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0 + ); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 1360-1363"); + return EXIT_FAILURE; + } + + int progress_status_rc = + bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (progress_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1360 did not reach WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + vector completed = target_only_completed(cluster); + int completed_rc = sim.topology_update(state.topology_endpoints, completed); + if (completed_rc != EXIT_SUCCESS) { + diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology for wHG 1360"); + return EXIT_FAILURE; + } + + int reader_status_rc = + bgd_wait_for_status(admin, hg, "READER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (reader_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1360 did not reach READER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + ok(true, "target-only SWITCHOVER_COMPLETED sets BGD status for wHG 1360 to READER_SWITCHOVER_IN_PROGRESS"); + return EXIT_SUCCESS; +} + +/** + * Refresh the BGD configuration during reader switchover. + * + * - Change check_timeout_ms for wHG 1360 and load it to runtime. + * - Verify that the refresh starts with a blue-writer table check. + * - Verify that blue-writer metadata follows the table check. + * - Verify BGD status returns to READER_SWITCHOVER_IN_PROGRESS. + */ +int test_config_refresh_after_writer_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before refreshing wHG 1360"); + return EXIT_FAILURE; + } + + string update_query = + "UPDATE mysql_aws_rds_bgd_hostgroups SET check_timeout_ms=950 WHERE writer_hostgroup=" + + to_string(hg.blue_writer); + vector queries { + update_query, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + int refresh_rc = execute_all(admin, queries); + if (refresh_rc != EXIT_SUCCESS) { + diag("Error: failed to refresh check_timeout_ms for wHG 1360"); + return EXIT_FAILURE; + } + + auto [table_rc, table] = + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + if (table_rc != EXIT_SUCCESS) { + diag("Error: post-completion refresh did not start with a blue-writer table check for wHG 1360"); + return EXIT_FAILURE; + } + + auto [blue_rc, blue] = sim.wait_for_probe_log( + table.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + ); + if (blue_rc != EXIT_SUCCESS) { + diag("Error: post-completion refresh did not probe blue-writer metadata for wHG 1360"); + return EXIT_FAILURE; + } + + bool probe_order = table.sequence_id < blue.sequence_id; + ok(probe_order, "post-completion refresh checks the table before blue-writer metadata for wHG 1360"); + + int status_rc = + bgd_wait_for_status(admin, hg, "READER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: refreshed wHG 1360 did not republish READER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + ok(true, "post-completion refresh republishes READER_SWITCHOVER_IN_PROGRESS for wHG 1360"); + return EXIT_SUCCESS; +} + +int main() { + plan(3); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish SWITCHOVER_IN_PROGRESS, then target-only SWITCHOVER_COMPLETED. + // ProxySQL: configure BGD hostgroups 1360-1363. + // Verify: BGD status for wHG 1360 reports READER_SWITCHOVER_IN_PROGRESS. + if (test_reader_switchover_in_progress(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: change check_timeout_ms and load the BGD configuration to runtime. + // Verify: refresh runs blue table check, then blue metadata, and republishes READER_SWITCHOVER_IN_PROGRESS. + if (test_config_refresh_after_writer_completion(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_worker_config_refresh-t.cpp b/test/tap/tests/test_rds_bgd_worker_config_refresh-t.cpp new file mode 100644 index 0000000000..1643200102 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_worker_config_refresh-t.cpp @@ -0,0 +1,589 @@ +/** + * @file test_rds_bgd_worker_config_refresh-t.cpp + * @brief Refreshing an active BGD worker after server and scalar configuration changes. + * + * Steps: + * + * 1. Configure BGD hostgroups 1370-1373 and reach `AVAILABLE`. + * 2. Change `weight` and `comment` and verify that discovery does not restart. + * 3. Change green-writer TLS and verify that metadata probes use the new value. + * 4. Replace green-reader membership and verify that discovery does not restart. + * 5. Move the green writer offline and online and verify that direct metadata + * probing stops and resumes without a table-check restart. + * 6. Change `check_interval_ms` and verify the metadata probe cadence. + */ + +#include +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; +const uint32_t kNegativeProbeTimeoutMs = 800; +const uint32_t kRefreshedCheckIntervalMs = 1000; +const uint32_t kMinimumProbeIntervalMs = 500; +const uint32_t kMaximumProbeIntervalMs = 1500; + +struct TestState { + RDS_BGD_Cluster cluster { bgd_cluster_2_init() }; + BGD_Hostgroups hostgroups { 1370, 1371, 1372, 1373 }; + vector topology_endpoints { cluster.get_endpoints() }; + uint64_t available_probe_sequence { 0 }; + string topology_discovery_interval {}; + bool topology_discovery_interval_saved { false }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int restore_topology_discovery_interval(MYSQL* admin, TestState& state) { + if (!state.topology_discovery_interval_saved) { + return EXIT_SUCCESS; + } + + vector queries { + "SET mysql-monitor_aws_rds_topology_discovery_interval=" + state.topology_discovery_interval, + "LOAD MYSQL VARIABLES TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + + int discovery_rc = restore_topology_discovery_interval(admin, state); + if (discovery_rc != EXIT_SUCCESS) { + diag("Error: failed to restore mysql-monitor_aws_rds_topology_discovery_interval"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || discovery_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int disable_topology_discovery_probes(MYSQL* admin, TestState& state) { + string query = + "SELECT variable_value FROM runtime_global_variables " + "WHERE variable_name='mysql-monitor_aws_rds_topology_discovery_interval'"; + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + diag("Error: failed to read mysql-monitor_aws_rds_topology_discovery_interval"); + return EXIT_FAILURE; + } + + state.topology_discovery_interval = rows[0][0]; + state.topology_discovery_interval_saved = true; + + vector queries { + "SET mysql-monitor_aws_rds_topology_discovery_interval=0", + "LOAD MYSQL VARIABLES TO RUNTIME", + }; + + int disable_rc = execute_all(admin, queries); + if (disable_rc != EXIT_SUCCESS) { + diag("Error: failed to disable automatic AWS topology discovery"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int wait_for_server_status(MYSQL* admin, int hostgroup, RDS_BGD_Host& host, string status) { + string query = + "SELECT COUNT(*)=1 FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hostgroup) + + " AND hostname=" + bgd_sql_quote(host.hostname) + " AND port=" + to_string(host.port) + + " AND status=" + bgd_sql_quote(status); + + int rc = bgd_wait_for_condition(admin, query, kTimeoutSeconds); + return rc; +} + +/** + * Configure BGD hostgroups 1370-1373 and change ignored server fields. + * + * - Set `read_only=0` for the simulated blue and green writers. + * - Publish `AVAILABLE` topology and configure the explicit BGD row. + * - Verify that the runtime BGD row reaches `AVAILABLE`. + * - Change blue-writer `weight` and `comment`. + * - Verify that the active worker does not restart with a table check. + */ +int test_irrelevant_server_fields(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Set read_only=0 for the simulated blue and green writers. + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated writer read_only values"); + return EXIT_FAILURE; + } + + // Publish AVAILABLE topology. + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology"); + return EXIT_FAILURE; + } + + // Configure mysql_servers and mysql_aws_rds_bgd_hostgroups. + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure mysql_servers and mysql_aws_rds_bgd_hostgroups"); + return EXIT_FAILURE; + } + + // Disable automatic AWS topology discovery so its metadata queries are not + // mistaken for probes from the explicitly configured BGD worker. + int discovery_rc = disable_topology_discovery_probes(admin, state); + if (discovery_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + // Wait for the runtime BGD row to report AVAILABLE. + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: runtime BGD status did not reach AVAILABLE"); + return EXIT_FAILURE; + } + + ok(true, "BGD status for wHG 1370 reports AVAILABLE"); + + // Record a green metadata probe before changing ignored server fields. + auto [probe_seq_rc, probe_seq] = sim.probe_log_last_sequence(); + if (probe_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before the AVAILABLE metadata probe"); + return EXIT_FAILURE; + } + + auto [probe_rc, probe] = sim.wait_for_probe_log( + probe_seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + ); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: green writer did not receive the AVAILABLE metadata probe"); + return EXIT_FAILURE; + } + state.available_probe_sequence = probe.sequence_id; + + // Change weight and comment, which are not BGD worker inputs. + string update_server = + "UPDATE mysql_servers SET weight=weight+7,comment='BGD TAP ignored fields' WHERE hostgroup_id=" + + to_string(hg.blue_writer) + " AND hostname=" + bgd_sql_quote(cluster.blue_writer.hostname) + + " AND port=" + to_string(cluster.blue_writer.port); + vector queries { + update_server, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int update_rc = execute_all(admin, queries); + if (update_rc != EXIT_SUCCESS) { + diag("Error: failed to update blue-writer weight and comment"); + return EXIT_FAILURE; + } + + int no_table_rc = bgd_expect_no_table_check(sim, state.available_probe_sequence, state.topology_endpoints, kNegativeProbeTimeoutMs); + if (no_table_rc != EXIT_SUCCESS) { + diag("Error: weight or comment change restarted BGD discovery"); + return EXIT_FAILURE; + } + + ok(true, "weight and comment changes do not restart BGD discovery for wHG 1370"); + return EXIT_SUCCESS; +} + +/** + * Refresh green-writer TLS for writer hostgroup 1370. + * + * - Set `use_ssl=1` on the configured green writer. + * - Load `mysql_servers` to runtime. + * - Verify that the next green-writer metadata probe uses TLS. + * - Verify that discovery does not restart. + */ +int test_tls_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Record the probe sequence before changing green-writer TLS. + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before the TLS refresh"); + return EXIT_FAILURE; + } + + // Set use_ssl=1 on the green writer and load mysql_servers to runtime. + string update_tls = + "UPDATE mysql_servers SET use_ssl=1 WHERE hostgroup_id=" + to_string(hg.green_writer) + + " AND hostname=" + bgd_sql_quote(cluster.green_writer.hostname) + + " AND port=" + to_string(cluster.green_writer.port); + vector queries { + update_tls, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int update_rc = execute_all(admin, queries); + if (update_rc != EXIT_SUCCESS) { + diag("Error: failed to set use_ssl=1 for the green writer"); + return EXIT_FAILURE; + } + + // Wait for the active worker to use TLS without starting a table check. + auto [probe_rc, probe] = sim.wait_for_probe_log( + seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + ); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: green-writer metadata probe did not use TLS after refresh"); + return EXIT_FAILURE; + } + + int no_table_rc = bgd_expect_no_table_check(sim, seq, state.topology_endpoints, kNegativeProbeTimeoutMs); + if (no_table_rc != EXIT_SUCCESS) { + diag("Error: TLS refresh restarted BGD discovery"); + return EXIT_FAILURE; + } + + ok(true, "use_ssl=1 refreshes green-writer metadata probes without a table-check restart"); + return EXIT_SUCCESS; +} + +/** + * Replace green-reader membership for writer hostgroup 1370. + * + * - Add the second green reader to hostgroup 1373. + * - Delete the first green reader from hostgroup 1373. + * - Verify that discovery does not restart. + */ +int test_green_membership_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Record the probe sequence before changing green-reader membership. + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before the membership refresh"); + return EXIT_FAILURE; + } + + // Replace the first green reader with the second green reader. + string add_reader = + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + + to_string(hg.green_reader) + "," + bgd_sql_quote(cluster.green_readers[1].hostname) + + "," + to_string(cluster.green_readers[1].port) + ",'ONLINE',0,'BGD TAP green reader')"; + string delete_reader = + "DELETE FROM mysql_servers WHERE hostgroup_id=" + to_string(hg.green_reader) + + " AND hostname=" + bgd_sql_quote(cluster.green_readers[0].hostname) + + " AND port=" + to_string(cluster.green_readers[0].port); + vector queries { + add_reader, + delete_reader, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int update_rc = execute_all(admin, queries); + if (update_rc != EXIT_SUCCESS) { + diag("Error: failed to replace green-reader membership"); + return EXIT_FAILURE; + } + + int no_table_rc = bgd_expect_no_table_check(sim, seq, state.topology_endpoints, kNegativeProbeTimeoutMs); + if (no_table_rc != EXIT_SUCCESS) { + diag("Error: green-reader membership refresh restarted BGD discovery"); + return EXIT_FAILURE; + } + + ok(true, "green-reader membership refresh does not restart BGD discovery for wHG 1370"); + return EXIT_SUCCESS; +} + +/** + * Refresh green-writer eligibility for writer hostgroup 1370. + * + * - Move the green writer to `OFFLINE_SOFT`. + * - Verify that direct metadata probing stops without a table-check restart. + * - Return the green writer to `ONLINE`. + * - Verify that TLS metadata probing resumes without a table-check restart. + */ +int test_server_eligibility_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Record the probe sequence before changing green-writer eligibility. + auto [refresh_seq_rc, refresh_seq] = sim.probe_log_last_sequence(); + if (refresh_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before the OFFLINE_SOFT refresh"); + return EXIT_FAILURE; + } + + // Move the green writer to OFFLINE_SOFT. + string set_offline = + "UPDATE mysql_servers SET status='OFFLINE_SOFT' WHERE hostgroup_id=" + to_string(hg.green_writer) + + " AND hostname=" + bgd_sql_quote(cluster.green_writer.hostname) + + " AND port=" + to_string(cluster.green_writer.port); + vector offline_queries { + set_offline, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int offline_rc = execute_all(admin, offline_queries); + if (offline_rc != EXIT_SUCCESS) { + diag("Error: failed to set the green writer OFFLINE_SOFT"); + return EXIT_FAILURE; + } + + int status_rc = wait_for_server_status(admin, hg.green_writer, cluster.green_writer, "OFFLINE_SOFT"); + if (status_rc != EXIT_SUCCESS) { + diag("Error: runtime green writer did not reach OFFLINE_SOFT"); + return EXIT_FAILURE; + } + + // Wait for the worker to apply the refreshed server list and return to its eligible blue writer. + auto [blue_probe_rc, blue_probe] = sim.wait_for_probe_log( + refresh_seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + ); + if (blue_probe_rc != EXIT_SUCCESS) { + diag("Error: OFFLINE_SOFT refresh did not return metadata probing to the blue writer"); + return EXIT_FAILURE; + } + + int no_metadata_rc = + bgd_expect_no_metadata_probe(sim, blue_probe.sequence_id, cluster.green_writer.endpoint(), kNegativeProbeTimeoutMs); + if (no_metadata_rc != EXIT_SUCCESS) { + diag("Error: OFFLINE_SOFT green writer continued receiving metadata probes"); + return EXIT_FAILURE; + } + + int offline_no_table_rc = + bgd_expect_no_table_check(sim, refresh_seq, state.topology_endpoints, kNegativeProbeTimeoutMs); + if (offline_no_table_rc != EXIT_SUCCESS) { + diag("Error: OFFLINE_SOFT refresh restarted BGD discovery"); + return EXIT_FAILURE; + } + + ok(true, "OFFLINE_SOFT stops green-writer metadata probes without a table-check restart"); + + // Return the green writer to ONLINE. + auto [online_seq_rc, online_seq] = sim.probe_log_last_sequence(); + if (online_seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before the ONLINE refresh"); + return EXIT_FAILURE; + } + + string set_online = + "UPDATE mysql_servers SET status='ONLINE' WHERE hostgroup_id=" + to_string(hg.green_writer) + + " AND hostname=" + bgd_sql_quote(cluster.green_writer.hostname) + + " AND port=" + to_string(cluster.green_writer.port); + vector online_queries { + set_online, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int online_rc = execute_all(admin, online_queries); + if (online_rc != EXIT_SUCCESS) { + diag("Error: failed to return the green writer to ONLINE"); + return EXIT_FAILURE; + } + + auto [probe_rc, probe] = sim.wait_for_probe_log( + online_seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + ); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: ONLINE green writer did not resume TLS metadata probes"); + return EXIT_FAILURE; + } + + int online_no_table_rc = bgd_expect_no_table_check(sim, online_seq, state.topology_endpoints, kNegativeProbeTimeoutMs); + if (online_no_table_rc != EXIT_SUCCESS) { + diag("Error: ONLINE refresh restarted BGD discovery"); + return EXIT_FAILURE; + } + + ok(true, "ONLINE resumes green-writer TLS metadata probes without a table-check restart"); + return EXIT_SUCCESS; +} + +/** + * Refresh the configured polling interval for writer hostgroup 1370. + * + * - Set `check_interval_ms=1000`. + * - Load the BGD configuration to runtime. + * - Publish empty topology so the worker uses its configured baseline interval. + * - Verify that the next metadata probe occurs between 500 and 1500 milliseconds. + * - Verify that the configuration refresh does not restart with a table check. + */ +int test_check_interval_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + + // Record the probe sequence before changing check_interval_ms. + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before the check_interval_ms refresh"); + return EXIT_FAILURE; + } + + // Set check_interval_ms=1000 and load the BGD configuration to runtime. + string update_bgd = + "UPDATE mysql_aws_rds_bgd_hostgroups SET check_interval_ms=" + + to_string(kRefreshedCheckIntervalMs) + " WHERE writer_hostgroup=" + + to_string(hg.blue_writer); + vector queries { + update_bgd, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int update_rc = execute_all(admin, queries); + if (update_rc != EXIT_SUCCESS) { + diag("Error: failed to update check_interval_ms"); + return EXIT_FAILURE; + } + + // Publish empty topology so the worker leaves AVAILABLE and uses check_interval_ms. + int topology_rc = sim.topology_delete(state.topology_endpoints); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish empty topology before checking the probe interval"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "NONE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status did not reach NONE before checking the probe interval"); + return EXIT_FAILURE; + } + + auto [baseline_rc, baseline] = sim.probe_log_last_sequence(); + if (baseline_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence after BGD reached NONE"); + return EXIT_FAILURE; + } + + // Consume the immediate refresh probe and the first blue probe after the worker reaches NONE. + auto [first_rc, first_probe] = + sim.wait_for_probe_log(baseline, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, -1); + if (first_rc != EXIT_SUCCESS) { + diag("Error: failed to observe the first blue metadata probe after the check_interval_ms refresh"); + return EXIT_FAILURE; + } + + auto [settled_rc, settled_probe] = + sim.wait_for_probe_log(first_probe.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, -1); + if (settled_rc != EXIT_SUCCESS) { + diag("Error: failed to observe the settled blue metadata probe after the check_interval_ms refresh"); + return EXIT_FAILURE; + } + + // Measure the steady-state interval between consecutive blue metadata probes. + unsigned long long interval_start = monotonic_time(); + auto [next_rc, next_probe] = sim.wait_for_probe_log( + settled_probe.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kMaximumProbeIntervalMs, -1 + ); + if (next_rc != EXIT_SUCCESS) { + diag("Error: metadata probing did not occur within 1.5 times check_interval_ms"); + return EXIT_FAILURE; + } + + unsigned long long elapsed_ms = (monotonic_time() - interval_start) / 1000; + if (elapsed_ms < kMinimumProbeIntervalMs) { + diag("Error: consecutive metadata probes occurred before half of check_interval_ms elapsed"); + return EXIT_FAILURE; + } + + int no_table_rc = bgd_expect_no_table_check(sim, seq, state.topology_endpoints, kNegativeProbeTimeoutMs); + if (no_table_rc != EXIT_SUCCESS) { + diag("Error: check_interval_ms refresh restarted discovery"); + return EXIT_FAILURE; + } + + ok(true, "check_interval_ms=1000 schedules the next metadata probe between 500 and 1500 milliseconds"); + return EXIT_SUCCESS; +} + +int main() { + plan(7); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: set blue/green writers to read_only=0 and publish AVAILABLE topology. + // ProxySQL: configure BGD hostgroups 1370-1373, then change blue-writer weight and comment. + // Verify: BGD reaches AVAILABLE and ignored fields do not start a new table check. + if (test_irrelevant_server_fields(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: set use_ssl=1 for the green writer in hostgroup 1372. + // Verify: the next green-writer metadata probe uses TLS without restarting discovery. + if (test_tls_refresh(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: replace the green reader in hostgroup 1373 and load mysql_servers to runtime. + // Verify: green-reader membership refresh does not restart BGD discovery. + if (test_green_membership_refresh(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: move the green writer OFFLINE_SOFT and then ONLINE. + // Verify: direct metadata probes stop and resume without a table-check restart. + if (test_server_eligibility_refresh(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: set check_interval_ms=1000 for wHG 1370 and publish empty topology. + // Verify: metadata probing follows the configured interval without restarting discovery. + if (test_check_interval_refresh(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim, state) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_worker_hostgroup_refresh-t.cpp b/test/tap/tests/test_rds_bgd_worker_hostgroup_refresh-t.cpp new file mode 100644 index 0000000000..6bd3d7ac64 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_worker_hostgroup_refresh-t.cpp @@ -0,0 +1,359 @@ +/** + * @file test_rds_bgd_worker_hostgroup_refresh-t.cpp + * @brief Refreshing hostgroups and the mapped blue writer during writer switchover. + * + * Steps: + * + * 1. Configure BGD hostgroups 1380-1383 and reach + * `WRITER_SWITCHOVER_IN_PROGRESS`. + * 2. Change the reader and green hostgroups to 1384-1386. + * 3. Verify that the BGD status remains in progress and runtime placement + * moves to the configured reader hostgroup. + * 4. Verify metadata probes use TLS configured only in the refreshed green + * hostgroups. + * 5. Move a blue reader into writer hostgroup 1380 and publish topology that + * maps it to a different green target. + * 6. Verify that the previous writer returns to hostgroup 1380, the newly + * mapped writer moves to hostgroup 1384, uses TLS from green writer + * hostgroup 1385, and stops probing the stale target. + */ + +#include +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; +const uint32_t kNegativeProbeTimeoutMs = 800; + +struct TestState { + RDS_BGD_Cluster cluster { bgd_cluster_3_init() }; + BGD_Hostgroups hostgroups { 1380, 1381, 1382, 1383 }; + BGD_Hostgroups refreshed_hostgroups { 1380, 1384, 1385, 1386 }; + vector topology_endpoints { cluster.get_endpoints() }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_reader_as_writer(RDS_BGD_Cluster& cluster) { + RDS_BGD_Host& blue_writer = cluster.blue_readers[0]; + RDS_BGD_Host& green_writer = cluster.green_readers[0]; + + vector rows { + { blue_writer.hostname, blue_writer.hostname, blue_writer.port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", "SWITCHOVER_IN_PROGRESS" }, + { green_writer.hostname, green_writer.hostname, green_writer.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", "SWITCHOVER_IN_PROGRESS" }, + }; + return rows; +} + +/** + * Refresh the reader and green hostgroups during writer switchover. + * + * - Configure BGD hostgroups 1380-1383. + * - Publish `SWITCHOVER_IN_PROGRESS` and require the blue writer in hostgroup + * 1381. + * - Change the reader and green hostgroups to 1384-1386. + * - Verify that `WRITER_SWITCHOVER_IN_PROGRESS` is preserved. + * - Verify writer placement in hostgroup 1384. + * - Verify metadata probes use TLS from green hostgroups 1385 and 1386. + */ +int test_hostgroup_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.hostgroups; + BGD_Hostgroups& refreshed_hg = state.refreshed_hostgroups; + + // Set read_only=0 for the simulated blue and green writers. + int writer_rc = bgd_set_writer_read_only_0(sim, cluster); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated writer read_only values"); + return EXIT_FAILURE; + } + + // Publish SWITCHOVER_IN_PROGRESS topology. + vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology"); + return EXIT_FAILURE; + } + + // Configure mysql_servers and mysql_aws_rds_bgd_hostgroups. + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0], cluster.blue_readers[1] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0], cluster.green_readers[1] }; + + int admin_rc = bgd_admin_setup(admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 1380-1383"); + return EXIT_FAILURE; + } + + // Require the in-progress status and blue-writer demotion before changing hostgroups. + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1380 did not reach WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + int placement_rc = bgd_wait_for_server_placement(admin, hg.blue_writer, hg.blue_reader, cluster.blue_writer, true, kTimeoutSeconds); + if (placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer did not move from hostgroup 1380 to 1381"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before refreshing hostgroups"); + return EXIT_FAILURE; + } + + // Move blue readers and TLS-enabled green servers into the refreshed hostgroups. + string update_replication = + "UPDATE mysql_replication_hostgroups SET reader_hostgroup=" + to_string(refreshed_hg.blue_reader) + + " WHERE writer_hostgroup=" + to_string(refreshed_hg.blue_writer); + string move_blue = + "UPDATE mysql_servers SET hostgroup_id=" + to_string(refreshed_hg.blue_reader) + + " WHERE hostgroup_id=" + to_string(hg.blue_reader); + string move_green_writer = + "UPDATE mysql_servers SET hostgroup_id=" + to_string(refreshed_hg.green_writer) + ",use_ssl=1" + + " WHERE hostgroup_id=" + to_string(hg.green_writer); + string move_green_readers = + "UPDATE mysql_servers SET hostgroup_id=" + to_string(refreshed_hg.green_reader) + ",use_ssl=1" + + " WHERE hostgroup_id=" + to_string(hg.green_reader); + string update_bgd = + "UPDATE mysql_aws_rds_bgd_hostgroups SET reader_hostgroup=" + to_string(refreshed_hg.blue_reader) + + ",green_writer_hostgroup=" + to_string(refreshed_hg.green_writer) + + ",green_reader_hostgroup=" + to_string(refreshed_hg.green_reader) + + " WHERE writer_hostgroup=" + to_string(refreshed_hg.blue_writer); + vector queries { + update_replication, + move_blue, + move_green_writer, + move_green_readers, + update_bgd, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int refresh_rc = execute_all(admin, queries); + if (refresh_rc != EXIT_SUCCESS) { + diag("Error: failed to refresh BGD hostgroups from 1381-1383 to 1384-1386"); + return EXIT_FAILURE; + } + + // Verify that the runtime BGD status is preserved. + int refreshed_status_rc = bgd_wait_for_status(admin, refreshed_hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (refreshed_status_rc != EXIT_SUCCESS) { + diag("Error: hostgroup refresh did not preserve WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + ok(true, "BGD status for wHG 1380 remains WRITER_SWITCHOVER_IN_PROGRESS after hostgroup refresh"); + + // Verify that the blue writer uses the refreshed reader hostgroup. + int refreshed_placement_rc = bgd_wait_for_server_placement( + admin, refreshed_hg.blue_writer, refreshed_hg.blue_reader, cluster.blue_writer, true, kTimeoutSeconds + ); + if (refreshed_placement_rc != EXIT_SUCCESS) { + diag("Error: blue writer did not move from reader hostgroup 1381 to 1384"); + return EXIT_FAILURE; + } + + ok(true, "hostgroup refresh moves the demoted blue writer from hostgroup 1381 to 1384"); + + // Require TLS from the green writer row in refreshed green writer hostgroup 1385. + auto [probe_rc, probe] = sim.wait_for_probe_log( + seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + ); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: metadata probe did not use TLS from green writer hostgroup 1385"); + return EXIT_FAILURE; + } + + ok(true, "hostgroup refresh uses TLS from the green writer in hostgroup 1385"); + return EXIT_SUCCESS; +} + +/** + * Refresh the mapped blue writer during writer switchover. + * + * - Move the first blue reader from hostgroup 1384 to writer hostgroup 1380. + * - Publish `SWITCHOVER_IN_PROGRESS` topology that maps it to the first green + * reader. + * - Move that green target from reader hostgroup 1386 to writer hostgroup + * 1385. + * - Verify that the previous writer returns to hostgroup 1380. + * - Verify that the newly mapped writer moves to hostgroup 1384. + * - Verify that metadata probing uses TLS from the new green writer target. + */ +int test_mapped_writer_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& cluster = state.cluster; + BGD_Hostgroups& hg = state.refreshed_hostgroups; + RDS_BGD_Host& previous_writer = cluster.blue_writer; + RDS_BGD_Host& mapped_writer = cluster.blue_readers[0]; + RDS_BGD_Host& mapped_target = cluster.green_readers[0]; + + // Publish topology that maps the first blue reader to the first green reader. + vector topology = topology_with_reader_as_writer(cluster); + int topology_rc = sim.topology_update(state.topology_endpoints, topology); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for the new mapped writer"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before changing the mapped writer"); + return EXIT_FAILURE; + } + + // Move the new blue/green writer pair into writer hostgroups 1380 and 1385. + string move_writer = + "UPDATE mysql_servers SET hostgroup_id=" + to_string(hg.blue_writer) + + " WHERE hostgroup_id=" + to_string(hg.blue_reader) + + " AND hostname=" + bgd_sql_quote(mapped_writer.hostname) + + " AND port=" + to_string(mapped_writer.port); + string move_target = + "UPDATE mysql_servers SET hostgroup_id=" + to_string(hg.green_writer) + + " WHERE hostgroup_id=" + to_string(hg.green_reader) + + " AND hostname=" + bgd_sql_quote(mapped_target.hostname) + + " AND port=" + to_string(mapped_target.port); + vector queries { + move_writer, + move_target, + "LOAD MYSQL SERVERS TO RUNTIME", + }; + + int refresh_rc = execute_all(admin, queries); + if (refresh_rc != EXIT_SUCCESS) { + diag("Error: failed to move the new mapped writer pair into hostgroups 1380 and 1385"); + return EXIT_FAILURE; + } + + // Require the preserved BGD status and the new target metadata probe. + int status_rc = bgd_wait_for_status(admin, hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: mapped-writer refresh did not preserve WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + auto [probe_rc, probe] = sim.wait_for_probe_log( + seq, mapped_target.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + ); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: metadata probing did not use TLS from green writer hostgroup 1385"); + return EXIT_FAILURE; + } + + // Verify that the previous writer was restored to writer hostgroup 1380. + int previous_writer_rc = bgd_wait_for_server_placement( + admin, hg.blue_writer, hg.blue_reader, previous_writer, false, kTimeoutSeconds + ); + if (previous_writer_rc != EXIT_SUCCESS) { + diag("Error: previous blue writer was not restored to hostgroup 1380"); + return EXIT_FAILURE; + } + + ok(true, "mapped-writer refresh restores the previous blue writer from hostgroup 1384 to 1380"); + + // Verify that the newly mapped writer was demoted to reader hostgroup 1384. + int mapped_writer_rc = bgd_wait_for_server_placement( + admin, hg.blue_writer, hg.blue_reader, mapped_writer, true, kTimeoutSeconds + ); + if (mapped_writer_rc != EXIT_SUCCESS) { + diag("Error: newly mapped writer did not move to reader hostgroup 1384"); + return EXIT_FAILURE; + } + + ok(true, "mapped-writer refresh moves the new blue writer from hostgroup 1380 to 1384"); + + // Verify that the previous green target receives no metadata probes after the new target. + int stale_probe_rc = + bgd_expect_no_metadata_probe(sim, probe.sequence_id, cluster.green_writer.endpoint(), kNegativeProbeTimeoutMs); + if (stale_probe_rc != EXIT_SUCCESS) { + diag("Error: previous green target continued receiving metadata probes"); + return EXIT_FAILURE; + } + + ok(true, "mapped-writer refresh uses TLS from hostgroup 1385 and stops probing the previous target"); + return EXIT_SUCCESS; +} + +int main() { + plan(6); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish SWITCHOVER_IN_PROGRESS for the blue/green writers. + // ProxySQL: configure BGD hostgroups 1380-1383, then change reader/green hostgroups to 1384-1386. + // Verify: BGD status stays in progress, writer placement uses 1384, and target probing uses TLS from 1385. + if (test_hostgroup_refresh(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish SWITCHOVER_IN_PROGRESS with the first reader pair as the writer pair. + // ProxySQL: move the first blue reader from hostgroup 1384 to writer hostgroup 1380. + // Verify: the previous writer is restored, the new writer is demoted, and probing uses TLS from hostgroup 1385. + if (test_mapped_writer_refresh(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} From 55449f214f97e469b6111a367cfaa1fefeec5ff6 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 28 Jul 2026 07:10:35 +0000 Subject: [PATCH 72/81] test: cover repeated and concurrent RDS BGD lifecycles - Verify repeated deployment processing does not retain stale state. - Verify concurrent BGD definitions remain isolated by hostgroup. - Register and lint the focused repeat and concurrency tests. Signed-off-by: Wazir Ahmed --- test/tap/groups/groups.json | 2 + .../test_rds_bgd_concurrent_isolation-t.cpp | 495 ++++++++++++++++++ .../test_rds_bgd_repeated_deployment-t.cpp | 447 ++++++++++++++++ 3 files changed, 944 insertions(+) create mode 100644 test/tap/tests/test_rds_bgd_concurrent_isolation-t.cpp create mode 100644 test/tap/tests/test_rds_bgd_repeated_deployment-t.cpp diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index 24b5f0af6b..abaa5af6ca 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -431,6 +431,7 @@ "test_query_rules_routing-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g4","mysql90-g4","mysql95-g4" ], "test_query_timeout-t" : [ "legacy-g9","mariadb10-galera-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql84-gr-g9","mysql90-g4","mysql95-g4" ], "test_rds_bgd_automatic_discovery-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_concurrent_isolation-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_config_refresh_after_writer_completion-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_configuration_persistence-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_disable_during_switchover-t" : [ "cluster_sim_rds_bgd-g1" ], @@ -443,6 +444,7 @@ "test_rds_bgd_reader_policy-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_reader_switchover_cleanup-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_remove_during_switchover-t" : [ "cluster_sim_rds_bgd-g1" ], + "test_rds_bgd_repeated_deployment-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_rollback-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_smoke-t" : [ "cluster_sim_rds_bgd-g1" ], "test_rds_bgd_topology_empty_absent-t" : [ "cluster_sim_rds_bgd-g1" ], diff --git a/test/tap/tests/test_rds_bgd_concurrent_isolation-t.cpp b/test/tap/tests/test_rds_bgd_concurrent_isolation-t.cpp new file mode 100644 index 0000000000..14329cd04f --- /dev/null +++ b/test/tap/tests/test_rds_bgd_concurrent_isolation-t.cpp @@ -0,0 +1,495 @@ +/** + * @file test_rds_bgd_concurrent_isolation-t.cpp + * @brief Isolating three concurrent BGD workers in hostgroups 1410-1433. + * + * Steps: + * + * 1. Configure three BGD rows with separate hostgroups, topology, and green + * metadata targets. + * 2. Move each worker to a different writer-switchover phase and verify that + * the other two workers keep their status and blue-writer placement. + * 3. Replace only cluster 1 green membership with a TLS-enabled deployment. + * 4. Verify that cluster 1 uses the new target while clusters 2 and 3 keep + * their own phases, placement, metadata targets, and TLS values. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; +const uint32_t kNegativeProbeTimeoutMs = 500; + +struct TestState { + RDS_BGD_Cluster cluster_1 { bgd_cluster_init() }; + RDS_BGD_Cluster cluster_1_b { bgd_cluster_1_deployment_b_init() }; + RDS_BGD_Cluster cluster_2 { bgd_cluster_2_init() }; + RDS_BGD_Cluster cluster_3 { bgd_cluster_3_init() }; + BGD_Hostgroups cluster_1_hg { 1410, 1411, 1412, 1413 }; + BGD_Hostgroups cluster_2_hg { 1420, 1421, 1422, 1423 }; + BGD_Hostgroups cluster_3_hg { 1430, 1431, 1432, 1433 }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + rows.push_back({ + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + rows.push_back({ + cluster.green_readers[0].hostname, + cluster.green_readers[0].hostname, + cluster.green_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + return rows; +} + +int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_0(sim, cluster.green_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.green_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int configure_available( + MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, int green_use_ssl) +{ + if (configure_read_only_values(sim, cluster) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector topology = topology_with_reader_pair(cluster, "AVAILABLE"); + if (sim.topology_update(cluster.get_endpoints(), topology) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector blue_servers { cluster.blue_writer, cluster.blue_readers[0] }; + vector green_servers { cluster.green_writer, cluster.green_readers[0] }; + int admin_rc = bgd_admin_setup( + admin, cluster, hg, BGD_Admin_Mode::explicit_configuration, + blue_servers, green_servers, 0, green_use_ssl + ); + if (admin_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + return status_rc; +} + +bool worker_matches(MYSQL* admin, BGD_Hostgroups& hg, RDS_BGD_Cluster& cluster, string status, bool demoted) { + string writer_count = demoted ? "0" : "1"; + string reader_count = demoted ? "1" : "0"; + string query = "SELECT " + "(SELECT COUNT(*) FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE writer_hostgroup=" + + to_string(hg.blue_writer) + " AND status=" + bgd_sql_quote(status) + ")=1 AND " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.blue_writer) + + " AND hostname=" + bgd_sql_quote(cluster.blue_writer.hostname) + " AND port=3306)=" + writer_count + " AND " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.blue_reader) + + " AND hostname=" + bgd_sql_quote(cluster.blue_writer.hostname) + " AND port=3306)=" + reader_count; + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool matches = rows[0][0] == "1"; + return matches; +} + +bool runtime_green_membership_matches( + MYSQL* admin, BGD_Hostgroups& hg, RDS_BGD_Cluster& present, RDS_BGD_Cluster& absent) +{ + string query = "SELECT " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.green_writer) + + " AND hostname=" + bgd_sql_quote(present.green_writer.hostname) + " AND port=3306 AND use_ssl=1)=1 AND " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.green_reader) + + " AND hostname=" + bgd_sql_quote(present.green_readers[0].hostname) + " AND port=3306 AND use_ssl=1)=1 AND " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.green_writer) + + " AND hostname=" + bgd_sql_quote(absent.green_writer.hostname) + " AND port=3306)=0 AND " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.green_reader) + + " AND hostname=" + bgd_sql_quote(absent.green_readers[0].hostname) + " AND port=3306)=0"; + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool matches = rows[0][0] == "1"; + return matches; +} + +int replace_cluster_1_green_membership(MYSQL* admin, TestState& state) { + RDS_BGD_Cluster& old_deployment = state.cluster_1; + RDS_BGD_Cluster& new_deployment = state.cluster_1_b; + BGD_Hostgroups& hg = state.cluster_1_hg; + + vector delete_queries { + "DELETE FROM mysql_servers WHERE hostgroup_id=" + to_string(hg.green_writer) + + " AND hostname=" + bgd_sql_quote(old_deployment.green_writer.hostname) + " AND port=3306", + "DELETE FROM mysql_servers WHERE hostgroup_id=" + to_string(hg.green_reader) + + " AND hostname=" + bgd_sql_quote(old_deployment.green_readers[0].hostname) + " AND port=3306", + }; + if (execute_all(admin, delete_queries) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector green_servers { new_deployment.green_writer, new_deployment.green_readers[0] }; + if (bgd_admin_add_servers(admin, new_deployment, hg, green_servers, true, 1) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector load_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int rc = execute_all(admin, load_queries); + return rc; +} + +/** + * Start three BGD workers in AVAILABLE. + * + * - Configure hostgroups 1410-1413, 1420-1423, and 1430-1433. + * - Use plaintext green metadata for clusters 1 and 3 and TLS for cluster 2. + * - Verify that each BGD row reaches AVAILABLE through its own green writer. + */ +int test_three_workers_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before starting three BGD workers"); + return EXIT_FAILURE; + } + + int cluster_1_rc = configure_available(admin, sim, state.cluster_1, state.cluster_1_hg, 0); + if (cluster_1_rc != EXIT_SUCCESS) { + diag("Error: failed to configure AVAILABLE for BGD wHG 1410"); + return EXIT_FAILURE; + } + + int cluster_2_rc = configure_available(admin, sim, state.cluster_2, state.cluster_2_hg, 1); + if (cluster_2_rc != EXIT_SUCCESS) { + diag("Error: failed to configure AVAILABLE for BGD wHG 1420"); + return EXIT_FAILURE; + } + + int cluster_3_rc = configure_available(admin, sim, state.cluster_3, state.cluster_3_hg, 0); + if (cluster_3_rc != EXIT_SUCCESS) { + diag("Error: failed to configure AVAILABLE for BGD wHG 1430"); + return EXIT_FAILURE; + } + + auto [cluster_1_probe_rc, cluster_1_probe] = sim.wait_for_probe_log( + seq, state.cluster_1.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + ); + if (cluster_1_probe_rc != EXIT_SUCCESS) { + diag("Error: BGD wHG 1410 did not probe its plaintext green writer"); + return EXIT_FAILURE; + } + ok(true, "BGD wHG 1410 reports AVAILABLE from its own plaintext green writer"); + + auto [cluster_2_probe_rc, cluster_2_probe] = sim.wait_for_probe_log( + seq, state.cluster_2.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + ); + if (cluster_2_probe_rc != EXIT_SUCCESS) { + diag("Error: BGD wHG 1420 did not probe its TLS green writer"); + return EXIT_FAILURE; + } + ok(true, "BGD wHG 1420 reports AVAILABLE from its own TLS green writer"); + + auto [cluster_3_probe_rc, cluster_3_probe] = sim.wait_for_probe_log( + seq, state.cluster_3.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + ); + if (cluster_3_probe_rc != EXIT_SUCCESS) { + diag("Error: BGD wHG 1430 did not probe its plaintext green writer"); + return EXIT_FAILURE; + } + ok(true, "BGD wHG 1430 reports AVAILABLE from its own plaintext green writer"); + return EXIT_SUCCESS; +} + +/** + * Move each BGD worker to a different writer-switchover phase. + * + * - Move wHG 1410 to WRITER_SWITCHOVER_IN_PROGRESS. + * - Move wHG 1420 to WRITER_SWITCHOVER_POST_PROCESSING. + * - Move wHG 1430 to WRITER_SWITCHOVER_INITIATED. + * - After each change, verify that the other two statuses and blue-writer + * placements remain unchanged. + */ +int test_independent_phase_changes(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + vector cluster_1_topology = + topology_with_reader_pair(state.cluster_1, "SWITCHOVER_IN_PROGRESS"); + if (sim.topology_update(state.cluster_1.get_endpoints(), cluster_1_topology) != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS for BGD wHG 1410"); + return EXIT_FAILURE; + } + + int cluster_1_status_rc = + bgd_wait_for_status(admin, state.cluster_1_hg, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds); + if (cluster_1_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1410 did not reach WRITER_SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + + int cluster_1_placement_rc = bgd_wait_for_server_placement( + admin, state.cluster_1_hg.blue_writer, state.cluster_1_hg.blue_reader, + state.cluster_1.blue_writer, true, kTimeoutSeconds + ); + if (cluster_1_placement_rc != EXIT_SUCCESS) { + diag("Error: BGD wHG 1410 did not move its blue writer to reader hostgroup 1411"); + return EXIT_FAILURE; + } + ok(true, "advancing wHG 1410 moves only its blue writer from hostgroup 1410 to 1411"); + + bool cluster_2_available = worker_matches(admin, state.cluster_2_hg, state.cluster_2, "AVAILABLE", false); + bool cluster_3_available = worker_matches(admin, state.cluster_3_hg, state.cluster_3, "AVAILABLE", false); + ok(cluster_2_available && cluster_3_available, + "advancing wHG 1410 leaves wHG 1420 and wHG 1430 in AVAILABLE with unchanged blue placement"); + + vector cluster_2_topology = + topology_with_reader_pair(state.cluster_2, "SWITCHOVER_IN_POST_PROCESSING"); + if (sim.topology_update(state.cluster_2.get_endpoints(), cluster_2_topology) != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING for BGD wHG 1420"); + return EXIT_FAILURE; + } + + int cluster_2_status_rc = + bgd_wait_for_status(admin, state.cluster_2_hg, "WRITER_SWITCHOVER_POST_PROCESSING", kTimeoutSeconds); + if (cluster_2_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1420 did not reach WRITER_SWITCHOVER_POST_PROCESSING"); + return EXIT_FAILURE; + } + + int cluster_2_placement_rc = bgd_wait_for_server_placement( + admin, state.cluster_2_hg.blue_writer, state.cluster_2_hg.blue_reader, + state.cluster_2.blue_writer, false, kTimeoutSeconds + ); + if (cluster_2_placement_rc != EXIT_SUCCESS) { + diag("Error: BGD wHG 1420 did not retain its blue writer in hostgroup 1420"); + return EXIT_FAILURE; + } + ok(true, "advancing wHG 1420 applies post-processing only to its blue writer"); + + bool cluster_1_in_progress = + worker_matches(admin, state.cluster_1_hg, state.cluster_1, "WRITER_SWITCHOVER_IN_PROGRESS", true); + bool cluster_3_still_available = worker_matches(admin, state.cluster_3_hg, state.cluster_3, "AVAILABLE", false); + ok(cluster_1_in_progress && cluster_3_still_available, + "advancing wHG 1420 preserves wHG 1410 progress and wHG 1430 availability"); + + vector cluster_3_topology = + topology_with_reader_pair(state.cluster_3, "SWITCHOVER_INITIATED"); + if (sim.topology_update(state.cluster_3.get_endpoints(), cluster_3_topology) != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_INITIATED for BGD wHG 1430"); + return EXIT_FAILURE; + } + + int cluster_3_status_rc = + bgd_wait_for_status(admin, state.cluster_3_hg, "WRITER_SWITCHOVER_INITIATED", kTimeoutSeconds); + if (cluster_3_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1430 did not reach WRITER_SWITCHOVER_INITIATED"); + return EXIT_FAILURE; + } + + int cluster_3_placement_rc = bgd_wait_for_server_placement( + admin, state.cluster_3_hg.blue_writer, state.cluster_3_hg.blue_reader, + state.cluster_3.blue_writer, false, kTimeoutSeconds + ); + if (cluster_3_placement_rc != EXIT_SUCCESS) { + diag("Error: BGD wHG 1430 did not retain its blue writer in hostgroup 1430"); + return EXIT_FAILURE; + } + ok(true, "advancing wHG 1430 records INITIATED without changing its blue writer placement"); + + bool cluster_1_still_in_progress = + worker_matches(admin, state.cluster_1_hg, state.cluster_1, "WRITER_SWITCHOVER_IN_PROGRESS", true); + bool cluster_2_post = worker_matches( + admin, state.cluster_2_hg, state.cluster_2, "WRITER_SWITCHOVER_POST_PROCESSING", false + ); + ok(cluster_1_still_in_progress && cluster_2_post, + "advancing wHG 1430 preserves wHG 1410 progress and wHG 1420 post-processing"); + return EXIT_SUCCESS; +} + +/** + * Refresh only cluster 1 green membership. + * + * - Replace cluster 1 green rows with TLS-enabled deployment B rows. + * - Keep wHG 1410 in WRITER_SWITCHOVER_IN_PROGRESS. + * - Verify that cluster 1 stops probing its removed target while clusters 2 + * and 3 keep their status, placement, metadata target, and TLS value. + */ +int test_independent_config_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + if (configure_read_only_values(sim, state.cluster_1_b) != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for cluster 1 deployment B"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before refreshing BGD wHG 1410"); + return EXIT_FAILURE; + } + + int replace_rc = replace_cluster_1_green_membership(admin, state); + if (replace_rc != EXIT_SUCCESS) { + diag("Error: failed to replace green membership for BGD wHG 1410"); + return EXIT_FAILURE; + } + + vector topology = + topology_with_reader_pair(state.cluster_1_b, "SWITCHOVER_IN_PROGRESS"); + if (sim.topology_update(state.cluster_1_b.get_endpoints(), topology) != EXIT_SUCCESS) { + diag("Error: failed to publish deployment B topology for BGD wHG 1410"); + return EXIT_FAILURE; + } + + auto [cluster_1_probe_rc, cluster_1_probe] = sim.wait_for_probe_log( + seq, state.cluster_1_b.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + ); + if (cluster_1_probe_rc != EXIT_SUCCESS) { + diag("Error: refreshed BGD wHG 1410 did not probe its TLS deployment B green writer"); + return EXIT_FAILURE; + } + + int stale_probe_rc = bgd_expect_no_metadata_probe( + sim, cluster_1_probe.sequence_id, state.cluster_1.green_writer.endpoint(), kNegativeProbeTimeoutMs + ); + if (stale_probe_rc != EXIT_SUCCESS) { + diag("Error: refreshed BGD wHG 1410 continued probing its removed green writer"); + return EXIT_FAILURE; + } + + auto [cluster_2_probe_rc, cluster_2_probe] = sim.wait_for_probe_log( + cluster_1_probe.sequence_id, state.cluster_2.green_writer.endpoint(), + RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + ); + if (cluster_2_probe_rc != EXIT_SUCCESS) { + diag("Error: BGD wHG 1420 did not continue probing its TLS green writer"); + return EXIT_FAILURE; + } + + auto [cluster_3_probe_rc, cluster_3_probe] = sim.wait_for_probe_log( + cluster_1_probe.sequence_id, state.cluster_3.green_writer.endpoint(), + RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + ); + if (cluster_3_probe_rc != EXIT_SUCCESS) { + diag("Error: BGD wHG 1430 did not continue probing its plaintext green writer"); + return EXIT_FAILURE; + } + + bool cluster_1_membership = + runtime_green_membership_matches(admin, state.cluster_1_hg, state.cluster_1_b, state.cluster_1); + bool cluster_1_phase = + worker_matches(admin, state.cluster_1_hg, state.cluster_1_b, "WRITER_SWITCHOVER_IN_PROGRESS", true); + bool cluster_2_phase = + worker_matches(admin, state.cluster_2_hg, state.cluster_2, "WRITER_SWITCHOVER_POST_PROCESSING", false); + bool cluster_3_phase = + worker_matches(admin, state.cluster_3_hg, state.cluster_3, "WRITER_SWITCHOVER_INITIATED", false); + ok(cluster_1_membership && cluster_1_phase && cluster_2_phase && cluster_3_phase, + "refreshing wHG 1410 changes only its target while wHG 1420 and wHG 1430 keep their phases and probes"); + return EXIT_SUCCESS; +} + +int main() { + plan(10); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish separate AVAILABLE topology for clusters 1, 2, and 3. + // ProxySQL: configure BGD wHGs 1410, 1420, and 1430 with distinct green targets and TLS values. + // Verify: each BGD row reports AVAILABLE through its own configured green writer. + if (test_three_workers_available(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish SWITCHOVER_IN_PROGRESS for wHG 1410, SWITCHOVER_IN_POST_PROCESSING for wHG 1420, + // and SWITCHOVER_INITIATED for wHG 1430. + // Verify: each BGD status and blue-writer placement changes without affecting the other two workers. + if (test_independent_phase_changes(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: keep cluster 1 in progress with deployment B topology. + // ProxySQL: replace only wHG 1410 green membership with TLS-enabled deployment B rows. + // Verify: wHG 1410 uses the new target while wHGs 1420 and 1430 keep their phases, targets, and TLS. + if (test_independent_config_refresh(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_rds_bgd_repeated_deployment-t.cpp b/test/tap/tests/test_rds_bgd_repeated_deployment-t.cpp new file mode 100644 index 0000000000..b1852dedf8 --- /dev/null +++ b/test/tap/tests/test_rds_bgd_repeated_deployment-t.cpp @@ -0,0 +1,447 @@ +/** + * @file test_rds_bgd_repeated_deployment-t.cpp + * @brief Reusing BGD hostgroups 1400-1403 for a second deployment. + * + * Steps: + * + * 1. Configure deployment A, complete writer and reader switchover, and publish + * empty topology. + * 2. Replace the configured green servers with TLS-enabled deployment B. + * 3. Verify that only deployment B membership, probes, and routing are used + * during the second lifecycle. + */ + +#include +#include +#include +#include + +#include "command_line.h" +#include "rds_bgd_tap.h" +#include "utils.h" + +const uint32_t kTimeoutSeconds = 3; +const uint32_t kProbeTimeoutMs = 3000; +const uint32_t kNegativeProbeTimeoutMs = 500; + +struct TestState { + RDS_BGD_Cluster deployment_a { bgd_cluster_init() }; + RDS_BGD_Cluster deployment_b { bgd_cluster_1_deployment_b_init() }; + BGD_Hostgroups hostgroups { 1400, 1401, 1402, 1403 }; + vector topology_endpoints { deployment_a.get_endpoints() }; + + TestState() { + vector deployment_b_green = deployment_b.get_green_endpoints(); + topology_endpoints.insert(topology_endpoints.end(), deployment_b_green.begin(), deployment_b_green.end()); + } +}; + +int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + + if (sim.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the SQLite3-server simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { + int admin_rc = bgd_admin_cleanup(admin); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to clean ProxySQL BGD test state"); + } + mysql_close(admin); + + int simulator_rc = sim.cleanup(); + if (simulator_rc != EXIT_SUCCESS) { + diag("Error: failed to clean SQLite3-server simulator state"); + } + + if (admin_rc != EXIT_SUCCESS || simulator_rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); + rows.push_back({ + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].hostname, + cluster.blue_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + status, + }); + rows.push_back({ + cluster.green_readers[0].hostname, + cluster.green_readers[0].hostname, + cluster.green_readers[0].port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + status, + }); + return rows; +} + +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ + cluster.green_writer.hostname, + cluster.green_writer.hostname, + cluster.green_writer.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + "SWITCHOVER_COMPLETED", + }}; + return rows; +} + +int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { + if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_0(sim, cluster.green_writer) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.blue_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + if (bgd_set_host_read_only_1(sim, cluster.green_readers[0]) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + vector queries { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }; + + int rc = execute_all(admin, queries); + return rc; +} + +rc_t connect_and_echo(CommandLine& cl) { + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + rc_t result { EXIT_FAILURE, {} }; + return result; + } + + auto result = bgd_backend_ip_echo(client); + mysql_close(client); + return result; +} + +bool runtime_green_membership_matches( + MYSQL* admin, BGD_Hostgroups& hg, RDS_BGD_Cluster& present, RDS_BGD_Cluster& absent, int use_ssl) +{ + string query = "SELECT " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.green_writer) + + " AND hostname=" + bgd_sql_quote(present.green_writer.hostname) + " AND port=3306 AND use_ssl=" + + to_string(use_ssl) + ")=1 AND " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.green_reader) + + " AND hostname=" + bgd_sql_quote(present.green_readers[0].hostname) + " AND port=3306 AND use_ssl=" + + to_string(use_ssl) + ")=1 AND " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.green_writer) + + " AND hostname=" + bgd_sql_quote(absent.green_writer.hostname) + " AND port=3306)=0 AND " + "(SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + to_string(hg.green_reader) + + " AND hostname=" + bgd_sql_quote(absent.green_readers[0].hostname) + " AND port=3306)=0"; + + auto [rc, rows] = mysql_query_ext_rows(admin, query); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows[0].size() != 1) { + return false; + } + + bool matches = rows[0][0] == "1"; + return matches; +} + +int replace_green_membership( + MYSQL* admin, BGD_Hostgroups& hg, RDS_BGD_Cluster& old_deployment, RDS_BGD_Cluster& new_deployment) +{ + vector delete_queries { + "DELETE FROM mysql_servers WHERE hostgroup_id=" + to_string(hg.green_writer) + + " AND hostname=" + bgd_sql_quote(old_deployment.green_writer.hostname) + " AND port=3306", + "DELETE FROM mysql_servers WHERE hostgroup_id=" + to_string(hg.green_reader) + + " AND hostname=" + bgd_sql_quote(old_deployment.green_readers[0].hostname) + " AND port=3306", + }; + if (execute_all(admin, delete_queries) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector green_servers { new_deployment.green_writer, new_deployment.green_readers[0] }; + if (bgd_admin_add_servers(admin, new_deployment, hg, green_servers, true, 1) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector load_queries { "LOAD MYSQL SERVERS TO RUNTIME" }; + int rc = execute_all(admin, load_queries); + return rc; +} + +int publish_writer_lifecycle( + MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state, RDS_BGD_Cluster& cluster, string deployment) +{ + vector initiated = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); + if (sim.topology_update(cluster.get_endpoints(), initiated) != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_INITIATED topology for deployment %s", deployment.c_str()); + return EXIT_FAILURE; + } + + if (bgd_wait_for_status(admin, state.hostgroups, "WRITER_SWITCHOVER_INITIATED", kTimeoutSeconds) != EXIT_SUCCESS) { + diag("Error: BGD status for deployment %s did not reach WRITER_SWITCHOVER_INITIATED", deployment.c_str()); + return EXIT_FAILURE; + } + + vector progress = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); + if (sim.topology_update(cluster.get_endpoints(), progress) != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for deployment %s", deployment.c_str()); + return EXIT_FAILURE; + } + + if (bgd_wait_for_status(admin, state.hostgroups, "WRITER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds) != EXIT_SUCCESS) { + diag("Error: BGD status for deployment %s did not reach WRITER_SWITCHOVER_IN_PROGRESS", deployment.c_str()); + return EXIT_FAILURE; + } + + vector post = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); + if (sim.topology_update(cluster.get_endpoints(), post) != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING topology for deployment %s", deployment.c_str()); + return EXIT_FAILURE; + } + + int post_status_rc = + bgd_wait_for_status(admin, state.hostgroups, "WRITER_SWITCHOVER_POST_PROCESSING", kTimeoutSeconds); + if (post_status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for deployment %s did not reach WRITER_SWITCHOVER_POST_PROCESSING", deployment.c_str()); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int publish_reader_cleanup(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state, RDS_BGD_Cluster& cluster, + string deployment) { + vector completed = target_only_completed(cluster); + if (sim.topology_update(cluster.get_endpoints(), completed) != EXIT_SUCCESS) { + diag("Error: failed to publish target-only SWITCHOVER_COMPLETED for deployment %s", deployment.c_str()); + return EXIT_FAILURE; + } + + if (bgd_wait_for_status(admin, state.hostgroups, "READER_SWITCHOVER_IN_PROGRESS", kTimeoutSeconds) != EXIT_SUCCESS) { + diag("Error: BGD status for deployment %s did not reach READER_SWITCHOVER_IN_PROGRESS", deployment.c_str()); + return EXIT_FAILURE; + } + + if (sim.topology_delete(state.topology_endpoints) != EXIT_SUCCESS) { + diag("Error: failed to publish empty topology for deployment %s", deployment.c_str()); + return EXIT_FAILURE; + } + + if (bgd_wait_for_status(admin, state.hostgroups, "NONE", kTimeoutSeconds) != EXIT_SUCCESS) { + diag("Error: BGD status for deployment %s did not reach NONE", deployment.c_str()); + return EXIT_FAILURE; + } + + if (bgd_wait_for_server_placement( + admin, state.hostgroups.blue_writer, state.hostgroups.blue_reader, + cluster.blue_writer, false, kTimeoutSeconds + ) != EXIT_SUCCESS) { + diag("Error: deployment %s did not restore the blue writer to hostgroup 1400", deployment.c_str()); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +/** + * Complete deployment A before reusing its BGD hostgroups. + * + * - Configure BGD hostgroups 1400-1403 with deployment A. + * - Complete writer and reader switchover, then publish empty topology. + * - Verify NONE and baseline blue-writer placement. + */ +int test_deployment_a(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& deployment = state.deployment_a; + BGD_Hostgroups& hg = state.hostgroups; + + int read_only_rc = configure_read_only_values(sim, deployment); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for deployment A"); + return EXIT_FAILURE; + } + + vector available = topology_with_reader_pair(deployment, "AVAILABLE"); + int topology_rc = sim.topology_update(deployment.get_endpoints(), available); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for deployment A"); + return EXIT_FAILURE; + } + + vector blue_servers { deployment.blue_writer, deployment.blue_readers[0] }; + vector green_servers { deployment.green_writer, deployment.green_readers[0] }; + int admin_rc = bgd_admin_setup( + admin, deployment, hg, BGD_Admin_Mode::explicit_configuration, blue_servers, green_servers, 0, 0 + ); + if (admin_rc != EXIT_SUCCESS) { + diag("Error: failed to configure BGD hostgroups 1400-1403 for deployment A"); + return EXIT_FAILURE; + } + + int available_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (available_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1400 did not reach AVAILABLE for deployment A"); + return EXIT_FAILURE; + } + + int writer_rc = publish_writer_lifecycle(admin, sim, state, deployment, "A"); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to complete writer switchover for deployment A"); + return EXIT_FAILURE; + } + + int reader_rc = publish_reader_cleanup(admin, sim, state, deployment, "A"); + if (reader_rc != EXIT_SUCCESS) { + diag("Error: failed to complete reader switchover cleanup for deployment A"); + return EXIT_FAILURE; + } + + ok(true, "deployment A cleanup sets BGD status for wHG 1400 to NONE and restores blue writer placement"); + return EXIT_SUCCESS; +} + +/** + * Reuse BGD hostgroups 1400-1403 for deployment B. + * + * - Replace deployment A green rows with TLS-enabled deployment B rows. + * - Verify that metadata probes and runtime rows use only deployment B. + * - Complete deployment B and verify that routing uses deployment B without + * recreating deployment A rows. + */ +int test_deployment_b(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { + RDS_BGD_Cluster& deployment_a = state.deployment_a; + RDS_BGD_Cluster& deployment_b = state.deployment_b; + BGD_Hostgroups& hg = state.hostgroups; + + int read_only_rc = configure_read_only_values(sim, deployment_b); + if (read_only_rc != EXIT_SUCCESS) { + diag("Error: failed to configure simulated read_only values for deployment B"); + return EXIT_FAILURE; + } + + auto [seq_rc, seq] = sim.probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS) { + diag("Error: failed to read the probe sequence before configuring deployment B"); + return EXIT_FAILURE; + } + + int replace_rc = replace_green_membership(admin, hg, deployment_a, deployment_b); + if (replace_rc != EXIT_SUCCESS) { + diag("Error: failed to replace deployment A green rows with deployment B rows"); + return EXIT_FAILURE; + } + + vector available = topology_with_reader_pair(deployment_b, "AVAILABLE"); + int topology_rc = sim.topology_update(deployment_b.get_endpoints(), available); + if (topology_rc != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology for deployment B"); + return EXIT_FAILURE; + } + + int status_rc = bgd_wait_for_status(admin, hg, "AVAILABLE", kTimeoutSeconds); + if (status_rc != EXIT_SUCCESS) { + diag("Error: BGD status for wHG 1400 did not reach AVAILABLE for deployment B"); + return EXIT_FAILURE; + } + + bool membership_matches = runtime_green_membership_matches(admin, hg, deployment_b, deployment_a, 1); + ok(membership_matches, "runtime_mysql_servers contains only TLS-enabled deployment B green rows"); + + auto [probe_rc, probe] = sim.wait_for_probe_log( + seq, deployment_b.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + ); + if (probe_rc != EXIT_SUCCESS) { + diag("Error: deployment B green writer did not receive a TLS metadata probe"); + return EXIT_FAILURE; + } + + int stale_probe_rc = + bgd_expect_no_metadata_probe(sim, probe.sequence_id, deployment_a.green_writer.endpoint(), kNegativeProbeTimeoutMs); + ok(stale_probe_rc == EXIT_SUCCESS, + "deployment B metadata probing does not return to the removed deployment A green writer"); + + int writer_rc = publish_writer_lifecycle(admin, sim, state, deployment_b, "B"); + if (writer_rc != EXIT_SUCCESS) { + diag("Error: failed to complete writer switchover for deployment B"); + return EXIT_FAILURE; + } + + int user_rc = set_default_hostgroup(admin, hg.blue_writer); + if (user_rc != EXIT_SUCCESS) { + diag("Error: failed to route testuser through writer hostgroup 1400"); + return EXIT_FAILURE; + } + + auto [route_rc, route] = connect_and_echo(cl); + bool route_matches = route_rc == EXIT_SUCCESS && route.find(deployment_b.green_writer.ip) != string::npos; + ok(route_matches, "deployment B post-processing routes new connections to deployment B"); + + int reader_rc = publish_reader_cleanup(admin, sim, state, deployment_b, "B"); + if (reader_rc != EXIT_SUCCESS) { + diag("Error: failed to complete reader switchover cleanup for deployment B"); + return EXIT_FAILURE; + } + + bool final_membership = runtime_green_membership_matches(admin, hg, deployment_b, deployment_a, 1); + ok(final_membership, "deployment B cleanup retains deployment B green rows without restoring deployment A rows"); + return EXIT_SUCCESS; +} + +int main() { + plan(5); + + CommandLine cl {}; + MYSQL* admin = nullptr; + RDS_BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish deployment A topology through writer and reader completion, then delete it. + // ProxySQL: configure BGD hostgroups 1400-1403 for deployment A. + // Verify: deployment A cleanup reaches NONE and restores blue-writer placement. + if (test_deployment_a(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish AVAILABLE through completion for deployment B on the same blue writer. + // ProxySQL: replace deployment A green rows with TLS-enabled deployment B rows. + // Verify: only deployment B membership, probes, and routing are used by the second lifecycle. + if (test_deployment_b(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean the BGD TAP state"); + return EXIT_FAILURE; + } + return exit_status(); +} From 6304c1825a7b2893e309a4879a60035ea57cba80 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 28 Jul 2026 07:11:00 +0000 Subject: [PATCH 73/81] docs: align RDS BGD simulator documentation - Align the documented helper APIs, cleanup flow, and readiness configuration with the implemented suite. - Document the shared cluster-simulator CI and final BGD group execution workflow. Signed-off-by: Wazir Ahmed --- doc/AWS_Blue_Green/RDS_BGD_Simulator.md | 271 +++++++++--------------- 1 file changed, 97 insertions(+), 174 deletions(-) diff --git a/doc/AWS_Blue_Green/RDS_BGD_Simulator.md b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md index 283df64949..7c228a93bc 100644 --- a/doc/AWS_Blue_Green/RDS_BGD_Simulator.md +++ b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md @@ -1,9 +1,9 @@ # AWS RDS Blue/Green Deployment Simulator -**Document status:** SIMULATOR IMPLEMENTED; GITHUB WORKFLOW FOLLOW-UP +**Document status:** IMPLEMENTED **Applies to:** `TEST_RDS_BGD`, the SQLite3-server simulation surface, BGD TAP -helpers, the local Docker runner, and supported simulator coverage +helpers, local and GitHub runners, and supported simulator coverage **Related monitor contract:** [RDS_BGD_Monitor.md](RDS_BGD_Monitor.md) @@ -12,8 +12,7 @@ helpers, the local Docker runner, and supported simulator coverage This document defines the simulator used to test ProxySQL's AWS RDS Blue/Green Deployment monitor. It combines the behavioral contract, SQLite3-server changes, TAP helper API, network fixture, local runner, and supported coverage -into one implementation specification. GitHub workflow execution is defined as -follow-up work. +into one implementation specification. ## Architecture @@ -212,7 +211,7 @@ handling does not consult `RDS_BGD_CONTROL` or write `RDS_BGD_PROBE_LOG`. The API follows existing TAP conventions: write methods return `EXIT_SUCCESS` or `EXIT_FAILURE`, and read methods return the existing `rc_t` type. The -signatures below are the initial API and may grow with reviewed test cases. +interfaces below form the simulator design surface used by BGD scenarios. ### Common Endpoint @@ -280,16 +279,21 @@ public: std::vector green_readers; std::vector get_writers(); - std::vector get_writer_hosts(); + std::vector get_blue_endpoints(); + std::vector get_green_endpoints(); + std::vector get_endpoints(); std::vector get_topology(std::string status); }; ``` Each TAP test owns and initializes the cluster fixtures it uses. A fixture keeps the selected `/etc/hosts` mapping together. `get_writers()` returns the -selected blue and green writer IPs, `get_writer_hosts()` returns their configured -hostnames, and `get_topology(status)` returns the standard two-row SOURCE/TARGET -topology using the writer hostnames and the provided status. +blue and green writer IP endpoints. `get_blue_endpoints()` and +`get_green_endpoints()` include the writer and readers for one deployment, +while `get_endpoints()` returns the complete cluster. `get_topology(status)` +returns the standard two-row SOURCE/TARGET writer topology using the configured +hostnames and status. Tests add reader rows explicitly when the scenario needs +reader mapping. ### BGD Topology Operations @@ -306,6 +310,8 @@ int topology_error( std::vector backends, int error_code, std::string error_msg); + +int cleanup(); ``` `topology_update()` marks the table present, clears any configured error, and @@ -317,6 +323,10 @@ with `Table 'mysql.rds_topology' doesn't exist`. `topology_error()` requires a nonzero code; 1146 marks topology absent, while any other code marks it present and leaves existing rows unchanged. +`cleanup()` removes read-only state, topology rows, control rows, and probe-log +rows. Tests call it together with their ProxySQL Admin cleanup so scenarios do +not inherit simulator state from an earlier binary. + ### Probe-Log Operations ```cpp @@ -355,54 +365,34 @@ int main() { plan(3); CommandLine cl {}; - if (cl.getEnv()) BAIL_OUT("failed to load TAP environment"); + MYSQL* admin = nullptr; + RDS_BGD_Simulator simulator {}; - MYSQL* admin = init_mysql_conn( - cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); - if (!admin) BAIL_OUT("failed to connect to ProxySQL Admin"); + if (setup(cl, admin, simulator) != EXIT_SUCCESS) + return exit_status(); - RDS_BGD_Cluster cluster = bgd_cluster_init(); - RDS_BGD_Simulator simulator {}; - if (simulator.connect(cl.host, 3306, cl.username, cl.password) != EXIT_SUCCESS) - BAIL_OUT("failed to connect to SQLite3 server"); - for (Endpoint& writer : cluster.get_writer_hosts()) { - if (simulator.read_only_update(writer, false) != EXIT_SUCCESS) - BAIL_OUT("failed to configure writer read_only state"); - } - - auto [seq_rc, last_seq] = simulator.probe_log_last_sequence(); - if (seq_rc != EXIT_SUCCESS) - BAIL_OUT("failed to read the last probe-log sequence"); - - const int update_rc = simulator.topology_update( - cluster.get_writers(), cluster.get_topology("AVAILABLE")); - ok(update_rc == EXIT_SUCCESS, "publish topology to both writer IPs"); - if (update_rc != EXIT_SUCCESS) - BAIL_OUT("failed to publish topology"); - - if (configure_proxysql_for_bgd(admin, cluster) != EXIT_SUCCESS) - BAIL_OUT("failed to configure ProxySQL"); - - ok(wait_for_cond(admin, - "SELECT status='AVAILABLE' FROM runtime_mysql_aws_rds_bgd_hostgroups " - "WHERE writer_hostgroup=10", 5) == EXIT_SUCCESS, - "ProxySQL enters AVAILABLE"); - - auto [probe_rc, green_log] = simulator.wait_for_probe_log( - last_seq, - cluster.green_writer.endpoint(), - RDS_BGD_Probe_Kind::metadata, - 5000); - ok(probe_rc == EXIT_SUCCESS, - "ProxySQL probes the green writer IP directly"); - - mysql_close(admin); + TestState state {}; + + if (publish_available_topology(simulator, state) != EXIT_SUCCESS) + goto exit_cleanup; + + if (configure_bgd_available(admin, state) != EXIT_SUCCESS) + goto exit_cleanup; + + if (test_plaintext_green_writer_probe(simulator, state) != EXIT_SUCCESS) + goto exit_cleanup; + +exit_cleanup: + if (cleanup(admin, simulator) != EXIT_SUCCESS) + return EXIT_FAILURE; return exit_status(); } ``` -ProxySQL configuration remains test-local. The simulator changes backend -responses and reads probe evidence; assertions against ProxySQL use Admin SQL. +Each test defines small setup, scenario, and cleanup functions around this +control flow. ProxySQL configuration remains test-local. The simulator changes +backend responses and reads probe evidence; assertions against ProxySQL use +Admin SQL. Cleanup removes both Admin and simulator state before returning. ## Build Integration @@ -418,9 +408,10 @@ build_deps_debug -> build_lib_test_rds_bgd -> build_src_test_rds_bgd -> TAP debug build ``` -`test_rds_bgd` is the single entry point. Do not invoke +`test_rds_bgd` is the focused local entry point. Do not invoke `build_tap_test_debug` afterward because its `build_src_debug` dependency -selects the normal debug daemon. `testall` includes `-DTEST_RDS_BGD` as well. +selects the normal debug daemon. `testall` includes `-DTEST_RDS_BGD` and is +used by the shared cluster-simulator CI build. ## Local CI Group @@ -429,7 +420,7 @@ The `test/tap/groups/cluster_sim_rds_bgd/` group executes as | File | BGD-specific content | |---|---| -| `env.sh` | Set `CLUSTER_SIM_HOST_FILE` and `SKIP_CLUSTER_START=1`. | +| `env.sh` | Set the fixed host map, wait for the SQLite3-server port, and skip backend cluster startup. | | `add-hosts` | Define the fixed hostname/IP map below. | | `pre-proxysql.bash` | Keep the existing short startup wait before Admin writes. | | `pre-proxysql.sql` | Add the simulator user and move the SQLite3 server to port 3306. | @@ -439,6 +430,7 @@ The group has no `infras.lst`, `CLUSTER_SIM_BINARY_PATH`, or ```bash export CLUSTER_SIM_HOST_FILE="${WORKSPACE}/test/tap/groups/cluster_sim_rds_bgd/add-hosts" +export PROXYSQL_READY_PORTS_EXTRA="3306" export SKIP_CLUSTER_START=1 ``` @@ -510,19 +502,17 @@ simulator transitions. From the TAP container, the control connection uses ### Group Registration and Local Run -Each BGD TAP binary is registered in `test/tap/groups/groups.json`: +The BGD TAP binaries are registered in `test/tap/groups/groups.json` under +`cluster_sim_rds_bgd-g1`. The registry is the source of truth for both local +execution and GitHub CI. The simulator table in `test/infra/README.md` records +the group and its `make test_rds_bgd` requirement. -```json -"test_rds_bgd_smoke-t": [ "cluster_sim_rds_bgd-g1" ] -``` - -The simulator table in `test/infra/README.md` records the group and its -`make test_rds_bgd` requirement. Clean when switching compile flavors because -Make does not track changed preprocessor flags: +Clean when switching compile flavors because Make does not track changed +preprocessor flags: ```bash make clean -PROXYSQL40=1 make -j"$(nproc)" test_rds_bgd +make -j"$(nproc)" test_rds_bgd export INFRA_ID="rds-bgd-$(date +%s)" export TAP_GROUP="cluster_sim_rds_bgd-g1" @@ -537,128 +527,60 @@ The existing runner injects the host aliases, starts ProxySQL with and collects logs. No BGD branch is required in `ensure-infras.bash`, `start-proxysql-isolated.bash`, or `run-tests-isolated.bash`. -## GitHub Workflow Follow-up - -A separate follow-up adds `.github/workflows/CI-rds-bgd-simulator.yml`; the -workflow is not part of the simulator implementation described above. It runs on -`workflow_dispatch` and after a successful `CI-trigger`, follows the repository's -existing concurrency/cancellation pattern, and checks out the exact triggering -SHA. - -The regular Ubuntu TAP cache contains a daemon built without `TEST_RDS_BGD` and -must not be used as the BGD executable. The workflow therefore has a BGD build -job and a dependent execution job. - -The build job checks out the triggering SHA, installs or reuses the normal -Ubuntu TAP build dependencies, and runs: - -```bash -PROXYSQL40=1 make -j"$(nproc)" test_rds_bgd -``` - -After verifying `src/proxysql` and `test/tap/tests/test_rds_bgd_smoke-t`, it saves the -build output as two BGD-specific cache entries, following the existing CI -separation between daemon and test artifacts: +## GitHub CI -```text -${SHA}_ubuntu22-tap-rds-bgd_src -> src/ -${SHA}_ubuntu22-tap-rds-bgd_test -> test/ -``` +`.github/workflows/CI-cluster-simulator.yml` builds and executes every registered +`cluster_sim_*` group. It discovers groups and their TAP binaries from +`test/tap/groups/groups.json`, so registration in `cluster_sim_rds_bgd-g1` +places the complete BGD suite in the workflow matrix without BGD-specific YAML. +The workflow runs for pull requests and `workflow_dispatch`. -The cache keys are exact and include the BGD build flavor. Do not configure -`restore-keys`: falling back to the normal Ubuntu TAP cache could execute a -daemon compiled without `TEST_RDS_BGD`. The workflow must grant the cache-save -permission required by its `workflow_run` context. +The build job uses `test/infra/control/cluster-simulator-ci.bash` to build +`testall`, the cluster-simulator binary, the TAP library, and every registered +simulation binary. `testall` is intentional: one ProxySQL executable contains +all simulation flags, including `TEST_RDS_BGD`, and is shared by the matrix +jobs. The verified runtime is staged in an exact-SHA cache. -The execution job depends on the build job, checks out the same SHA, and -restores both entries with `fail-on-cache-miss: true`. It verifies the restored -executables before building the runner image and starting the test group. One -producer can therefore supply the same flagged artifacts to additional BGD -execution jobs without rebuilding ProxySQL. +Each matrix job restores and verifies that runtime for its selected group, +builds the common runner image, and executes `ensure-infras.bash` followed by +`run-tests-isolated.bash`. Cleanup always stops ProxySQL and destroys the +isolated runner; failure logs are archived by group and SHA. -| Execution-job step | Required behavior | -|---|---| -| Checkout | Check out the triggering SHA, not the default branch tip. | -| Restore `src` | Restore the exact BGD `_src` key into `src/`; fail on a miss. | -| Restore `test` | Restore the exact BGD `_test` key into `test/`; fail on a miss. | -| Verify artifacts | Confirm `src/proxysql` and `test/tap/tests/test_rds_bgd_smoke-t` are executable. | -| Build runner image | Build `test/infra/docker-base` as `proxysql-ci-base:latest`. | -| Start | Export the shared variables below and run `ensure-infras.bash`. | -| Test | Run `run-tests-isolated.bash`; this execution, not compilation alone, is the required check. | -| Cleanup | With `if: always()`, stop ProxySQL and run `destroy-infras.bash`; cleanup failures must not hide the test result. | -| Logs | On failure, upload `ci_infra_logs/` with the workflow name, SHA, and run number in the artifact name. | - -The start, test, and cleanup steps use the same values: - -```bash -export WORKSPACE="${GITHUB_WORKSPACE}" -export INFRA_ID="rds-bgd-${GITHUB_RUN_ID}" -export TAP_GROUP="cluster_sim_rds_bgd-g1" -source test/infra/common/env.sh -``` - -The job starts no backend infrastructure and never invokes -`test/deps/cluster_simulator`. Its pass condition is: the flagged build -succeeds, the BGD TAP group executes, every TAP test exits successfully, and -the standard runner reports no infrastructure or test failure. +The shared runtime includes `test/deps/cluster_simulator` for groups that need +it. The BGD group sets `SKIP_CLUSTER_START=1`, starts no backend infrastructure, +and drives ProxySQL's SQLite3-server simulator directly. ## Supported Test Coverage -### Simulator Acceptance - -The simulator implementation includes one end-to-end smoke test, not a separate -unit-test suite for every helper method. `test_rds_bgd_smoke-t` proves that the -`TEST_RDS_BGD` daemon accepts TAP-controlled topology, ProxySQL observes an -`AVAILABLE` deployment, and the green-IP probe is logged. The isolated local -runner executes the test without `test/deps/cluster_simulator`. - -The follow-up configuration and lifecycle tests below exercise the remaining -helper and SQLite3-server paths through BGD behavior. Before changing simulator -state, each test reads the last probe-log sequence; failures report the -configured backend state, last ProxySQL runtime state, and later probe rows. +The suite is behavior-driven rather than a unit test for every helper method. +Tests publish backend observations through the simulator and verify the BGD +monitor through Admin runtime tables, server placement, connection-pool state, +backend routing, read-only logs, and the simulator probe log. ### Configuration and Discovery -Configuration tests keep topology at `AVAILABLE` until the expected runtime row -and worker generation are stable. A relevant case then continues through a -switchover, proving that the configuration adopted during setup is the one used -by the FSM. - -| Case | Configuration sequence | Expected observations | -|---|---|---| -| Available topology before blue writer | Publish `AVAILABLE`, enable automatic discovery, then add the blue writer and its replication-hostgroup mapping. | The read-only discovery path creates one runtime BGD row with derived blue hostgroups, NULL green hostgroups, and `auto_generated=1`; its worker begins probing. | -| Blue deployment before BGD exists | Add the blue writer and readers while topology is absent, then publish `AVAILABLE`. | No BGD row is created before discovery; topology appearance creates the auto-generated runtime row and starts its worker. | -| Blue readers added after discovery | Start from an auto-generated row with only the blue writer, then add one or more blue readers and load servers to runtime. | The host checksum changes, the worker generation is replaced, probing resumes, and later reader actions use the new reader set. | -| Explicit BGD row before servers | Disable automatic discovery, load an explicit BGD hostgroup row, then add the blue writer, blue readers, green writer, and green readers. | The row remains `auto_generated=0`; no worker runs without an eligible blue server, and each relevant server change is incorporated by the replacement worker. | -| Servers before explicit BGD row | Add blue and green servers first with automatic discovery disabled, then load the explicit BGD hostgroup row. | No BGD worker runs before the row exists; loading it starts a worker that uses the existing server membership. | -| Blue first, green later | Configure blue servers, publish `AVAILABLE`, then add explicit green writer and reader rows before starting switchover. | Green membership changes replace the worker and rebuild its mapping; existing rows are not duplicated and the explicit green writer supplies its configured TLS mode. | -| Existing explicit green TLS | Configure the blue writer with `use_ssl=0` and the exact green TARGET row at the supported port with `use_ssl=1`, then publish `AVAILABLE`. | The direct metadata probe targets the resolved green writer IP with `encrypted=1`. | -| Discovered green TLS defaults | Configure the blue writer with `use_ssl=0`, leave the configured green writer hostgroup empty, and set its `servers_defaults.use_ssl=1`, then publish `AVAILABLE`. | Discovery adds the exact TARGET row with runtime `use_ssl=1`, and the subsequent green-IP metadata probe has `encrypted=1`. | -| Green timing variants | With explicit green hostgroups, add green nodes before `AVAILABLE`, after discovery, or after the worker starts but before switchover. | Each ordering converges on the same runtime membership and blue/green mapping before the FSM advances. | -| Automatic to explicit configuration | Allow discovery to create an automatic row, then load a user row with explicit green hostgroups. | The runtime row becomes user-defined with `auto_generated=0`, explicit green hostgroups replace NULLs, and a replacement worker uses the new configuration. | -| Configuration mutation | Change `active`, hostgroup IDs, `writer_is_also_reader`, check interval/timeout, server status, or `use_ssl`; also cover row disablement and removal. | Relevant checksum changes stop the old worker, run phase-appropriate cleanup, and start or suppress a worker from the new active configuration. | -| Persistence and validation | Save automatic and explicit runtime state, and attempt invalid persistent rows. | Auto-generated rows are not persisted; explicit rows are retained; missing or mixed green hostgroups and other schema-invalid configurations are rejected. | - -Configuration assertions use `runtime_mysql_aws_rds_bgd_hostgroups`, -`runtime_mysql_servers`, probe-log destinations, and subsequent hostgroup -effects. They do not depend only on worker log messages. +| Behavior | Coverage | +|---|---| +| Automatic discovery ordering | Topology before blue configuration and blue configuration before topology both converge on one runtime-only automatic BGD row. | +| Explicit startup ordering | A worker starts only after both an explicit BGD row and an eligible blue server exist, regardless of which is loaded first. | +| Green membership ordering | Configured green membership may arrive before `AVAILABLE`, after discovery, or after the worker starts. | +| Configuration ownership | Automatic rows remain runtime-only; explicit rows persist; invalid partial green-hostgroup configuration is rejected; automatic discovery does not overwrite administrator-owned rows. | +| Probe destination and TLS | Automatic and explicit configurations select the mapped writer tuple, apply the correct TLS source, and probe table check, blue metadata, and green metadata in order. | +| Active configuration refresh | Server TLS, membership, status, interval, timeout, hostgroups, and mapped-writer changes are incorporated while preserving the applicable BGD phase and probe policy. | +| Disablement and removal | Disabling or deleting an active BGD row performs phase-appropriate rollback and suppresses or removes the runtime worker. | ### Switchover, Rollback, and Cleanup -| Case | Simulator/configuration transition | Expected observations | +| Behavior | Coverage | |---|---|---| -| Normal lifecycle | Advance AVAILABLE → INITIATED → IN_PROGRESS → POST_PROCESSING → COMPLETED, then make topology empty or absent. | Runtime status follows every phase; writer/reader placement, server status, DNS effects, and connection-pool changes occur at their defined boundaries; final cleanup returns status to `NONE`. | -| Cancellation rollback | Move from INITIATED or IN_PROGRESS back to `AVAILABLE`. | Accumulated effects are rolled back, the blue writer and reader policy are restored, probe pinning is rebuilt for AVAILABLE, and the deployment remains monitorable. | -| Pre-completion topology loss | Delete or drop topology before writer completion. | Empty and absent observations remain distinguishable, but both select rollback rather than successful finalization. | -| Configuration change during switchover | Add/remove servers, disable/remove the BGD row, or change relevant configuration while a non-NONE phase is active. | The old worker performs one-shot phase-appropriate rollback before its replacement uses the new configuration; stale mappings do not drive later actions. | -| Rollback postconditions | Trigger rollback after writer demotion, reader shunning, or DNS pinning has occurred. | Blue writer service and configured reader membership are restored, BGD-shunned readers are unshunned, pins and direct-probe state are cleared or rebuilt, runtime status resets, green rows remain, and green connections are not drained. | -| Successful cleanup | Complete writer switchover, enter reader switchover, then drain topology. | Eligible green connections are drained while green rows and statuses remain; readers are reconciled and the worker returns to `NONE`. | -| Late entry | Start a fresh worker with INITIATED, IN_PROGRESS, POST_PROCESSING, or COMPLETED already published. | The worker reconstructs only the state supported by that observation and applies the defined phase actions without requiring earlier samples. | -| Direct-probe policy | Vary blue/green IP and blue versus explicit-green `use_ssl` while every endpoint uses port 3306. | Probe-log rows identify the correct green writer destination at port 3306 and the correct automatic or explicit TLS source. | -| Reader and offline handling | Use matched, unmatched, and `OFFLINE_SOFT`/`OFFLINE_HARD` blue and green readers. | Only eligible pairs are mapped or drained; unmatched readers follow BGD shun policy and offline nodes are excluded. | -| Metadata failures | Return an empty result, error 1146, or another configured query error from selected writers. | Absence, empty metadata, and generic query failure remain distinct and never masquerade as a successful switchover. | -| Repeated switchover | Complete cluster-1 deployment A, reset on empty/absent topology, replace its green rows with deployment B, and run again. | The second lifecycle uses deployment B without stale mapping, probe, or simulator state from deployment A. | -| Concurrent switchovers | Publish independent topology for clusters 1, 2, or 3 and advance them independently. | Multiple workers make isolated progress; configuration or topology changes in one cluster do not alter another. | +| Acceptance | An explicitly configured worker consumes TAP-controlled `AVAILABLE` topology and probes the green writer directly. | +| Writer switchover | `AVAILABLE`, `SWITCHOVER_INITIATED`, `SWITCHOVER_IN_PROGRESS`, and `SWITCHOVER_IN_POST_PROCESSING` drive the defined status, placement, suppression, pool-drain, and routing effects. Repeated post-processing does not redrain a post-cutover pool. | +| Reader switchover and cleanup | Target-only `SWITCHOVER_COMPLETED` enters reader switchover; terminal empty or absent topology restores reader policy, drains eligible green pools, retains configured green rows, and returns to `NONE`. | +| Cancellation rollback | Returning from initiated or in-progress topology to `AVAILABLE` restores blue routing and monitoring without removing explicit green rows or draining green pools. | +| Topology loss and errors | Empty topology, absent topology, metadata error 1146, and generic metadata errors retain their distinct effects before and after writer completion. | +| Late entry | Fresh workers starting at initiated, in-progress, post-processing, or completed observations apply only the state supported by the first observation. | +| Reader and pool policy | Matched and unmatched readers, writer fallback, and `ONLINE`, `SHUNNED`, `OFFLINE_SOFT`, and `OFFLINE_HARD` green pools follow their routing and cleanup policies. | +| Repeated and concurrent deployments | A second deployment reuses hostgroups without stale membership or probes, while three simultaneous workers retain independent topology, phase, placement, and TLS state. | The simulator does not claim to validate application traffic, AWS control-plane timing, mutable DNS propagation, packet loss, or exact post-switchover address @@ -667,16 +589,17 @@ assertion depends on them. ## Code Boundaries -| Area | Current boundary | +| Area | Boundary | |---|---| | `Makefile` | Provides the BGD build targets and includes `TEST_RDS_BGD` in `testall`. | -| `include/SQLite3_Server.h` | Defines the BGD tables and shared simulator helpers. | +| `include/SQLite3_Server.h` | Declares the TEST-mode table ownership and shared read-only simulator members. | | `src/SQLite3_Server.cpp` | Handles endpoint extraction, table creation, BGD/read-only interception, and probe logging. | | `test/tap` helpers | Provide the common simulator and BGD-specific API defined above. | | `test/tap/groups/cluster_sim_rds_bgd` | Defines the fixed host map and SQLite3-server group configuration. | | `test/tap/groups/groups.json` | Registers BGD TAP binaries in `cluster_sim_rds_bgd-g1`. | | `test/infra/README.md` | Documents the group and its required `test_rds_bgd` build target. | -| GitHub workflow | Follow-up work builds the flagged flavor and executes the BGD simulator group. | +| `.github/workflows/CI-cluster-simulator.yml` | Builds the combined simulation flavor and executes each registered simulator group in its own matrix job. | +| `test/infra/control/cluster-simulator-ci.bash` | Discovers groups and binaries, builds and verifies the shared runtime, and stages the exact-SHA cache payload. | | BGD production monitor | Reuses existing query constants without simulator query decoration or a test initializer. | Existing simulator builds retain their behavior. The scenario, not the helper, From 1688214bc09b27e9bd12e3ad35750ae6c80dbdb2 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 28 Jul 2026 09:48:34 +0000 Subject: [PATCH 74/81] fix: stabilize RDS BGD read_only suppression - Track and clear the exact endpoints suppressed by each BGD worker. - Ignore stale read_only results without blocking BGD-owned writer placement. Signed-off-by: Wazir Ahmed --- include/MySQL_HostGroups_Manager.h | 3 +- include/MySQL_Monitor.hpp | 12 +++---- lib/MySQL_HostGroups_Manager.cpp | 21 +++++++++-- lib/MySQL_Monitor.cpp | 56 +++++++++++++++++------------- 4 files changed, 57 insertions(+), 35 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 2dce864181..c226422f32 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -1055,8 +1055,9 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { * mysql_servers table and checksum are regenerated. * * @param mysql_servers Servers and their observed/read-only state. + * @param ignore_aws_bgd True to apply the result while BGD switchover is in progress. */ - void read_only_action_v2(const std::list& mysql_servers); + void read_only_action_v2(const std::list& mysql_servers, bool ignore_aws_bgd = false); unsigned int get_servers_table_version(); void wait_servers_table_version(unsigned, unsigned); bool shun_and_killall(char *hostname, int port); diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index e2a5f5ed0d..ad599493a5 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -575,7 +575,8 @@ struct AWS_RDS_BGD_State { std::vector probe_hosts; ///< hosts eligible for topology probes std::vector shunned_readers; ///< readers we shunned - AWS_RDS_BGD_Status bgd_status = AWS_RDS_BGD_Status::NONE; ///< drives the FSM and the deferred cleanup + std::vector read_only_check_disabled; ///< servers whose read_only checks this worker disabled + AWS_RDS_BGD_Status bgd_status = AWS_RDS_BGD_Status::NONE; ///< drives the FSM and the deferred cleanup bool bgd_in_progress_set = false; ///< deployment's servers flagged in aws_rds_bgd_server_status bool config_refresh_pending = false; ///< bg_map must be rebuilt from the next topology result @@ -835,14 +836,13 @@ class MySQL_Monitor { * @brief Flag/unflag every server in BGD hostgroups as switchover-in-progress. * * @details Called by the BGD worker at switchover initiation (INITIATED / IN_PROGRESS / - * POST_PROCESSING) and cleared after SWITCHOVER_COMPLETED. Iterates the writer and reader - * hostgroups and marks all member servers in the shared aws_rds_bgd_server_status map. + * POST_PROCESSING) and cleared after SWITCHOVER_COMPLETED. Saves the marked servers in the + * worker state so cleanup does not depend on the current hostgroup configuration. * - * @param writer_hg Writer hostgroup for the deployment. - * @param reader_hg Reader hostgroup for the deployment. + * @param st BGD worker state. * @param in_progress true to flag servers, false to clear. */ - void set_aws_rds_bgd_server_in_progress(unsigned int writer_hg, unsigned int reader_hg, bool in_progress); + void set_aws_rds_bgd_server_in_progress(AWS_RDS_BGD_State& st, bool in_progress); void * monitor_replication_lag(); void * monitor_dns_cache(); diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index a021d4f0f3..309363e8eb 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -3588,14 +3588,29 @@ SQLite3_result * MySQL_HostGroups_Manager::SQL3_Connection_Pool(bool _reset, int * mysql_servers table and checksum are regenerated. * * @param mysql_servers Servers and their observed/read-only state. + * @param ignore_aws_bgd True to apply the result while BGD switchover is in progress. */ -void MySQL_HostGroups_Manager::read_only_action_v2(const std::list& mysql_servers) { +void MySQL_HostGroups_Manager::read_only_action_v2(const std::list& mysql_servers, bool ignore_aws_bgd) { + // Skip read_only results for servers flagged as AWS RDS BGD switchover in progress. + std::list filtered_servers; + for (const auto& server : mysql_servers) { + const std::string& hostname = std::get(server); + const int port = std::get(server); + + if (!ignore_aws_bgd && GloMyMon->is_aws_rds_bgd_server_in_progress(hostname, port)) { + proxy_debug(PROXY_DEBUG_MONITOR, 5, + "Ignoring read_only result for '%s:%d' because AWS RDS BGD switchover is in progress\n", + hostname.c_str(), port); + continue; + } + filtered_servers.push_back(server); + } bool update_mysql_servers_table = false; unsigned long long curtime1 = monotonic_time(); wrlock(); - for (const auto& server : mysql_servers) { + for (const auto& server : filtered_servers) { bool is_writer = false; const std::string& hostname = std::get(server); const int port = std::get(server); @@ -3731,7 +3746,7 @@ void MySQL_HostGroups_Manager::read_only_action_v2(const std::listset_aws_rds_bgd_server_in_progress(st.writer_hg, st.reader_hg, true); + GloMyMon->set_aws_rds_bgd_server_in_progress(st, true); st.bgd_in_progress_set = true; proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: switchover in progress, suspending read_only monitor checks on writer/reader hostgroups until SWITCHOVER_COMPLETED\n", st.writer_hg, st.reader_hg); @@ -6706,7 +6706,7 @@ static void aws_rds_bgd_clear_bgd_in_progress(AWS_RDS_BGD_State& st) { return; } - GloMyMon->set_aws_rds_bgd_server_in_progress(st.writer_hg, st.reader_hg, false); + GloMyMon->set_aws_rds_bgd_server_in_progress(st, false); st.bgd_in_progress_set = false; proxy_info("AWS RDS BGD [wHG=%u rHG=%u]: switchover completed, resuming read_only monitor checks on writer/reader hostgroups\n", st.writer_hg, st.reader_hg); @@ -7486,12 +7486,12 @@ void MySQL_Monitor::aws_rds_bgd_config_refresh_action(AWS_RDS_BGD_State& st, AWS if (writer_changed && had_old_writer) { MyHGM->read_only_action_v2(std::list { read_only_server_t { old_writer.host, (port_t)old_writer.port, 0 } - }); + }, true); } if (has_new_writer) { MyHGM->read_only_action_v2(std::list { read_only_server_t { new_writer.host, (port_t)new_writer.port, 1 } - }); + }, true); } } } @@ -7635,7 +7635,7 @@ void MySQL_Monitor::handle_aws_rds_bgd(AWS_RDS_BGD_State& st, AWS_RDS_Topology_R for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { if (p.is_writer) { auto srv = read_only_server_t{ p.blue_host, (port_t)p.port, 1 }; - MyHGM->read_only_action_v2(std::list{srv}); + MyHGM->read_only_action_v2(std::list{srv}, true); break; } } @@ -7848,7 +7848,7 @@ void MySQL_Monitor::handle_aws_rds_bgd_post_switchover(AWS_RDS_BGD_State& st, bo for (const AWS_RDS_BlueGreenPair& p : st.bg_map) { if (p.is_writer) { auto srv = read_only_server_t{ p.blue_host, (port_t)p.port, 0 }; - MyHGM->read_only_action_v2(std::list{srv}); + MyHGM->read_only_action_v2(std::list{srv}, true); break; } } @@ -8001,40 +8001,46 @@ bool MySQL_Monitor::is_aws_rds_bgd_server_in_progress(const std::string& hostnam * @brief Flag/unflag every server in BGD hostgroups as switchover-in-progress. * * @details Called by the BGD worker at switchover initiation (INITIATED / IN_PROGRESS / -* POST_PROCESSING) and cleared after SWITCHOVER_COMPLETED. Iterates the writer and reader -* hostgroups and marks all member servers in the shared aws_rds_bgd_server_status map. +* POST_PROCESSING) and cleared after SWITCHOVER_COMPLETED. Saves the marked servers in the +* worker state so cleanup does not depend on the current hostgroup configuration. * -* @param writer_hg Writer hostgroup for the deployment. -* @param reader_hg Reader hostgroup for the deployment. +* @param st BGD worker state. * @param in_progress true to flag servers, false to clear. */ -void MySQL_Monitor::set_aws_rds_bgd_server_in_progress(unsigned int writer_hg, unsigned int reader_hg, bool in_progress) { - std::vector keys; - MyHGM->wrlock(); - unsigned int hgs[2] = { writer_hg, reader_hg }; - for (unsigned int i = 0; i < 2; i++) { - MyHGC* myhgc = MyHGM->MyHGC_find(hgs[i]); - if (myhgc == nullptr || myhgc->mysrvs == nullptr) { - continue; - } - for (unsigned int j = 0; j < myhgc->mysrvs->cnt(); j++) { - MySrvC* s = myhgc->mysrvs->idx(j); - keys.push_back(std::string(s->address) + ":::" + std::to_string(s->port)); +void MySQL_Monitor::set_aws_rds_bgd_server_in_progress(AWS_RDS_BGD_State& st, bool in_progress) { + if (in_progress) { + st.read_only_check_disabled.clear(); + + MyHGM->wrlock(); + unsigned int hgs[2] = { st.writer_hg, st.reader_hg }; + for (unsigned int i = 0; i < 2; i++) { + MyHGC* myhgc = MyHGM->MyHGC_find(hgs[i]); + if (myhgc == nullptr || myhgc->mysrvs == nullptr) { + continue; + } + for (unsigned int j = 0; j < myhgc->mysrvs->cnt(); j++) { + MySrvC* s = myhgc->mysrvs->idx(j); + st.read_only_check_disabled.push_back(std::string(s->address) + ":::" + std::to_string(s->port)); + } } + MyHGM->wrunlock(); } - MyHGM->wrunlock(); pthread_mutex_lock(&aws_rds_bgd_mutex); if (in_progress) { - for (const auto& k : keys) { + for (const auto& k : st.read_only_check_disabled) { aws_rds_bgd_server_status[k] = AWS_RDS_BGD_Server_Status::IN_PROGRESS; } } else { - for (const auto& k : keys) { + for (const auto& k : st.read_only_check_disabled) { aws_rds_bgd_server_status.erase(k); } } pthread_mutex_unlock(&aws_rds_bgd_mutex); + + if (!in_progress) { + st.read_only_check_disabled.clear(); + } } /** From 1995c8492a4573ef701a583d1337a46aa4710bf0 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Tue, 28 Jul 2026 09:54:16 +0000 Subject: [PATCH 75/81] docs: finalize RDS BGD monitor documentation - Replace the review contract with the implemented feature contract. - Document current topology, lifecycle, cleanup, configuration, and worker behavior. - Record overlapping endpoint assignments as an unsupported configuration. Signed-off-by: Wazir Ahmed --- doc/AWS_Blue_Green/RDS_BGD_Monitor.md | 1537 ++++++++++--------------- 1 file changed, 612 insertions(+), 925 deletions(-) diff --git a/doc/AWS_Blue_Green/RDS_BGD_Monitor.md b/doc/AWS_Blue_Green/RDS_BGD_Monitor.md index edfadd25c5..0ac0555823 100644 --- a/doc/AWS_Blue_Green/RDS_BGD_Monitor.md +++ b/doc/AWS_Blue_Green/RDS_BGD_Monitor.md @@ -1,167 +1,149 @@ # AWS RDS Blue/Green Monitor -**Document status:** AUTHOR VALIDATION COMPLETE; IMPLEMENTATION CONFORMANCE OPEN +**Document status:** FEATURE CONTRACT; IMPLEMENTED -**Applies to:** Amazon RDS Multi-AZ DB instance blue/green deployment monitoring +**Applies to:** Amazon RDS Multi-AZ DB instance blue/green deployment +monitoring -**Primary monitor entry points:** `include/MySQL_Monitor.hpp`, +**Primary implementation:** `include/MySQL_Monitor.hpp` and `lib/MySQL_Monitor.cpp` -**Simulator design:** [RDS_BGD_Simulator.md](RDS_BGD_Simulator.md) +**Simulator specification:** [RDS_BGD_Simulator.md](RDS_BGD_Simulator.md) -**Related implementation:** `include/DNS_Cache.hpp`, `lib/DNS_Cache.cpp`, -`include/MySQL_HostGroups_Manager.h`, `lib/MySQL_HostGroups_Manager.cpp`, -`include/mysql_connection.h`, `lib/mysql_connection.cpp`, -`lib/MySrvConnList.cpp`, `lib/ProxySQL_Admin.cpp`, `lib/ProxySQL_Config.cpp`, and -`include/ProxySQL_Admin_Tables_Definitions.h`. This is the non-exhaustive -side-effect/configuration surface referenced by later `SOURCE-CODE` sections. +## Purpose And Scope -## Purpose +This document defines the operational contract for ProxySQL monitoring of +Amazon RDS Multi-AZ DB instance blue/green deployments. It describes the AWS +topology observations consumed by the monitor, the resulting ProxySQL state +transitions and external effects, configuration and worker lifetime, +connection-retirement and cleanup semantics, and the verification surface for +the feature introduced by +[PR #5861](https://github.com/sysown/proxysql/pull/5861). -This document defines the AWS observations, current ProxySQL behavior, and -safety requirements for the AWS RDS blue/green deployment monitor introduced -by [PR #5861](https://github.com/sysown/proxysql/pull/5861). +The contract distinguishes among behavior defined by AWS, behavior observed in +a bounded deployment trace, intentional ProxySQL policy, and implementation +mechanics. A scoped observation is not promoted to a universal AWS guarantee, +and a ProxySQL policy is not presented as an AWS property. -The document is intentionally explicit about evidence. AWS behavior for which -this review has not recorded author evidence is not presented as an operational -guarantee. This evidence status does not imply that the author originally -inferred, assumed, or failed to observe the behavior. - -## Evidence Labels - -| Label | Dimension | Meaning | -|---|---|---| -| `SOURCE-CODE` | Provenance | Direct description of current implementation; it does not validate an external AWS claim. | -| `AUTHOR-VALIDATED` | External evidence | AWS behavior confirmed by the feature author. The statement must identify whether it is an AWS-provided contract or scoped observation. | -| `AUTHOR-ACCEPTED-POLICY` | Intent | ProxySQL behavior explicitly accepted by the feature author, including a deliberate policy choice made under an external uncertainty. | -| `REVIEW-VALIDATION-PENDING` | Review evidence | An external claim present in the PR, source comments, or implementation contract for which this review has not yet recorded the author's evidence or correction. It does not characterize how the author derived the claim. | -| `IMPLEMENTATION-CONFORMANCE-OPEN` | Review finding | The evidence or policy decision is resolved, but current source does not implement it or lacks verification. | -| `PROPOSED-POLICY` | Intent | Reviewer-proposed hardening that is not part of the current implementation or an author-accepted production contract unless separately promoted to `AUTHOR-ACCEPTED-POLICY`. | - -Labels may be combined. `SOURCE-CODE, REVIEW-VALIDATION-PENDING` means the -current code or comments encode an external claim whose supporting evidence has -not yet been recorded in this review. `SOURCE-CODE` alone must be used only for -internal mechanics and never promotes an external claim. - -A `REVIEW-VALIDATION-PENDING` claim must be promoted to `AUTHOR-VALIDATED`, -replaced by an explicit `AUTHOR-ACCEPTED-POLICY`, or corrected before the -author-validation gate closes. Closing that evidence gate does not imply that -the implementation conforms to the recorded decision or that a reviewer has -accepted the operational risk. - -## Author Evidence Record - -The feature author supplied the following evidence in the -[author-validation response](https://github.com/sysown/proxysql/pull/5934#issuecomment-4972444890): - -- An [AWS-provided RDS topology metadata document](https://github.com/user-attachments/files/30019110/RDS_Topology_metadata.md) - describing the `mysql.rds_topology` schema, roles, statuses, switchover - stages, traffic availability, and polling guidance. -- A [timestamped topology trace](https://github.com/user-attachments/files/30019175/aws-rds-topology-watch.txt) - from one complete switchover, polled at approximately 250 ms. -- Source-code references and additional author observations for cancellation, - reader behavior, and green-hostname retirement. - -The captured deployment used RDS MySQL 8.4.x, a Multi-AZ DB instance with two -read replicas, and `eu-north-1`. The trace covers one complete switchover; the -author separately observed one cancellation. A statement supported only by -that trace or an unrecorded author observation is scoped accordingly and is not -promoted to a universal AWS guarantee. - -The author then answered the eight remaining decisions in the -[counter-review response](https://github.com/sysown/proxysql/pull/5934#issuecomment-4989347968). -That response explicitly: - -- Defines the matched blue writer's configured port as the direct green-probe - port and accepts that a source/target pair using different ports is not - supported. -- Defines explicit-mode TLS from the matched green writer's `mysql_servers` - row and automatic-mode TLS from the matched blue writer's row. -- Makes green hostgroup membership persistent until an administrator removes - it, including membership created automatically at runtime. -- Accepts one-shot cleanup and loss of per-effect completion state rather than - a retained or durable cleanup ledger. -- Records the same-phase DNS-resolution failure for later per-pair - reconciliation. -- Accepts cleanup-on-worker-exit followed by fresh worker state, and a - no-persistence fresh start after a full ProxySQL process restart. - -The source changes reviewed with that response are commits `7d272074c` through -`ac4167cd0`, based on `0a37316c9`. A stated policy is recorded as resolved even -when implementation conformance is still open; those cases are called out -explicitly below. - -The author subsequently clarified two ProxySQL-internal contracts during the -review: - -- User configuration in the persistent `mysql_aws_rds_bgd_hostgroups` table - requires both green hostgroup values. Nullable green hostgroups belong only - to runtime rows generated by automatic discovery; those rows carry - `auto_generated=1` and are skipped when runtime state is saved back to the - persistent configuration table. -- Reads of connection state without an additional per-connection lock are an - accepted project-level risk. For BGD connection retirement, `healthy=false` - is to become a terminal marker: `MySQL_Connection::reset()` must not restore - it, and both local and global pool-return paths must destroy an unhealthy - connection instead of caching it. - -The BGD test foundation uses ProxySQL's SQLite3 server, compiled under -`TEST_RDS_BGD` and controlled directly by each TAP test. The simulator -foundation and the BGD scenario suite are deliberately separate follow-up PRs. -Registration in `groups.json` is not considered CI integration by itself; the -BGD simulator group must be executed by an automatic PR check. - -## Scope - -This document covers: +The monitor contract covers: - Detection of blue/green topology through `mysql.rds_topology`. +- Interpretation of source and target roles and switchover statuses. - Mapping of configured blue writer and reader servers to green servers. - Green address resolution and direct topology probing. - Writer and reader switchover handling. -- DNS pinning, hostgroup changes, and backend connection draining. -- Successful finalization, cancellation rollback, and worker replacement. +- DNS pinning, hostgroup placement, reader shunning, and connection draining. +- Successful completion, cancellation rollback, and topology disappearance. - Explicit and automatic green-hostgroup configuration. +- In-place configuration refresh, worker detach and recreation, and process + restart. +- The simulator, TAP, unit-test, and CI surface used to verify the contract. +- Documented configuration limitations and administrator responsibilities. + +This document does not define: + +- Amazon Aurora, Group Replication, Galera, or PostgreSQL monitoring. +- RDS behavior that is not consumed by this monitor. +- A durable effect ledger or a replacement controller architecture. +- Persistence of in-progress BGD state across a ProxySQL process restart. +- Detailed simulator implementation, which is specified in + [RDS_BGD_Simulator.md](RDS_BGD_Simulator.md). + +## Contract Basis + +The external behavior in this contract is based on: + +- An + [AWS-provided RDS topology metadata document](https://github.com/user-attachments/files/30019110/RDS_Topology_metadata.md) + describing `mysql.rds_topology`, roles, statuses, switchover stages, traffic + availability, and polling guidance. +- A + [timestamped topology trace](https://github.com/user-attachments/files/30019175/aws-rds-topology-watch.txt) + from one complete switchover, sampled at approximately 250 ms. +- A separately observed cancellation and operational observations supplied by + the feature author. +- The source implementation and tests referenced in this document. -This document does not define Aurora monitoring, Group Replication monitoring, -Galera monitoring, or PostgreSQL behavior. +The captured deployment used RDS MySQL 8.4.x, a Multi-AZ DB instance with two +read replicas, and `eu-north-1`. Timing and row-lifecycle statements derived +only from this trace apply to that observation. They do not establish fixed +timing or universal AWS behavior. -## Terminology +The following language identifies the authority for a statement: -The table includes current implementation terms and proposed hardening -concepts; proposed concepts are explicitly labeled. +| Wording | Meaning | +|---|---| +| **AWS defines** | The AWS-provided metadata document specifies the behavior. | +| **The supplied trace observed** | The behavior occurred in the bounded trace described above. | +| **The author observed** | The feature author supplied an operational observation outside that trace. | +| **ProxySQL policy** | The behavior is an intentional product decision, including a decision made where AWS does not provide a stronger guarantee. | +| **The implementation** | The statement describes the source behavior on the feature branch. | + +## Terminology -| Term | Definition | +| Term | Meaning | |---|---| | Blue | The source deployment before switchover. | | Green | The target deployment before switchover. | -| Observation | The result of one topology query or lifecycle event. | -| Controller state | `PROPOSED-POLICY`: ProxySQL's progress and policy across observations. | -| External effect | A DNS, hostgroup, monitor, or connection change visible outside controller bookkeeping. | -| Effect ledger | `PROPOSED-POLICY`: The identities and results of external effects requiring retry or cleanup. | -| Finalization | `PROPOSED-POLICY`: Cleanup on the proposed successful-completion path after observed writer completion and the accepted reader-completion signal. | -| Rollback | `PROPOSED-POLICY`: Restoration after cancellation or topology disappearance before observed writer completion. | +| Topology observation | The result of one `mysql.rds_topology` existence check or metadata query. | +| Blue/green pair | A configured blue server and its name-matched green counterpart, together with the connection attributes needed by the monitor. | +| Direct probe | A topology query sent to the resolved green writer IP rather than to a configured blue hostname. | +| Topology drain | A successful metadata query returning no rows, or the topology table becoming unavailable. | +| Rollback cleanup | One-shot cleanup invoked for topology drain outside the reader phase, a recognized backward transition, configuration refresh at or after post-processing, or worker exit from any active phase. | +| Successful cleanup | One-shot cleanup selected when topology drains after the monitor has observed writer completion. | +| Configuration refresh | An in-place update of a running worker after its deployment checksum changes. | +| Worker detach | Termination of a worker because the deployment is disabled, removed, or no longer has an eligible blue writer. | + +## Configuration Model + +Each active deployment identifies a blue writer hostgroup and a blue reader +hostgroup. It also carries `writer_is_also_reader`, a baseline check interval, +and a check timeout. One worker owns the monitor state for one blue writer +hostgroup. + +Green hostgroup nullability depends on the origin of the row. It is not a +user-selectable mixed configuration: -## Topology Shape +| Row origin and storage | Green writer hostgroup | Green reader hostgroup | Semantics | +|---|---|---|---| +| User row in persistent Admin configuration | Required | Required | Explicit green-hostgroup mode. The persistent table declares both columns `NOT NULL`. | +| User row materialized into runtime/HGM | Value | Value | The configured values are retained with `auto_generated=0`. | +| Runtime row created by automatic discovery | `NULL` | `NULL` | Automatic mode. The row carries `auto_generated=1` and exists only in runtime/HGM state. | +| User row with one or both values missing | Invalid | Invalid | A user cannot select automatic handling for only one green role. | + +The runtime Admin and Hostgroup Manager schemas permit nullable green +hostgroups so that they can represent automatically generated rows. This +runtime representation does not make a `NULL` green hostgroup valid in the +persistent user table. + +Saving runtime BGD configuration to the persistent Admin table skips every row +whose runtime `auto_generated` value is nonzero. An automatically generated row +with two `NULL` green hostgroups is therefore never inserted into the +persistent `NOT NULL` columns. User rows contain both values and are saved +normally. -`SOURCE-CODE`: `parse_aws_rds_topology` sets `blue_green` once, from the first -fetched row, when the role and status columns exist and that row's cells for -both columns are non-NULL. Empty strings still meet this current non-NULL test. -Later rows do not re-evaluate or reverse the classification. +`OFFLINE_SOFT` and `OFFLINE_HARD` servers are not eligible for blue/green +mapping or for connection-drain actions. An explicit green writer in either +offline state is not selected as the active green endpoint. -`SOURCE-CODE, AUTHOR-VALIDATED (AWS-PROVIDED CONTRACT)`: Actual RDS blue/green -rows use the source and target role values and recognized target status values -listed below. The AWS-provided metadata document defines these values, and all -five statuses appeared in the supplied trace. +## AWS Topology -Role values: +### Topology Recognition + +`parse_aws_rds_topology()` classifies a result as blue/green topology from the +first fetched row. The result is classified as blue/green when the `role` and +`status` columns exist and both first-row cells are non-`NULL`. Empty strings +still satisfy this non-`NULL` test. Later rows do not change the initial +classification. + +AWS defines the following blue/green role values: ```text BLUE_GREEN_DEPLOYMENT_SOURCE BLUE_GREEN_DEPLOYMENT_TARGET ``` -Target status values: +AWS defines the following target status values: ```text AVAILABLE @@ -171,48 +153,14 @@ SWITCHOVER_IN_POST_PROCESSING SWITCHOVER_COMPLETED ``` -`SOURCE-CODE`: If there is no row, either column is absent, or either first-row -cell is NULL, `blue_green` remains false. Later malformed rows do not change a -true first-row classification. Other RDS topology shapes may be processed by -the Multi-AZ Cluster discovery path. - -`AUTHOR-VALIDATED (SCOPED OBSERVATION)`: While both rows were present in the -supplied trace, the source and target rows carried the same status. The trace -does not by itself establish that equality as a universal contract. +If the query returns no rows, either column is absent, or either first-row cell +is `NULL`, `blue_green` remains false. The parser is shared with the Multi-AZ +Cluster discovery path, so other RDS topology shapes may be handled outside the +BGD state machine. -## Observation Model +### Observed Lifecycle -`PROPOSED-POLICY`: The hardened controller design distinguishes the following -observation and lifecycle-event vocabulary. `SOURCE-CODE`: The -`TOPOLOGY_ABSENT` inputs correspond to the current table-existence query and -`ER_NO_SUCH_TABLE` metadata-fetch paths, but these observations are not the -current implementation state enum. - -| Observation | Meaning | -|---|---| -| `TOPOLOGY_ABSENT` | The existence query returns zero rows, or a metadata fetch reports `ER_NO_SUCH_TABLE`. The table is not available; this is not a generic failure. | -| `TOPOLOGY_EMPTY` | A successful metadata query returns zero topology rows. | -| `AVAILABLE` | The target reports the recognized `AVAILABLE` status. | -| `WRITER_INITIATED` | The target reports the recognized `SWITCHOVER_INITIATED` status. | -| `WRITER_IN_PROGRESS` | The target reports the recognized `SWITCHOVER_IN_PROGRESS` status. | -| `WRITER_POST_PROCESSING` | The target reports the recognized `SWITCHOVER_IN_POST_PROCESSING` status. | -| `WRITER_COMPLETED` | The target reports the recognized `SWITCHOVER_COMPLETED` status. | -| `UNKNOWN_STATUS` | The target has a non-empty, unrecognized status. | -| `MALFORMED_TOPOLOGY` | The target, endpoint, role, status, or required identity is missing. | -| `QUERY_FAILED` | The query times out, the connection fails, or SQL reports an error other than documented absence. | -| `CONFIG_CHANGED` | The monitor result-set checksum or generation changes and may replace the worker without losing deployment state. | -| `CONFIG_DISABLED` | The deployment remains configured but is disabled; outstanding effects require phase-appropriate settlement. | -| `CONFIG_REMOVED` | The deployment configuration is removed; outstanding effects require phase-appropriate settlement before the context is removed. | -| `WORKER_RESTARTED` | A replacement worker attaches to and resumes the existing context. | - -`PROPOSED-POLICY`: `TOPOLOGY_ABSENT`, `TOPOLOGY_EMPTY`, and `QUERY_FAILED` are -not interchangeable. Query failure never proves cancellation or completion. - -## Validated Lifecycle Evidence - -`SOURCE-CODE, AUTHOR-VALIDATED (AWS-PROVIDED CONTRACT AND SCOPED OBSERVATION)`: -The AWS-provided metadata document defines the five forward phases. The supplied -trace observed the following row lifecycle: +The supplied trace observed this row and status sequence: ```text Two rows: @@ -221,48 +169,167 @@ Two rows: status = AVAILABLE Two rows: - repeated observations of SWITCHOVER_INITIATED - -> repeated observations of SWITCHOVER_IN_PROGRESS - -> repeated observations of SWITCHOVER_IN_POST_PROCESSING + repeated SWITCHOVER_INITIATED observations + -> repeated SWITCHOVER_IN_PROGRESS observations + -> repeated SWITCHOVER_IN_POST_PROCESSING observations -One target row: - repeated observations of SWITCHOVER_COMPLETED +One row: + TARGET = green + repeated SWITCHOVER_COMPLETED observations Zero rows: - observed approximately 44 seconds after SWITCHOVER_COMPLETED in this trace + observed approximately 44 seconds after SWITCHOVER_COMPLETED ``` -`AUTHOR-VALIDATED (AWS-PROVIDED CONTRACT)`: `SWITCHOVER_COMPLETED` means writer -DNS propagation completed and the original source endpoint points to the -target. The status sequence permits cancellation during `SWITCHOVER_INITIATED` -and `SWITCHOVER_IN_PROGRESS`; rollback is no longer allowed in -`SWITCHOVER_IN_POST_PROCESSING`. - -`AUTHOR-VALIDATED (SCOPED OBSERVATION)`: The trace observed monotonic forward -phase changes, with repeated identical observations while each phase remained -active. At `SWITCHOVER_COMPLETED`, the source row disappeared and the target -row remained for approximately 44 seconds before the table became empty. The -duration is not fixed. The table remained present in `information_schema`; an -`ER_NO_SUCH_TABLE` outcome was not observed. - -`AUTHOR-ACCEPTED-POLICY`: After writer completion has been observed, -`TOPOLOGY_EMPTY` is the accepted reader-cleanup signal. The author observed -reader errors before the table drained and normal reader behavior afterward. -The metadata table contains writer topology only, the trace did not measure -reader DNS timing, and AWS does not document table emptiness as proof of reader -DNS propagation. This policy therefore records an explicitly accepted -operational correlation, not an AWS guarantee. - -`AUTHOR-ACCEPTED-POLICY`: `TOPOLOGY_ABSENT` remains distinct from -`TOPOLOGY_EMPTY` in diagnostics but selects the same phase-specific policy: -rollback before observed writer completion and reader cleanup afterward. Only -the empty-table outcome was observed. - -## Current ProxySQL State Machine - -`SOURCE-CODE`: This is the nominal ordering encoded by the enum names and the -lifecycle currently described by the implementation. The arrows are not -enforced transition edges: +Both rows carried the same status while both were present in the supplied +trace. The trace establishes that this occurred in the captured deployment; it +does not establish source/target status equality as a universal AWS guarantee. + +The trace also observed monotonic forward transitions and repeated observations +within each phase. At `SWITCHOVER_COMPLETED`, the source row disappeared and +the target row remained for approximately 44 seconds before the table became +empty. The 44-second duration is not fixed. The table remained present in +`information_schema`; the trace did not observe `ER_NO_SUCH_TABLE`. + +### AWS Completion And Cancellation Boundaries + +AWS defines `SWITCHOVER_COMPLETED` as completion of writer DNS propagation: the +original source endpoint points to the promoted target. + +AWS permits cancellation during `SWITCHOVER_INITIATED` and +`SWITCHOVER_IN_PROGRESS`. Rollback is no longer allowed after the deployment +enters `SWITCHOVER_IN_POST_PROCESSING`. The author separately observed a +cancellation returning the deployment to `AVAILABLE`. + +The author observed that the green hostname stopped resolving after completion +while the promoted IP remained reachable. The monitor consequently retains a +complete direct-probe target while direct probing is required rather than +depending on the continued resolvability of the green hostname. + +### Reader-Completion Policy + +The metadata table describes writer topology; it does not expose a reader +switchover status. The supplied trace did not measure reader DNS propagation +directly. The author observed reader errors before the table drained and normal +reader behavior afterward. + +ProxySQL therefore uses the following explicit policy: + +- `SWITCHOVER_COMPLETED` is the writer-completion signal. +- A topology drain after writer completion is the reader-cleanup signal. +- A topology drain before writer completion is a cancellation or rollback + signal. + +Using topology drain as the reader-cleanup signal is an accepted operational +correlation. It is not an AWS guarantee that an empty topology result proves +reader DNS propagation. + +### Topology Outcomes + +The worker distinguishes the following query outcomes: + +| Outcome | Detection | Monitor behavior | +|---|---|---| +| Table absent | The existence query returns no rows. | Apply phase-specific topology-drain handling. | +| Table vanished | A metadata query returns `ER_NO_SUCH_TABLE`. | Reset the query state to the table check, restore the baseline interval, and apply phase-specific topology-drain handling. | +| Table empty | A successful metadata query returns no rows. | Apply phase-specific topology-drain handling. | +| Metadata available | A successful metadata query returns rows. | Parse and pass the topology to the BGD state machine. | +| Query failure | The connection, timeout, existence query, or metadata query fails for another reason. | Log the error and retain the state for a later poll. A generic query failure is not a completion or cancellation signal. | + +Table absence and an empty table remain diagnostically distinct. ProxySQL +intentionally applies the same phase boundary to both because only the empty +table was present in the supplied lifecycle trace. + +## Monitor Architecture + +### Worker And Polling Model + +The dispatcher creates one BGD worker for each active deployment that has an +eligible blue writer. The worker keeps its state on its own stack and performs +the following polling cycle: + +```text +check whether mysql.rds_topology exists + -> fetch topology metadata + -> classify the observation + -> update the deployment state machine + -> apply phase actions + -> wait for the effective check interval +``` + +After the table has been observed, the worker normally continues with metadata +fetches. `ER_NO_SUCH_TABLE` returns it to the table-existence check. + +The configured check interval is the baseline. The state machine uses 250 ms +while the deployment is `AVAILABLE` and 100 ms during the active writer +switchover phases. It returns to the configured baseline after writer +completion or state cleanup. + +### Blue/Green Mapping + +The worker builds a `bg_map` from current runtime configuration and the +discovered topology: + +- The blue writer is matched to the target writer using the RDS green-hostname + naming relationship. +- In explicit mode, eligible configured green readers are name-matched to blue + readers. +- The topology contains writer endpoints only, so complete reader mapping is + not assumed. +- Blue readers without a mapped green counterpart are handled independently + and may be shunned during post-processing. +- `OFFLINE_SOFT` and `OFFLINE_HARD` servers do not participate. + +Each pair retains the blue hostname and connection attributes, the green +hostname, the resolved green IP and its expiry, and whether DNS pinning and +connection draining have already completed for that pair. + +### Direct-Probe Tuple + +The direct green-writer probe always uses one coherent host, port, and TLS +tuple derived from the matched writer pair: + +| Mode | Host or IP | Port | TLS | +|---|---|---|---| +| Automatic | Resolved IP of the target endpoint from `mysql.rds_topology`. | Configured port of the matched blue writer. | `use_ssl` from the matched blue writer. | +| Explicit | Resolved IP of the exact configured target writer. | Configured port of the matched blue writer. | `use_ssl` from the exact eligible green writer row. | + +The monitor does not consume or validate the target topology row's port. +Source/target pairs that use different ports are unsupported. This is a +ProxySQL constraint, not an AWS guarantee that the ports are always equal. +Different configured pairs may use different ports. + +Automatic mode has no independent green `mysql_servers` row from which to +derive TLS configuration, so it uses the matched blue writer's value. + +Explicit mode selects the green writer by exact target hostname and the +matched-blue port. It copies `use_ssl` from an existing eligible row. If +discovery creates a missing row or restores an `OFFLINE_HARD` row, the +successful add path obtains `use_ssl` from the resulting exact row after +hostgroup defaults are applied. An `OFFLINE_SOFT` row remains ineligible. + +### Direct-Probe Lifetime + +The worker resolves each eligible green endpoint through the DNS cache or a +live DNS lookup. Once the green writer IP is available, the worker directs +subsequent topology polls to that IP. This keeps the observation path available +through the source connectivity gap and the retirement of the green DNS name. + +The worker retries unresolved pairs on every eligible equal-phase observation +from `AVAILABLE` through `WRITER_SWITCHOVER_POST_PROCESSING`. During repeated +post-processing observations, it pins and drains only pairs whose green IP is +available and whose `green_ip_pinned` flag is false. A completed pair is not +drained again during that worker lifetime; unresolved pairs remain eligible +for a later retry. + +If three consecutive topology polls to the direct green IP fail, the worker +clears the direct target, removes its mapped blue DNS pins, purges the +corresponding monitor connections, and falls back to polling through the blue +configuration. + +## ProxySQL State Machine + +The monitor uses the following ordered states: ```text NONE @@ -276,118 +343,100 @@ NONE -> NONE ``` -`SOURCE-CODE`: The current handler converts the target status and compares enum -ordering with the stored status. A lower value is treated as a backward -transition: the handler runs the current rollback cleanup, and only an observed -`AVAILABLE` phase is then re-entered and initialized. Forward transitions do -not require their immediate predecessor, so a worker can still first observe -`WRITER_SWITCHOVER_POST_PROCESSING` and perform setup in that phase. +The five states from `AVAILABLE` through +`WRITER_SWITCHOVER_COMPLETED` correspond to AWS target statuses. +`READER_SWITCHOVER_IN_PROGRESS` and `SWITCHOVER_COMPLETED` are ProxySQL states: -`SOURCE-CODE`: An unknown non-empty target status converts to `NONE`. From an -active higher-valued state this is treated as a backward transition and invokes -rollback cleanup. An empty target status takes an earlier path that directly -sets `NONE` and does not invoke rollback cleanup. A special guard ignores a -repeated raw `WRITER_SWITCHOVER_COMPLETED` while local state is -`READER_SWITCHOVER_IN_PROGRESS`. A newly observed -`WRITER_SWITCHOVER_COMPLETED` is first stored and then immediately advanced to -`READER_SWITCHOVER_IN_PROGRESS` in the same handler call. +- `READER_SWITCHOVER_IN_PROGRESS` records that writer completion has been + observed and defers reader cleanup until topology drains. +- `SWITCHOVER_COMPLETED` is a short-lived successful-cleanup state before the + worker returns to `NONE`. -`SOURCE-CODE`: `READER_SWITCHOVER_IN_PROGRESS` and `SWITCHOVER_COMPLETED` are -ProxySQL-inferred and cleanup states, not raw AWS status strings. +The arrows show nominal lifecycle order, not mandatory predecessor edges. +Forward transitions may skip states. A worker that first observes +`SWITCHOVER_IN_POST_PROCESSING`, for example, builds the required mapping and +performs post-processing setup directly. -`SOURCE-CODE`: Phase transitions currently perform these actions: +### Phase Actions -| Phase or observation | Current action | +| State or observation | Action | |---|---| -| `AVAILABLE` | Set the next-check interval to 250 ms; build the blue/green mapping; resolve green IPs; optionally add the green writer to its configured hostgroup. | -| `WRITER_SWITCHOVER_INITIATED` | On transition, set the next-check interval to 100 ms; invoke mapping, green-IP resolution, and optional green-writer setup; suppress read-only checks for the deployment hostgroups. | -| `WRITER_SWITCHOVER_IN_PROGRESS` | On transition, set the next-check interval to 100 ms; invoke the same setup; suppress read-only checks; demote the mapped blue writer to read-only. | -| `WRITER_SWITCHOVER_POST_PROCESSING` | On transition, set the next-check interval to 100 ms; invoke the same setup; enable or sustain read-only suppression; pin mapped blue names to resolved green IPs; drain matching connections; configure writer placement; shun unmapped readers. | -| `WRITER_SWITCHOVER_COMPLETED` | Enter the inferred reader phase; remove the writer DNS-cache entry; retain mapped reader pins until the topology drains; reset the next-check interval override to the baseline value of `0`. | -| `READER_SWITCHOVER_IN_PROGRESS` plus empty or absent topology | Run successful cleanup: reconcile configured writer membership in the reader hostgroup; unshun recorded readers; remove DNS-cache entries and purge monitor-pool connections for recorded shunned readers and all mapped pairs; drain configured green hostgroups; clear worker bookkeeping; transition through `SWITCHOVER_COMPLETED` to `NONE`. | -| Empty or absent topology from any other non-`NONE` state | Run rollback cleanup: conditionally restore a writer demoted during `WRITER_SWITCHOVER_IN_PROGRESS` or `WRITER_SWITCHOVER_POST_PROCESSING`; reconcile blue reader membership and shuns; remove blue DNS-cache entries; purge related blue monitor-pool connections; leave green rows, statuses, DNS entries, and connections unchanged; clear worker bookkeeping; then enter `NONE`. | -| Recognized backward status transition | Run rollback cleanup. If the new raw status is `AVAILABLE`, re-enter `AVAILABLE`, set the 250 ms interval, and rebuild mapping, resolution, and optional green-writer placement. Other backward statuses leave the worker in `NONE` after cleanup. | -| Worker exit with non-`NONE` state | Run rollback cleanup before destroying the worker-local state. A replacement worker starts with a new state instance. | - -### Current Probe Tuple - -`SOURCE-CODE, AUTHOR-ACCEPTED-POLICY`: When direct probing is active, the -worker takes the port from the writer pair in `bg_map`, not from an arbitrary -blue polling row. That pair records the configured blue writer's port. The -implementation does not consume or validate the TARGET topology row's port; -the author explicitly accepts that a matched source/target pair with different -ports is unsupported. - -`SOURCE-CODE, AUTHOR-ACCEPTED-POLICY`: Automatic mode has no independent green -`mysql_servers` row from which to read TLS configuration, so it intentionally -uses the matched blue writer's `use_ssl` value. - -`SOURCE-CODE, AUTHOR-ACCEPTED-POLICY`: Explicit mode selects an eligible green -writer row by exact TARGET hostname and the matched blue writer's port. Map -construction copies `use_ssl` from an existing row. When discovery creates a -missing row or restores an `OFFLINE_HARD` row, the successful add path copies -the exact row's resolved `use_ssl` after hostgroup defaults are applied. A -valid initially empty configured green writer hostgroup produces no warning. -An `OFFLINE_SOFT` row remains ineligible and retains the matched-blue TLS -fallback. Simulator coverage for both row paths is assigned to PR6. - -### Current Phase-Equality Behavior - -`SOURCE-CODE`: After status conversion, when the converted status equals the -stored status, the handler retries green-IP resolution from `AVAILABLE` through -`WRITER_SWITCHOVER_POST_PROCESSING`. It then returns without rebuilding the -configuration-derived pair map or rerunning the full phase action. - -`SOURCE-CODE`: During an equal `WRITER_SWITCHOVER_POST_PROCESSING` observation, -the handler pins and drains only pairs whose green IP is available and whose -worker-local `green_ip_pinned` flag is false. Unresolved pairs remain eligible -for the next observation, while completed pairs are skipped for the remainder -of that worker generation. - -### Current Connection-Retirement Behavior - -`SOURCE-CODE`: `MySrvConnList::mark_connections_unhealthy()` marks every used -connection selected by a BGD drain with both `healthy=false` and -`reusable=false`. Free connections are deleted immediately. The intended used -connection lifecycle is therefore retirement after its current owner releases -it, not cancellation of an in-flight query solely because the drain began. - -`SOURCE-CODE`: `MySQL_Connection::reset()` currently assigns both -`healthy=true` and `reusable=true`. The author observes that the two backend -reset call sites, `handler_again___status_RESETTING_CONNECTION` and -`handler_again___status_CHANGING_USER_SERVER`, are surrounded by logic that -destroys rather than reuses the affected backend connection. That observation -reduces the known exposure, but the terminal nature of a BGD drain remains -implicit and distributed across callers. - -`AUTHOR-ACCEPTED-POLICY, IMPLEMENTATION-CONFORMANCE-OPEN`: The follow-up uses -the existing `healthy` field rather than introducing a second retirement flag: - -1. `MySQL_Connection::reset()` resets session state but does not change - `healthy` from false to true. -2. `MySQL_Thread::push_MyConn_local()` checks `healthy` before adding a - connection to the thread-local cache. An unhealthy connection is sent to - the global destruction path and cannot enter `cached_connections`. -3. `MySQL_HostGroups_Manager::push_MyConn_to_pool()` checks `healthy` after - removing the connection from `ConnectionsUsed` and destroys an unhealthy - connection before any optimization or insertion into `ConnectionsFree`. -4. `push_MyConn_to_pool_array()` remains covered because reusable entries - delegate to `push_MyConn_to_pool()`; the existing `reusable=false` branch - already destroys a drained connection directly. - -The accepted state flow is: +| `AVAILABLE` | Set the next-check interval to 250 ms, build the blue/green map, resolve green IPs, and optionally add the green writer to its configured hostgroup. | +| `WRITER_SWITCHOVER_INITIATED` | Set the interval to 100 ms, rebuild the map, resolve green IPs, optionally add the green writer, and suppress ordinary read-only monitoring for the deployment servers. | +| `WRITER_SWITCHOVER_IN_PROGRESS` | Perform initiated-phase setup, sustain read-only suppression, and demote the mapped blue writer to read-only. | +| `WRITER_SWITCHOVER_POST_PROCESSING` | Perform setup even after late entry, sustain read-only suppression, pin mapped blue names to resolved green IPs, retire matching connections, configure writer placement, and shun unmapped blue readers. | +| `WRITER_SWITCHOVER_COMPLETED` | Advance immediately to `READER_SWITCHOVER_IN_PROGRESS`, remove the writer DNS-cache entry, retain mapped reader pins, and restore the baseline check interval. | +| Topology drain in `READER_SWITCHOVER_IN_PROGRESS` | Run successful cleanup, transition briefly through `SWITCHOVER_COMPLETED`, and return to `NONE`. | +| Topology drain in another active state | Run rollback cleanup and return to `NONE`. | +| Worker exit in an active state | Run rollback cleanup before discarding worker-local state. | + +### Equal And Repeated Phases + +When the converted target status equals the stored state, the worker does not +rebuild the configuration-derived map or repeat the complete phase action. +It performs only the eligible same-phase reconciliation: + +- Retry unresolved green IPs from `AVAILABLE` through post-processing. +- During post-processing, pin and drain newly resolved pairs once. +- Ignore repeated raw `SWITCHOVER_COMPLETED` observations after the local state + has advanced to `READER_SWITCHOVER_IN_PROGRESS`. + +### Backward And Unrecognized Phases + +A recognized target status whose enum value is lower than the stored state is a +backward transition. The worker runs rollback cleanup. If the new status is +`AVAILABLE`, it then re-enters `AVAILABLE`, restores the 250 ms interval, and +rebuilds mapping, resolution, and optional green-writer placement. Other +backward statuses leave the worker in `NONE`. + +An unknown nonempty target status maps to `NONE`. From an active higher state, +that value follows the backward-transition path and invokes rollback. An empty +target status, a missing target row, or a topology result not classified as +blue/green sets the phase to `NONE` and resets the interval without calling the +rollback helper. The latter path does not perform the rollback helper's map, +DNS, connection, or reader cleanup. + +## Switchover Effects + +### DNS Pinning And Reader Handling + +During writer post-processing, ProxySQL pins each mapped blue hostname to the +resolved green IP and retires existing connections for that blue endpoint. +New connections through the stable blue hostname then reach the green +instance. + +The metadata topology does not provide a complete reader mapping. During +post-processing, ProxySQL identifies eligible blue readers with no mapped +green counterpart and marks them `SHUNNED_AWS_BGD`. If shunning all blue +readers would leave the reader hostgroup empty, the worker temporarily makes +the writer available to the reader hostgroup. Final cleanup restores placement +according to `writer_is_also_reader` and unshuns the readers recorded by that +worker. + +After writer completion, the writer DNS-cache entry is removed immediately so +normal DNS resolution can resume for the stable writer name. Reader pins remain +until the topology drains and successful cleanup runs. + +### Connection Retirement + +`MySrvConnList::mark_connections_unhealthy()` deletes matching free +connections immediately. It marks matching used connections with +`healthy=false` and `reusable=false`; those connections finish their current +ownership and are destroyed when released. + +The retirement state flow is: ```text ACTIVE_BACKEND healthy=true, reusable=true | - | BGD drain marks a used connection + | BGD drain selects a used connection v RETIRE_ON_RELEASE healthy=false, reusable=false | - | optional connection/session reset - | (healthy remains false) + | optional connection or session reset + | healthy remains false v POOL_RETURN | @@ -395,636 +444,274 @@ POOL_RETURN | `--> push_MyConn_to_pool: unhealthy -> delete -No transition returns RETIRE_ON_RELEASE to a free or local pool. +No transition returns RETIRE_ON_RELEASE to a local or global free pool. ``` -`AUTHOR-ACCEPTED-POLICY`: Reads of `healthy` in these paths use the same -unlocked connection-field convention used elsewhere in ProxySQL. The author -accepts that race model for this focused fix. This decision does not assert -that a C++ data race is generally safe or introduce a broader locking policy. - -### Current Topology-Absence Behavior - -`SOURCE-CODE`: If local state is `READER_SWITCHOVER_IN_PROGRESS`, -`aws_rds_bgd_handle_topology_absent` runs the full current successful cleanup. -For every other non-`NONE` state, it calls the same cleanup helper with -`rollback=true`. - -`SOURCE-CODE`: Rollback conditionally moves a writer demoted during -`WRITER_SWITCHOVER_IN_PROGRESS` or `WRITER_SWITCHOVER_POST_PROCESSING` back to -the writer role. It then runs the common completion hostgroup action, unshuns -recorded readers, removes DNS-cache entries and purges monitor-pool connections -for recorded shunned readers and all mapped blue pairs, clears the worker -bookkeeping, and enters `NONE`, which clears read-only suppression. Rollback -does not drain green connections, remove green DNS entries, change green -statuses, or remove green rows. - -`SOURCE-CODE, AUTHOR-ACCEPTED-POLICY`: Successful cleanup drains connections -for every server in the configured green writer and reader hostgroups except -`OFFLINE_SOFT` and `OFFLINE_HARD` servers. It also removes those green -hostnames from the DNS and monitor connection caches. It leaves all green -server rows and statuses unchanged. The author assigns membership cleanup to -the administrator, including for a row automatically added to runtime by BGD. - -`SOURCE-CODE`: The current rollback is a one-shot best-effort procedure. Its -effect operations do not return an action result to this controller, there is no -owned effect ledger, and the worker state is cleared even when external state -has not been verified. - -`AUTHOR-ACCEPTED-POLICY`: This loss of per-effect completion and retry state is -intentional. The author accepts the possibility that process termination during -cleanup prevents the controller from proving that every intended postcondition -was reached. The retained-ledger design below remains a reviewer proposal, not -accepted follow-up work. - -`SOURCE-CODE`: The interval result depends on the caller path: - -- A metadata fetch reporting `ER_NO_SUCH_TABLE` sets - `next_check_interval_ms` to `0` before calling the helper. -- Successful or rollback cleanup resets the interval as part of state cleanup. -- If the helper is called while state is already `NONE`, it performs no cleanup; - an existence query or successful empty metadata query does not independently - reset an existing interval override in that case. - -`SOURCE-CODE`: PR 1 documents this current behavior. PR 1 does not change this. - -### Current Worker Lifetime - -`SOURCE-CODE`: State lives on the per-writer-hostgroup worker stack. The worker -and dispatcher compare a generation checksum that combines eligible blue and -green runtime rows. An Admin `mysql_servers` commit refreshes the checksum; -when its value changes, the old worker exits and the dispatcher creates a -replacement. If old state is non-`NONE`, the exit path runs one-shot rollback -cleanup before discarding it. The mapping, shunned-reader records, probe -target, and cleanup identities are not transferred; the replacement starts -with fresh state and rebuilds its map from current runtime configuration. - -`SOURCE-CODE`: The combined checksum fixes the earlier case in which an Admin -commit adding an eligible green row could leave a nonempty partial map alive. -It is a configuration-generation signal, not a per-effect result ledger and -not a retry trigger for DNS recovery. In-process BGD calls to -`publish_mysql_servers_to_runtime()` do not refresh this generation checksum; -that permits the current worker's own hostgroup actions to continue without -self-replacement. - -`AUTHOR-ACCEPTED-POLICY`: Cleanup-on-detach followed by fresh worker state is -the selected worker-replacement contract. A replacement whose first -observation is `SWITCHOVER_COMPLETED` does not reconstruct the prior worker's -map or effect ownership; it enters `READER_SWITCHOVER_IN_PROGRESS` and waits -for topology drain. - -### Current Source Anchors - -`SOURCE-CODE`: The source entry points for the current mechanics are -`parse_aws_rds_topology`, `handle_aws_rds_bgd`, -`aws_rds_bgd_handle_topology_absent`, -`handle_aws_rds_bgd_post_switchover`, and `monitor_RDS_BGD_thread_HG`. These -entry points should be reviewed with this document whenever behavior changes. - -## External Effects And Cleanup Ledger - -`PROPOSED-POLICY`: This section records the reviewer's stronger recovery model -for comparison and possible future reconsideration. The author explicitly -selected one-shot cleanup, worker-local state, and no durable BGD ledger. None -of the ledger states or invariants below is therefore an accepted requirement -for PR #5861 or the same-phase reconciliation follow-up. - -`PROPOSED-POLICY`: Every externally visible effect must have a stable identity -and a cleanup record before the effect is considered applied. - -`PROPOSED-POLICY`: Every mutable effect uses the same stable ownership key: -deployment ID, deployment generation, action ID, and resource ID. The resource -ID identifies the effect-specific resource and does not change when its value -changes. Mutable result data is recorded separately and includes the before -value, intended or applied value, last observed value, and command result. In -particular, a resolved IP, its resolution source, and its expiry are result -data, not stable identity. - -`PROPOSED-POLICY`: An effect has one of these states: - -- `PENDING`: The intended value is not yet verified as applied. -- `APPLIED`: The intended value is verified, but cleanup or handoff remains. -- `REVERTED`: Compare-and-restore verified the owned effect was undone. -- `COMMITTED`: An irreversible effect was verified complete, or ownership of a - retained effect was explicitly handed off to desired runtime configuration. -- `CONFLICT`: The resource no longer has the value applied by this owner, so - automatic restoration would overwrite a newer value or owner. - -`PROPOSED-POLICY`: The active cleanup ledger contains only unsettled, -controller-owned effects in `PENDING`, `APPLIED`, or `CONFLICT`. `REVERTED` and -`COMMITTED` entries leave the active ledger; they may remain as audit -tombstones outside it. - -`PROPOSED-POLICY`: Cleanup uses compare-and-restore. An inverse action is -applied only when the resource still equals the applied value owned by the -ledger entry. Otherwise the effect becomes `CONFLICT`, remains in the active -ledger, and exposes `FAULTED`; cleanup does not overwrite newer configuration -or another owner. - -`PROPOSED-POLICY`: A transitional effect superseded by the accepted desired -post-success state becomes `COMMITTED` only after the controller verifies an -explicit handoff of the resource and its intended value into desired runtime -configuration. A conflict or permanent inability to complete that handoff -enters `FAULTED` with the active ledger retained. - -| Effect | Required identity | Successful postcondition | Required recovery | -|---|---|---|---| -| Blue/green mapping | Common owner key; resource identity is the blue endpoint, green endpoint, and role within the deployment. | All required endpoints are mapped; endpoint and role values are recorded as result data. | Clear the owned mapping or reconstruct it from a new complete observation. | -| Green resolution | Common owner key; resource identity is the green hostname and probe purpose. | A complete writer probe target is available; resolved IP, source, and expiry are recorded as result data. | Retry resolution or clear the incomplete owned result. | -| Direct probe | Common owner key; resource identity is the deployment probe slot, with host or IP, port, and SSL mode recorded as intended and applied result data. | Topology checks use the mapped writer endpoint. | Compare-and-restore the configured blue probe candidates. | -| Green writer placement | Common owner key; resource identity is the hostgroup and server. | The intended server is present with the mapped options, recorded as the applied value. | Compare-and-restore the prior placement, or mark `COMMITTED` only after explicit handoff of the retained placement into desired runtime configuration. | -| Monitor suppression | Common owner key; resource identity is the affected hostgroup and monitor scope. | Read-only monitoring skips only the intended servers. | Compare-and-restore the prior suppression value. | -| Blue writer demotion | Common owner key; resource identity is the server and affected placement; original and applied role and placement are result data. | The temporary role and placement are visible. | During rollback, compare-and-restore the original role and placement. After observed writer completion, successful finalization or `SAFE_TEARDOWN` reconciles and hands off the role and placement to accepted desired post-switchover runtime configuration, then marks the effect `COMMITTED`; it never restores obsolete blue solely because completion was observed or configuration was removed. A conflict or permanent inability to settle enters `FAULTED` with the active ledger retained. | -| DNS pin | Common owner key; resource identity is the hostname; pinned IP is applied result data. | Lookup returns the owned pinned IP. | Compare-and-restore only when the action owner still owns the pin. | -| Connection drain | Common owner key; resource identity is the server and drain generation. | Every connection predating the drain generation is verified retired and cannot be reused. | Never revert; mark `COMMITTED` only after every connection predating the generation is verified retired. | -| Reader shun | Common owner key; resource identity is the hostgroup, hostname, and port; previous status is before-value result data. | The intended reader is `SHUNNED_AWS_BGD`. | Compare-and-restore the recorded prior status. | -| Writer reader-hostgroup membership | Common owner key; resource identity is the writer server and reader hostgroup; original and applied membership are result data. | Membership matches the transition policy. | Compare-and-restore the configured membership. | - -`PROPOSED-POLICY`: The effect ledger is stored as keyed sets or maps. Repeated -observations cannot create duplicate records or ambiguous effect ownership. - -## Required Controller Invariants - -`PROPOSED-POLICY`: The hardened controller must maintain the following safety -and liveness invariants. They describe intended behavior, not the current -implementation. - -### Safety - -1. `IDLE` has no unsettled temporary effects in the active cleanup ledger; - settled audit tombstones may remain outside it. -2. Every visible controller-owned effect remains in the active cleanup ledger - until it is `REVERTED` or `COMMITTED`. -3. Worker termination cannot destroy the only cleanup record for an effect. -4. Repeated observations retry incomplete actions. -5. Completed actions are not reapplied without a new action generation. -6. An unknown or malformed observation, or a query failure, preserves the last - safe state and causes no destructive transition. -7. Topology disappearance before observed writer completion selects rollback. -8. Successful finalization requires observed writer completion. -9. A drained connection cannot become reusable. -10. A direct target includes the writer host or IP, port, and SSL mode. -11. Persistent configuration, runtime configuration, the hostgroup manager, - and exported configuration agree on nullability. -12. Rollback and successful finalization are idempotent. -13. A stale worker cannot apply a result to a newer deployment generation. -14. No deployment-registry lock is held during DNS, SQLite, hostgroup-manager, - connection-pool, or socket operations. - -### Liveness - -`PROPOSED-POLICY`: Liveness holds under fair scheduling, eventual recovery of -retryable dependencies, and unchanged effect ownership. A permanent failure or -ownership conflict converges to externally visible `FAULTED` with the active -ledger retained, rather than an unsafe overwrite or infinite silent retry. - -1. Transient DNS failure remains retryable. -2. Under these liveness conditions, a cancelled deployment eventually restores - the blue configuration; a permanent failure or ownership conflict instead - exposes `FAULTED` while retaining its active ledger. -3. Under these liveness conditions, successful completion eventually removes - temporary pins and shuns; a permanent failure or ownership conflict instead - exposes `FAULTED` while retaining its active ledger. -4. Worker replacement resumes the deployment or safely rolls it back. -5. Failure for one blue/green pair does not hide other pairs. -6. Fast polling is bounded and has an observable reason. -7. Configuration removal cannot abandon outstanding effects. - -## Proposed Controller Model - -`PROPOSED-POLICY`: Controller state is separate from the raw AWS status. The -following state diagram is a reviewer-proposed alternative; it is not the -current implementation enum or an author-accepted implementation contract. +`MySQL_Connection::reset()` resets session state without restoring +`healthy=true`. `MySQL_Thread::push_MyConn_local()` rejects an unhealthy +connection before adding it to the thread-local cache. +`MySQL_HostGroups_Manager::push_MyConn_to_pool()` removes the connection from +the used list and destroys it before any free-pool insertion. Array pool return +is covered because reusable entries delegate to the same global return path, +while the existing non-reusable branch destroys the connection directly. + +These paths read `healthy` using ProxySQL's existing unlocked +connection-field convention. The feature accepts that project-level race model +and does not introduce a separate retirement flag or a broader locking policy. + +## Completion And Rollback + +### Phase Selection + +`aws_rds_bgd_handle_topology_absent()` selects cleanup from the last monitor +state: ```text -IDLE - -> TRACKING - -> PREPARING - -> CUTOVER - -> REPOINTING - -> AWAITING_READER_DNS - -> FINALIZING_SUCCESS - -> IDLE only when the active cleanup ledger is settled and empty - -Any active state + topology disappears before observed writer completion - -> ROLLING_BACK - -> IDLE only when the active cleanup ledger is settled and empty - -Any state at or after observed writer completion + configuration removal - -> SAFE_TEARDOWN - -> IDLE only when the active cleanup ledger is settled and empty - -Any state + permanent failure or unrecoverable inconsistency - -> FAULTED with the active cleanup ledger and deployment context retained +topology drains + | + +--> state is READER_SWITCHOVER_IN_PROGRESS + | -> successful cleanup + | + +--> state is another non-NONE state + | -> rollback cleanup + | + `--> state is NONE + -> no cleanup ``` -### Observation-Driven Transitions - -`PROPOSED-POLICY`: The controller applies the following transitions from -observations and lifecycle events. The accepted reader-completion signal is a -symbolic policy input pending author validation; it may ultimately be defined -as empty or absent topology after observed writer completion. - -`PROPOSED-POLICY`: Each deployment context has a configuration-management mode -orthogonal to its raw AWS phase. `ACTIVE` means validated configuration still -manages the deployment. `CONFIG_DISABLED` is the explicit-disable form of the -existing `CONFIG_REMOVED` lifecycle event. Either event latches -`REMOVAL_REQUESTED` as a cleanup request. Raw topology observations never clear -`REMOVAL_REQUESTED`. Only an explicit, validated re-add or re-enable of the -same deployment under a new configuration generation may request a return to -`ACTIVE`, and only after ownership and configuration reconciliation succeeds. - -`PROPOSED-POLICY`: Permanent-failure and ownership-conflict rules have highest -precedence. The latched management mode and `ROLLING_BACK` rules are evaluated -next, before repeated, regressed, or generic forward-phase mappings. An -eligible forward-mapping state therefore excludes `ROLLING_BACK`, `FAULTED`, -`FINALIZING_SUCCESS`, and `SAFE_TEARDOWN`. - -`PROPOSED-POLICY`: The normal recognized-phase mappings below apply only to an -initial or forward observation. An observation equal to the recorded phase -uses the repeated-phase rule. An observation lower than the highest trusted -completion evidence uses the regression rule and never causes a reverse -transition. For a newly observed deployment, its initial `IDLE` context is a -pre-completion nonterminal context for these mappings. - -| Controller state and input | Required transition or action | -|---|---| -| Any + permanent failure or ownership conflict | Enter `FAULTED` with the active ledger and deployment context retained. | -| Any + `QUERY_FAILED` | Preserve state and effects, then retry. | -| Any + `UNKNOWN_STATUS` or `MALFORMED_TOPOLOGY` | Preserve state, expose the input, and make no destructive transition; action-result or error policy may enter `FAULTED` when the condition is classified permanent. | -| `ROLLING_BACK` + any pre-completion recognized phase | Remain in `ROLLING_BACK` and record the observation for diagnostics; do not resume cutover unless validated configuration explicitly re-enables the deployment and ownership reconciliation accepts it. | -| `ROLLING_BACK` + `WRITER_COMPLETED` | Stop or cancel pending rollback commands that would restore obsolete blue or remove the promoted target, latch writer-completion evidence, and enter `SAFE_TEARDOWN` when management mode is `REMOVAL_REQUESTED`; otherwise select an author-validated post-completion recovery or finalization path. | -| `ROLLING_BACK` + `TOPOLOGY_ABSENT` or `TOPOLOGY_EMPTY` | Remain in `ROLLING_BACK` and continue reconciling rollback effects. | -| `REMOVAL_REQUESTED` + any observation at or after writer completion | Enter or remain in `SAFE_TEARDOWN`; never apply a generic `AWAITING_READER_DNS` transition or restore blue solely from the raw phase. | -| Any + repeated observation of the same phase | Keep the controller state and reconcile incomplete effects. | -| Any + a regressed recognized phase | Preserve the highest trusted completion evidence and active effects, expose the regression, and make no reverse destructive transition until an author-validated policy decides how to handle it. | -| `IDLE` or `TRACKING` + `AVAILABLE` | Enter or remain in `TRACKING`; reconcile mapping, resolution, and preparation without applying cutover effects. | -| Any eligible forward-mapping state + `WRITER_INITIATED` | Enter `PREPARING`; record the latest observation and reconstruct or reconcile prerequisites. | -| Any eligible forward-mapping state + `WRITER_IN_PROGRESS` | Enter `CUTOVER`; record the latest observation and reconstruct or reconcile prerequisites and required cutover actions. | -| Any eligible forward-mapping state + `WRITER_POST_PROCESSING` | Enter `REPOINTING`; record the latest observation and reconstruct or reconcile all unmet prerequisites and repoint actions. | -| Any eligible forward-mapping state + `WRITER_COMPLETED` | Enter `AWAITING_READER_DNS`; record writer-completion evidence, then reconstruct and verify every unmet prerequisite effect or establish that it is obsolete under an author-validated policy; never infer that a skipped action succeeded. | -| Any pre-completion state + `TOPOLOGY_ABSENT` or `TOPOLOGY_EMPTY` | Enter `ROLLING_BACK`. | -| `ACTIVE` + `AWAITING_READER_DNS` + `ACCEPTED_READER_COMPLETION_SIGNAL` | Enter `FINALIZING_SUCCESS`. | -| `ACTIVE` + `CONFIG_CHANGED` | Preserve the deployment context and active ledger, then reconcile validated new configuration. | -| Any pre-completion state + `CONFIG_REMOVED` or `CONFIG_DISABLED` | Latch `REMOVAL_REQUESTED` and enter `ROLLING_BACK`. | -| Any state at or after observed writer completion + `CONFIG_REMOVED` or `CONFIG_DISABLED` | Latch `REMOVAL_REQUESTED` and enter `SAFE_TEARDOWN`; never restore blue solely because configuration was removed or disabled. | -| `REMOVAL_REQUESTED` + ordinary `CONFIG_CHANGED` or raw topology | Preserve `REMOVAL_REQUESTED`; do not resume generic forward processing. | -| `REMOVAL_REQUESTED` + explicit validated re-add or re-enable | Start a new configuration generation, reconcile ownership and configuration, and return to `ACTIVE` at the controller state appropriate to retained trusted evidence only after reconciliation accepts ownership. | -| Any + `WORKER_RESTARTED` | Attach the new worker generation to the same deployment context, state, and active ledger. | -| `ROLLING_BACK` + active ledger settled and empty | Enter `IDLE`. | -| `FINALIZING_SUCCESS` or `SAFE_TEARDOWN` + active ledger settled and empty | Enter `IDLE`. | - -`PROPOSED-POLICY`: Every action execution returns one classified result: - -- `SUCCEEDED`: The command changed the owned resource and verification observed - the intended value. -- `ALREADY_SATISFIED`: Verification found the owned intended value without - needing to repeat the command. -- `RETRYABLE_FAILURE`: The effect remains pending for a later reconciliation. -- `PERMANENT_FAILURE`: Policy cannot safely complete or retry the effect; enter - `FAULTED` with the active ledger retained. -- `OWNERSHIP_CONFLICT`: The resource does not equal the value applied by this - owner; retain the entry as `CONFLICT` and enter `FAULTED` without overwriting - it. -- `STALE_RESULT`: The result belongs to an older deployment or worker - generation and must not mutate current state or effects. - -`PROPOSED-POLICY`: Repetition count alone does not make a failure permanent. -The executor classification and author-validated error policy determine -whether a failure is retryable or permanent. - -### Reconciliation - -`PROPOSED-POLICY`: Phase changes update controller status and emit an -observable log record, but actions are reconciled on every poll. -Actions returning `RETRYABLE_FAILURE` remain pending. Completed actions return -`ALREADY_SATISFIED` instead of repeating the effect, and stale results are -discarded. Advancing an observation does not prove that its actions completed. - -### Successful Finalization - -`PROPOSED-POLICY`: Successful finalization is selected only when management -mode remains `ACTIVE` after observed writer completion and the controller has -the reader-completion signal accepted by policy after author validation. It -then removes temporary DNS pins, reconciles reader status and writer -reader-hostgroup membership, reconciles writer role and placement with the -accepted desired post-switchover runtime configuration, drains obsolete -green-hostgroup connections, clears the direct probe and monitor suppression, -and clears mapping and resolution records. Successfully restored reversible -entries become `REVERTED`. Connection drains become `COMMITTED` only after -every connection predating the drain generation is verified retired. Any -retained green placement becomes `COMMITTED` only after explicit handoff into -desired runtime configuration. The blue-writer-demotion record becomes -`COMMITTED` only after the controller verifies handoff of writer role and -placement into the accepted desired post-switchover runtime configuration; it -never restores obsolete blue solely after observed writer completion. A -conflict or permanent inability to complete that handoff enters `FAULTED` with -the active ledger retained. The controller enters `IDLE` only when the active -cleanup ledger is settled and empty; settled audit tombstones may remain -outside it. - -`PROPOSED-POLICY`: Successful finalization uses the ordered per-effect -settlement checklist in **Safe Teardown** wherever it applies. Its terminal -desired configuration is the accepted `ACTIVE` post-switchover configuration, -rather than removal intent, but it uses the same evidence gate, ownership -checks, dependency ordering, and `FAULTED` behavior. - -### Safe Teardown - -`PROPOSED-POLICY`: `SAFE_TEARDOWN` is selected when management mode is -`REMOVAL_REQUESTED` at or after observed writer completion. It settles effects -against the accepted terminal post-switchover removal intent and never restores -blue merely because configuration was removed or disabled. - -`PROPOSED-POLICY`: Reader-related DNS pins, reader shuns, writer -reader-hostgroup membership, and direct-probe protection cannot be cleared -before the accepted reader-completion evidence or an author-validated -equivalent is observed. The dispatcher-owned cleanup executor continues -observing through a retained complete probe target and deployment context even -after the configured worker is removed. If accepted evidence cannot be -obtained, or an effect cannot be settled safely because of permanent failure or -ownership conflict, the controller enters externally visible `FAULTED`, -retains the active ledger and context, and does not clear effects merely to -reach `IDLE`. - -`PROPOSED-POLICY`: After the evidence gate is satisfied, the controller settles -effects in this order: - -1. DNS pins: Remove each pin only when the action owner still owns its applied - value. -2. Reader shuns: Compare-and-restore each prior value or hand it off to the - accepted terminal desired configuration. A missing resource that removal - intent deliberately deleted may be `ALREADY_SATISFIED` only after ownership - validation. -3. Writer reader-hostgroup membership, blue writer demotion, and - green writer placement: Reconcile and hand them off to the accepted terminal - post-switchover configuration or removal intent, then mark them `COMMITTED`. - An ownership conflict or permanent settlement failure enters `FAULTED`. -4. Monitor suppression: Clear it only after DNS, reader, and routing-placement - protection are settled. -5. Direct probe: Clear it only after it is no longer needed to obtain evidence - or complete cleanup. -6. Connection drain: Mark it `COMMITTED` only after every connection predating - the drain generation is verified retired. -7. Blue/green mapping and green resolution records: Commit or clear them only - after every dependent effect is settled. - -`PROPOSED-POLICY`: The deployment context enters `IDLE` and becomes eligible -for removal only when the active cleanup ledger is empty. - -### Cancellation Rollback - -`PROPOSED-POLICY`: Topology absence before observed writer completion enters -rollback. Rollback removes owned DNS pins; restores the configured blue probe -candidates, recorded reader statuses, and recorded hostgroup placement; -compare-and-restores the original blue writer role and placement; clears -monitor suppression; removes temporary green placement according to the -validated placement policy; and clears mapping and resolution records. The -controller marks a reversible entry `REVERTED` only after compare-and-restore -verifies the inverse. An irreversible drain or explicitly handed-off retained -placement becomes `COMMITTED` under the ledger rules. The controller enters -`IDLE` only when the active cleanup ledger is settled and empty; a conflict -instead retains the active ledger in `FAULTED`. - -### Query And Data Errors - -`PROPOSED-POLICY`: Query and data errors select the following recovery -behavior; they do not imply successful completion. - -| Condition | Required controller behavior | -|---|---| -| Query timeout or connection failure | Preserve state and retry. | -| Unknown status | Preserve state, expose the unknown value, and make no destructive transition. | -| Malformed topology | Preserve state, expose the malformed input, and make no destructive transition. | -| DNS failure | Keep the resolution action pending. | -| Required mapping missing | Keep the mapping action pending. | -| One reader action fails | Retain completed reader actions and retry the failed reader action. | -| Permanent action failure | Enter `FAULTED` with the active cleanup ledger retained. | -| Ownership conflict | Enter `FAULTED` with the conflicting entry and active ledger retained; do not overwrite the resource. | -| Repeated rollback failure | Remain in `ROLLING_BACK` while classified retryable, or enter `FAULTED` when classified permanent; repetition count alone is not permanent, and the controller never enters `IDLE` with an unsettled entry. | +The same rollback helper also runs for a recognized backward phase and for +worker exit from an active state. + +### Rollback Cleanup + +Rollback performs the following one-shot actions: + +- Restore a blue writer demoted during + `WRITER_SWITCHOVER_IN_PROGRESS` or + `WRITER_SWITCHOVER_POST_PROCESSING`. +- Reconcile writer membership in the reader hostgroup according to + `writer_is_also_reader`. +- Unshun readers recorded by the worker. +- Remove DNS-cache entries and purge monitor-pool connections for recorded + shunned readers and mapped blue endpoints. +- Purge direct-probe monitor connections keyed by resolved green IPs. +- Clear worker mapping, probe, interval, and phase bookkeeping. +- Invoke read-only suppression cleanup and return to `NONE`. + +Rollback intentionally does not: + +- Drain application connections in configured green hostgroups. +- Remove green DNS entries. +- Change green server status. +- Remove user-configured or automatically added green rows. + +Purging a direct-probe monitor connection keyed by a green IP cleans up the +monitor's observation channel. It is not equivalent to draining application +connections from a green hostgroup. + +### Successful Cleanup + +Successful cleanup does not restore the obsolete blue writer. It reconciles +writer membership in the reader hostgroup according to +`writer_is_also_reader`, unshuns the readers recorded by the worker, removes +mapped and recorded-reader DNS entries, purges the corresponding monitor-pool +connections, invokes read-only suppression cleanup, and clears worker +bookkeeping. It also drains application connections for every eligible server +in the configured green writer and reader hostgroups and removes those green +hostnames from the DNS and monitor connection caches. + +`OFFLINE_SOFT` and `OFFLINE_HARD` green servers are excluded from that drain. +Successful cleanup leaves every green server row and status unchanged. +Green-hostgroup membership remains runtime configuration until an +administrator removes it, including membership added automatically by the BGD +monitor. + +### One-Shot Cleanup Policy + +Cleanup is best effort and one shot. The worker does not maintain a per-effect +completion ledger, does not verify every external postcondition, and clears its +local state after invoking the cleanup operations. + +This is an intentional ProxySQL policy. Process termination or an individual +operation failure can prevent the monitor from proving that every action +completed. The feature does not require a retained cleanup executor or durable +effect ownership. + +The final interval depends on the caller: + +- `ER_NO_SUCH_TABLE` restores the baseline interval before cleanup. +- Successful and rollback cleanup reset the interval. +- Topology absence observed while already in `NONE` performs no cleanup and + does not independently reset an existing interval override. ## Worker And Configuration Lifetime -`PROPOSED-POLICY`: Controller state and its effect ledger have deployment -lifetime, not worker-stack lifetime. +### Deployment Checksum -```text -dispatcher creates or retrieves deployment context - -> worker generation N attaches - -> configuration checksum changes - -> generation N detaches; deployment context remains - -> worker generation N+1 attaches and resumes reconciliation -``` +The dispatcher calculates a per-deployment checksum from the active BGD row and +eligible blue and green runtime server rows. An Admin `mysql_servers` commit +refreshes that checksum. -`PROPOSED-POLICY`: Configuration changes update the deployment context without -clearing it. Disabling or removing configuration before observed writer -completion latches `REMOVAL_REQUESTED` and requests rollback. At or after -observed writer completion, removal latches `REMOVAL_REQUESTED` and enters -`SAFE_TEARDOWN`, including when `FINALIZING_SUCCESS` had already begun; it never -restores blue solely because configuration was removed. - -`PROPOSED-POLICY`: Post-completion configuration removal follows the evidence -gate and ordered settlement checklist in **Safe Teardown**. The terminal desired -configuration reflects validated removal intent. The same checklist reconciles -writer role and placement and marks the demotion record `COMMITTED` only after -verified handoff; it never compare-and-restores obsolete blue after observed -writer completion. - -`PROPOSED-POLICY`: When a configuration worker is removed, the dispatcher -retains or starts an independent cleanup executor until the active ledger is -settled or an externally visible `FAULTED` state is reached. The dispatcher -removes the deployment context only after it reaches `IDLE` with a settled, -empty active ledger. It does not erase a context in `FAULTED`. - -`PROPOSED-POLICY`: Durable SQLite persistence is unnecessary only if every -controller-owned effect is proven to disappear, revert, or be reconstructable -after a full ProxySQL process restart. This includes DNS pins, runtime shuns, -monitor suppression, backend connections and their drain generations, -green writer placement, blue writer demotion, -writer reader-hostgroup membership, and the direct probe target. If any effect -does not meet that condition, the controller persists its ledger or provides -deterministic startup recovery. -The author instead accepts a no-persistence fresh start and the loss of -per-effect ownership across process restart. The condition above is therefore -a reviewer hardening criterion, not a pending author-validation question or a -guarantee of current behavior. +A checksum change for an active deployment signals the existing worker through +`AWS_RDS_BGD_Worker::current_checksum`. The worker captures a consistent +candidate configuration and applies it in place. A checksum change alone does +not stop, join, or replace the worker thread. -## Configuration Model +The checksum is a configuration-generation signal. It is not an effect ledger +and is not the mechanism used to retry transient DNS resolution. Runtime +publishes initiated by the BGD worker do not refresh the checksum, preventing +the worker's own hostgroup actions from triggering a configuration refresh. -The feature has blue writer and reader hostgroups. Green hostgroup nullability -depends on row origin; it is not a user-selectable mixed-mode configuration. +### In-Place Refresh -| Row origin and storage | Green writer hostgroup | Green reader hostgroup | Semantics | -|---|---|---|---| -| User row in persistent Admin configuration | Value required | Value required | Explicit green hostgroups. The persistent schema declares both columns `NOT NULL`; a user `NULL` insert is rejected by SQLite. | -| User row materialized into runtime/HGM | Value | Value | The values from persistent configuration are retained with `auto_generated=0`. | -| Runtime row created by automatic discovery | `NULL` | `NULL` | Automatic green handling. The row carries `auto_generated=1` and exists only in runtime/HGM state. | -| Any user mixed combination | Invalid | Invalid | A user row cannot select automatic handling for only one green role. | - -`SOURCE-CODE, AUTHOR-VALIDATED`: The runtime Admin and HGM schemas allow the -two green columns to be nullable because they must represent auto-generated -rows. That storage capability does not make `NULL` valid in the persistent -user table. Defensive `NULL` binding while materializing or dumping HGM rows -likewise does not expand the user configuration contract. - -`SOURCE-CODE, AUTHOR-VALIDATED`: Saving runtime BGD hostgroups to the persistent -Admin table skips every row whose runtime `auto_generated` field is nonzero. -Consequently, a runtime auto-generated row with two `NULL` green hostgroups is -not inserted into the persistent `NOT NULL` table. User rows have both values -and are saved normally. - -`PROPOSED-POLICY`: Configuration updates have generations. Configuration -removal latches `REMOVAL_REQUESTED`. Re-adding or re-enabling the deployment -creates a new validated generation and must reconcile ownership before -resuming controller processing. - -## Author Validation Checklist - -The author response is recorded below. `RESOLVED` means the external evidence -has been scoped correctly or the author explicitly accepted the policy or -limitation. An implementation can still fail to conform to a resolved policy; -that is tracked separately rather than reopening the evidence decision. - -| ID | Recorded author evidence or decision | Review disposition | -|---|---|---| -| AWS-01a | One trace and the AWS examples show the source row present through all pre-completion phases. | `RESOLVED`: This is scoped observation, not a universal prohibition. Missing source identity remains `MALFORMED_TOPOLOGY` and causes no destructive transition. | -| AWS-01b | The trace shows the source row disappearing at `SWITCHOVER_COMPLETED`, leaving one target row. | `RESOLVED`: Source-row absence after completion is expected in the observed lifecycle but is not independently the reader-completion signal. | -| AWS-01c | One trace and the AWS examples show the target row present in every nonempty result. | `RESOLVED`: This is scoped observation. A missing target remains `MALFORMED_TOPOLOGY`. | -| AWS-01d | `TOPOLOGY_EMPTY` was observed only after writer completion. | `RESOLVED`: The author accepts rollback before completion and reader cleanup afterward. The pre-completion impossibility is not stated as an AWS guarantee. | -| AWS-01e | `TOPOLOGY_ABSENT` was not observed; the table remained present and empty. | `RESOLVED AS POLICY`: Preserve the distinct observation but select the same phase boundary as `TOPOLOGY_EMPTY`. | -| AWS-02a | The AWS-provided contract permits rollback during initiated and in-progress; the author separately observed cancellation returning to `AVAILABLE`. | `RESOLVED`: Pre-completion cancellation selects the current one-shot rollback path. Retained settlement is a reviewer proposal, not accepted policy. | -| AWS-02b | The AWS-provided contract says rollback is no longer allowed during post-processing. | `RESOLVED`: At or after writer completion, never restore obsolete blue solely because of cancellation or removal. | -| AWS-03 | The author accepts the same phase-specific policy for empty and absent topology while retaining distinct diagnostics. | `RESOLVED AS POLICY`. | -| AWS-04 | The contract defines `SWITCHOVER_COMPLETED` as writer DNS completion; the trace observes source-row removal in that completed snapshot. | `RESOLVED`: Use the status, not row count alone, as writer-DNS evidence. | -| AWS-05a | The target existed through every phase and lingered about 44 seconds after completion in one trace. | `RESOLVED`: Record the duration only as variable, single-observation evidence. | -| AWS-05b | The author accepts `TOPOLOGY_EMPTY` after observed writer completion as the reader-cleanup signal despite no AWS guarantee or direct reader-DNS timestamp. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: The observational risk is explicit. | -| AWS-06 | The author observed the green hostname stop resolving after completion while the promoted IP survived. | `RESOLVED AS SCOPED OBSERVATION`: Retain a complete probe target while it is needed. | -| AWS-07 | The author agrees the evidence does not establish universal source/target port equality. Commit `20247dcf0` takes the probe port from the matched blue writer pair. Pair-specific port mismatch is explicitly unsupported; different pairs may use different ports. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: Use the matched blue writer's configured port and accept failure for a target using a different port. Do not present equality as an AWS guarantee. | -| AWS-08 | The author agrees `use_ssl` is ProxySQL configuration. Automatic mode uses the matched blue writer's value; explicit mode must use the matched green writer row's value. | `RESOLVED AS AUTHOR-ACCEPTED POLICY; IMPLEMENTED`: Existing rows are selected by exact TARGET hostname and matched-blue port; successfully created or restored rows supply their resolved `use_ssl`. Simulator coverage remains assigned to PR6. | -| AWS-09 | The topology contains writer endpoints only; incomplete explicit reader mapping is expected and unmatched blue readers are shunned. | `RESOLVED AS POLICY`: Track and reconcile readers independently. | -| AWS-10 | Commits `cdffd77ee` and `ac4167cd0` retain auto-added and user-configured green rows on rollback and success. Rollback leaves green connections untouched; success drains eligible green connections but leaves rows and statuses unchanged. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: Green membership is persistent runtime configuration, not a temporary owned effect. Administrative cleanup is required even for an auto-added row. | -| AWS-11a | The author explicitly chooses one-shot worker-exit/configuration-change cleanup and no retained retry ledger. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: Loss of the cleanup context, including when a process terminates during cleanup, is accepted. The stronger retained rollback model is not PR2 scope. | -| AWS-11b | The author explicitly applies the same one-shot choice after completion and relies on the current phase-specific cleanup path. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: No retained `SAFE_TEARDOWN` executor or per-effect settlement record is required. This acceptance does not prove each one-shot operation succeeds. | -| AWS-12a | Commits `727b2166b` and `d45c953d2` combine eligible blue/green rows into the worker generation checksum and refresh it after Admin `mysql_servers` commits. The author assigned failed same-phase DNS setup to a per-pair follow-up. | `RESOLVED; IMPLEMENTED`: Eligible same-phase observations retry green-IP resolution, and worker-local `green_ip_pinned` state prevents repeated pin/drain work for completed pairs. The existing simulator cannot reproduce mutable DNS recovery, so simulator verification is unavailable for this case. | -| AWS-12b | The author selects cleanup-on-worker-exit and fresh replacement state. Persistent green membership and rollback-time green connections have no worker ownership under AWS-10. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: A replacement first observing COMPLETED may enter the inferred reader phase without reconstructing the prior map or effects. | -| AWS-13 | The author separates worker replacement from full restart. Replacement performs one-shot rollback then starts fresh. Full restart rebuilds DNS cache, pools, suppression, maps, probe target, and FSM; configured state reloads, while an unsynchronized auto-added runtime green row disappears. | `RESOLVED AS AUTHOR-ACCEPTED POLICY`: No durable BGD progress or ownership persistence is required. This is an accepted fresh-start contract, not a traced per-effect guarantee. | -| CFG-01a | User-configured rows require both green hostgroup values. The persistent Admin table declares both columns `NOT NULL`. | `RESOLVED AS AUTHOR-VALIDATED PROXYSQL CONTRACT`: A user `NULL` or mixed row is invalid; no configuration-nullability follow-up is required. | -| CFG-01b | Automatic discovery creates runtime/HGM rows with both green hostgroups `NULL` and `auto_generated=1`; runtime-to-persistent save skips those rows. | `RESOLVED AS AUTHOR-VALIDATED PROXYSQL CONTRACT`: Runtime nullability is intentional and does not conflict with persistent user constraints. | - -### Probe Target Validation Matrix - -| Mode | Host/IP source | Port source | SSL source | Author decision/evidence | -|---|---|---|---|---| -| Automatic | `AUTHOR-VALIDATED`: Resolved IP of the TARGET endpoint from `mysql.rds_topology`. | `AUTHOR-ACCEPTED-POLICY`: Matched blue writer's configured port. A different TARGET port is unsupported and is not forbidden by the recorded AWS evidence. | `AUTHOR-ACCEPTED-POLICY`: Matched blue writer's `use_ssl`, because no independent green row exists. | Policy resolved and implemented by writer-pair selection in `20247dcf0`. | -| Explicit | `AUTHOR-VALIDATED`: Resolved IP of the TARGET endpoint from `mysql.rds_topology`; the configured green writer must identify that target. | `AUTHOR-ACCEPTED-POLICY`: Matched blue writer's configured port. A different TARGET or explicit-green port is unsupported. | `AUTHOR-ACCEPTED-POLICY`: Exact matching green writer row's resolved `use_ssl`. | Policy resolved and implemented; PR6 owns simulator coverage. | - -`AUTHOR-ACCEPTED-POLICY`: A direct probe target is a complete host or IP, port, -and SSL tuple derived from the matched writer pair, never from an arbitrary -monitor row. The author accepts the blue-port constraint above. Explicit TLS -must be looked up by exact green writer identity, including the supported port, -rather than by applying a blue-to-green matcher to two green names. - -### Restart Validation Matrix - -`AUTHOR-ACCEPTED-POLICY`: Worker replacement and full process restart are -different fresh-start events. Neither recovers a durable BGD ledger. The table -records the selected behavior, not a claim that every one-shot operation has -been traced or verified under crash injection. - -| Effect | Worker replacement in the same process | Full ProxySQL process restart | -|---|---|---| -| Blue DNS pins | The exiting worker attempts to remove mapped blue DNS entries and purge their monitor-pool connections before discarding state. No result is retained for the replacement. | DNS cache and monitor connection pools are recreated; no BGD pin ownership is recovered. | -| Reader shuns | The exiting worker attempts to unshun only readers recorded in its local `shunned_readers` list. The replacement receives no list. | Runtime-only BGD shuns are discarded; server status is rebuilt from administrator configuration. | -| Monitor suppression | Entering `NONE` clears the worker's in-progress suppression entries for its current hostgroup members. | Suppression state is recreated empty. | -| Active connections and drains | Rollback purges mapped blue and recorded-reader monitor-pool connections. Green connections are intentionally untouched. No drain generation or completion result transfers. | Connection pools are recreated; no connection or drain-generation record survives. | -| Green placement | Auto-added and administrator-configured green rows remain in same-process runtime state. They are intentionally not worker-owned. | Administrator-configured rows reload. An auto-added runtime-only row disappears unless independently configured or synchronized into restart input. | -| Blue demotion | Exit cleanup attempts to restore a writer demoted in `WRITER_SWITCHOVER_IN_PROGRESS` or `WRITER_SWITCHOVER_POST_PROCESSING`. The replacement trusts current runtime placement. | Writer status and placement rebuild from administrator configuration. | -| Writer reader-hostgroup membership | Exit cleanup runs the current completion hostgroup action using local map and configuration values, then discards the map. | Membership rebuilds from administrator configuration. | -| Direct probe | The local direct-probe IP and failure counter are discarded. The replacement derives a new target from its first observation and current map, except that a first COMPLETED observation does not rebuild prior effects. | Probe state is recreated empty and derived from newly observed topology. | -| Mapping and resolution | Local pairs and resolved IPs are discarded after one-shot exit cleanup. The replacement builds a new map when its observed phase runs setup. | Pair map and resolution results are recreated from configuration and topology. | - -`AUTHOR-ACCEPTED-POLICY`: If the process terminates during cleanup, no durable -record proves which operations completed. The author accepts that uncertainty -because the relevant in-memory structures are expected to be rebuilt at full -restart. This explicitly rejects the stronger recovery requirement proposed in -**External Effects And Cleanup Ledger** for the current feature and accepted -follow-up scope. - -## Test Mapping For Later PRs - -The six response commits add no automated test. The following cases exercise -the code and policy changed by those commits without assuming the declined -durable-ledger design. - -| Requirement | Named unit/simulator case | Named Admin/TAP case | Observable postcondition | -|---|---|---|---| -| Matched writer probe destination | `writer_tuple_not_first_poll_row` | `matched_writer_destination` | With a reader first in the polling result and every endpoint at port 3306, the direct probe uses the mapped green writer destination and never `hpa[0]`. | -| Automatic TLS source | `auto_green_inherits_writer_ssl` | `automatic_green_tls` | With no explicit green row, the direct probe and auto-added green writer use the matched blue writer's `use_ssl`. | -| Explicit TLS source | `explicit_green_ssl_override` | `explicit_green_tls_differs_from_blue` | With every endpoint at port 3306, blue `use_ssl=0`, and explicit green `use_ssl=1`, the direct IP probe enables TLS. This guards the exact explicit-green TLS selection. | -| Eligible green generation checksum | `green_checksum_matrix` | `admin_green_add_remove_ssl_status` | Add/remove, `use_ssl`, and transitions into or out of `OFFLINE_SOFT`/`OFFLINE_HARD` change the checksum and replace workers; irrelevant changes do not. | -| Admin commit during active phase | `config_change_exits_worker` | `load_mysql_servers_mid_switchover` | The old worker runs one-shot rollback, the dispatcher joins it, and the replacement builds a new map from the committed runtime rows. | -| Green membership persistence | `green_row_persists_cancel_and_success` | `green_row_lifecycle` | Auto-added and user rows remain after rollback and success; no existing status is changed. | -| Green drain policy | `green_drain_status_matrix` | `green_hg_cleanup` | Rollback drains no green connections. Success drains eligible non-offline green servers while leaving `OFFLINE_SOFT` and `OFFLINE_HARD` untouched. Rows remain present. | -| Offline status exclusions | `offline_servers_not_acted_on` | `offline_soft_hard_servers` | Blue servers in either offline status do not participate in mapping or unmatched-reader shunning; green servers in either status are not drained. | -| Terminal connection retirement | `unhealthy_survives_reset` | `drained_used_connection_not_repooled` | After a drain marks a used connection unhealthy, reset does not revive it and neither local nor global pool return can place it in a free cache. | -| Persistent/user green hostgroups | `user_green_hostgroups_not_null` | `user_configuration_requires_both_green_hgs` | Persistent user inserts with either green hostgroup `NULL` fail; a row with both values loads with `auto_generated=0`. | -| Automatic runtime row persistence | `auto_generated_null_green_hgs` | `save_runtime_skips_auto_generated_bgd` | Auto-discovery creates a runtime row with both green hostgroups `NULL` and `auto_generated=1`; saving runtime to memory/disk does not persist that row. | -| First observation COMPLETED | `fresh_worker_first_completed` | `replace_worker_at_completed` | Fresh state advances to the inferred reader phase without reconstructing a prior map, then finishes on topology drain. | -| Full restart fresh start | `restart_discards_bgd_state` | `proxysql_restart_fixture` | DNS cache, pools, suppression, mapping, probe target, and FSM are recreated; configured rows reload; an unsynchronized auto-added runtime-only green row does not. | -| Same-phase DNS retry follow-up | — | — | Implemented in source: the unresolved pair retries while the phase is unchanged, successful pairs are not redrained, and the recovered pair is pinned and drained once. Mutable DNS recovery is outside the existing simulator contract. | -| Partial pair progress follow-up | — | — | Implemented in source: successful pair state is retained worker-locally and only the unresolved pair retries. | - -`PROPOSED-POLICY`: The simulator cases previously proposed for durable effect -ownership, compare-and-restore, retained `FAULTED` state, cleanup across stale -worker generations, and a persistent restart ledger remain useful reviewer -hardening ideas. They are not author-accepted follow-up requirements after AWS-11a, -AWS-11b, AWS-12b, and AWS-13. Implementing them would require a new policy -decision rather than treating this document as approval. - -## Review Gate - -The author has answered all 23 validation IDs. No external -`REVIEW-VALIDATION-PENDING` claim remains. The evidence gate is therefore -closed, with observational scope and accepted operational risks preserved in -the checklist rather than promoted to AWS guarantees. - -The source review remains open on implementation and verification: - -1. **COMPLETED:** Select explicit green TLS from the exact supported writer row, - including a row created or restored during discovery. Simulator coverage - for the existing-row and discovered-row paths remains part of PR6. -2. **COMPLETED:** Make unhealthy connection retirement terminal across reset, - local pool return, and global pool return. The follow-up uses the existing - `healthy` field and does not add a second flag. `connection_unhealthy_unit-t` - verifies that unhealthy connections remain terminal across reset and cannot - enter either free pool. -3. **COMPLETED:** Retry green-IP resolution during eligible same-phase - observations and use worker-local per-pair completion state to prevent - repeated pin/drain work in POST_PROCESSING. The existing simulator cannot - verify mutable DNS failure and recovery; the author accepts this test - coverage limitation. -4. Add focused simulator and TAP coverage for the response commits and these - follow-ups. Registration in `test/tap/groups/groups.json` is insufficient: - an automatic PR check must build the BGD test flavor and execute the BGD - simulator group. - -### Follow-up PR Sequence - -PR #5861 remains the live umbrella PR into `v3.0`. Every implementation and -test follow-up below targets `feature/aws-rds-monitor`, so each accepted change -becomes part of #5861 rather than replacing or closing it. The originally -proposed broad durable-ledger/controller PR is not part of this sequence. - -| Review PR | Scope | Dependency and completion signal | -|---|---|---| -| PR1: #5934 | This document only: evidence, accepted risks, current behavior, and follow-up contract. | Ready for author approval; merge into `feature/aws-rds-monitor` before implementation follow-ups so their scope is stable. | -| PR2: BGD simulator foundation — COMPLETED | Add the TAP-controlled SQLite3-server simulator defined in [RDS_BGD_Simulator.md](RDS_BGD_Simulator.md): the `TEST_RDS_BGD` build mode, IP-keyed topology responses, common and BGD TAP helpers, a simulator group, and an end-to-end acceptance smoke test. | Completed after the isolated local Docker group passed. Provides the reusable harness required by PR6; GitHub workflow execution remains separate follow-up work under the review gate above. | -| PR3: probe target and explicit TLS (**complete**) | Correct AWS-08 by selecting the exact supported explicit green writer row and its resolved `use_ssl`, including a row created or restored during discovery, while retaining the matched blue writer port and automatic-mode blue TLS fallback. | **Completed:** production behavior conforms to AWS-08. Existing-row and discovered-row simulator coverage remains part of PR6. | -| PR4: terminal connection retirement (**complete**) | Preserve `healthy=false` across `MySQL_Connection::reset()` and destroy unhealthy connections in local and global pool-return paths. Do not introduce another flag or a new locking policy. | **Completed:** `connection_unhealthy_unit-t` proves a drained used connection cannot enter either free pool after reset or release. | -| PR5: same-phase per-pair reconciliation (**complete**) | Retry green-IP resolution on eligible equal-phase observations and never redrain a pair completed in the current worker generation. Configuration-derived pair mapping remains transition/generation driven. | **Completed:** production behavior conforms to AWS-12a. The existing simulator cannot verify mutable DNS recovery. | -| PR6: simulator-driven BGD scenario suite | Use PR2's simulator to cover configuration and discovery order, automatic and explicit rows, worker replacement, normal lifecycle, late entry, cancellation and rollback, topology drain, direct probe destination/TLS for existing and discovered explicit green rows, offline exclusions, and terminal connection retirement where observable. | Depends on PR2 and should normally follow PR3-PR4 so the suite validates final behavior rather than encoding known failures. All payloads run in the BGD simulator GitHub workflow. | - -Any retained cleanup ledger, durable restart ownership, or alternative -controller state machine requires a new author policy decision. The simulator -contract and integration design consumed by PR2 and PR6 are defined in -[RDS_BGD_Simulator.md](RDS_BGD_Simulator.md). +Refresh behavior depends on the phase: + +1. Before `WRITER_SWITCHOVER_POST_PROCESSING`, the worker preserves its phase, + applies the candidate scalar and hostgroup configuration, clears the direct + probe and failure counter, and marks the map for reconstruction. The next + topology result rebuilds configuration-derived state. +2. If the reader hostgroup changes while read-only suppression is active, the + worker clears suppression for the old hostgroup and enables it for the new + hostgroup. +3. During `WRITER_SWITCHOVER_IN_PROGRESS`, map reconstruction restores a + replaced old writer and demotes the newly mapped writer. +4. At or after `WRITER_SWITCHOVER_POST_PROCESSING`, replacing only the map + cannot safely reconcile existing DNS pins, reader shuns, monitor + connections, and placement. The same worker runs one-shot rollback, applies + the candidate configuration, resets topology polling to the table check, + and restarts the state machine from `NONE`. + +The fourth case is rollback and state-machine restart within the same worker, +not worker replacement. + +### Worker Detach And Recreation + +If a deployment is disabled, removed, or loses its eligible blue writer, the +dispatcher requests worker stop. A worker exiting from a non-`NONE` state runs +one-shot rollback before discarding its local map, reader list, probe target, +and phase. + +If configuration later makes the deployment eligible again, the dispatcher +creates a new worker with fresh state. A fresh worker does not recover the +prior worker's effects or cleanup results. If its first observation is +`SWITCHOVER_COMPLETED`, it enters `READER_SWITCHOVER_IN_PROGRESS` without +reconstructing a prior map and waits for topology drain. + +### Process Restart + +A full ProxySQL process restart is a fresh start: + +- Worker state, blue/green maps, direct-probe targets, suppression state, and + cleanup bookkeeping are recreated empty. +- DNS cache and connection pools are recreated. +- Runtime placement and status are rebuilt from administrator configuration. +- User-configured BGD and server rows reload through the normal configuration + path. +- An automatically added runtime-only green row disappears unless it was + independently configured or synchronized into another restart input. + +No durable BGD progress or effect-ownership state is recovered. A +simulator-driven full process-restart fixture is not required by this contract. + +## Read-Only Suppression + +BGD suppresses ordinary read-only monitoring while it intentionally changes +writer and reader placement. Suppression is keyed by +`hostname:::port` in the shared `aws_rds_bgd_server_status` map. + +The required lifetime is: + +```text +UNSUPPRESSED + | + | BGD enters INITIATED, IN_PROGRESS, or POST_PROCESSING + | record every endpoint key owned by this deployment + v +SUPPRESSED + | + +--> new read-only work for an owned key is not admitted + | + +--> an earlier result revalidates suppression before applying placement + | + | BGD returns to NONE or changes configuration + v +OWNED_KEYS_REMOVED + | + `--> other deployments' keys remain unchanged +``` + +Suppression ownership is deployment-specific. Cleanup must erase the exact keys +inserted for that deployment even if the corresponding server has already been +removed from its current hostgroup. Concurrent deployments using distinct +endpoints must not clear each other's keys. + +Each server endpoint must belong to only one active blue/green deployment. +Administrators are responsible for avoiding overlapping endpoint assignments; +the monitor does not validate or protect against this unsupported configuration. + +Before a read-only result changes placement through +`read_only_action_v2()`, result application must revalidate that the endpoint is +not suppressed for an active BGD transition. Admission-time validation alone +is insufficient because a task may have been admitted before suppression was +enabled. + +## Verification + +The BGD verification surface consists of: + +- The SQLite3-server simulator compiled under `TEST_RDS_BGD`. +- The `cluster_sim_rds_bgd-g1` TAP group registered in + `test/tap/groups/groups.json`. +- The 22 `test_rds_bgd_*-t` scenario binaries under `test/tap/tests`. +- `connection_unhealthy_unit-t`, which verifies terminal retirement across + reset, local pool return, and global pool return. +- `.github/workflows/CI-cluster-simulator.yml`, which discovers registered + simulator groups, builds the combined simulator flavor, and runs each group + as an automatic pull-request check. + +The scenario suite covers: + +- Explicit startup and automatic discovery. +- Probe destination and TLS selection. +- Writer and reader switchover. +- Late entry into writer phases and first observation at completion. +- Cancellation, rollback, topology empty/absent, and query errors. +- Green membership persistence and green-pool cleanup. +- Offline server exclusions and reader policy. +- Configuration persistence, in-place refresh, disablement, removal, and + worker hostgroup changes. +- Repeated deployments and concurrent deployment isolation. + +The simulator cannot reproduce mutable DNS failure followed by recovery, so +same-phase DNS recovery is verified by source review rather than a mutable-DNS +simulator case. Full ProxySQL process restart is an accepted fresh-start +assumption and intentionally has no simulator fixture. + +## Implementation Anchors + +The monitor behavior is primarily implemented by: + +- `parse_aws_rds_topology()` +- `monitor_RDS_BGD_thread_HG()` +- `handle_aws_rds_bgd()` +- `aws_rds_bgd_resolve_green_ips()` +- `aws_rds_bgd_refresh_worker_config()` +- `aws_rds_bgd_config_refresh_action()` +- `aws_rds_bgd_handle_topology_absent()` +- `handle_aws_rds_bgd_post_switchover()` +- `aws_rds_bgd_drain_green_hg()` + +The broader configuration and effect surface includes: + +- `include/DNS_Cache.hpp` and `lib/DNS_Cache.cpp` +- `include/MySQL_HostGroups_Manager.h` and + `lib/MySQL_HostGroups_Manager.cpp` +- `include/mysql_connection.h` and `lib/mysql_connection.cpp` +- `lib/MySrvConnList.cpp` +- `lib/ProxySQL_Admin.cpp` +- `lib/ProxySQL_Config.cpp` +- `include/ProxySQL_Admin_Tables_Definitions.h` + +Changes to these entry points or their state, configuration, DNS, hostgroup, or +connection semantics should be reviewed against this contract and the +simulator specification. From 13cd483bba7366e19ed1c4b1652bfab43469f433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Wed, 29 Jul 2026 03:35:56 +0200 Subject: [PATCH 76/81] Add BGD cluster sync plumbing, unit test, and fix int/bool type inconsistency - Add CLUSTER_QUERY_MYSQL_AWS_RDS_BGD query and incoming_servers_t field for cluster sync of mysql_aws_rds_bgd_hostgroups - Implement save-to-database logic (save_server_changes_to_database_rds_bgd) - Register cluster sync handler and admin shell variable for background sync - Add unit test (cluster_sync_unit-t) with 19 assertions - Change aws_blue_green_deployment_auto_discovery from bool to int for consistency with thread-local plumbing --- include/Base_HostGroups_Manager.h | 1 + include/MySQL_HostGroups_Manager.h | 1 + include/MySQL_Thread.h | 2 +- include/ProxySQL_Cluster.hpp | 5 + include/proxysql_admin.h | 3 +- lib/Admin_Handler.cpp | 2 + lib/MySQL_HostGroups_Manager.cpp | 3 + lib/MySQL_Thread.cpp | 6 +- lib/ProxySQL_Admin.cpp | 9 +- lib/ProxySQL_Cluster.cpp | 92 +++++++++++++++- test/tap/groups/groups.json | 1 + test/tap/tests/unit/Makefile | 3 +- test/tap/tests/unit/cluster_sync_unit-t.cpp | 116 ++++++++++++++++++++ 13 files changed, 231 insertions(+), 13 deletions(-) create mode 100644 test/tap/tests/unit/cluster_sync_unit-t.cpp diff --git a/include/Base_HostGroups_Manager.h b/include/Base_HostGroups_Manager.h index 7e0984defd..9a47dcc68e 100644 --- a/include/Base_HostGroups_Manager.h +++ b/include/Base_HostGroups_Manager.h @@ -610,6 +610,7 @@ class MySQL_HostGroups_Manager { MYSQL_AWS_AURORA_HOSTGROUPS, MYSQL_HOSTGROUP_ATTRIBUTES, MYSQL_SERVERS_SSL_PARAMS, + MYSQL_AWS_RDS_BGD_HOSTGROUPS, MYSQL_SERVERS, HGM_TABLES_SIZE_ diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index c226422f32..6edc571723 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -533,6 +533,7 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { MYSQL_AWS_AURORA_HOSTGROUPS, MYSQL_HOSTGROUP_ATTRIBUTES, MYSQL_SERVERS_SSL_PARAMS, + MYSQL_AWS_RDS_BGD_HOSTGROUPS, MYSQL_SERVERS, HGM_TABLES_SIZE_ diff --git a/include/MySQL_Thread.h b/include/MySQL_Thread.h index 67de3fb97c..47b6de5222 100644 --- a/include/MySQL_Thread.h +++ b/include/MySQL_Thread.h @@ -456,7 +456,7 @@ class MySQL_Threads_Handler //! Monitor aws rds topology discovery interval. Unit: 'one discovery check per X monitor_read_only checks'. int monitor_aws_rds_topology_discovery_interval; //! Auto-generate runtime aws_rds_bgd_hostgroups entries when the read_only monitor detects a blue/green deployment. - bool aws_blue_green_deployment_auto_discovery; + int aws_blue_green_deployment_auto_discovery; //! Monitor read only timeout. Unit: 'ms'. int monitor_read_only_interval; //! Monitor read only timeout. Unit: 'ms'. diff --git a/include/ProxySQL_Cluster.hpp b/include/ProxySQL_Cluster.hpp index d81485151f..eaccc4bbb6 100644 --- a/include/ProxySQL_Cluster.hpp +++ b/include/ProxySQL_Cluster.hpp @@ -72,6 +72,9 @@ /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_aws_aurora_hostgroups'. See top comment for details. */ #define CLUSTER_QUERY_MYSQL_AWS_AURORA "PROXY_SELECT writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment FROM runtime_mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup" +/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_aws_rds_bgd_hostgroups'. See top comment for details. */ +#define CLUSTER_QUERY_MYSQL_AWS_RDS_BGD "PROXY_SELECT writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active, writer_is_also_reader, check_interval_ms, check_timeout_ms, comment, auto_generated, status FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE auto_generated=0 ORDER BY writer_hostgroup" + /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_galera_hostgroups'. See top comment for details. */ #define CLUSTER_QUERY_MYSQL_GALERA "PROXY_SELECT writer_hostgroup, backup_writer_hostgroup, reader_hostgroup, offline_hostgroup, active, max_writers, writer_is_also_reader, max_transactions_behind, comment FROM runtime_mysql_galera_hostgroups ORDER BY writer_hostgroup" @@ -446,6 +449,8 @@ struct p_cluster_counter { pulled_mysql_servers_hostgroup_attributes_failure, pulled_mysql_servers_ssl_params_success, pulled_mysql_servers_ssl_params_failure, + pulled_mysql_servers_aws_rds_bgd_hostgroups_success, + pulled_mysql_servers_aws_rds_bgd_hostgroups_failure, pulled_mysql_servers_runtime_checks_success, pulled_mysql_servers_runtime_checks_failure, diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 5ef4c980ca..a4b947bfb4 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -160,10 +160,11 @@ struct incoming_servers_t { SQLite3_result* incoming_aurora_hostgroups = NULL; SQLite3_result* incoming_hostgroup_attributes = NULL; SQLite3_result* incoming_mysql_servers_ssl_params = NULL; + SQLite3_result* incoming_aws_rds_bgd_hostgroups = NULL; SQLite3_result* runtime_mysql_servers = NULL; incoming_servers_t(); - incoming_servers_t(SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*); + incoming_servers_t(SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*); }; // Separate structs for runtime mysql server and mysql server v2 to avoid human error diff --git a/lib/Admin_Handler.cpp b/lib/Admin_Handler.cpp index c8fa27d51d..8946c6d388 100644 --- a/lib/Admin_Handler.cpp +++ b/lib/Admin_Handler.cpp @@ -3206,6 +3206,8 @@ void admin_session_handler(S* sess, void *_pa, PtrSize_t *pkt) { tn = "mysql_hostgroup_attributes"; } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_SERVERS_SSL_PARAMS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_SERVERS_SSL_PARAMS))) { tn = "mysql_servers_ssl_params"; + } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_AWS_RDS_BGD, query_no_space, strlen(CLUSTER_QUERY_MYSQL_AWS_RDS_BGD))) { + tn = "mysql_aws_rds_bgd_hostgroups"; } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_SERVERS_V2, query_no_space, strlen(CLUSTER_QUERY_MYSQL_SERVERS_V2))) { tn = "mysql_servers_v2"; } diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 309363e8eb..5a5ef1ad4f 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1002,6 +1002,7 @@ void MySQL_HostGroups_Manager::commit_update_checksums_from_tables(SpookyHash& m CUCFT1(myhash,init,"mysql_aws_aurora_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_AWS_AURORA_HOSTGROUPS]); CUCFT1(myhash,init,"mysql_hostgroup_attributes","hostgroup_id", table_resultset_checksum[HGM_TABLES::MYSQL_HOSTGROUP_ATTRIBUTES]); CUCFT1(myhash,init,"mysql_servers_ssl_params","hostname,port,username", table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS_SSL_PARAMS]); + CUCFT1(myhash,init,"mysql_aws_rds_bgd_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_AWS_RDS_BGD_HOSTGROUPS]); } /** @@ -3180,6 +3181,8 @@ SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_table(const string& return this->incoming_hostgroup_attributes; } else if (name == "mysql_servers_ssl_params") { return this->incoming_mysql_servers_ssl_params; + } else if (name == "mysql_aws_rds_bgd_hostgroups") { + return this->incoming_aws_rds_bgd_hostgroups; } else if (name == "cluster_mysql_servers") { return this->runtime_mysql_servers; } else if (name == "mysql_servers_v2") { diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index b3debb77ec..c068006b4f 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -1297,7 +1297,7 @@ MySQL_Threads_Handler::MySQL_Threads_Handler() { variables.monitor_ping_max_failures=3; variables.monitor_ping_timeout=1000; variables.monitor_aws_rds_topology_discovery_interval=0; - variables.aws_blue_green_deployment_auto_discovery=true; + variables.aws_blue_green_deployment_auto_discovery=1; variables.monitor_read_only_interval=1000; variables.monitor_read_only_timeout=800; variables.monitor_read_only_max_timeout_count=3; @@ -2650,7 +2650,7 @@ char ** MySQL_Threads_Handler::get_variables_list() { VariablesPointers_bool["log_mysql_warnings_enabled"] = make_tuple(&variables.log_mysql_warnings_enabled, false); VariablesPointers_bool["log_unhealthy_connections"] = make_tuple(&variables.log_unhealthy_connections, false); VariablesPointers_bool["monitor_enabled"] = make_tuple(&variables.monitor_enabled, false); - VariablesPointers_bool["aws_blue_green_deployment_auto_discovery"] = make_tuple(&variables.aws_blue_green_deployment_auto_discovery, false); + VariablesPointers_int["aws_blue_green_deployment_auto_discovery"] = make_tuple(&variables.aws_blue_green_deployment_auto_discovery, 0, 1, false); VariablesPointers_bool["monitor_replication_lag_group_by_host"] = make_tuple(&variables.monitor_replication_lag_group_by_host, false); VariablesPointers_bool["monitor_wait_timeout"] = make_tuple(&variables.monitor_wait_timeout, false); VariablesPointers_bool["monitor_writer_is_also_reader"] = make_tuple(&variables.monitor_writer_is_also_reader, false); @@ -4799,7 +4799,7 @@ void MySQL_Thread::refresh_variables() { REFRESH_VARIABLE_INT(monitor_ping_max_failures); REFRESH_VARIABLE_INT(monitor_ping_timeout); REFRESH_VARIABLE_INT(monitor_aws_rds_topology_discovery_interval); - REFRESH_VARIABLE_BOOL(aws_blue_green_deployment_auto_discovery); + REFRESH_VARIABLE_INT(aws_blue_green_deployment_auto_discovery); REFRESH_VARIABLE_INT(monitor_read_only_interval); REFRESH_VARIABLE_INT(monitor_read_only_timeout); REFRESH_VARIABLE_INT(monitor_read_only_max_timeout_count); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 7704cf8f79..84c7215e2d 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -876,6 +876,7 @@ incoming_servers_t::incoming_servers_t( SQLite3_result* incoming_aurora_hostgroups, SQLite3_result* incoming_hostgroup_attributes, SQLite3_result* incoming_mysql_servers_ssl_params, + SQLite3_result* incoming_aws_rds_bgd_hostgroups, SQLite3_result* runtime_mysql_servers ) : incoming_mysql_servers_v2(incoming_mysql_servers_v2), @@ -885,6 +886,7 @@ incoming_servers_t::incoming_servers_t( incoming_aurora_hostgroups(incoming_aurora_hostgroups), incoming_hostgroup_attributes(incoming_hostgroup_attributes), incoming_mysql_servers_ssl_params(incoming_mysql_servers_ssl_params), + incoming_aws_rds_bgd_hostgroups(incoming_aws_rds_bgd_hostgroups), runtime_mysql_servers(runtime_mysql_servers) {} @@ -7978,6 +7980,7 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc SQLite3_result* incoming_aurora_hostgroups = incoming_servers.incoming_aurora_hostgroups; SQLite3_result* incoming_hostgroup_attributes = incoming_servers.incoming_hostgroup_attributes; SQLite3_result* incoming_mysql_servers_ssl_params = incoming_servers.incoming_mysql_servers_ssl_params; + SQLite3_result* incoming_aws_rds_bgd_hostgroups = incoming_servers.incoming_aws_rds_bgd_hostgroups; SQLite3_result* incoming_mysql_servers_v2 = incoming_servers.incoming_mysql_servers_v2; const char *query=(char *)"SELECT hostgroup_id,hostname,port,gtid_port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM main.mysql_servers ORDER BY hostgroup_id, hostname, port"; @@ -8134,7 +8137,11 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc // support for AWS RDS, table mysql_aws_rds_bgd_hostgroups query=(char *)"SELECT a.* FROM mysql_aws_rds_bgd_hostgroups a LEFT JOIN mysql_aws_rds_bgd_hostgroups b ON (a.writer_hostgroup=b.reader_hostgroup) WHERE b.reader_hostgroup IS NULL ORDER BY writer_hostgroup"; proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); - admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_aws_rds_bgd); + if (incoming_aws_rds_bgd_hostgroups == nullptr) { + admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_aws_rds_bgd); + } else { + resultset_aws_rds_bgd = incoming_aws_rds_bgd_hostgroups; + } if (error) { proxy_error("Error on %s : %s\n", query, error); } else { diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 1e54d0b361..9fe540c361 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -91,6 +91,7 @@ namespace SQLQueries { const char* const DELETE_MYSQL_AWS_AURORA_HOSTGROUPS = "DELETE FROM mysql_aws_aurora_hostgroups"; const char* const DELETE_MYSQL_HOSTGROUP_ATTRIBUTES = "DELETE FROM mysql_hostgroup_attributes"; const char* const DELETE_MYSQL_SERVERS_SSL_PARAMS = "DELETE FROM mysql_servers_ssl_params"; + const char* const DELETE_MYSQL_AWS_RDS_BGD_HOSTGROUPS = "DELETE FROM mysql_aws_rds_bgd_hostgroups"; const char* const DELETE_PGSQL_SERVERS = "DELETE FROM pgsql_servers"; const char* const DELETE_PGSQL_REPLICATION_HOSTGROUPS = "DELETE FROM pgsql_replication_hostgroups"; const char* const DELETE_PGSQL_HOSTGROUP_ATTRIBUTES = "DELETE FROM pgsql_hostgroup_attributes"; @@ -1821,12 +1822,14 @@ int ProxySQL_Cluster::fetch_and_store(MYSQL* conn, const fetch_query& f_query, M /** * @brief Generates a hash from the received resultsets from executing the following queries in the specified * order: - * - CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS. + * - CLUSTER_QUERY_MYSQL_SERVERS_V2. * - CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS. * - CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS. * - CLUSTER_QUERY_MYSQL_GALERA. * - CLUSTER_QUERY_MYSQL_AWS_AURORA. * - CLUSTER_QUERY_MYSQL_HOSTGROUP_ATTRIBUTES. + * - CLUSTER_QUERY_MYSQL_SERVERS_SSL_PARAMS. + * - CLUSTER_QUERY_MYSQL_AWS_RDS_BGD. * * IMPORTANT: It's assumed that the previous queries were successful and that the resultsets are received in * the specified order. @@ -1871,6 +1874,7 @@ incoming_servers_t convert_mysql_servers_resultsets(const std::vector results(8,nullptr); + std::vector results(9,nullptr); // servers messages std::string fetch_servers_done = ""; @@ -2122,6 +2126,12 @@ void ProxySQL_Cluster::pull_mysql_servers_v2_from_peer(const mysql_servers_v2_ch std::string fetch_mysql_servers_ssl_params_err = ""; string_format("Cluster: Fetching 'MySQL Servers SSL Params' from peer %s:%d failed: \n", fetch_mysql_servers_ssl_params_err, hostname, port); + // AWS RDS BGD hostgroups messages + std::string fetch_aws_rds_bgd_start = ""; + string_format("Cluster: Fetching 'MySQL AWS RDS BGD Hostgroups' from peer %s:%d\n", fetch_aws_rds_bgd_start, hostname, port); + std::string fetch_aws_rds_bgd_err = ""; + string_format("Cluster: Fetching 'MySQL AWS RDS BGD Hostgroups' from peer %s:%d failed: \n", fetch_aws_rds_bgd_err, hostname, port); + // Create fetching queries /** @@ -2170,6 +2180,12 @@ void ProxySQL_Cluster::pull_mysql_servers_v2_from_peer(const mysql_servers_v2_ch p_cluster_counter::pulled_mysql_servers_ssl_params_success, p_cluster_counter::pulled_mysql_servers_ssl_params_failure, { fetch_mysql_servers_ssl_params_start, "", fetch_mysql_servers_ssl_params_err } + }, + { + CLUSTER_QUERY_MYSQL_AWS_RDS_BGD, + p_cluster_counter::pulled_mysql_servers_aws_rds_bgd_hostgroups_success, + p_cluster_counter::pulled_mysql_servers_aws_rds_bgd_hostgroups_failure, + { fetch_aws_rds_bgd_start, "", fetch_aws_rds_bgd_err } } }; @@ -2205,22 +2221,22 @@ void ProxySQL_Cluster::pull_mysql_servers_v2_from_peer(const mysql_servers_v2_ch MYSQL_RES* fetch_res = nullptr; if (fetch_and_store(conn, query, &fetch_res) == 0) { - results[7] = fetch_res; + results[8] = fetch_res; } else { fetching_error = true; } } if (fetching_error == false) { - const uint64_t servers_hash = compute_servers_tables_raw_checksum(results, 7); // ignore runtime_mysql_servers in checksum calculation + const uint64_t servers_hash = compute_servers_tables_raw_checksum(results, 8); // ignore runtime_mysql_servers in checksum calculation const string computed_checksum{ get_checksum_from_hash(servers_hash) }; proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Computed checksum for MySQL Servers v2 from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); proxy_info("Cluster: Computed checksum for MySQL Servers v2 from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); bool runtime_checksum_matches = true; - if (results[7]) { - const uint64_t runtime_mysql_server_hash = mysql_raw_checksum(results[7]); + if (results[8]) { + const uint64_t runtime_mysql_server_hash = mysql_raw_checksum(results[8]); const std::string runtime_mysql_server_computed_checksum = get_checksum_from_hash(runtime_mysql_server_hash); proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Computed checksum for MySQL Servers from peer %s:%d : %s\n", hostname, port, runtime_mysql_server_computed_checksum.c_str()); proxy_info("Cluster: Computed checksum for MySQL Servers from peer %s:%d : %s\n", hostname, port, runtime_mysql_server_computed_checksum.c_str()); @@ -2484,6 +2500,49 @@ void ProxySQL_Cluster::pull_mysql_servers_v2_from_peer(const mysql_servers_v2_ch resultset->dump_to_stderr(); delete resultset; + // sync mysql_aws_rds_bgd_hostgroups + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Writing mysql_aws_rds_bgd_hostgroups table\n"); + proxy_info("Cluster: Writing mysql_aws_rds_bgd_hostgroups table\n"); + GloAdmin->admindb->execute(SQLQueries::DELETE_MYSQL_AWS_RDS_BGD_HOSTGROUPS); + { + const char* q = (const char*)"INSERT INTO mysql_aws_rds_bgd_hostgroups (" + "writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, " + "active, writer_is_also_reader, check_interval_ms, check_timeout_ms, comment) " + "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)"; + auto [rc, statement1_unique] = GloAdmin->admindb->prepare_v2(q); + ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + sqlite3_stmt *statement1 = statement1_unique.get(); + + while ((row = mysql_fetch_row(results[7]))) { + rc=(*proxy_sqlite3_bind_int64)(statement1, 1, atol(row[0])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // writer_hostgroup + rc=(*proxy_sqlite3_bind_int64)(statement1, 2, atol(row[1])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // reader_hostgroup + if (row[2]) { + rc=(*proxy_sqlite3_bind_int64)(statement1, 3, atol(row[2])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // green_writer_hostgroup + } else { + rc=(*proxy_sqlite3_bind_null)(statement1, 3); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + } + if (row[3]) { + rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atol(row[3])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // green_reader_hostgroup + } else { + rc=(*proxy_sqlite3_bind_null)(statement1, 4); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + } + rc=(*proxy_sqlite3_bind_int64)(statement1, 5, atol(row[4])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // active + rc=(*proxy_sqlite3_bind_int64)(statement1, 6, atol(row[5])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // writer_is_also_reader + rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atol(row[6])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // check_interval_ms + rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atol(row[7])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // check_timeout_ms + rc=(*proxy_sqlite3_bind_text)(statement1, 9, row[8], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // comment + SAFE_SQLITE3_STEP2(statement1); + rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + } + } + + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Dumping fetched 'mysql_aws_rds_bgd_hostgroups'\n"); + proxy_info("Dumping fetched 'mysql_aws_rds_bgd_hostgroups'\n"); + GloAdmin->admindb->execute_statement((char*)"SELECT * FROM mysql_aws_rds_bgd_hostgroups", &error, &cols, &affected_rows, &resultset); + resultset->dump_to_stderr(); + delete resultset; + proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Loading to runtime MySQL Servers v2 from peer %s:%d\n", hostname, port); proxy_info("Cluster: Loading to runtime MySQL Servers v2 from peer %s:%d\n", hostname, port); GloAdmin->load_mysql_servers_to_runtime(incoming_servers, peer_runtime_mysql_server, peer_mysql_server_v2); @@ -4982,6 +5041,27 @@ cluster_metrics_map = std::make_tuple( ), // ==================================================================== + // ==================================================================== + std::make_tuple ( + p_cluster_counter::pulled_mysql_servers_aws_rds_bgd_hostgroups_success, + "proxysql_cluster_pulled_total", + "Number of times a 'module' have been pulled from a peer.", + metric_tags { + { "module_name", "mysql_servers_aws_rds_bgd_hostgroups" }, + { "status", "success" } + } + ), + std::make_tuple ( + p_cluster_counter::pulled_mysql_servers_aws_rds_bgd_hostgroups_failure, + "proxysql_cluster_pulled_total", + "Number of times a 'module' have been pulled from a peer.", + metric_tags { + { "module_name", "mysql_servers_aws_rds_bgd_hostgroups" }, + { "status", "failure" } + } + ), + // ==================================================================== + // ==================================================================== std::make_tuple ( p_cluster_counter::pulled_mysql_servers_runtime_checks_success, diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index 649f83059f..c8432a8bb0 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -22,6 +22,7 @@ "config_write_unit-t" : [ "unit-tests-g1" ], "connection_pool_unit-t" : [ "unit-tests-g1" ], "connection_unhealthy_unit-t" : [ "unit-tests-g1" ], + "cluster_sync_unit-t" : [ "unit-tests-g1" ], "deprecate_eof_cache-t" : [ "legacy-g4","mariadb10-galera-g4","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g4","mysql84-gr-g4","mysql90-g4","mysql95-g4" ], "envvars-t" : [ "legacy-g1","mariadb10-galera-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql84-gr-g1","mysql90-g1","mysql90-gr-g1","mysql93-g1","mysql93-gr-g1","mysql95-g1","mysql95-gr-g1" ], "eof_cache_mixed_flags-t" : [ "legacy-g4","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g4","mysql90-g4","mysql95-g4" ], diff --git a/test/tap/tests/unit/Makefile b/test/tap/tests/unit/Makefile index 06071e962b..b96734651e 100644 --- a/test/tap/tests/unit/Makefile +++ b/test/tap/tests/unit/Makefile @@ -421,7 +421,8 @@ UNIT_TESTS := smoke_test-t query_cache_unit-t query_processor_unit-t \ admin_disk_upgrade_unit-t \ glovars_unit-t \ pgsql_servers_ssl_params_unit-t \ - connection_unhealthy_unit-t + connection_unhealthy_unit-t \ + cluster_sync_unit-t # Plugin-chassis + mysqlx-plugin unit tests — built only when # libproxysql.a was compiled with -DPROXYSQL40 (autodetected higher up diff --git a/test/tap/tests/unit/cluster_sync_unit-t.cpp b/test/tap/tests/unit/cluster_sync_unit-t.cpp new file mode 100644 index 0000000000..a6bc86ca7b --- /dev/null +++ b/test/tap/tests/unit/cluster_sync_unit-t.cpp @@ -0,0 +1,116 @@ +#include "tap.h" +#include "test_globals.h" +#include "test_init.h" + +#include "proxysql.h" + +#include +#include +#include + +// The incoming_servers_t struct is defined in proxysql_admin.h, which cannot +// be included in this test due to circular include dependencies. Instead we +// re-declare the struct here to verify its layout. The canonical definition +// is the source of truth; this copy must match it exactly. +// +// WARNING: If the struct in proxysql_admin.h changes, this test will +// silently drift. Keep in sync. +struct incoming_servers_t { + void* incoming_mysql_servers_v2 = nullptr; + void* incoming_replication_hostgroups = nullptr; + void* incoming_group_replication_hostgroups = nullptr; + void* incoming_galera_hostgroups = nullptr; + void* incoming_aurora_hostgroups = nullptr; + void* incoming_hostgroup_attributes = nullptr; + void* incoming_mysql_servers_ssl_params = nullptr; + void* incoming_aws_rds_bgd_hostgroups = nullptr; + void* runtime_mysql_servers = nullptr; +}; + +static_assert(sizeof(incoming_servers_t) == 9 * sizeof(void*), + "incoming_servers_t must have exactly 9 pointer-sized fields"); + +static void test_incoming_servers_t_size() { + size_t expected = 9; + size_t actual = sizeof(incoming_servers_t) / sizeof(void*); + ok(actual == expected, + "sizeof(incoming_servers_t)/sizeof(void*) == %zu (expected %zu)", + actual, expected); +} + +static void test_incoming_servers_t_field_positions() { + incoming_servers_t s; + s.incoming_mysql_servers_v2 = (void*)1; + s.incoming_replication_hostgroups = (void*)2; + s.incoming_group_replication_hostgroups = (void*)3; + s.incoming_galera_hostgroups = (void*)4; + s.incoming_aurora_hostgroups = (void*)5; + s.incoming_hostgroup_attributes = (void*)6; + s.incoming_mysql_servers_ssl_params = (void*)7; + s.incoming_aws_rds_bgd_hostgroups = (void*)8; + s.runtime_mysql_servers = (void*)9; + + ok(s.incoming_mysql_servers_v2 == (void*)1, "field 0 set correctly"); + ok(s.incoming_replication_hostgroups == (void*)2, "field 1 set correctly"); + ok(s.incoming_group_replication_hostgroups == (void*)3, "field 2 set correctly"); + ok(s.incoming_galera_hostgroups == (void*)4, "field 3 set correctly"); + ok(s.incoming_aurora_hostgroups == (void*)5, "field 4 set correctly"); + ok(s.incoming_hostgroup_attributes == (void*)6, "field 5 set correctly"); + ok(s.incoming_mysql_servers_ssl_params == (void*)7, "field 6 set correctly"); + ok(s.incoming_aws_rds_bgd_hostgroups == (void*)8, "field 7 (BGD) set correctly"); + ok(s.runtime_mysql_servers == (void*)9, "field 8 set correctly"); +} + +// CLUSTER_QUERY_MYSQL_AWS_RDS_BGD is defined in ProxySQL_Cluster.hpp which +// has the same include dependency issue. Define it locally instead. +#define CLUSTER_QUERY_MYSQL_AWS_RDS_BGD \ + "PROXY_SELECT writer_hostgroup, reader_hostgroup, green_writer_hostgroup, " \ + "green_reader_hostgroup, active, writer_is_also_reader, check_interval_ms, " \ + "check_timeout_ms, comment, auto_generated, status " \ + "FROM runtime_mysql_aws_rds_bgd_hostgroups " \ + "WHERE auto_generated=0 ORDER BY writer_hostgroup" + +static void test_cluster_query_rds_bgd() { + const char* query = CLUSTER_QUERY_MYSQL_AWS_RDS_BGD; + ok(strncmp(query, "PROXY_SELECT", 12) == 0, + "CLUSTER_QUERY_MYSQL_AWS_RDS_BGD starts with PROXY_SELECT"); + ok(strstr(query, "auto_generated=0") != nullptr, + "CLUSTER_QUERY_MYSQL_AWS_RDS_BGD filters auto_generated=0"); + ok(strstr(query, "green_writer_hostgroup") != nullptr, + "CLUSTER_QUERY_MYSQL_AWS_RDS_BGD includes green_writer_hostgroup"); + ok(strstr(query, "green_reader_hostgroup") != nullptr, + "CLUSTER_QUERY_MYSQL_AWS_RDS_BGD includes green_reader_hostgroup"); + ok(strstr(query, "runtime_mysql_aws_rds_bgd_hostgroups") != nullptr, + "CLUSTER_QUERY_MYSQL_AWS_RDS_BGD queries runtime_mysql_aws_rds_bgd_hostgroups"); + ok(strstr(query, "ORDER BY writer_hostgroup") != nullptr, + "CLUSTER_QUERY_MYSQL_AWS_RDS_BGD has ORDER BY writer_hostgroup"); +} + +static void test_convert_size_check() { + std::vector v8(8, nullptr); + std::vector v9(9, nullptr); + std::vector v10(10, nullptr); + + size_t expected_struct_ptrs = sizeof(incoming_servers_t) / sizeof(void*); + + ok(v9.size() == expected_struct_ptrs, + "9-element vector matches sizeof(incoming_servers_t)/sizeof(void*) (%zu)", + expected_struct_ptrs); + ok(v8.size() != expected_struct_ptrs, + "8-element vector does NOT match (%zu vs %zu)", + v8.size(), expected_struct_ptrs); + ok(v10.size() != expected_struct_ptrs, + "10-element vector does NOT match (%zu vs %zu)", + v10.size(), expected_struct_ptrs); +} + +int main() { + plan(19); + + test_incoming_servers_t_size(); + test_incoming_servers_t_field_positions(); + test_cluster_query_rds_bgd(); + test_convert_size_check(); + + return exit_status(); +} From 1d8be511cdba23839bfbc95db0eda7530e52299b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Wed, 29 Jul 2026 08:19:53 +0200 Subject: [PATCH 77/81] Fix groups.json sorting for cluster_sync_unit-t --- test/tap/groups/groups.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index c8432a8bb0..3c00cadabc 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -18,11 +18,11 @@ "charset_find_unit-t" : [ "unit-tests-g1" ], "charset_unsigned_int-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql90-g1","mysql95-g1" ], "clickhouse_php_conn-t" : [ "legacy-clickhouse-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1" ], + "cluster_sync_unit-t" : [ "unit-tests-g1" ], "config_validation_unit-t" : [ "unit-tests-g1" ], "config_write_unit-t" : [ "unit-tests-g1" ], "connection_pool_unit-t" : [ "unit-tests-g1" ], "connection_unhealthy_unit-t" : [ "unit-tests-g1" ], - "cluster_sync_unit-t" : [ "unit-tests-g1" ], "deprecate_eof_cache-t" : [ "legacy-g4","mariadb10-galera-g4","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g4","mysql84-gr-g4","mysql90-g4","mysql95-g4" ], "envvars-t" : [ "legacy-g1","mariadb10-galera-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql84-gr-g1","mysql90-g1","mysql90-gr-g1","mysql93-g1","mysql93-gr-g1","mysql95-g1","mysql95-gr-g1" ], "eof_cache_mixed_flags-t" : [ "legacy-g4","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g4","mysql90-g4","mysql95-g4" ], From 74ecfcb294169b62e200e9063f956932fac23caa Mon Sep 17 00:00:00 2001 From: Rene Cannao Date: Wed, 29 Jul 2026 07:15:51 +0000 Subject: [PATCH 78/81] Fix MySQL integer variable initialization --- lib/MySQL_Thread.cpp | 2 +- test/tap/groups/groups.json | 2 ++ test/tap/tests/unit/Makefile | 1 + .../tap/tests/unit/mysql_variables_unit-t.cpp | 27 +++++++++++++++++++ 4 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 test/tap/tests/unit/mysql_variables_unit-t.cpp diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index c068006b4f..fd07a5e00c 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -2650,7 +2650,6 @@ char ** MySQL_Threads_Handler::get_variables_list() { VariablesPointers_bool["log_mysql_warnings_enabled"] = make_tuple(&variables.log_mysql_warnings_enabled, false); VariablesPointers_bool["log_unhealthy_connections"] = make_tuple(&variables.log_unhealthy_connections, false); VariablesPointers_bool["monitor_enabled"] = make_tuple(&variables.monitor_enabled, false); - VariablesPointers_int["aws_blue_green_deployment_auto_discovery"] = make_tuple(&variables.aws_blue_green_deployment_auto_discovery, 0, 1, false); VariablesPointers_bool["monitor_replication_lag_group_by_host"] = make_tuple(&variables.monitor_replication_lag_group_by_host, false); VariablesPointers_bool["monitor_wait_timeout"] = make_tuple(&variables.monitor_wait_timeout, false); VariablesPointers_bool["monitor_writer_is_also_reader"] = make_tuple(&variables.monitor_writer_is_also_reader, false); @@ -2693,6 +2692,7 @@ char ** MySQL_Threads_Handler::get_variables_list() { // it is safe to do it here because get_variables_list() is the first function called during start time if (VariablesPointers_int.size() == 0) { // Monitor variables + VariablesPointers_int["aws_blue_green_deployment_auto_discovery"] = make_tuple(&variables.aws_blue_green_deployment_auto_discovery, 0, 1, false); VariablesPointers_int["monitor_history"] = make_tuple(&variables.monitor_history, 1000, 7*24*3600*1000, false); VariablesPointers_int["monitor_connect_interval"] = make_tuple(&variables.monitor_connect_interval, 100, 7*24*3600*1000, false); diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index 3c00cadabc..028563807c 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -107,6 +107,7 @@ "mysql-watchdog_test-t" : [ "legacy-g4","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g4","mysql90-g4","mysql95-g4" ], "mysql-zstd_compression_level-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql90-g1","mysql95-g1" ], "mysql-zstd_compression_level_libmysql-t" : [ "mysql84-g1","mysql90-g1","mysql95-g1" ], + "mysql_decompress_payload_unit-t" : [ "unit-tests-g1" ], "mysql_encode_unit-t" : [ "unit-tests-g1" ], "mysql_error_classifier_unit-t" : [ "unit-tests-g1" ], "mysql_hostgroup_attributes-servers_defaults-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql90-g1","mysql95-g1" ], @@ -117,6 +118,7 @@ "mysql_resolution_unit-t" : [ "unit-tests-g1" ], "mysql_stmt_send_long_data-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql90-g1","mysql95-g1" ], "mysql_stmt_send_long_data_large-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql90-g1","mysql95-g1" ], + "mysql_variables_unit-t" : [ "unit-tests-g1" ], "mysqlx_admin_commands_unit-t" : [ "unit-tests-g1","@proxysql_min_version:4.0" ], "mysqlx_admin_disk_commands_unit-t" : [ "mysqlx-tsan-g1","unit-tests-g1","@proxysql_min_version:4.0" ], "mysqlx_admin_schema_unit-t" : [ "mysqlx-tsan-g1","unit-tests-g1","@proxysql_min_version:4.0" ], diff --git a/test/tap/tests/unit/Makefile b/test/tap/tests/unit/Makefile index b96734651e..a053f80ffd 100644 --- a/test/tap/tests/unit/Makefile +++ b/test/tap/tests/unit/Makefile @@ -397,6 +397,7 @@ UNIT_TESTS := smoke_test-t query_cache_unit-t query_processor_unit-t \ mysql_error_classifier_unit-t \ backend_sync_unit-t \ mysql_encode_unit-t \ + mysql_variables_unit-t \ mysql_resolution_unit-t \ pgsql_variables_validator_unit-t \ proxysql_utils_unit-t \ diff --git a/test/tap/tests/unit/mysql_variables_unit-t.cpp b/test/tap/tests/unit/mysql_variables_unit-t.cpp new file mode 100644 index 0000000000..9475b7716d --- /dev/null +++ b/test/tap/tests/unit/mysql_variables_unit-t.cpp @@ -0,0 +1,27 @@ +#include "tap.h" +#include "test_globals.h" + +#include "MySQL_Thread.h" + +static void test_mysql_integer_variables_are_registered() { + test_globals_init(); + MySQL_Threads_Handler handler; + char **variables = handler.get_variables_list(); + + ok(handler.get_variable_int("aws_blue_green_deployment_auto_discovery") == 1, + "aws_blue_green_deployment_auto_discovery is registered as an integer variable"); + ok(handler.get_variable_int("session_track_variables") == 0, + "session_track_variables is registered as an integer variable"); + + if (variables) { + for (char **p = variables; *p != nullptr; ++p) free(*p); + free(variables); + } + test_globals_cleanup(); +} + +int main() { + plan(2); + test_mysql_integer_variables_are_registered(); + return exit_status(); +} From 67d4a20606f0c63c6637874d62069d0496a34f3a Mon Sep 17 00:00:00 2001 From: Rene Cannao Date: Wed, 29 Jul 2026 08:01:16 +0000 Subject: [PATCH 79/81] Fix duplicate groups entry --- test/tap/groups/groups.json | 1 - 1 file changed, 1 deletion(-) diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index 96c114bcc2..028563807c 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -109,7 +109,6 @@ "mysql-zstd_compression_level_libmysql-t" : [ "mysql84-g1","mysql90-g1","mysql95-g1" ], "mysql_decompress_payload_unit-t" : [ "unit-tests-g1" ], "mysql_encode_unit-t" : [ "unit-tests-g1" ], - "mysql_decompress_payload_unit-t" : [ "unit-tests-g1" ], "mysql_error_classifier_unit-t" : [ "unit-tests-g1" ], "mysql_hostgroup_attributes-servers_defaults-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql90-g1","mysql95-g1" ], "mysql_hostgroup_attributes_config_file-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql90-g1","mysql95-g1" ], From c59fd8acdfa207b1c16a0cf6bf0e9be88b4a5e49 Mon Sep 17 00:00:00 2001 From: Rene Cannao Date: Wed, 29 Jul 2026 10:50:39 +0000 Subject: [PATCH 80/81] Fix RDS BGD simulator workflows --- lib/MySQL_Thread.cpp | 12 ++++- .../galera_tests_payloads/test_template.json | 16 +++--- test/tap/tap/rds_bgd_tap.h | 51 +++++++++++++++++++ .../test_rds_bgd_automatic_discovery-t.cpp | 4 +- ...st_rds_bgd_configuration_persistence-t.cpp | 8 +-- test/tap/tests/test_rds_bgd_probe_tls-t.cpp | 4 +- .../tap/tests/unit/mysql_variables_unit-t.cpp | 31 +++++++++-- 7 files changed, 109 insertions(+), 17 deletions(-) diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index fd07a5e00c..801cfeac3a 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -2067,7 +2067,17 @@ bool MySQL_Threads_Handler::set_variable(char *name, const char *value) { // thi } bool special_variable = std::get<3>(it->second); // if special_variable is true, min and max values are ignored, and more input validation is needed if (special_variable == false) { - int intv=atoi(value); + // This option is stored as an integer for compatibility with the + // existing variable interface, but is documented and commonly set + // using the same true/false spelling as boolean variables. + int intv; + if (nameS == "aws_blue_green_deployment_auto_discovery" && strcasecmp(value, "true") == 0) { + intv = 1; + } else if (nameS == "aws_blue_green_deployment_auto_discovery" && strcasecmp(value, "false") == 0) { + intv = 0; + } else { + intv = atoi(value); + } if (intv >= std::get<1>(it->second) && intv <= std::get<2>(it->second)) { int * v = std::get<0>(it->second); *v = intv; diff --git a/test/deps/cluster_simulator/tests/galera_tests_payloads/test_template.json b/test/deps/cluster_simulator/tests/galera_tests_payloads/test_template.json index 465eda711c..fb5a01a98e 100644 --- a/test/deps/cluster_simulator/tests/galera_tests_payloads/test_template.json +++ b/test/deps/cluster_simulator/tests/galera_tests_payloads/test_template.json @@ -90,14 +90,14 @@ } ], "proxysql_init_state": [ - { "hostgroup_id":2271,"hostname":"127.1.1.11","port":3306,"status":"SHUNNED" }, - { "hostgroup_id":2271,"hostname":"127.1.1.12","port":3306,"status":"SHUNNED" }, - { "hostgroup_id":2271,"hostname":"127.1.1.13","port":3306,"status":"ONLINE" }, - { "hostgroup_id":2272,"hostname":"127.1.1.11","port":3306,"status":"ONLINE" }, - { "hostgroup_id":2272,"hostname":"127.1.1.12","port":3306,"status":"ONLINE" }, - { "hostgroup_id":2273,"hostname":"127.1.1.11","port":3306,"status":"ONLINE" }, - { "hostgroup_id":2273,"hostname":"127.1.1.12","port":3306,"status":"ONLINE" }, - { "hostgroup_id":2273,"hostname":"127.1.1.13","port":3306,"status":"ONLINE" } + { "hostgroup_id":2271,"hostname":"127.1.1.11","port":3306,"status":"SHUNNED","comment":"node_127.1.1.11" }, + { "hostgroup_id":2271,"hostname":"127.1.1.12","port":3306,"status":"SHUNNED","comment":"node_127.1.1.12" }, + { "hostgroup_id":2271,"hostname":"127.1.1.13","port":3306,"status":"ONLINE","comment":"node_127.1.1.13" }, + { "hostgroup_id":2272,"hostname":"127.1.1.11","port":3306,"status":"ONLINE","comment":"node_127.1.1.11" }, + { "hostgroup_id":2272,"hostname":"127.1.1.12","port":3306,"status":"ONLINE","comment":"node_127.1.1.12" }, + { "hostgroup_id":2273,"hostname":"127.1.1.11","port":3306,"status":"ONLINE","comment":"node_127.1.1.11" }, + { "hostgroup_id":2273,"hostname":"127.1.1.12","port":3306,"status":"ONLINE","comment":"node_127.1.1.12" }, + { "hostgroup_id":2273,"hostname":"127.1.1.13","port":3306,"status":"ONLINE","comment":"node_127.1.1.13" } ], "proxysql_final_state": [ { "hostgroup_id":2271,"hostname":"127.1.1.11","port":3306,"status":"SHUNNED","weight":1 }, diff --git a/test/tap/tap/rds_bgd_tap.h b/test/tap/tap/rds_bgd_tap.h index 8dd80847bf..619450c7af 100644 --- a/test/tap/tap/rds_bgd_tap.h +++ b/test/tap/tap/rds_bgd_tap.h @@ -333,8 +333,59 @@ inline rc_t bgd_probe_count_since( return result; } +inline void bgd_diag_runtime_state(MYSQL* admin) { + const string runtime_query = + "SELECT writer_hostgroup,reader_hostgroup,IFNULL(green_writer_hostgroup,'NULL')," + "IFNULL(green_reader_hostgroup,'NULL'),auto_generated,status " + "FROM runtime_mysql_aws_rds_bgd_hostgroups ORDER BY writer_hostgroup"; + auto [runtime_rc, runtime_rows] = mysql_query_ext_rows(admin, runtime_query); + if (runtime_rc != EXIT_SUCCESS) { + diag("RDS BGD diagnostic query failed with error %d", runtime_rc); + } else if (runtime_rows.empty()) { + diag("RDS BGD runtime hostgroup table is empty"); + } else { + diag("RDS BGD runtime hostgroup rows:"); + for (const mysql_res_row& row : runtime_rows) { + string row_text {}; + for (size_t i = 0; i < row.size(); ++i) { + if (i != 0) { + row_text += ","; + } + row_text += row[i]; + } + diag(" %s", row_text.c_str()); + } + } + + const string monitor_query = + "SELECT hostname,port,IFNULL(read_only,'NULL'),IFNULL(error,'') FROM mysql_server_read_only_log " + "ORDER BY time_start_us DESC LIMIT 10"; + auto [monitor_rc, monitor_rows] = mysql_query_ext_rows(admin, monitor_query); + if (monitor_rc != EXIT_SUCCESS) { + diag("RDS read_only diagnostic query failed with error %d", monitor_rc); + } else if (monitor_rows.empty()) { + diag("RDS read_only log has no rows"); + } else { + diag("Latest RDS read_only monitor rows:"); + for (const mysql_res_row& row : monitor_rows) { + string row_text {}; + for (size_t i = 0; i < row.size(); ++i) { + if (i != 0) { + row_text += ","; + } + row_text += row[i]; + } + diag(" %s", row_text.c_str()); + } + } +} + inline int bgd_wait_for_condition(MYSQL* admin, string query, uint32_t timeout_seconds) { int rc = wait_for_cond(admin, query, timeout_seconds); + if (rc != EXIT_SUCCESS) { + diag("RDS BGD wait timed out or failed for condition: %s", query.c_str()); + bgd_diag_runtime_state(admin); + } return rc; } diff --git a/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp b/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp index 30c30c3be2..f7ac46e125 100644 --- a/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp +++ b/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp @@ -21,7 +21,9 @@ #include "rds_bgd_tap.h" #include "utils.h" -const uint32_t kTimeoutSeconds = 3; +// Automatic discovery is asynchronous and starts after the monitor observes the +// runtime server. Allow the monitor and the BGD worker to become ready on slower CI runners. +const uint32_t kTimeoutSeconds = 15; const uint32_t kProbeTimeoutMs = 3000; struct TestState { diff --git a/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp b/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp index 7638e03c0a..2d01d467b4 100644 --- a/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp +++ b/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp @@ -21,7 +21,9 @@ #include "rds_bgd_tap.h" #include "utils.h" -const uint32_t kTimeoutSeconds = 3; +// Automatic discovery is asynchronous and starts after the monitor observes the +// runtime server. Allow the monitor and the BGD worker to become ready on slower CI runners. +const uint32_t kTimeoutSeconds = 15; const uint32_t kProbeTimeoutMs = 3000; struct TestState { @@ -99,7 +101,7 @@ int configure_monitor(MYSQL* admin, BGD_Hostgroups& hg, bool automatic) { return rc; } -int insert_explicit_bgd_row(MYSQL* admin, BGD_Hostgroups& hg, string comment, int active = 1) { +int insert_explicit_bgd_row(MYSQL* admin, BGD_Hostgroups& hg, const string& comment, int active = 1) { string query = "INSERT INTO mysql_aws_rds_bgd_hostgroups(" "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," @@ -207,7 +209,7 @@ rc_t> runtime_bgd_ownership_snapshot(MYSQL* admin, int wri return result; } -rc_t> green_server_snapshot(MYSQL* admin, string table, BGD_Hostgroups& hg) { +rc_t> green_server_snapshot(MYSQL* admin, const string& table, BGD_Hostgroups& hg) { string query = "SELECT hostgroup_id,hostname,port,status,use_ssl,weight,max_connections FROM " + table + " WHERE hostgroup_id IN (" + to_string(hg.green_writer) + "," + to_string(hg.green_reader) + diff --git a/test/tap/tests/test_rds_bgd_probe_tls-t.cpp b/test/tap/tests/test_rds_bgd_probe_tls-t.cpp index 4751272c62..f6c616306b 100644 --- a/test/tap/tests/test_rds_bgd_probe_tls-t.cpp +++ b/test/tap/tests/test_rds_bgd_probe_tls-t.cpp @@ -22,7 +22,9 @@ #include "rds_bgd_tap.h" #include "utils.h" -const uint32_t kTimeoutSeconds = 3; +// Automatic discovery is asynchronous and starts after the monitor observes the +// runtime server. Allow the monitor and the BGD worker to become ready on slower CI runners. +const uint32_t kTimeoutSeconds = 15; const uint32_t kProbeTimeoutMs = 3000; const uint32_t kNegativeProbeTimeoutMs = 1200; diff --git a/test/tap/tests/unit/mysql_variables_unit-t.cpp b/test/tap/tests/unit/mysql_variables_unit-t.cpp index 9475b7716d..fcf42acfcb 100644 --- a/test/tap/tests/unit/mysql_variables_unit-t.cpp +++ b/test/tap/tests/unit/mysql_variables_unit-t.cpp @@ -14,14 +14,39 @@ static void test_mysql_integer_variables_are_registered() { "session_track_variables is registered as an integer variable"); if (variables) { - for (char **p = variables; *p != nullptr; ++p) free(*p); - free(variables); + for (char **p = variables; *p != nullptr; ++p) { + free(*p); + } + free(reinterpret_cast(variables)); + } + test_globals_cleanup(); +} + +static void test_mysql_integer_boolean_aliases() { + test_globals_init(); + MySQL_Threads_Handler handler; + char **variables = handler.get_variables_list(); + char variable_name[] = "aws_blue_green_deployment_auto_discovery"; + + ok(handler.set_variable(variable_name, "true") && + handler.get_variable_int(variable_name) == 1, + "aws_blue_green_deployment_auto_discovery accepts true"); + ok(handler.set_variable(variable_name, "false") && + handler.get_variable_int(variable_name) == 0, + "aws_blue_green_deployment_auto_discovery accepts false"); + + if (variables) { + for (char **p = variables; *p != nullptr; ++p) { + free(*p); + } + free(reinterpret_cast(variables)); } test_globals_cleanup(); } int main() { - plan(2); + plan(4); test_mysql_integer_variables_are_registered(); + test_mysql_integer_boolean_aliases(); return exit_status(); } From a899fd9ba36440f3fcd0631b67a3fafbcf1e29af Mon Sep 17 00:00:00 2001 From: Rene Cannao Date: Wed, 29 Jul 2026 11:36:05 +0000 Subject: [PATCH 81/81] Fix Galera simulator expected state comments --- .../galera_tests_payloads/test_template.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/deps/cluster_simulator/tests/galera_tests_payloads/test_template.json b/test/deps/cluster_simulator/tests/galera_tests_payloads/test_template.json index fb5a01a98e..465eda711c 100644 --- a/test/deps/cluster_simulator/tests/galera_tests_payloads/test_template.json +++ b/test/deps/cluster_simulator/tests/galera_tests_payloads/test_template.json @@ -90,14 +90,14 @@ } ], "proxysql_init_state": [ - { "hostgroup_id":2271,"hostname":"127.1.1.11","port":3306,"status":"SHUNNED","comment":"node_127.1.1.11" }, - { "hostgroup_id":2271,"hostname":"127.1.1.12","port":3306,"status":"SHUNNED","comment":"node_127.1.1.12" }, - { "hostgroup_id":2271,"hostname":"127.1.1.13","port":3306,"status":"ONLINE","comment":"node_127.1.1.13" }, - { "hostgroup_id":2272,"hostname":"127.1.1.11","port":3306,"status":"ONLINE","comment":"node_127.1.1.11" }, - { "hostgroup_id":2272,"hostname":"127.1.1.12","port":3306,"status":"ONLINE","comment":"node_127.1.1.12" }, - { "hostgroup_id":2273,"hostname":"127.1.1.11","port":3306,"status":"ONLINE","comment":"node_127.1.1.11" }, - { "hostgroup_id":2273,"hostname":"127.1.1.12","port":3306,"status":"ONLINE","comment":"node_127.1.1.12" }, - { "hostgroup_id":2273,"hostname":"127.1.1.13","port":3306,"status":"ONLINE","comment":"node_127.1.1.13" } + { "hostgroup_id":2271,"hostname":"127.1.1.11","port":3306,"status":"SHUNNED" }, + { "hostgroup_id":2271,"hostname":"127.1.1.12","port":3306,"status":"SHUNNED" }, + { "hostgroup_id":2271,"hostname":"127.1.1.13","port":3306,"status":"ONLINE" }, + { "hostgroup_id":2272,"hostname":"127.1.1.11","port":3306,"status":"ONLINE" }, + { "hostgroup_id":2272,"hostname":"127.1.1.12","port":3306,"status":"ONLINE" }, + { "hostgroup_id":2273,"hostname":"127.1.1.11","port":3306,"status":"ONLINE" }, + { "hostgroup_id":2273,"hostname":"127.1.1.12","port":3306,"status":"ONLINE" }, + { "hostgroup_id":2273,"hostname":"127.1.1.13","port":3306,"status":"ONLINE" } ], "proxysql_final_state": [ { "hostgroup_id":2271,"hostname":"127.1.1.11","port":3306,"status":"SHUNNED","weight":1 },